From e939def66e7a64ada5237e46827242abb497ccb9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 25 Sep 2024 08:53:48 -0500 Subject: [PATCH 01/56] feat: gh-app poc --- api/build/plan.go | 2 +- api/step/plan.go | 18 +++++-- api/step/update.go | 13 +++++ api/types/repo.go | 27 +++++++++++ cmd/vela-server/scm.go | 2 + database/repo/table.go | 2 + database/step/table.go | 2 + go.mod | 5 ++ go.sum | 8 +++- scm/flags.go | 12 +++++ scm/github/github.go | 92 +++++++++++++++++++++++++++++++++-- scm/github/opts.go | 24 ++++++++++ scm/github/repo.go | 106 +++++++++++++++++++++++++++++++++++++++++ scm/service.go | 4 ++ scm/setup.go | 5 ++ 15 files changed, 311 insertions(+), 11 deletions(-) diff --git a/api/build/plan.go b/api/build/plan.go index fcdcd2dff..e8ed0b643 100644 --- a/api/build/plan.go +++ b/api/build/plan.go @@ -57,7 +57,7 @@ func PlanBuild(ctx context.Context, database database.Interface, scm scm.Service } // plan all steps for the build - steps, err := step.PlanSteps(ctx, database, scm, p, b) + steps, err := step.PlanSteps(ctx, database, scm, p, b, r) if err != nil { // clean up the objects from the pipeline in the database CleanBuild(ctx, database, b, services, steps, err) diff --git a/api/step/plan.go b/api/step/plan.go index cfb55652e..bfacc5136 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -20,7 +20,7 @@ import ( // PlanSteps is a helper function to plan all steps // in the build for execution. This creates the steps // for the build. -func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build) ([]*library.Step, error) { +func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build, r *types.Repo) ([]*library.Step, error) { // variable to store planned steps steps := []*library.Step{} @@ -29,7 +29,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service // iterate through all steps for each pipeline stage for _, step := range stage.Steps { // create the step object - s, err := planStep(ctx, database, scm, b, step, stage.Name) + s, err := planStep(ctx, database, scm, b, r, step, stage.Name) if err != nil { return steps, err } @@ -40,7 +40,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service // iterate through all pipeline steps for _, step := range p.Steps { - s, err := planStep(ctx, database, scm, b, step, "") + s, err := planStep(ctx, database, scm, b, r, step, "") if err != nil { return steps, err } @@ -51,7 +51,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service return steps, nil } -func planStep(ctx context.Context, database database.Interface, scm scm.Service, b *types.Build, c *pipeline.Container, stage string) (*library.Step, error) { +func planStep(ctx context.Context, database database.Interface, scm scm.Service, b *types.Build, r *types.Repo, c *pipeline.Container, stage string) (*library.Step, error) { // create the step object s := new(library.Step) s.SetBuildID(b.GetID()) @@ -64,6 +64,16 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetReportAs(c.ReportAs) s.SetCreated(time.Now().UTC().Unix()) + if c.ReportStatus { + id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName(), b.GetEvent()) + if err != nil { + // TODO: make this error more meaningful + return nil, err + } + + s.SetCheckID(id) + } + // send API call to create the step s, err := database.CreateStep(ctx, s) if err != nil { diff --git a/api/step/update.go b/api/step/update.go index c543de2c8..7d3ebcddd 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -154,6 +154,19 @@ func UpdateStep(c *gin.Context) { return } + if s.GetCheckID() != 0 { + s.SetReport(input.GetReport()) + + err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit(), b.GetEvent()) + if err != nil { + retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + c.JSON(http.StatusOK, s) // check if the build is in a "final" state diff --git a/api/types/repo.go b/api/types/repo.go index bfad93623..01e9f03ea 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -32,6 +32,7 @@ type Repo struct { PipelineType *string `json:"pipeline_type,omitempty"` PreviousName *string `json:"previous_name,omitempty"` ApproveBuild *string `json:"approve_build,omitempty"` + InstallID *int64 `json:"install_id,omitempty"` } // Environment returns a list of environment variables @@ -345,6 +346,19 @@ func (r *Repo) GetApproveBuild() string { return *r.ApproveBuild } +// GetInstallID returns the InstallID field. +// +// When the provided Repo type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Repo) GetInstallID() int64 { + // return zero value if Repo type or InstallID field is nil + if r == nil || r.InstallID == nil { + return 0 + } + + return *r.InstallID +} + // SetID sets the ID field. // // When the provided Repo type is nil, it @@ -618,6 +632,19 @@ func (r *Repo) SetApproveBuild(v string) { r.ApproveBuild = &v } +// SetInstallID sets the InstallID field. +// +// When the provided Repo type is nil, it +// will set nothing and immediately return. +func (r *Repo) SetInstallID(v int64) { + // return if Repo type is nil + if r == nil { + return + } + + r.InstallID = &v +} + // String implements the Stringer interface for the Repo type. func (r *Repo) String() string { return fmt.Sprintf(`{ diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index 7124761f0..9edc5fcd6 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -26,6 +26,8 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) { WebUIAddress: c.String("webui-addr"), Scopes: c.StringSlice("scm.scopes"), Tracing: tc, + GithubAppID: c.Int64("scm.app.id"), + GithubAppPrivateKey: c.String("scm.app.private_key"), } // setup the scm diff --git a/database/repo/table.go b/database/repo/table.go index 0b16bae94..0db3c61c1 100644 --- a/database/repo/table.go +++ b/database/repo/table.go @@ -35,6 +35,7 @@ repos ( pipeline_type TEXT, previous_name VARCHAR(100), approve_build VARCHAR(20), + install_id INTEGER, UNIQUE(full_name) ); ` @@ -65,6 +66,7 @@ repos ( pipeline_type TEXT, previous_name TEXT, approve_build TEXT, + install_id INTEGER, UNIQUE(full_name) ); ` diff --git a/database/step/table.go b/database/step/table.go index c7560326d..b76733896 100644 --- a/database/step/table.go +++ b/database/step/table.go @@ -30,6 +30,7 @@ steps ( host VARCHAR(250), runtime VARCHAR(250), distribution VARCHAR(250), + check_id INTEGER, report_as VARCHAR(250), UNIQUE(build_id, number) ); @@ -56,6 +57,7 @@ steps ( host TEXT, runtime TEXT, distribution TEXT, + check_id INTEGER, report_as TEXT, UNIQUE(build_id, number) ); diff --git a/go.mod b/go.mod index 53fc2e4b1..dc23e3cf3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/go-vela/server go 1.23.1 +replace github.com/go-vela/types => ../types + require ( github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb github.com/DATA-DOG/go-sqlmock v1.5.2 @@ -10,6 +12,7 @@ require ( github.com/adhocore/gronx v1.19.0 github.com/alicebob/miniredis/v2 v2.33.0 github.com/aws/aws-sdk-go v1.55.5 + github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 github.com/distribution/reference v0.6.0 github.com/drone/envsubst v1.0.3 github.com/ghodss/yaml v1.0.0 @@ -18,6 +21,7 @@ require ( github.com/go-vela/types v0.25.0-rc1 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/go-cmp v0.6.0 + github.com/google/go-github/v62 v62.0.0 github.com/google/go-github/v65 v65.0.0 github.com/google/uuid v1.6.0 github.com/goware/urlx v0.3.2 @@ -85,6 +89,7 @@ require ( github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/go.sum b/go.sum index 4f741e0d4..b0369b357 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53AL4ED5MzgEMwGTZag= +github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -100,12 +102,12 @@ github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27 github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-vela/types v0.25.0-rc1 h1:5pCV4pVt1bm6YYUdkNglRDa3PcFX3qGtf5rrmkUvdOc= -github.com/go-vela/types v0.25.0-rc1/go.mod h1:fLv2pbzIy6puAV6Cgh5ixUcchTUHT4D3xX05zIhkA9I= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -115,6 +117,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= +github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= github.com/google/go-github/v65 v65.0.0 h1:pQ7BmO3DZivvFk92geC0jB0q2m3gyn8vnYPgV7GSLhQ= github.com/google/go-github/v65 v65.0.0/go.mod h1:DvrqWo5hvsdhJvHd4WyVF9ttANN3BniqjP8uTFMNb60= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= diff --git a/scm/flags.go b/scm/flags.go index 2a64f7e14..fecb944ce 100644 --- a/scm/flags.go +++ b/scm/flags.go @@ -67,4 +67,16 @@ var Flags = []cli.Flag{ "is behind a Firewall or NAT, or when using something like ngrok to forward webhooks. " + "(defaults to VELA_ADDR).", }, + &cli.Int64Flag{ + EnvVars: []string{"VELA_SCM_APP_ID", "SCM_APP_ID"}, + FilePath: "/vela/scm/app_id", + Name: "scm.app.id", + Usage: "(optional & experimental) ID for the GitHub App", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY", "SCM_APP_PRIVATE_KEY"}, + FilePath: "/vela/scm/app_private_key", + Name: "scm.app.private_key", + Usage: "(optional & experimental) path to private key for the GitHub App", + }, } diff --git a/scm/github/github.go b/scm/github/github.go index dfbffe53a..af7f6f782 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -4,10 +4,18 @@ package github import ( "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" "fmt" + "net/http" "net/http/httptrace" "net/url" + "strings" + "github.com/bradleyfalzon/ghinstallation/v2" + api "github.com/go-vela/server/api/types" + "github.com/google/go-github/v62/github" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" @@ -49,13 +57,17 @@ type config struct { WebUIAddress string // specifies the OAuth scopes to use for the GitHub client Scopes []string + // optional and experimental + GithubAppID int64 + GithubAppPrivateKey string } type client struct { - config *config - OAuth *oauth2.Config - AuthReq *github.AuthorizationRequest - Tracing *tracing.Client + config *config + OAuth *oauth2.Config + AuthReq *github.AuthorizationRequest + Tracing *tracing.Client + AppsTransport *ghinstallation.AppsTransport // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -114,6 +126,30 @@ func New(opts ...ClientOpt) (*client, error) { Scopes: githubScopes, } + if c.config.GithubAppID != 0 && len(c.config.GithubAppPrivateKey) > 0 { + c.Logger.Infof("sourcing private key from path: %s", c.config.GithubAppPrivateKey) + + decodedPEM, err := base64.StdEncoding.DecodeString(c.config.GithubAppPrivateKey) + if err != nil { + return nil, fmt.Errorf("error decoding base64: %w", err) + } + + block, _ := pem.Decode(decodedPEM) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse RSA private key: %w", err) + } + + transport := ghinstallation.NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.GithubAppID, privateKey) + + transport.BaseURL = c.config.API + c.AppsTransport = transport + } + return c, nil } @@ -179,3 +215,51 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien return github } + +// helper function to return the GitHub App token. +func (c *client) newGithubAppToken(r *api.Repo) (*github.Client, error) { + // create a github client based off the existing GitHub App configuration + client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) + if err != nil { + return nil, err + } + + // if repo has an install ID, use it to create an installation token + if r.GetInstallID() != 0 { + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(context.Background(), r.GetInstallID(), &github.InstallationTokenOptions{}) + if err != nil { + panic(err) + } + + return c.newClientToken(t.GetToken()), nil + } + + // list all installations (a.k.a. orgs) where the GitHub App is installed + installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) + if err != nil { + return nil, err + } + + var id int64 + // iterate through the list of installations + for _, install := range installations { + // find the installation that matches the org for the repo + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + id = install.GetID() + } + } + + // failsafe in case the repo does not belong to an org where the GitHub App is installed + if id == 0 { + return nil, err + } + + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(context.Background(), id, &github.InstallationTokenOptions{}) + if err != nil { + panic(err) + } + + return c.newClientToken(t.GetToken()), nil +} diff --git a/scm/github/opts.go b/scm/github/opts.go index bb7385827..7856a681e 100644 --- a/scm/github/opts.go +++ b/scm/github/opts.go @@ -160,3 +160,27 @@ func WithTracing(tracing *tracing.Client) ClientOpt { return nil } } + +// WithGithubAppID sets the ID for the GitHub App in the scm client. +func WithGithubAppID(id int64) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring ID for GitHub App in github scm client") + + // set the ID for the GitHub App in the github client + c.config.GithubAppID = id + + return nil + } +} + +// WithGithubPrivateKey sets the private key for the GitHub App in the scm client. +func WithGithubPrivateKey(key string) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring private key for GitHub App in github scm client") + + // set the private key for the GitHub App in the github client + c.config.GithubAppPrivateKey = key + + return nil + } +} diff --git a/scm/github/repo.go b/scm/github/repo.go index f608b5199..c253aea0d 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -667,3 +667,109 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str return data.GetName(), data.GetCommit().GetSHA(), nil } + +// CreateChecks defines a function that does stuff... +func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { + // create client from GitHub App + client, err := c.newGithubAppToken(r) + if err != nil { + return 0, err + } + + opts := github.CreateCheckRunOptions{ + Name: fmt.Sprintf("vela-%s-%s", event, step), + HeadSHA: commit, + } + + check, _, err := client.Checks.CreateCheckRun(ctx, r.GetOrg(), r.GetName(), opts) + if err != nil { + return 0, err + } + + return check.GetID(), nil +} + +// UpdateChecks defines a function that does stuff... +func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { + // create client from GitHub App + client, err := c.newGithubAppToken(r) + if err != nil { + return err + } + + var ( + conclusion string + status string + ) + // set the conclusion and status for the step check depending on what the status of the step is + switch s.GetStatus() { + case constants.StatusPending: + conclusion = "neutral" + status = "queued" + case constants.StatusPendingApproval: + conclusion = "action_required" + status = "queued" + case constants.StatusRunning: + conclusion = "neutral" + status = "in_progress" + case constants.StatusSuccess: + conclusion = "success" + status = "completed" + case constants.StatusFailure: + conclusion = "failure" + status = "completed" + case constants.StatusCanceled: + conclusion = "cancelled" + status = "completed" + case constants.StatusKilled: + conclusion = "cancelled" + status = "completed" + case constants.StatusSkipped: + conclusion = "skipped" + status = "completed" + default: + conclusion = "neutral" + status = "completed" + } + + var annotations []*github.CheckRunAnnotation + + for _, reportAnnotation := range s.GetReport().GetAnnotations() { + annotation := &github.CheckRunAnnotation{ + Path: github.String(reportAnnotation.GetPath()), + StartLine: github.Int(reportAnnotation.GetStartLine()), + EndLine: github.Int(reportAnnotation.GetEndLine()), + StartColumn: github.Int(reportAnnotation.GetStartColumn()), + EndColumn: github.Int(reportAnnotation.GetEndColumn()), + AnnotationLevel: github.String(reportAnnotation.GetAnnotationLevel()), + Message: github.String(reportAnnotation.GetMessage()), + Title: github.String(reportAnnotation.GetTitle()), + RawDetails: github.String(reportAnnotation.GetRawDetails()), + } + + annotations = append(annotations, annotation) + } + + output := &github.CheckRunOutput{ + Title: github.String(s.GetReport().GetTitle()), + Summary: github.String(s.GetReport().GetSummary()), + Text: github.String(s.GetReport().GetText()), + AnnotationsCount: github.Int(s.GetReport().GetAnnotationsCount()), + AnnotationsURL: github.String(s.GetReport().GetAnnotationsURL()), + Annotations: annotations, + } + + opts := github.UpdateCheckRunOptions{ + Name: fmt.Sprintf("vela-%s-%s", event, s.GetName()), + Conclusion: github.String(conclusion), + Status: github.String(status), + Output: output, + } + + _, _, err = client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) + if err != nil { + return err + } + + return nil +} diff --git a/scm/service.go b/scm/service.go index 697bcf8ef..8472b9e85 100644 --- a/scm/service.go +++ b/scm/service.go @@ -142,6 +142,10 @@ type Service interface { // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) + // TODO: add comments + CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) + UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error + // Webhook SCM Interface Functions // ProcessWebhook defines a function that diff --git a/scm/setup.go b/scm/setup.go index c32a0cf66..98a672289 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -39,6 +39,9 @@ type Setup struct { Scopes []string // specifies OTel tracing configurations Tracing *tracing.Client + // specifies GitHub App installation configurations + GithubAppID int64 + GithubAppPrivateKey string } // Github creates and returns a Vela service capable of @@ -59,6 +62,8 @@ func (s *Setup) Github() (Service, error) { github.WithWebUIAddress(s.WebUIAddress), github.WithScopes(s.Scopes), github.WithTracing(s.Tracing), + github.WithGithubAppID(s.GithubAppID), + github.WithGithubPrivateKey(s.GithubAppPrivateKey), ) } From e032e0bd338c88378e2399a86b7758af8d12ddaa Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 4 Oct 2024 09:25:27 -0500 Subject: [PATCH 02/56] chore: merge with main --- scm/github/github.go | 7 +++---- scm/github/repo.go | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scm/github/github.go b/scm/github/github.go index af7f6f782..99c6c5425 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -15,7 +15,6 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" api "github.com/go-vela/server/api/types" - "github.com/google/go-github/v62/github" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" @@ -217,7 +216,7 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien } // helper function to return the GitHub App token. -func (c *client) newGithubAppToken(r *api.Repo) (*github.Client, error) { +func (c *client) newGithubAppToken(ctx context.Context, r *api.Repo) (*github.Client, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) if err != nil { @@ -232,7 +231,7 @@ func (c *client) newGithubAppToken(r *api.Repo) (*github.Client, error) { panic(err) } - return c.newClientToken(t.GetToken()), nil + return c.newClientToken(ctx, t.GetToken()), nil } // list all installations (a.k.a. orgs) where the GitHub App is installed @@ -261,5 +260,5 @@ func (c *client) newGithubAppToken(r *api.Repo) (*github.Client, error) { panic(err) } - return c.newClientToken(t.GetToken()), nil + return c.newClientToken(ctx, t.GetToken()), nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index c253aea0d..0b03cf03c 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -671,7 +671,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App - client, err := c.newGithubAppToken(r) + client, err := c.newGithubAppToken(ctx, r) if err != nil { return 0, err } @@ -692,7 +692,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { // create client from GitHub App - client, err := c.newGithubAppToken(r) + client, err := c.newGithubAppToken(ctx, r) if err != nil { return err } From cacaaf9de51b9989cdbf9ce4a3da82b7ebbcfcd4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 4 Oct 2024 14:39:23 -0500 Subject: [PATCH 03/56] wip: debug files, todos, random code --- .vscode/launch.json | 12 ++++++++++++ Dockerfile | 11 +++++++++-- api/repo/repair.go | 6 ++++++ api/step/plan.go | 12 +++++++----- api/types/build.go | 37 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 2 ++ go.mod | 2 +- go.sum | 2 -- scm/github/github.go | 4 ++++ scm/github/repo.go | 23 +++++++++++++++++++++++ 10 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..ecd3df700 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "port": 4000, + "host": "127.0.0.1", + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1bc26a794..071b8e451 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367eff RUN apk add --update --no-cache ca-certificates -FROM scratch +# FROM scratch +FROM golang COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt @@ -14,4 +15,10 @@ ENV GODEBUG=netdns=go ADD release/vela-server /bin/ -CMD ["/bin/vela-server"] +# CMD ["/bin/vela-server"] + +# dlv wrapper + +EXPOSE 4000 +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest +CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] diff --git a/api/repo/repair.go b/api/repo/repair.go index 2fd99f9e2..027c00a8e 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -72,6 +72,12 @@ func RepairRepo(c *gin.Context) { l.Debugf("repairing repo %s", r.GetFullName()) + // todo: get org app installation + // doesnt exist? redirect them and wait... + + // todo: from org installation, check if this repo is visible/enabled + // no? use scm api to add the repo to the org + // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to remove the webhook diff --git a/api/step/plan.go b/api/step/plan.go index bfacc5136..46ea3fff1 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -64,14 +64,16 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetReportAs(c.ReportAs) s.SetCreated(time.Now().UTC().Unix()) - if c.ReportStatus { + if len(c.ReportAs) > 0 { + // todo: is this okay if checks already exist? id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName(), b.GetEvent()) if err != nil { - // TODO: make this error more meaningful - return nil, err + // todo: need better error-handling + // in a perfect world we warn the user that they need to install the github app to get this to work + logrus.Warnf("unable to create checks for step: %v", err) + } else { + s.SetCheckID(id) } - - s.SetCheckID(id) } // send API call to create the step diff --git a/api/types/build.go b/api/types/build.go index 10ccc31e2..a7f770125 100644 --- a/api/types/build.go +++ b/api/types/build.go @@ -8,6 +8,8 @@ import ( "time" "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" ) @@ -1205,3 +1207,38 @@ func (b *Build) String() string { b.GetTitle(), ) } + +// StepFromBuildContainer creates a new Step based on a Build and pipeline Container. +func StepFromBuildContainer(build *Build, ctn *pipeline.Container) *library.Step { + // create new step type we want to return + s := new(library.Step) + + // default status to Pending + s.SetStatus(constants.StatusPending) + + // copy fields from build + if build != nil { + // set values from the build + s.SetHost(build.GetHost()) + s.SetRuntime(build.GetRuntime()) + s.SetDistribution(build.GetDistribution()) + } + + // copy fields from container + if ctn != nil && ctn.Name != "" { + // set values from the container + s.SetName(ctn.Name) + s.SetNumber(ctn.Number) + s.SetImage(ctn.Image) + s.SetReportAs(ctn.ReportAs) + + // check if the VELA_STEP_STAGE environment variable exists + value, ok := ctn.Environment["VELA_STEP_STAGE"] + if ok { + // set the Stage field to the value from environment variable + s.SetStage(value) + } + } + + return s +} diff --git a/docker-compose.yml b/docker-compose.yml index a96b4b87f..28e235860 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,8 @@ services: restart: always ports: - '8080:8080' + # dlv + - '4000:4000' depends_on: postgres: condition: service_healthy diff --git a/go.mod b/go.mod index e25496458..04f9ae923 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/go-vela/types v0.25.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/go-cmp v0.6.0 - github.com/google/go-github/v62 v62.0.0 github.com/google/go-github/v65 v65.0.0 github.com/google/uuid v1.6.0 github.com/goware/urlx v0.3.2 @@ -91,6 +90,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect + github.com/google/go-github/v62 v62.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/css v1.0.1 // indirect diff --git a/go.sum b/go.sum index d1ecc4746..d0b72b0ae 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,6 @@ github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27 github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-vela/types v0.25.0 h1:5jSXgW8uf2ODbhOiWdVmKtbznF/CfNIzkZSYuNQIars= -github.com/go-vela/types v0.25.0/go.mod h1:gyKVRQjNosAJy4AJ164CnEF6jIkwd1y6Cm5pZ6M20ZM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/scm/github/github.go b/scm/github/github.go index 99c6c5425..441b39705 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -126,6 +126,7 @@ func New(opts ...ClientOpt) (*client, error) { } if c.config.GithubAppID != 0 && len(c.config.GithubAppPrivateKey) > 0 { + // todo: this log isnt accurate, it reads it directly as a string c.Logger.Infof("sourcing private key from path: %s", c.config.GithubAppPrivateKey) decodedPEM, err := base64.StdEncoding.DecodeString(c.config.GithubAppPrivateKey) @@ -234,6 +235,7 @@ func (c *client) newGithubAppToken(ctx context.Context, r *api.Repo) (*github.Cl return c.newClientToken(ctx, t.GetToken()), nil } + // todo: this panics internally? // list all installations (a.k.a. orgs) where the GitHub App is installed installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) if err != nil { @@ -250,6 +252,8 @@ func (c *client) newGithubAppToken(ctx context.Context, r *api.Repo) (*github.Cl } // failsafe in case the repo does not belong to an org where the GitHub App is installed + // todo: should this be an error? + // in reality we should warn them that they should install this app to their org and add this repo if id == 0 { return nil, err } diff --git a/scm/github/repo.go b/scm/github/repo.go index 0b03cf03c..af21879a0 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -4,6 +4,7 @@ package github import ( "context" + "errors" "fmt" "net/http" "strconv" @@ -96,6 +97,14 @@ func (c *client) Config(ctx context.Context, u *api.User, r *api.Repo, ref strin // Disable deactivates a repo by deleting the webhook. func (c *client) Disable(ctx context.Context, u *api.User, org, name string) error { + // todo: remove repo from github app installation + + // todo: if there are no other repos in the org github app installation, should we uninstall it from the org? + return c.DestroyWebhook(ctx, u, org, name) +} + +// DestroyWebhook deletes a repo's webhook. +func (c *client) DestroyWebhook(ctx context.Context, u *api.User, org, name string) error { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": name, @@ -151,6 +160,16 @@ func (c *client) Disable(ctx context.Context, u *api.User, org, name string) err // Enable activates a repo by creating the webhook. func (c *client) Enable(ctx context.Context, u *api.User, r *api.Repo, h *api.Hook) (*api.Hook, string, error) { + // todo: check for org installation + // todo: if org installation does not exist, we need to redirec the user + // todo: use cli vs web redirect logic + // todo: ensure repo is visible/enabled in org installation + + return c.CreateWebhook(ctx, u, r, h) +} + +// CreateWebhook creates a repo's webhook. +func (c *client) CreateWebhook(ctx context.Context, u *api.User, r *api.Repo, h *api.Hook) (*api.Hook, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -676,6 +695,10 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev return 0, err } + if client == nil { + return 0, errors.New("unable to make github app token client") + } + opts := github.CreateCheckRunOptions{ Name: fmt.Sprintf("vela-%s-%s", event, step), HeadSHA: commit, From 7e8a6762c8ef418b89a46ef19d158c16d6ce042e Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 4 Oct 2024 14:42:08 -0500 Subject: [PATCH 04/56] refactor: required changes from api build types migration --- api/types/build.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/api/types/build.go b/api/types/build.go index 10ccc31e2..a7f770125 100644 --- a/api/types/build.go +++ b/api/types/build.go @@ -8,6 +8,8 @@ import ( "time" "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" ) @@ -1205,3 +1207,38 @@ func (b *Build) String() string { b.GetTitle(), ) } + +// StepFromBuildContainer creates a new Step based on a Build and pipeline Container. +func StepFromBuildContainer(build *Build, ctn *pipeline.Container) *library.Step { + // create new step type we want to return + s := new(library.Step) + + // default status to Pending + s.SetStatus(constants.StatusPending) + + // copy fields from build + if build != nil { + // set values from the build + s.SetHost(build.GetHost()) + s.SetRuntime(build.GetRuntime()) + s.SetDistribution(build.GetDistribution()) + } + + // copy fields from container + if ctn != nil && ctn.Name != "" { + // set values from the container + s.SetName(ctn.Name) + s.SetNumber(ctn.Number) + s.SetImage(ctn.Image) + s.SetReportAs(ctn.ReportAs) + + // check if the VELA_STEP_STAGE environment variable exists + value, ok := ctn.Environment["VELA_STEP_STAGE"] + if ok { + // set the Stage field to the value from environment variable + s.SetStage(value) + } + } + + return s +} From 37b30a185244da59bc7c9eda8441fcd1e968d9a8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 8 Oct 2024 10:07:35 -0500 Subject: [PATCH 05/56] feat: cli auth repo install flow --- api/auth/get_token.go | 29 ++++++- api/install.go | 161 +++++++++++++++++++++++++++++++++++ api/repo/install_html_url.go | 86 +++++++++++++++++++ api/types/repo.go | 10 +++ cmd/vela-server/server.go | 2 +- go.mod | 2 +- router/repo.go | 1 + router/router.go | 3 + scm/github/github.go | 16 +++- scm/github/repo.go | 68 ++++++++++++++- scm/service.go | 4 +- 11 files changed, 373 insertions(+), 9 deletions(-) create mode 100644 api/install.go create mode 100644 api/repo/install_html_url.go diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 07df70b0c..afe3d2c28 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -9,8 +9,10 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + "github.com/go-vela/server/api" "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" + "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -37,6 +39,10 @@ import ( // name: redirect_uri // description: The URL where the user will be sent after authorization // type: string +// - in: query +// name: setup_action +// description: The specific setup action callback identifier +// type: string // responses: // '200': // description: Successfully authenticated @@ -60,16 +66,35 @@ import ( // process a user logging in to Vela from // the API or UI. func GetAuthToken(c *gin.Context) { - var err error - // capture middleware values tm := c.MustGet("token-manager").(*token.Manager) + m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + // GitHub App and OAuth share the same callback URL, + // so we need to differentiate between the two using setup_action + if c.Request.FormValue("setup_action") == "install" { + redirect, err := api.GetAppInstallRedirectURL(ctx, l, m, c.Request.URL.Query()) + if err != nil { + retErr := fmt.Errorf("unable to get app install redirect URL: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.Redirect(http.StatusTemporaryRedirect, redirect) + + return + } + // capture the OAuth state if present oAuthState := c.Request.FormValue("state") + var err error + // capture the OAuth code if present code := c.Request.FormValue("code") if len(code) == 0 { diff --git a/api/install.go b/api/install.go new file mode 100644 index 000000000..641f7ea53 --- /dev/null +++ b/api/install.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/internal" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /install install Install +// +// Start SCM app installation flow and redirect to the external SCM destination +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: type +// description: The type of installation flow, either 'cli' or 'web' +// type: string +// - in: query +// name: port +// description: The local server port used during 'cli' flow +// type: string +// - in: query +// name: org_scm_id +// description: The SCM org id +// type: string +// - in: query +// name: repo_scm_id +// description: The SCM repo id +// type: string +// responses: +// '307': +// description: Redirected for installation +// '400': +// description: Invalid request payload +// schema: +// "$ref": "#/definitions/Error" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '503': +// description: Service unavailable +// schema: +// "$ref": "#/definitions/Error" + +// Install represents the API handler to +// process an SCM app installation for Vela from +// the API or UI. +func Install(c *gin.Context) { + // capture middleware values + l := c.MustGet("logger").(*logrus.Entry) + scm := scm.FromContext(c) + + l.Debug("redirecting to SCM to complete app flow installation") + + orgSCMID, err := strconv.Atoi(util.FormParameter(c, "org_scm_id")) + if err != nil { + retErr := fmt.Errorf("unable to parse org_scm_id to integer: %v", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + repoSCMID, err := strconv.Atoi(util.FormParameter(c, "repo_scm_id")) + if err != nil { + retErr := fmt.Errorf("unable to parse repo_scm_id to integer: %v", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // type cannot be empty + t := util.FormParameter(c, "type") + if len(t) == 0 { + retErr := errors.New("no type query provided") + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // port can be empty when using web flow + p := util.FormParameter(c, "port") + + // capture query params + ri := &types.RepoInstall{ + Type: t, + Port: p, + OrgSCMID: int64(orgSCMID), + RepoSCMID: int64(repoSCMID), + } + + // construct the repo installation url + redirectURL, err := scm.GetRepoInstallURL(c.Request.Context(), ri) + if err != nil { + l.Errorf("unable to get repo install url: %v", err) + + return + } + + c.Redirect(http.StatusTemporaryRedirect, redirectURL) +} + +// GetAppInstallRedirectURL is a helper function to generate the redirect URL for completing an app installation flow. +func GetAppInstallRedirectURL(ctx context.Context, l *logrus.Entry, m *internal.Metadata, q url.Values) (string, error) { + // extract state that is passed along during the installation process + pairs := strings.Split(q.Get("state"), ",") + + values := make(map[string]string) + + for _, pair := range pairs { + parts := strings.SplitN(pair, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + + value := strings.TrimSpace(parts[1]) + + values[key] = value + } + } + + t, p := values["type"], values["port"] + + // default redirect location if a user ended up here + // by providing an unsupported type + r := fmt.Sprintf("%s/install", m.Vela.Address) + + switch t { + // cli auth flow + case "cli": + r = fmt.Sprintf("http://127.0.0.1:%s", p) + // web auth flow + case "web": + r = fmt.Sprintf("%s%s", m.Vela.WebAddress, m.Vela.WebOauthCallbackPath) + } + + // append the code and state values + r = fmt.Sprintf("%s?%s", r, q.Encode()) + + l.Debug("redirecting for final app installation flow") + + return r, nil +} diff --git a/api/repo/install_html_url.go b/api/repo/install_html_url.go new file mode 100644 index 000000000..95e416105 --- /dev/null +++ b/api/repo/install_html_url.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package repo + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/internal" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/install/html_url repos GetInstallHTMLURL +// +// Repair a hook for a repository in Vela and the configured SCM +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the organization +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repository +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully constructed the repo installation HTML URL +// schema: +// type: string +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Not found +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// GetInstallHTMLURL represents the API handler to retrieve the +// SCM installation HTML URL for a particular repo and Vela server. +func GetInstallHTMLURL(c *gin.Context) { + // capture middleware values + m := c.MustGet("metadata").(*internal.Metadata) + l := c.MustGet("logger").(*logrus.Entry) + u := user.Retrieve(c) + r := repo.Retrieve(c) + scm := scm.FromContext(c) + + l.Debug("constructing repo install url") + + ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r.GetOrg(), r.GetName()) + if err != nil { + retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // todo: use url.values etc + appInstallURL := fmt.Sprintf( + "%s/install?org_scm_id=%d&repo_scm_id=%d", + m.Vela.Address, + ri.OrgSCMID, ri.RepoSCMID, + ) + + c.JSON(http.StatusOK, fmt.Sprintf("%s", appInstallURL)) +} diff --git a/api/types/repo.go b/api/types/repo.go index 01e9f03ea..063454ccd 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -7,6 +7,16 @@ import ( "strings" ) +// RepoInstall is the configuration for installing a repo into the SCM. +// +// swagger:model RepoInstall +type RepoInstall struct { + Type string + Port string + OrgSCMID int64 + RepoSCMID int64 +} + // Repo is the API representation of a repo. // // swagger:model Repo diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 7733b9c15..0c1d5cb4e 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -132,7 +132,7 @@ func server(c *cli.Context) error { metadata.Vela.OpenIDIssuer = oidcIssuer tm.Issuer = oidcIssuer - jitter := wait.Jitter(5*time.Second, 2.0) + jitter := wait.Jitter(0*time.Second, 2.0) logrus.Infof("retrieving initial platform settings after %v delay", jitter) diff --git a/go.mod b/go.mod index 04f9ae923..7da3f69a8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/go-vela/server -go 1.23.1 +go 1.23.2 replace github.com/go-vela/types => ../types diff --git a/router/repo.go b/router/repo.go index 3cd7ecc9f..c87341467 100644 --- a/router/repo.go +++ b/router/repo.go @@ -73,6 +73,7 @@ func RepoHandlers(base *gin.RouterGroup) { _repo.DELETE("", perm.MustAdmin(), repo.DeleteRepo) _repo.PATCH("/repair", perm.MustAdmin(), repo.RepairRepo) _repo.PATCH("/chown", perm.MustAdmin(), repo.ChownRepo) + _repo.GET("/install/html_url", repo.GetInstallHTMLURL) // Build endpoints // * Service endpoints diff --git a/router/router.go b/router/router.go index 2e7aebbb2..36ed24be3 100644 --- a/router/router.go +++ b/router/router.go @@ -102,6 +102,9 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { authenticate.POST("/token", auth.PostAuthToken) } + // Repo installation endpoint (GitHub App) + r.GET("/install", api.Install) + // API endpoints baseAPI := r.Group(base, claims.Establish(), user.Establish()) { diff --git a/scm/github/github.go b/scm/github/github.go index 441b39705..5f3cad127 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -216,8 +216,20 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien return github } -// helper function to return the GitHub App token. -func (c *client) newGithubAppToken(ctx context.Context, r *api.Repo) (*github.Client, error) { +// helper function to return the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. +func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) { + // todo: create transport using context to apply tracing + // create a github client based off the existing GitHub App configuration + client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) + if err != nil { + return nil, err + } + + return client, nil +} + +// helper function to return the GitHub App installation token. +func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo) (*github.Client, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) if err != nil { diff --git a/scm/github/repo.go b/scm/github/repo.go index af21879a0..530ac849e 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strconv" "strings" "time" @@ -690,7 +691,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App - client, err := c.newGithubAppToken(ctx, r) + client, err := c.newGithubAppInstallationToken(ctx, r) if err != nil { return 0, err } @@ -715,7 +716,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { // create client from GitHub App - client, err := c.newGithubAppToken(ctx, r) + client, err := c.newGithubAppInstallationToken(ctx, r) if err != nil { return err } @@ -796,3 +797,66 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, return nil } + +// GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. +func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, o string, r string) (*api.RepoInstall, error) { + c.Logger.WithFields(logrus.Fields{ + "org": o, + "user": u.GetName(), + }).Tracef("retrieving repo install information for %s", o) + + client := c.newClientToken(ctx, u.GetToken()) + + // send an API call to get the org info + repoInfo, resp, err := client.Repositories.Get(ctx, o, r) + + orgID := repoInfo.GetOwner().GetID() + + // if org is not found, return the personal org + if resp.StatusCode == http.StatusNotFound { + user, _, err := client.Users.Get(ctx, "") + if err != nil { + return nil, err + } + + orgID = user.GetID() + } else if err != nil { + return nil, err + } + + ri := &api.RepoInstall{ + OrgSCMID: orgID, + RepoSCMID: repoInfo.GetID(), + } + + return ri, nil +} + +// GetRepoInstallURL takes RepoInstall configurations and returns the SCM URL for installing the application. +func (c *client) GetRepoInstallURL(ctx context.Context, ri *api.RepoInstall) (string, error) { + client, err := c.newGithubAppClient(ctx) + if err != nil { + return "", err + } + + // retrieve the authenticated app information + // required for slug and HTML URL + app, _, err := client.Apps.Get(ctx, "") + if err != nil { + return "", err + } + + path := fmt.Sprintf( + "%s/installations/new/permissions", + app.GetHTMLURL()) + + // stored as state to retrieve from the post-install callback + state := fmt.Sprintf("type=%s,port=%s", ri.Type, ri.Port) + + v := &url.Values{} + v.Set("state", state) + v.Set("suggested_target_id", strconv.FormatInt(ri.OrgSCMID, 10)) + v.Set("repository_ids", strconv.FormatInt(ri.RepoSCMID, 10)) + + return fmt.Sprintf("%s?%s", path, v.Encode()), nil +} diff --git a/scm/service.go b/scm/service.go index 36965c59d..ac7bd9c1c 100644 --- a/scm/service.go +++ b/scm/service.go @@ -142,7 +142,9 @@ type Service interface { // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) - // TODO: add comments + GetRepoInstallInfo(context.Context, *api.User, string, string) (*api.RepoInstall, error) + GetRepoInstallURL(context.Context, *api.RepoInstall) (string, error) + CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error From e84299dcfc8b793378741e64835322fbae4d7ef2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Sat, 12 Oct 2024 08:57:04 -0500 Subject: [PATCH 06/56] feat: clone token --- api/auth/get_token.go | 28 ++++- api/build/compile_publish.go | 1 + api/install.go | 14 ++- api/repo/{install_html_url.go => install.go} | 12 +-- api/types/repo.go | 18 ++-- compiler/engine.go | 2 + compiler/native/compile_test.go | 106 +++++++++---------- compiler/native/environment.go | 38 +++++-- compiler/native/environment_test.go | 4 +- compiler/native/native.go | 16 ++- compiler/native/script_test.go | 4 +- compiler/native/transform_test.go | 16 +-- router/repo.go | 2 +- scm/github/github.go | 16 +-- scm/github/repo.go | 80 ++++++++++++-- scm/service.go | 3 +- 16 files changed, 251 insertions(+), 109 deletions(-) rename api/repo/{install_html_url.go => install.go} (85%) diff --git a/api/auth/get_token.go b/api/auth/get_token.go index afe3d2c28..2bd981088 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -3,6 +3,7 @@ package auth import ( + "errors" "fmt" "net/http" @@ -75,7 +76,24 @@ func GetAuthToken(c *gin.Context) { // GitHub App and OAuth share the same callback URL, // so we need to differentiate between the two using setup_action - if c.Request.FormValue("setup_action") == "install" { + setupAction := c.Request.FormValue("setup_action") + switch setupAction { + case "install": + case "update": + installID := c.Request.FormValue("installation_id") + if len(installID) == 0 { + retErr := errors.New("setup_action is install but installation_id is missing") + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // todo: if the repo is already added, then redirecting to the install url will try to add ALL repos... + + // todo: on "install" we also need to check if it was just a regular github ui manual installation + // todo: on "update" this might just be a regular ui update to the github app + // todo: we need to capture the installation ID and sync all the vela repos for that installation redirect, err := api.GetAppInstallRedirectURL(ctx, l, m, c.Request.URL.Query()) if err != nil { retErr := fmt.Errorf("unable to get app install redirect URL: %w", err) @@ -85,9 +103,17 @@ func GetAuthToken(c *gin.Context) { return } + if len(redirect) == 0 { + c.JSON(http.StatusOK, "installation completed") + + return + } + c.Redirect(http.StatusTemporaryRedirect, redirect) return + case "": + break } // capture the OAuth state if present diff --git a/api/build/compile_publish.go b/api/build/compile_publish.go index 63db6c895..5d18a9a55 100644 --- a/api/build/compile_publish.go +++ b/api/build/compile_publish.go @@ -269,6 +269,7 @@ func CompileAndPublish( WithRepo(repo). WithUser(u). WithLabels(cfg.Labels). + WithSCM(scm). Compile(ctx, pipelineFile) if err != nil { // format the error message with extra information diff --git a/api/install.go b/api/install.go index 641f7ea53..ee8ab87fa 100644 --- a/api/install.go +++ b/api/install.go @@ -87,6 +87,11 @@ func Install(c *gin.Context) { return } + // first, check if the org installation exists. + // if it does, just add the repo manually using the api and be done with it + // if it doesn't, then we need to start the installation flow + // but this came from the browser... it has NO auth to contact github api + // type cannot be empty t := util.FormParameter(c, "type") if len(t) == 0 { @@ -102,10 +107,12 @@ func Install(c *gin.Context) { // capture query params ri := &types.RepoInstall{ - Type: t, - Port: p, OrgSCMID: int64(orgSCMID), RepoSCMID: int64(repoSCMID), + InstallCallback: types.InstallCallback{ + Type: t, + Port: p, + }, } // construct the repo installation url @@ -141,7 +148,8 @@ func GetAppInstallRedirectURL(ctx context.Context, l *logrus.Entry, m *internal. // default redirect location if a user ended up here // by providing an unsupported type - r := fmt.Sprintf("%s/install", m.Vela.Address) + // this is ignored when empty + r := "" switch t { // cli auth flow diff --git a/api/repo/install_html_url.go b/api/repo/install.go similarity index 85% rename from api/repo/install_html_url.go rename to api/repo/install.go index 95e416105..306e8f6a1 100644 --- a/api/repo/install_html_url.go +++ b/api/repo/install.go @@ -54,9 +54,9 @@ import ( // schema: // "$ref": "#/definitions/Error" -// GetInstallHTMLURL represents the API handler to retrieve the +// GetInstallInfo represents the API handler to retrieve the // SCM installation HTML URL for a particular repo and Vela server. -func GetInstallHTMLURL(c *gin.Context) { +func GetInstallInfo(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) @@ -64,9 +64,9 @@ func GetInstallHTMLURL(c *gin.Context) { r := repo.Retrieve(c) scm := scm.FromContext(c) - l.Debug("constructing repo install url") + l.Debug("retrieving repo install information") - ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r.GetOrg(), r.GetName()) + ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r) if err != nil { retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) @@ -76,11 +76,11 @@ func GetInstallHTMLURL(c *gin.Context) { } // todo: use url.values etc - appInstallURL := fmt.Sprintf( + ri.InstallURL = fmt.Sprintf( "%s/install?org_scm_id=%d&repo_scm_id=%d", m.Vela.Address, ri.OrgSCMID, ri.RepoSCMID, ) - c.JSON(http.StatusOK, fmt.Sprintf("%s", appInstallURL)) + c.JSON(http.StatusOK, ri) } diff --git a/api/types/repo.go b/api/types/repo.go index 063454ccd..eaabe2d44 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -8,13 +8,19 @@ import ( ) // RepoInstall is the configuration for installing a repo into the SCM. -// -// swagger:model RepoInstall type RepoInstall struct { - Type string - Port string - OrgSCMID int64 - RepoSCMID int64 + OrgSCMID int64 + RepoSCMID int64 + AppInstalled bool + RepoAdded bool + InstallURL string + InstallCallback +} + +// InstallCallback is the callback configuration for the installation. +type InstallCallback struct { + Type string + Port string } // Repo is the API representation of a repo. diff --git a/compiler/engine.go b/compiler/engine.go index 931cef4ad..5aaf4ddd9 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -8,6 +8,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/internal" + "github.com/go-vela/server/scm" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" @@ -147,6 +148,7 @@ type Engine interface { // WithLabel defines a function that sets // the label(s) in the Engine. WithLabels([]string) Engine + WithSCM(scm.Service) Engine // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(context.Context, string, string) Engine diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 3670cae07..78282121f 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -54,21 +54,21 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, "") initEnv["HELLO"] = "Hello, Global Environment" - stageEnvInstall := environment(nil, m, nil, nil) + stageEnvInstall := environment(nil, m, nil, nil, "") stageEnvInstall["HELLO"] = "Hello, Global Environment" stageEnvInstall["GRADLE_USER_HOME"] = ".gradle" - stageEnvTest := environment(nil, m, nil, nil) + stageEnvTest := environment(nil, m, nil, nil, "") stageEnvTest["HELLO"] = "Hello, Global Environment" stageEnvTest["GRADLE_USER_HOME"] = "willBeOverwrittenInStep" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, "") cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, "") installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -76,7 +76,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, "") testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -84,7 +84,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, "") buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -92,7 +92,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, "") dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -479,13 +479,13 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, "") initEnv["HELLO"] = "Hello, Global Environment" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, "") cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, "") installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -493,7 +493,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, "") testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -501,7 +501,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, "") buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -509,7 +509,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, "") dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -690,11 +690,11 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, "") setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, "") installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -703,7 +703,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, "") testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -712,7 +712,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, "") buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -721,14 +721,14 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, "") dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil) + serviceEnv := environment(nil, m, nil, nil, "") serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -961,11 +961,11 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, "") setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, "") installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -974,7 +974,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, "") testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -983,7 +983,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, "") buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -992,14 +992,14 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, "") dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil) + serviceEnv := environment(nil, m, nil, nil, "") serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -1195,9 +1195,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, "") - helloEnv := environment(nil, m, nil, nil) + helloEnv := environment(nil, m, nil, nil, "") helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo sample"}) @@ -1316,9 +1316,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, "") - helloEnv := environment(nil, m, nil, nil) + helloEnv := environment(nil, m, nil, nil, "") helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo inline_templatename"}) @@ -1436,11 +1436,11 @@ func TestNative_Compile_InvalidType(t *testing.T) { }, } - gradleEnv := environment(nil, m, nil, nil) + gradleEnv := environment(nil, m, nil, nil, "") gradleEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" gradleEnv["GRADLE_USER_HOME"] = ".gradle" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, "") dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -1493,10 +1493,10 @@ func TestNative_Compile_Clone(t *testing.T) { }, } - fooEnv := environment(nil, m, nil, nil) + fooEnv := environment(nil, m, nil, nil, "") fooEnv["PARAMETER_REGISTRY"] = "foo" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, "") cloneEnv["PARAMETER_DEPTH"] = "5" wantFalse := &pipeline.Build{ @@ -1512,7 +1512,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: "#init", Name: "init", Number: 1, @@ -1543,7 +1543,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: "#init", Name: "init", Number: 1, @@ -1552,7 +1552,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_clone", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: defaultCloneImage, Name: "clone", Number: 2, @@ -1583,7 +1583,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: "#init", Name: "init", Number: 1, @@ -1687,10 +1687,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { }, } - defaultFooEnv := environment(nil, m, nil, nil) + defaultFooEnv := environment(nil, m, nil, nil, "") defaultFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultEnv := environment(nil, m, nil, nil) + defaultEnv := environment(nil, m, nil, nil, "") wantDefault := &pipeline.Build{ Version: "1", ID: "__0", @@ -1733,10 +1733,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { goPipelineType := "go" - goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil) + goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, "") goFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil) + defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, "") wantGo := &pipeline.Build{ Version: "1", ID: "__0", @@ -1779,10 +1779,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { starPipelineType := "starlark" - starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil) + starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, "") starlarkFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil) + defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, "") wantStarlark := &pipeline.Build{ Version: "1", ID: "__0", @@ -2039,13 +2039,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2072,13 +2072,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, ""), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2255,7 +2255,7 @@ func convertFileToGithubResponse(file string) (github.RepositoryContent, error) } func generateTestEnv(command string, m *internal.Metadata, pipelineType string) map[string]string { - output := environment(nil, m, nil, nil) + output := environment(nil, m, nil, nil, "") output["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{command}) output["HOME"] = "/root" output["SHELL"] = "/bin/sh" @@ -2312,15 +2312,15 @@ func Test_Compile_Inline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) - testEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, "") + testEnv := environment(nil, m, nil, nil, "") testEnv["FOO"] = "Hello, foo!" testEnv["HELLO"] = "Hello, Vela!" - stepEnv := environment(nil, m, nil, nil) + stepEnv := environment(nil, m, nil, nil, "") stepEnv["FOO"] = "Hello, foo!" stepEnv["HELLO"] = "Hello, Vela!" stepEnv["PARAMETER_FIRST"] = "foo" - golangEnv := environment(nil, m, nil, nil) + golangEnv := environment(nil, m, nil, nil, "") golangEnv["VELA_REPO_PIPELINE_TYPE"] = "go" type args struct { diff --git a/compiler/native/environment.go b/compiler/native/environment.go index 653984100..001a6ef73 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -3,6 +3,7 @@ package native import ( + "context" "fmt" "os" "strings" @@ -13,6 +14,7 @@ import ( "github.com/go-vela/types/library" "github.com/go-vela/types/raw" "github.com/go-vela/types/yaml" + "github.com/sirupsen/logrus" ) // EnvironmentStages injects environment variables @@ -35,7 +37,11 @@ func (c *client) EnvironmentStage(s *yaml.Stage, globalEnv raw.StringSliceMap) ( // make empty map of environment variables env := make(map[string]string) // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + t, err := c.scm.GetCloneToken(context.Background(), c.user, c.repo) + if err != nil { + logrus.Errorf("couldnt get clone token: %v", err) + } + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) // inject the declared global environment // WARNING: local env can override global @@ -89,7 +95,11 @@ func (c *client) EnvironmentStep(s *yaml.Step, stageEnv raw.StringSliceMap) (*ya // make empty map of environment variables env := make(map[string]string) // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + t, err := c.scm.GetCloneToken(context.Background(), c.user, c.repo) + if err != nil { + logrus.Errorf("couldnt get clone token: %v", err) + } + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) // inject the declared stage environment // WARNING: local env can override global + stage @@ -150,7 +160,11 @@ func (c *client) EnvironmentServices(s yaml.ServiceSlice, globalEnv raw.StringSl // make empty map of environment variables env := make(map[string]string) // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + t, err := c.scm.GetCloneToken(context.Background(), c.user, c.repo) + if err != nil { + logrus.Errorf("couldnt get clone token: %v", err) + } + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) // inject the declared global environment // WARNING: local env can override global @@ -190,7 +204,11 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic // make empty map of environment variables env := make(map[string]string) // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + t, err := c.scm.GetCloneToken(context.Background(), c.user, c.repo) + if err != nil { + logrus.Errorf("couldnt get clone token: %v", err) + } + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) // inject the declared global environment // WARNING: local env can override global @@ -248,7 +266,11 @@ func (c *client) EnvironmentBuild() map[string]string { // make empty map of environment variables env := make(map[string]string) // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + t, err := c.scm.GetCloneToken(context.Background(), c.user, c.repo) + if err != nil { + logrus.Errorf("couldnt get clone token: %v", err) + } + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) // inject the default environment // variables to the build @@ -282,7 +304,7 @@ func appendMap(originalMap, otherMap map[string]string) map[string]string { } // helper function that creates the standard set of environment variables for a pipeline. -func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User) map[string]string { +func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, netrcPassword string) map[string]string { // set default workspace workspace := constants.WorkspaceDefault notImplemented := "TODO" @@ -298,12 +320,14 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User) m env["VELA_DISTRIBUTION"] = notImplemented env["VELA_HOST"] = notImplemented env["VELA_NETRC_MACHINE"] = notImplemented - env["VELA_NETRC_PASSWORD"] = u.GetToken() + env["VELA_NETRC_PASSWORD"] = netrcPassword + logrus.Infof("using netrc password: %s", netrcPassword) env["VELA_NETRC_USERNAME"] = "x-oauth-basic" env["VELA_QUEUE"] = notImplemented env["VELA_RUNTIME"] = notImplemented env["VELA_SOURCE"] = notImplemented env["VELA_VERSION"] = notImplemented + env["VELA_VADER"] = "yes" env["CI"] = "true" // populate environment variables from metadata diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index 5f00c6088..52fb9b4b2 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -42,7 +42,7 @@ func TestNative_EnvironmentStages(t *testing.T) { }, } - env := environment(nil, nil, nil, nil) + env := environment(nil, nil, nil, nil, "") env["HELLO"] = "Hello, Global Message" want := yaml.StageSlice{ @@ -629,7 +629,7 @@ func TestNative_environment(t *testing.T) { // run test for _, test := range tests { - got := environment(test.b, test.m, test.r, test.u) + got := environment(test.b, test.m, test.r, test.u, "") if diff := cmp.Diff(got, test.want); diff != "" { t.Errorf("environment mismatch (-want +got):\n%s", diff) diff --git a/compiler/native/native.go b/compiler/native/native.go index db28375e5..846ca602b 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -17,6 +17,7 @@ import ( "github.com/go-vela/server/compiler/registry/github" "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/image" + "github.com/go-vela/server/scm" ) type ModificationConfig struct { @@ -27,9 +28,10 @@ type ModificationConfig struct { } type client struct { - Github registry.Service - PrivateGithub registry.Service - UsePrivateGithub bool + Github registry.Service + PrivateGithub registry.Service + UsePrivateGithub bool + ModificationService ModificationConfig settings.Compiler @@ -44,6 +46,7 @@ type client struct { repo *api.Repo user *api.User labels []string + scm scm.Service } // FromCLIContext returns a Pipeline implementation that integrates with the supported registries. @@ -231,3 +234,10 @@ func (c *client) WithLabels(labels []string) compiler.Engine { return c } + +// WithSCM sets the scm in the Engine. +func (c *client) WithSCM(_scm scm.Service) compiler.Engine { + c.scm = _scm + + return c +} diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 0f03e6865..c2ee8b586 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -19,7 +19,7 @@ func TestNative_ScriptStages(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - baseEnv := environment(nil, nil, nil, nil) + baseEnv := environment(nil, nil, nil, nil, "") s := yaml.StageSlice{ &yaml.Stage{ @@ -109,7 +109,7 @@ func TestNative_ScriptSteps(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - emptyEnv := environment(nil, nil, nil, nil) + emptyEnv := environment(nil, nil, nil, nil, "") baseEnv := emptyEnv baseEnv["HOME"] = "/root" diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index bcd318103..3074a1108 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -59,7 +59,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install", Pull: "always", @@ -72,7 +72,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -138,7 +138,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "__0_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install", Number: 1, @@ -194,7 +194,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "localOrg_localRepo_1_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install", Number: 1, @@ -297,14 +297,14 @@ func TestNative_TransformSteps(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install deps", Pull: "always", }, &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -365,7 +365,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step___0_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install deps", Number: 1, @@ -416,7 +416,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step_localOrg_localRepo_1_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, ""), Image: "openjdk:latest", Name: "install deps", Number: 1, diff --git a/router/repo.go b/router/repo.go index c87341467..eb58258a9 100644 --- a/router/repo.go +++ b/router/repo.go @@ -73,7 +73,7 @@ func RepoHandlers(base *gin.RouterGroup) { _repo.DELETE("", perm.MustAdmin(), repo.DeleteRepo) _repo.PATCH("/repair", perm.MustAdmin(), repo.RepairRepo) _repo.PATCH("/chown", perm.MustAdmin(), repo.ChownRepo) - _repo.GET("/install/html_url", repo.GetInstallHTMLURL) + _repo.GET("/install/info", perm.MustRead(), repo.GetInstallInfo) // Build endpoints // * Service endpoints diff --git a/scm/github/github.go b/scm/github/github.go index 5f3cad127..4226b3bc3 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -229,11 +229,11 @@ func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) } // helper function to return the GitHub App installation token. -func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo) (*github.Client, error) { +func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo) (string, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) if err != nil { - return nil, err + return "", err } // if repo has an install ID, use it to create an installation token @@ -241,17 +241,17 @@ func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo) // create installation token for the repo t, _, err := client.Apps.CreateInstallationToken(context.Background(), r.GetInstallID(), &github.InstallationTokenOptions{}) if err != nil { - panic(err) + return "", err } - return c.newClientToken(ctx, t.GetToken()), nil + return t.GetToken(), nil } // todo: this panics internally? // list all installations (a.k.a. orgs) where the GitHub App is installed installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) if err != nil { - return nil, err + return "", err } var id int64 @@ -267,14 +267,14 @@ func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo) // todo: should this be an error? // in reality we should warn them that they should install this app to their org and add this repo if id == 0 { - return nil, err + return "", nil } // create installation token for the repo t, _, err := client.Apps.CreateInstallationToken(context.Background(), id, &github.InstallationTokenOptions{}) if err != nil { - panic(err) + return "", err } - return c.newClientToken(ctx, t.GetToken()), nil + return t.GetToken(), nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index 530ac849e..ec72abb13 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -691,15 +691,17 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App - client, err := c.newGithubAppInstallationToken(ctx, r) + t, err := c.newGithubAppInstallationToken(ctx, r) if err != nil { return 0, err } - if client == nil { - return 0, errors.New("unable to make github app token client") + if len(t) == 0 { + return 0, errors.New("unable to get github app installation token") } + client := c.newClientToken(ctx, t) + opts := github.CreateCheckRunOptions{ Name: fmt.Sprintf("vela-%s-%s", event, step), HeadSHA: commit, @@ -716,11 +718,17 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { // create client from GitHub App - client, err := c.newGithubAppInstallationToken(ctx, r) + t, err := c.newGithubAppInstallationToken(ctx, r) if err != nil { return err } + if len(t) == 0 { + return errors.New("unable to get github app installation token") + } + + client := c.newClientToken(ctx, t) + var ( conclusion string status string @@ -798,18 +806,40 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, return nil } +// GetCloneToken returns a clone token using the repo's github app installation if it exists. +// If not, it defaults to the user OAuth token. +func (c *client) GetCloneToken(ctx context.Context, u *api.User, r *api.Repo) (string, error) { + logrus.Infof("getting clone token") + // the app might not be installed + // todo: pass in THIS repo to only get access to that repo + // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation + // maybe take an optional list of repos and permission set that is driven by yaml + t, err := c.newGithubAppInstallationToken(ctx, r) + if err != nil { + logrus.Errorf("unable to get github app installation token: %v", err) + } + if len(t) != 0 { + logrus.Infof("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) + return t, nil + } + logrus.Infof("using user oauth token for %s/%s", r.GetOrg(), r.GetName()) + + return u.GetToken(), nil +} + // GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. -func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, o string, r string) (*api.RepoInstall, error) { +func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, r *api.Repo) (*api.RepoInstall, error) { c.Logger.WithFields(logrus.Fields{ - "org": o, + "org": r.GetOrg(), "user": u.GetName(), - }).Tracef("retrieving repo install information for %s", o) + }).Tracef("retrieving repo install information for %s", r.GetFullName()) client := c.newClientToken(ctx, u.GetToken()) // send an API call to get the org info - repoInfo, resp, err := client.Repositories.Get(ctx, o, r) + repoInfo, resp, err := client.Repositories.Get(ctx, r.GetOrg(), r.GetName()) + // orgName := repoInfo.GetOwner().GetLogin() orgID := repoInfo.GetOwner().GetID() // if org is not found, return the personal org @@ -819,6 +849,7 @@ func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, o string, return nil, err } + // orgName = user.GetLogin() orgID = user.GetID() } else if err != nil { return nil, err @@ -829,6 +860,39 @@ func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, o string, RepoSCMID: repoInfo.GetID(), } + ghAppClient, err := c.newGithubAppClient(ctx) + if err != nil { + return nil, err + } + + // todo: pagination... + installations, resp, err := ghAppClient.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil && (resp == nil || resp.StatusCode != http.StatusNotFound) { + return nil, err + } + + // check if the app is installed on the org + var id int64 + for _, installation := range installations { + // app is installed to the org + if installation.GetAccount().GetID() == orgID { + ri.AppInstalled = true + id = installation.GetID() + } + } + + ghAppClient2, err := c.newGithubAppInstallationToken(ctx, r) + if err != nil { + return nil, err + } + + _ = ghAppClient2 + + _, _, err = client.Apps.AddRepository(ctx, id, repoInfo.GetID()) + if err != nil { + return nil, err + } + return ri, nil } diff --git a/scm/service.go b/scm/service.go index ac7bd9c1c..cb058e216 100644 --- a/scm/service.go +++ b/scm/service.go @@ -142,9 +142,10 @@ type Service interface { // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) - GetRepoInstallInfo(context.Context, *api.User, string, string) (*api.RepoInstall, error) + GetRepoInstallInfo(context.Context, *api.User, *api.Repo) (*api.RepoInstall, error) GetRepoInstallURL(context.Context, *api.RepoInstall) (string, error) + GetCloneToken(context.Context, *api.User, *api.Repo) (string, error) CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error From b2cb87abb463e5d78f2f8b2b1e3d50da2e1647be Mon Sep 17 00:00:00 2001 From: davidvader Date: Sat, 12 Oct 2024 09:46:09 -0500 Subject: [PATCH 07/56] feat: yaml driven git token access --- compiler/native/compile.go | 2 ++ compiler/native/environment.go | 10 +++++----- compiler/native/native.go | 9 +++++++++ compiler/types/pipeline/git.go | 14 ++++++++++++++ compiler/types/yaml/build.go | 15 +++++++++++++++ scm/github/github.go | 2 +- scm/github/repo.go | 12 +++--------- scm/service.go | 2 +- 8 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 compiler/types/pipeline/git.go diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 088dbca15..05bc21acd 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -306,6 +306,8 @@ func (c *client) compileInline(ctx context.Context, p *yaml.Build, depth int) (* func (c *client) compileSteps(ctx context.Context, p *yaml.Build, _pipeline *api.Pipeline, tmpls map[string]*yaml.Template, r *pipeline.RuleData) (*pipeline.Build, *api.Pipeline, error) { var err error + c.git = &p.Git + // check if the pipeline disabled the clone if p.Metadata.Clone == nil || *p.Metadata.Clone { // inject the clone step diff --git a/compiler/native/environment.go b/compiler/native/environment.go index 208ffb606..09d7e5d12 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -37,7 +37,7 @@ func (c *client) EnvironmentStage(s *yaml.Stage, globalEnv raw.StringSliceMap) ( // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo) + t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -97,7 +97,7 @@ func (c *client) EnvironmentStep(s *yaml.Step, stageEnv raw.StringSliceMap) (*ya // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo) + t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -164,7 +164,7 @@ func (c *client) EnvironmentServices(s yaml.ServiceSlice, globalEnv raw.StringSl // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo) + t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -210,7 +210,7 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo) + t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -274,7 +274,7 @@ func (c *client) EnvironmentBuild() map[string]string { // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo) + t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 846ca602b..4d184e78f 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/image" "github.com/go-vela/server/scm" @@ -47,6 +48,7 @@ type client struct { user *api.User labels []string scm scm.Service + git *yaml.Git } // FromCLIContext returns a Pipeline implementation that integrates with the supported registries. @@ -241,3 +243,10 @@ func (c *client) WithSCM(_scm scm.Service) compiler.Engine { return c } + +// WithGit sets the git access configurations in the Engine. +func (c *client) WithGit(g *yaml.Git) compiler.Engine { + c.git = g + + return c +} diff --git a/compiler/types/pipeline/git.go b/compiler/types/pipeline/git.go new file mode 100644 index 000000000..8799e4d97 --- /dev/null +++ b/compiler/types/pipeline/git.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +// Git is the pipeline representation of the git block for a pipeline. +// +// swagger:model PipelineGit +type Git struct { + Access *Access `json:"access,omitempty" yaml:"access,omitempty"` +} + +type Access struct { + Repositories []string `json:"repositories,omitempty" yaml:"repositories,omitempty"` +} diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index b2d918945..9d22d0e97 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -7,8 +7,17 @@ import ( "github.com/go-vela/server/compiler/types/raw" ) +type Git struct { + Access `yaml:"access,omitempty" json:"access,omitempty" jsonschema:"description=Provide the git token specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#access"` +} + +type Access struct { + Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` +} + // Build is the yaml representation of a build for a pipeline. type Build struct { + Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` Version string `yaml:"version,omitempty" json:"version,omitempty" jsonschema:"required,minLength=1,description=Provide syntax version used to evaluate the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/version/"` Metadata Metadata `yaml:"metadata,omitempty" json:"metadata,omitempty" jsonschema:"description=Pass extra information.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/"` Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide global environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` @@ -64,6 +73,7 @@ func (b *Build) ToPipelineAPI() *api.Pipeline { func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { // build we try unmarshalling to build := new(struct { + Git Git Version string Metadata Metadata Environment raw.StringSliceMap @@ -86,7 +96,12 @@ func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { build.Metadata.Environment = []string{"steps", "services", "secrets"} } + if build.Git.Repositories == nil || len(build.Git.Repositories) == 0 { + build.Git.Repositories = []string{} + } + // override the values + b.Git = build.Git b.Version = build.Version b.Metadata = build.Metadata b.Environment = build.Environment diff --git a/scm/github/github.go b/scm/github/github.go index 74c7eec37..27300700c 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -237,7 +237,7 @@ func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo, if err != nil { return "", err } - + logrus.Warnf("getting token with repos: %v", repos) opts := &github.InstallationTokenOptions{ Repositories: repos, } diff --git a/scm/github/repo.go b/scm/github/repo.go index 1d328e627..54867e40a 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -808,13 +808,13 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, // GetNetrcPassword returns a clone token using the repo's github app installation if it exists. // If not, it defaults to the user OAuth token. -func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo) (string, error) { +func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo, repositories []string) (string, error) { logrus.Infof("getting clone token") // the app might not be installed // todo: pass in THIS repo to only get access to that repo // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation // maybe take an optional list of repos and permission set that is driven by yaml - t, err := c.newGithubAppInstallationToken(ctx, r, []string{}, []string{}) + t, err := c.newGithubAppInstallationToken(ctx, r, repositories, []string{}) if err != nil { logrus.Errorf("unable to get github app installation token: %v", err) } @@ -881,13 +881,7 @@ func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, r *api.Rep } } - ghAppClient2, err := c.newGithubAppInstallationToken(ctx, r, []string{}, []string{}) - if err != nil { - return nil, err - } - - _ = ghAppClient2 - + // todo: remove all this, it doesnt work without a PAT, lol _, _, err = client.Apps.AddRepository(ctx, id, repoInfo.GetID()) if err != nil { return nil, err diff --git a/scm/service.go b/scm/service.go index bfd9fa88f..88ccbd9bd 100644 --- a/scm/service.go +++ b/scm/service.go @@ -145,7 +145,7 @@ type Service interface { GetRepoInstallInfo(context.Context, *api.User, *api.Repo) (*api.RepoInstall, error) GetRepoInstallURL(context.Context, *api.RepoInstall) (string, error) - GetNetrcPassword(context.Context, *api.User, *api.Repo) (string, error) + GetNetrcPassword(context.Context, *api.User, *api.Repo, []string) (string, error) CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error From 38e8db6311ba4e0165120e41523e8926e2413ce8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 14 Oct 2024 09:11:38 -0500 Subject: [PATCH 08/56] chore: move all web flow code into single file --- api/auth/get_token.go | 46 ------------ api/install.go | 161 ++++++++++++++++++++++++++++++++---------- api/repo/install.go | 86 ---------------------- router/repo.go | 3 +- 4 files changed, 124 insertions(+), 172 deletions(-) delete mode 100644 api/repo/install.go diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 2bd981088..f17963177 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -3,17 +3,14 @@ package auth import ( - "errors" "fmt" "net/http" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "github.com/go-vela/server/api" "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" - "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -69,53 +66,10 @@ import ( func GetAuthToken(c *gin.Context) { // capture middleware values tm := c.MustGet("token-manager").(*token.Manager) - m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) ctx := c.Request.Context() - // GitHub App and OAuth share the same callback URL, - // so we need to differentiate between the two using setup_action - setupAction := c.Request.FormValue("setup_action") - switch setupAction { - case "install": - case "update": - installID := c.Request.FormValue("installation_id") - if len(installID) == 0 { - retErr := errors.New("setup_action is install but installation_id is missing") - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // todo: if the repo is already added, then redirecting to the install url will try to add ALL repos... - - // todo: on "install" we also need to check if it was just a regular github ui manual installation - // todo: on "update" this might just be a regular ui update to the github app - // todo: we need to capture the installation ID and sync all the vela repos for that installation - redirect, err := api.GetAppInstallRedirectURL(ctx, l, m, c.Request.URL.Query()) - if err != nil { - retErr := fmt.Errorf("unable to get app install redirect URL: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - if len(redirect) == 0 { - c.JSON(http.StatusOK, "installation completed") - - return - } - - c.Redirect(http.StatusTemporaryRedirect, redirect) - - return - case "": - break - } - // capture the OAuth state if present oAuthState := c.Request.FormValue("state") diff --git a/api/install.go b/api/install.go index ee8ab87fa..5f1330061 100644 --- a/api/install.go +++ b/api/install.go @@ -14,50 +14,64 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/api/types" "github.com/go-vela/server/internal" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" "github.com/sirupsen/logrus" ) -// swagger:operation GET /install install Install -// -// Start SCM app installation flow and redirect to the external SCM destination -// -// --- -// produces: -// - application/json -// parameters: -// - in: query -// name: type -// description: The type of installation flow, either 'cli' or 'web' -// type: string -// - in: query -// name: port -// description: The local server port used during 'cli' flow -// type: string -// - in: query -// name: org_scm_id -// description: The SCM org id -// type: string -// - in: query -// name: repo_scm_id -// description: The SCM repo id -// type: string -// responses: -// '307': -// description: Redirected for installation -// '400': -// description: Invalid request payload -// schema: -// "$ref": "#/definitions/Error" -// '401': -// description: Unauthorized -// schema: -// "$ref": "#/definitions/Error" -// '503': -// description: Service unavailable -// schema: -// "$ref": "#/definitions/Error" +// HandleInstallCallback represents the API handler to +// process an SCM app installation for Vela. +func HandleInstallCallback(c *gin.Context) { + // capture middleware values + m := c.MustGet("metadata").(*internal.Metadata) + l := c.MustGet("logger").(*logrus.Entry) + + ctx := c.Request.Context() + + // GitHub App and OAuth share the same callback URL, + // so we need to differentiate between the two using setup_action + setupAction := c.Request.FormValue("setup_action") + switch setupAction { + case "install": + case "update": + installID := c.Request.FormValue("installation_id") + if len(installID) == 0 { + retErr := errors.New("setup_action is install but installation_id is missing") + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // todo: if the repo is already added, then redirecting to the install url will try to add ALL repos... + + // todo: on "install" we also need to check if it was just a regular github ui manual installation + // todo: on "update" this might just be a regular ui update to the github app + // todo: we need to capture the installation ID and sync all the vela repos for that installation + redirect, err := GetAppInstallRedirectURL(ctx, l, m, c.Request.URL.Query()) + if err != nil { + retErr := fmt.Errorf("unable to get app install redirect URL: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + if len(redirect) == 0 { + c.JSON(http.StatusOK, "installation completed") + + return + } + + c.Redirect(http.StatusTemporaryRedirect, redirect) + + return + case "": + break + } +} // Install represents the API handler to // process an SCM app installation for Vela from @@ -167,3 +181,72 @@ func GetAppInstallRedirectURL(ctx context.Context, l *logrus.Entry, m *internal. return r, nil } + +// swagger:operation GET /api/v1/repos/{org}/{repo}/install/html_url repos GetInstallHTMLURL +// +// Repair a hook for a repository in Vela and the configured SCM +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the organization +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repository +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully constructed the repo installation HTML URL +// schema: +// type: string +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Not found +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// GetInstallInfo represents the API handler to retrieve the +// SCM installation HTML URL for a particular repo and Vela server. +func GetInstallInfo(c *gin.Context) { + // capture middleware values + m := c.MustGet("metadata").(*internal.Metadata) + l := c.MustGet("logger").(*logrus.Entry) + u := user.Retrieve(c) + r := repo.Retrieve(c) + scm := scm.FromContext(c) + + l.Debug("retrieving repo install information") + + ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r) + if err != nil { + retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // todo: use url.values etc + ri.InstallURL = fmt.Sprintf( + "%s/install?org_scm_id=%d&repo_scm_id=%d", + m.Vela.Address, + ri.OrgSCMID, ri.RepoSCMID, + ) + + c.JSON(http.StatusOK, ri) +} diff --git a/api/repo/install.go b/api/repo/install.go deleted file mode 100644 index 306e8f6a1..000000000 --- a/api/repo/install.go +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package repo - -import ( - "fmt" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/internal" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/util" -) - -// swagger:operation GET /api/v1/repos/{org}/{repo}/install/html_url repos GetInstallHTMLURL -// -// Repair a hook for a repository in Vela and the configured SCM -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the organization -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repository -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully constructed the repo installation HTML URL -// schema: -// type: string -// '401': -// description: Unauthorized -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Not found -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unexpected server error -// schema: -// "$ref": "#/definitions/Error" - -// GetInstallInfo represents the API handler to retrieve the -// SCM installation HTML URL for a particular repo and Vela server. -func GetInstallInfo(c *gin.Context) { - // capture middleware values - m := c.MustGet("metadata").(*internal.Metadata) - l := c.MustGet("logger").(*logrus.Entry) - u := user.Retrieve(c) - r := repo.Retrieve(c) - scm := scm.FromContext(c) - - l.Debug("retrieving repo install information") - - ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r) - if err != nil { - retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // todo: use url.values etc - ri.InstallURL = fmt.Sprintf( - "%s/install?org_scm_id=%d&repo_scm_id=%d", - m.Vela.Address, - ri.OrgSCMID, ri.RepoSCMID, - ) - - c.JSON(http.StatusOK, ri) -} diff --git a/router/repo.go b/router/repo.go index eb58258a9..90c840bc4 100644 --- a/router/repo.go +++ b/router/repo.go @@ -5,6 +5,7 @@ package router import ( "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" "github.com/go-vela/server/api/build" "github.com/go-vela/server/api/repo" "github.com/go-vela/server/router/middleware" @@ -73,7 +74,7 @@ func RepoHandlers(base *gin.RouterGroup) { _repo.DELETE("", perm.MustAdmin(), repo.DeleteRepo) _repo.PATCH("/repair", perm.MustAdmin(), repo.RepairRepo) _repo.PATCH("/chown", perm.MustAdmin(), repo.ChownRepo) - _repo.GET("/install/info", perm.MustRead(), repo.GetInstallInfo) + _repo.GET("/install/info", perm.MustRead(), api.GetInstallInfo) // Build endpoints // * Service endpoints From cbe94e696ac9a2764a797a008683e93ab07ba806 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 14 Oct 2024 09:23:20 -0500 Subject: [PATCH 09/56] chore: moving more code to isolated file for future reference --- api/install.go | 86 ++++++++++++++++++++++++++++++++++++++--- api/types/repo.go | 1 + scm/github/repo.go | 96 +--------------------------------------------- scm/service.go | 3 -- 4 files changed, 84 insertions(+), 102 deletions(-) diff --git a/api/install.go b/api/install.go index 5f1330061..b2fc3d11c 100644 --- a/api/install.go +++ b/api/install.go @@ -16,8 +16,8 @@ import ( "github.com/go-vela/server/internal" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" "github.com/go-vela/server/util" + "github.com/google/go-github/v62/github" "github.com/sirupsen/logrus" ) @@ -79,7 +79,7 @@ func HandleInstallCallback(c *gin.Context) { func Install(c *gin.Context) { // capture middleware values l := c.MustGet("logger").(*logrus.Entry) - scm := scm.FromContext(c) + // scm := scm.FromContext(c) l.Debug("redirecting to SCM to complete app flow installation") @@ -130,7 +130,8 @@ func Install(c *gin.Context) { } // construct the repo installation url - redirectURL, err := scm.GetRepoInstallURL(c.Request.Context(), ri) + // todo: this would need a github client + redirectURL, err := GetRepoInstallURL(c.Request.Context(), nil, ri) if err != nil { l.Errorf("unable to get repo install url: %v", err) @@ -228,11 +229,12 @@ func GetInstallInfo(c *gin.Context) { l := c.MustGet("logger").(*logrus.Entry) u := user.Retrieve(c) r := repo.Retrieve(c) - scm := scm.FromContext(c) + // scm := scm.FromContext(c) l.Debug("retrieving repo install information") - ri, err := scm.GetRepoInstallInfo(c.Request.Context(), u, r) + // todo: this would need github clients + ri, err := GetRepoInstallInfo(c.Request.Context(), nil, nil, u, r) if err != nil { retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) @@ -250,3 +252,77 @@ func GetInstallInfo(c *gin.Context) { c.JSON(http.StatusOK, ri) } + +// GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. +func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClient *github.Client, u *types.User, r *types.Repo) (*types.RepoInstall, error) { + // client := c.newClientToken(ctx, u.GetToken()) + + // send an API call to get the org info + repoInfo, resp, err := userClient.Repositories.Get(ctx, r.GetOrg(), r.GetName()) + + orgID := repoInfo.GetOwner().GetID() + + // if org is not found, return the personal org + if resp.StatusCode == http.StatusNotFound { + user, _, err := userClient.Users.Get(ctx, "") + if err != nil { + return nil, err + } + orgID = user.GetID() + } else if err != nil { + return nil, err + } + + ri := &types.RepoInstall{ + OrgSCMID: orgID, + RepoSCMID: repoInfo.GetID(), + } + + // todo: pagination... + installations, resp, err := appClient.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil && (resp == nil || resp.StatusCode != http.StatusNotFound) { + return nil, err + } + + // check if the app is installed on the org + var id int64 + for _, installation := range installations { + // app is installed to the org + if installation.GetAccount().GetID() == orgID { + ri.AppInstalled = true + ri.InstallID = installation.GetID() + } + } + + // todo: remove all this, it doesnt work without a PAT, lol + _, _, err = appClient.Apps.AddRepository(ctx, id, repoInfo.GetID()) + if err != nil { + return nil, err + } + + return ri, nil +} + +// GetRepoInstallURL takes RepoInstall configurations and returns the SCM URL for installing the application. +func GetRepoInstallURL(ctx context.Context, appClient *github.Client, ri *types.RepoInstall) (string, error) { + // retrieve the authenticated app information + // required for slug and HTML URL + app, _, err := appClient.Apps.Get(ctx, "") + if err != nil { + return "", err + } + + path := fmt.Sprintf( + "%s/installations/new/permissions", + app.GetHTMLURL()) + + // stored as state to retrieve from the post-install callback + state := fmt.Sprintf("type=%s,port=%s", ri.Type, ri.Port) + + v := &url.Values{} + v.Set("state", state) + v.Set("suggested_target_id", strconv.FormatInt(ri.OrgSCMID, 10)) + v.Set("repository_ids", strconv.FormatInt(ri.RepoSCMID, 10)) + + return fmt.Sprintf("%s?%s", path, v.Encode()), nil +} diff --git a/api/types/repo.go b/api/types/repo.go index eaabe2d44..0099057a7 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -12,6 +12,7 @@ type RepoInstall struct { OrgSCMID int64 RepoSCMID int64 AppInstalled bool + InstallID int64 RepoAdded bool InstallURL string InstallCallback diff --git a/scm/github/repo.go b/scm/github/repo.go index 54867e40a..6536542c6 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "net/http" - "net/url" "strconv" "strings" "time" @@ -809,7 +808,8 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, // GetNetrcPassword returns a clone token using the repo's github app installation if it exists. // If not, it defaults to the user OAuth token. func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo, repositories []string) (string, error) { - logrus.Infof("getting clone token") + logrus.Infof("getting netrc password") + // the app might not be installed // todo: pass in THIS repo to only get access to that repo // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation @@ -826,95 +826,3 @@ func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo, return u.GetToken(), nil } - -// GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. -func (c *client) GetRepoInstallInfo(ctx context.Context, u *api.User, r *api.Repo) (*api.RepoInstall, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "user": u.GetName(), - }).Tracef("retrieving repo install information for %s", r.GetFullName()) - - client := c.newClientToken(ctx, u.GetToken()) - - // send an API call to get the org info - repoInfo, resp, err := client.Repositories.Get(ctx, r.GetOrg(), r.GetName()) - - // orgName := repoInfo.GetOwner().GetLogin() - orgID := repoInfo.GetOwner().GetID() - - // if org is not found, return the personal org - if resp.StatusCode == http.StatusNotFound { - user, _, err := client.Users.Get(ctx, "") - if err != nil { - return nil, err - } - - // orgName = user.GetLogin() - orgID = user.GetID() - } else if err != nil { - return nil, err - } - - ri := &api.RepoInstall{ - OrgSCMID: orgID, - RepoSCMID: repoInfo.GetID(), - } - - ghAppClient, err := c.newGithubAppClient(ctx) - if err != nil { - return nil, err - } - - // todo: pagination... - installations, resp, err := ghAppClient.Apps.ListInstallations(ctx, &github.ListOptions{}) - if err != nil && (resp == nil || resp.StatusCode != http.StatusNotFound) { - return nil, err - } - - // check if the app is installed on the org - var id int64 - for _, installation := range installations { - // app is installed to the org - if installation.GetAccount().GetID() == orgID { - ri.AppInstalled = true - id = installation.GetID() - } - } - - // todo: remove all this, it doesnt work without a PAT, lol - _, _, err = client.Apps.AddRepository(ctx, id, repoInfo.GetID()) - if err != nil { - return nil, err - } - - return ri, nil -} - -// GetRepoInstallURL takes RepoInstall configurations and returns the SCM URL for installing the application. -func (c *client) GetRepoInstallURL(ctx context.Context, ri *api.RepoInstall) (string, error) { - client, err := c.newGithubAppClient(ctx) - if err != nil { - return "", err - } - - // retrieve the authenticated app information - // required for slug and HTML URL - app, _, err := client.Apps.Get(ctx, "") - if err != nil { - return "", err - } - - path := fmt.Sprintf( - "%s/installations/new/permissions", - app.GetHTMLURL()) - - // stored as state to retrieve from the post-install callback - state := fmt.Sprintf("type=%s,port=%s", ri.Type, ri.Port) - - v := &url.Values{} - v.Set("state", state) - v.Set("suggested_target_id", strconv.FormatInt(ri.OrgSCMID, 10)) - v.Set("repository_ids", strconv.FormatInt(ri.RepoSCMID, 10)) - - return fmt.Sprintf("%s?%s", path, v.Encode()), nil -} diff --git a/scm/service.go b/scm/service.go index 88ccbd9bd..a793a707d 100644 --- a/scm/service.go +++ b/scm/service.go @@ -142,9 +142,6 @@ type Service interface { // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) - GetRepoInstallInfo(context.Context, *api.User, *api.Repo) (*api.RepoInstall, error) - GetRepoInstallURL(context.Context, *api.RepoInstall) (string, error) - GetNetrcPassword(context.Context, *api.User, *api.Repo, []string) (string, error) CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error From 8b022a3730f5d368fb3d3aece8f5688fcb5b832f Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 14 Oct 2024 09:27:29 -0500 Subject: [PATCH 10/56] chore: removing github app web flow install code --- api/install.go | 27 ++++++++++++++++++++++----- api/types/repo.go | 17 ----------------- router/repo.go | 2 -- router/router.go | 3 --- scm/github/repo.go | 8 -------- 5 files changed, 22 insertions(+), 35 deletions(-) diff --git a/api/install.go b/api/install.go index b2fc3d11c..d2c6170fb 100644 --- a/api/install.go +++ b/api/install.go @@ -21,6 +21,23 @@ import ( "github.com/sirupsen/logrus" ) +// RepoInstall is the configuration for installing a repo into the SCM. +type RepoInstall struct { + OrgSCMID int64 + RepoSCMID int64 + AppInstalled bool + InstallID int64 + RepoAdded bool + InstallURL string + InstallCallback +} + +// InstallCallback is the callback configuration for the installation. +type InstallCallback struct { + Type string + Port string +} + // HandleInstallCallback represents the API handler to // process an SCM app installation for Vela. func HandleInstallCallback(c *gin.Context) { @@ -120,10 +137,10 @@ func Install(c *gin.Context) { p := util.FormParameter(c, "port") // capture query params - ri := &types.RepoInstall{ + ri := &RepoInstall{ OrgSCMID: int64(orgSCMID), RepoSCMID: int64(repoSCMID), - InstallCallback: types.InstallCallback{ + InstallCallback: InstallCallback{ Type: t, Port: p, }, @@ -254,7 +271,7 @@ func GetInstallInfo(c *gin.Context) { } // GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. -func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClient *github.Client, u *types.User, r *types.Repo) (*types.RepoInstall, error) { +func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClient *github.Client, u *types.User, r *types.Repo) (*RepoInstall, error) { // client := c.newClientToken(ctx, u.GetToken()) // send an API call to get the org info @@ -273,7 +290,7 @@ func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClien return nil, err } - ri := &types.RepoInstall{ + ri := &RepoInstall{ OrgSCMID: orgID, RepoSCMID: repoInfo.GetID(), } @@ -304,7 +321,7 @@ func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClien } // GetRepoInstallURL takes RepoInstall configurations and returns the SCM URL for installing the application. -func GetRepoInstallURL(ctx context.Context, appClient *github.Client, ri *types.RepoInstall) (string, error) { +func GetRepoInstallURL(ctx context.Context, appClient *github.Client, ri *RepoInstall) (string, error) { // retrieve the authenticated app information // required for slug and HTML URL app, _, err := appClient.Apps.Get(ctx, "") diff --git a/api/types/repo.go b/api/types/repo.go index 0099057a7..01e9f03ea 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -7,23 +7,6 @@ import ( "strings" ) -// RepoInstall is the configuration for installing a repo into the SCM. -type RepoInstall struct { - OrgSCMID int64 - RepoSCMID int64 - AppInstalled bool - InstallID int64 - RepoAdded bool - InstallURL string - InstallCallback -} - -// InstallCallback is the callback configuration for the installation. -type InstallCallback struct { - Type string - Port string -} - // Repo is the API representation of a repo. // // swagger:model Repo diff --git a/router/repo.go b/router/repo.go index 90c840bc4..3cd7ecc9f 100644 --- a/router/repo.go +++ b/router/repo.go @@ -5,7 +5,6 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" "github.com/go-vela/server/api/build" "github.com/go-vela/server/api/repo" "github.com/go-vela/server/router/middleware" @@ -74,7 +73,6 @@ func RepoHandlers(base *gin.RouterGroup) { _repo.DELETE("", perm.MustAdmin(), repo.DeleteRepo) _repo.PATCH("/repair", perm.MustAdmin(), repo.RepairRepo) _repo.PATCH("/chown", perm.MustAdmin(), repo.ChownRepo) - _repo.GET("/install/info", perm.MustRead(), api.GetInstallInfo) // Build endpoints // * Service endpoints diff --git a/router/router.go b/router/router.go index 36ed24be3..2e7aebbb2 100644 --- a/router/router.go +++ b/router/router.go @@ -102,9 +102,6 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { authenticate.POST("/token", auth.PostAuthToken) } - // Repo installation endpoint (GitHub App) - r.GET("/install", api.Install) - // API endpoints baseAPI := r.Group(base, claims.Establish(), user.Establish()) { diff --git a/scm/github/repo.go b/scm/github/repo.go index 6536542c6..bab84276d 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -97,9 +97,6 @@ func (c *client) Config(ctx context.Context, u *api.User, r *api.Repo, ref strin // Disable deactivates a repo by deleting the webhook. func (c *client) Disable(ctx context.Context, u *api.User, org, name string) error { - // todo: remove repo from github app installation - - // todo: if there are no other repos in the org github app installation, should we uninstall it from the org? return c.DestroyWebhook(ctx, u, org, name) } @@ -160,11 +157,6 @@ func (c *client) DestroyWebhook(ctx context.Context, u *api.User, org, name stri // Enable activates a repo by creating the webhook. func (c *client) Enable(ctx context.Context, u *api.User, r *api.Repo, h *api.Hook) (*api.Hook, string, error) { - // todo: check for org installation - // todo: if org installation does not exist, we need to redirec the user - // todo: use cli vs web redirect logic - // todo: ensure repo is visible/enabled in org installation - return c.CreateWebhook(ctx, u, r, h) } From 7d5d47417c5347ab7a949de55734338e81400457 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 14 Oct 2024 22:42:39 -0500 Subject: [PATCH 11/56] chore: wip --- api/install.go | 345 ------------------------------------------- api/repo/repair.go | 6 - scm/github/github.go | 1 + 3 files changed, 1 insertion(+), 351 deletions(-) delete mode 100644 api/install.go diff --git a/api/install.go b/api/install.go deleted file mode 100644 index d2c6170fb..000000000 --- a/api/install.go +++ /dev/null @@ -1,345 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/api/types" - "github.com/go-vela/server/internal" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/util" - "github.com/google/go-github/v62/github" - "github.com/sirupsen/logrus" -) - -// RepoInstall is the configuration for installing a repo into the SCM. -type RepoInstall struct { - OrgSCMID int64 - RepoSCMID int64 - AppInstalled bool - InstallID int64 - RepoAdded bool - InstallURL string - InstallCallback -} - -// InstallCallback is the callback configuration for the installation. -type InstallCallback struct { - Type string - Port string -} - -// HandleInstallCallback represents the API handler to -// process an SCM app installation for Vela. -func HandleInstallCallback(c *gin.Context) { - // capture middleware values - m := c.MustGet("metadata").(*internal.Metadata) - l := c.MustGet("logger").(*logrus.Entry) - - ctx := c.Request.Context() - - // GitHub App and OAuth share the same callback URL, - // so we need to differentiate between the two using setup_action - setupAction := c.Request.FormValue("setup_action") - switch setupAction { - case "install": - case "update": - installID := c.Request.FormValue("installation_id") - if len(installID) == 0 { - retErr := errors.New("setup_action is install but installation_id is missing") - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // todo: if the repo is already added, then redirecting to the install url will try to add ALL repos... - - // todo: on "install" we also need to check if it was just a regular github ui manual installation - // todo: on "update" this might just be a regular ui update to the github app - // todo: we need to capture the installation ID and sync all the vela repos for that installation - redirect, err := GetAppInstallRedirectURL(ctx, l, m, c.Request.URL.Query()) - if err != nil { - retErr := fmt.Errorf("unable to get app install redirect URL: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - if len(redirect) == 0 { - c.JSON(http.StatusOK, "installation completed") - - return - } - - c.Redirect(http.StatusTemporaryRedirect, redirect) - - return - case "": - break - } -} - -// Install represents the API handler to -// process an SCM app installation for Vela from -// the API or UI. -func Install(c *gin.Context) { - // capture middleware values - l := c.MustGet("logger").(*logrus.Entry) - // scm := scm.FromContext(c) - - l.Debug("redirecting to SCM to complete app flow installation") - - orgSCMID, err := strconv.Atoi(util.FormParameter(c, "org_scm_id")) - if err != nil { - retErr := fmt.Errorf("unable to parse org_scm_id to integer: %v", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - repoSCMID, err := strconv.Atoi(util.FormParameter(c, "repo_scm_id")) - if err != nil { - retErr := fmt.Errorf("unable to parse repo_scm_id to integer: %v", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // first, check if the org installation exists. - // if it does, just add the repo manually using the api and be done with it - // if it doesn't, then we need to start the installation flow - // but this came from the browser... it has NO auth to contact github api - - // type cannot be empty - t := util.FormParameter(c, "type") - if len(t) == 0 { - retErr := errors.New("no type query provided") - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // port can be empty when using web flow - p := util.FormParameter(c, "port") - - // capture query params - ri := &RepoInstall{ - OrgSCMID: int64(orgSCMID), - RepoSCMID: int64(repoSCMID), - InstallCallback: InstallCallback{ - Type: t, - Port: p, - }, - } - - // construct the repo installation url - // todo: this would need a github client - redirectURL, err := GetRepoInstallURL(c.Request.Context(), nil, ri) - if err != nil { - l.Errorf("unable to get repo install url: %v", err) - - return - } - - c.Redirect(http.StatusTemporaryRedirect, redirectURL) -} - -// GetAppInstallRedirectURL is a helper function to generate the redirect URL for completing an app installation flow. -func GetAppInstallRedirectURL(ctx context.Context, l *logrus.Entry, m *internal.Metadata, q url.Values) (string, error) { - // extract state that is passed along during the installation process - pairs := strings.Split(q.Get("state"), ",") - - values := make(map[string]string) - - for _, pair := range pairs { - parts := strings.SplitN(pair, "=", 2) - if len(parts) == 2 { - key := strings.TrimSpace(parts[0]) - - value := strings.TrimSpace(parts[1]) - - values[key] = value - } - } - - t, p := values["type"], values["port"] - - // default redirect location if a user ended up here - // by providing an unsupported type - // this is ignored when empty - r := "" - - switch t { - // cli auth flow - case "cli": - r = fmt.Sprintf("http://127.0.0.1:%s", p) - // web auth flow - case "web": - r = fmt.Sprintf("%s%s", m.Vela.WebAddress, m.Vela.WebOauthCallbackPath) - } - - // append the code and state values - r = fmt.Sprintf("%s?%s", r, q.Encode()) - - l.Debug("redirecting for final app installation flow") - - return r, nil -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/install/html_url repos GetInstallHTMLURL -// -// Repair a hook for a repository in Vela and the configured SCM -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the organization -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repository -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully constructed the repo installation HTML URL -// schema: -// type: string -// '401': -// description: Unauthorized -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Not found -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unexpected server error -// schema: -// "$ref": "#/definitions/Error" - -// GetInstallInfo represents the API handler to retrieve the -// SCM installation HTML URL for a particular repo and Vela server. -func GetInstallInfo(c *gin.Context) { - // capture middleware values - m := c.MustGet("metadata").(*internal.Metadata) - l := c.MustGet("logger").(*logrus.Entry) - u := user.Retrieve(c) - r := repo.Retrieve(c) - // scm := scm.FromContext(c) - - l.Debug("retrieving repo install information") - - // todo: this would need github clients - ri, err := GetRepoInstallInfo(c.Request.Context(), nil, nil, u, r) - if err != nil { - retErr := fmt.Errorf("unable to get repo scm install info %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // todo: use url.values etc - ri.InstallURL = fmt.Sprintf( - "%s/install?org_scm_id=%d&repo_scm_id=%d", - m.Vela.Address, - ri.OrgSCMID, ri.RepoSCMID, - ) - - c.JSON(http.StatusOK, ri) -} - -// GetRepoInstallInfo retrieves the repo information required for installation, such as org and repo ID for the given org and repo name. -func GetRepoInstallInfo(ctx context.Context, userClient *github.Client, appClient *github.Client, u *types.User, r *types.Repo) (*RepoInstall, error) { - // client := c.newClientToken(ctx, u.GetToken()) - - // send an API call to get the org info - repoInfo, resp, err := userClient.Repositories.Get(ctx, r.GetOrg(), r.GetName()) - - orgID := repoInfo.GetOwner().GetID() - - // if org is not found, return the personal org - if resp.StatusCode == http.StatusNotFound { - user, _, err := userClient.Users.Get(ctx, "") - if err != nil { - return nil, err - } - orgID = user.GetID() - } else if err != nil { - return nil, err - } - - ri := &RepoInstall{ - OrgSCMID: orgID, - RepoSCMID: repoInfo.GetID(), - } - - // todo: pagination... - installations, resp, err := appClient.Apps.ListInstallations(ctx, &github.ListOptions{}) - if err != nil && (resp == nil || resp.StatusCode != http.StatusNotFound) { - return nil, err - } - - // check if the app is installed on the org - var id int64 - for _, installation := range installations { - // app is installed to the org - if installation.GetAccount().GetID() == orgID { - ri.AppInstalled = true - ri.InstallID = installation.GetID() - } - } - - // todo: remove all this, it doesnt work without a PAT, lol - _, _, err = appClient.Apps.AddRepository(ctx, id, repoInfo.GetID()) - if err != nil { - return nil, err - } - - return ri, nil -} - -// GetRepoInstallURL takes RepoInstall configurations and returns the SCM URL for installing the application. -func GetRepoInstallURL(ctx context.Context, appClient *github.Client, ri *RepoInstall) (string, error) { - // retrieve the authenticated app information - // required for slug and HTML URL - app, _, err := appClient.Apps.Get(ctx, "") - if err != nil { - return "", err - } - - path := fmt.Sprintf( - "%s/installations/new/permissions", - app.GetHTMLURL()) - - // stored as state to retrieve from the post-install callback - state := fmt.Sprintf("type=%s,port=%s", ri.Type, ri.Port) - - v := &url.Values{} - v.Set("state", state) - v.Set("suggested_target_id", strconv.FormatInt(ri.OrgSCMID, 10)) - v.Set("repository_ids", strconv.FormatInt(ri.RepoSCMID, 10)) - - return fmt.Sprintf("%s?%s", path, v.Encode()), nil -} diff --git a/api/repo/repair.go b/api/repo/repair.go index 027c00a8e..2fd99f9e2 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -72,12 +72,6 @@ func RepairRepo(c *gin.Context) { l.Debugf("repairing repo %s", r.GetFullName()) - // todo: get org app installation - // doesnt exist? redirect them and wait... - - // todo: from org installation, check if this repo is visible/enabled - // no? use scm api to add the repo to the org - // check if we should create the webhook if c.Value("webhookvalidation").(bool) { // send API call to remove the webhook diff --git a/scm/github/github.go b/scm/github/github.go index 27300700c..f7c7612bb 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -240,6 +240,7 @@ func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo, logrus.Warnf("getting token with repos: %v", repos) opts := &github.InstallationTokenOptions{ Repositories: repos, + Permissions: &github.InstallationPermissions{}, } // if repo has an install ID, use it to create an installation token From c1197eb374445a5fa0d369a697be410e88e460b1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 22 Oct 2024 10:18:27 -0500 Subject: [PATCH 12/56] feat: webhook handler for repo installation events --- api/auth/get_token.go | 11 +++ api/types/repo.go | 2 + api/webhook/post.go | 15 ++++ database/types/repo.go | 3 + internal/webhook.go | 19 +++-- scm/github/github.go | 2 +- scm/github/installation.go | 144 +++++++++++++++++++++++++++++++++++++ scm/github/webhook.go | 47 +++++++++++- scm/service.go | 4 ++ 9 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 scm/github/installation.go diff --git a/api/auth/get_token.go b/api/auth/get_token.go index f17963177..d123c2716 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -75,6 +75,17 @@ func GetAuthToken(c *gin.Context) { var err error + if c.Request.FormValue("setup_action") == "install" { + // todo: make this better... + // random todos: + // what if a repo is added to the installation before it exists in vela + // then we need to sync repo installID all the time. + // sadly, installID might change if it gets re-installed. + c.Redirect(http.StatusTemporaryRedirect, "https://git.target.com/") + + return + } + // capture the OAuth code if present code := c.Request.FormValue("code") if len(code) == 0 { diff --git a/api/types/repo.go b/api/types/repo.go index 01e9f03ea..345240173 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -657,6 +657,7 @@ func (r *Repo) String() string { Counter: %d, FullName: %s, ID: %d, + InstallID: %d, Link: %s, Name: %s, Org: %s, @@ -678,6 +679,7 @@ func (r *Repo) String() string { r.GetCounter(), r.GetFullName(), r.GetID(), + r.GetInstallID(), r.GetLink(), r.GetName(), r.GetOrg(), diff --git a/api/webhook/post.go b/api/webhook/post.go index 20576acd3..bd182c723 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -85,6 +85,7 @@ func PostWebhook(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) + db := database.FromContext(c) ctx := c.Request.Context() l.Debug("webhook received") @@ -134,6 +135,20 @@ func PostWebhook(c *gin.Context) { return } + if webhook.Installation != nil { + err = scm.FromContext(c).ProcessInstallation(ctx, c.Request, webhook, db) + if err != nil { + retErr := fmt.Errorf("unable to process installation: %w", err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.JSON(http.StatusOK, "handled installation event!") + + return + } + // check if the hook should be skipped if skip, skipReason := webhook.ShouldSkip(); skip { c.JSON(http.StatusOK, fmt.Sprintf("skipping build: %s", skipReason)) diff --git a/database/types/repo.go b/database/types/repo.go index 584d12923..4b65bc3fa 100644 --- a/database/types/repo.go +++ b/database/types/repo.go @@ -67,6 +67,7 @@ type Repo struct { PipelineType sql.NullString `sql:"pipeline_type"` PreviousName sql.NullString `sql:"previous_name"` ApproveBuild sql.NullString `sql:"approve_build"` + InstallID sql.NullInt64 `sql:"install_id"` Owner User `gorm:"foreignKey:UserID"` } @@ -250,6 +251,7 @@ func (r *Repo) ToAPI() *api.Repo { repo.SetPipelineType(r.PipelineType.String) repo.SetPreviousName(r.PreviousName.String) repo.SetApproveBuild(r.ApproveBuild.String) + repo.SetInstallID(r.InstallID.Int64) return repo } @@ -342,6 +344,7 @@ func RepoFromAPI(r *api.Repo) *Repo { PipelineType: sql.NullString{String: r.GetPipelineType(), Valid: true}, PreviousName: sql.NullString{String: r.GetPreviousName(), Valid: true}, ApproveBuild: sql.NullString{String: r.GetApproveBuild(), Valid: true}, + InstallID: sql.NullInt64{Int64: r.GetInstallID(), Valid: true}, } return repo.Nullify() diff --git a/internal/webhook.go b/internal/webhook.go index 3c6889a59..7171d7fce 100644 --- a/internal/webhook.go +++ b/internal/webhook.go @@ -26,11 +26,20 @@ type PullRequest struct { // the required data when processing webhook event // a for a source provider event. type Webhook struct { - Hook *api.Hook - Repo *api.Repo - Build *api.Build - PullRequest PullRequest - Deployment *api.Deployment + Hook *api.Hook + Repo *api.Repo + Build *api.Build + PullRequest PullRequest + Deployment *api.Deployment + Installation *Installation +} + +type Installation struct { + Action string + ID int64 + Org string + RepositoriesAdded []string + RepositoriesRemoved []string } // ShouldSkip uses the build information diff --git a/scm/github/github.go b/scm/github/github.go index f7c7612bb..901d42551 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -237,7 +237,7 @@ func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo, if err != nil { return "", err } - logrus.Warnf("getting token with repos: %v", repos) + opts := &github.InstallationTokenOptions{ Repositories: repos, Permissions: &github.InstallationPermissions{}, diff --git a/scm/github/installation.go b/scm/github/installation.go new file mode 100644 index 000000000..839e8dfff --- /dev/null +++ b/scm/github/installation.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "net/http" + "time" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +// ProcessInstallation takes a GitHub installation and processes the changes. +func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, webhook *internal.Webhook, db database.Interface) error { + c.Logger.Tracef("processing GitHub App installation") + + errs := []error{} + + // set install_id for repos added to the installation + for _, repo := range webhook.Installation.RepositoriesAdded { + r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + errs = append(errs, err) + } + + // skip repos that dont exist in vela + continue + } + + err = updateRepoInstallationID(ctx, webhook, r, db, webhook.Installation.ID) + if err != nil { + errs = append(errs, err) + } + } + + // set install_id for repos removed from the installation + for _, repo := range webhook.Installation.RepositoriesRemoved { + r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + errs = append(errs, err) + } + + // skip repos that dont exist in vela + continue + } + + err = updateRepoInstallationID(ctx, webhook, r, db, 0) + if err != nil { + errs = append(errs, err) + } + } + + // combine all errors + if len(errs) > 0 { + var combined error + for _, e := range errs { + if combined == nil { + combined = e + } else { + combined = errors.Wrap(combined, e.Error()) + } + } + return combined + } + + return nil +} + +// updateRepoInstallationID updates the installation ID for a repo. +func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r *types.Repo, db database.Interface, installID int64) error { + r.SetInstallID(installID) + + h := new(types.Hook) + h.SetNumber(webhook.Hook.GetNumber()) + h.SetSourceID(webhook.Hook.GetSourceID()) + h.SetWebhookID(webhook.Hook.GetWebhookID()) + h.SetCreated(webhook.Hook.GetCreated()) + h.SetHost(webhook.Hook.GetHost()) + h.SetEvent("installation") + h.SetStatus(webhook.Hook.GetStatus()) + + r, err := db.UpdateRepo(ctx, r) + if err != nil { + h.SetStatus(constants.StatusFailure) + h.SetError(err.Error()) + } + + h.Repo = r + + // number of times to retry + retryLimit := 3 + // implement a loop to process asynchronous operations with a retry limit + // + // Some operations taken during the webhook workflow can lead to race conditions + // failing to successfully process the request. This logic ensures we attempt our + // best efforts to handle these cases gracefully. + for i := 0; i < retryLimit; i++ { + // check if we're on the first iteration of the loop + if i > 0 { + // incrementally sleep in between retries + time.Sleep(time.Duration(i) * time.Second) + } + + // send API call to capture the last hook for the repo + lastHook, err := db.LastHookForRepo(ctx, r) + if err != nil { + // log the error for traceability + logrus.Error(err.Error()) + + // check if the retry limit has been exceeded + if i < retryLimit { + // continue to the next iteration of the loop + continue + } + + return err + } + + // set the Number field + if lastHook != nil { + h.SetNumber( + lastHook.GetNumber() + 1, + ) + } + + // send hook update to db + _, err = db.CreateHook(ctx, h) + if err != nil { + return err + } + + break + } + + return nil +} diff --git a/scm/github/webhook.go b/scm/github/webhook.go index d79deb718..9d3e8323d 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -60,7 +60,6 @@ func (c *client) ProcessWebhook(ctx context.Context, request *http.Request) (*in // parse the payload from the webhook event, err := github.ParseWebHook(github.WebHookType(request), payload) - if err != nil { return &internal.Webhook{Hook: h}, nil } @@ -77,6 +76,10 @@ func (c *client) ProcessWebhook(ctx context.Context, request *http.Request) (*in return c.processIssueCommentEvent(h, event) case *github.RepositoryEvent: return c.processRepositoryEvent(h, event) + case *github.InstallationEvent: + return c.processInstallationEvent(ctx, h, event) + case *github.InstallationRepositoriesEvent: + return c.processInstallationRepositoriesEvent(ctx, h, event) } return &internal.Webhook{Hook: h}, nil @@ -510,7 +513,6 @@ func (c *client) processIssueCommentEvent(h *api.Hook, payload *github.IssueComm } // processRepositoryEvent is a helper function to process the repository event. - func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryEvent) (*internal.Webhook, error) { logrus.Tracef("processing repository event GitHub webhook for %s", payload.GetRepo().GetFullName()) @@ -541,6 +543,47 @@ func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryE }, nil } +func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { + h.SetEvent(constants.EventRepository) + h.SetEventAction(payload.GetAction()) + + install := new(internal.Installation) + + install.Action = payload.GetAction() + install.ID = payload.GetInstallation().GetID() + install.Org = payload.GetInstallation().GetAccount().GetLogin() + + for _, repo := range payload.Repositories { + install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + } + + return &internal.Webhook{ + Hook: h, + Installation: install, + }, nil +} + +func (c *client) processInstallationRepositoriesEvent(ctx context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { + install := new(internal.Installation) + + install.Action = payload.GetAction() + install.ID = payload.GetInstallation().GetID() + install.Org = payload.GetInstallation().GetAccount().GetLogin() + + for _, repo := range payload.RepositoriesAdded { + install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + } + + for _, repo := range payload.RepositoriesRemoved { + install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) + } + + return &internal.Webhook{ + Hook: h, + Installation: install, + }, nil +} + // getDeliveryID gets the last 100 webhook deliveries for a repo and // finds the matching delivery id with the source id in the hook. func (c *client) getDeliveryID(ctx context.Context, ghClient *github.Client, h *api.Hook) (int64, error) { diff --git a/scm/service.go b/scm/service.go index a793a707d..d98d2c535 100644 --- a/scm/service.go +++ b/scm/service.go @@ -7,6 +7,7 @@ import ( "net/http" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" "github.com/go-vela/types/library" ) @@ -157,6 +158,9 @@ type Service interface { // RedeliverWebhook defines a function that // redelivers the webhook from the SCM. RedeliverWebhook(context.Context, *api.User, *api.Hook) error + // ProcessInstallation defines a function that + // processes an installation event. + ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error // TODO: Add convert functions to interface? } From f9e7d774aa0351fb93d13215689e38a6962b3272 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 22 Oct 2024 11:01:27 -0500 Subject: [PATCH 13/56] fix: remove install_id when app is deleted --- scm/github/installation.go | 12 +++++++++++- scm/github/webhook.go | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scm/github/installation.go b/scm/github/installation.go index 839e8dfff..3fe0f4185 100644 --- a/scm/github/installation.go +++ b/scm/github/installation.go @@ -22,6 +22,9 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, errs := []error{} + // if action is "deleted" then the RepositoriesAdded field will indicate the repositories that + // need to have install_id set to zero + // set install_id for repos added to the installation for _, repo := range webhook.Installation.RepositoriesAdded { r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) @@ -34,7 +37,14 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, continue } - err = updateRepoInstallationID(ctx, webhook, r, db, webhook.Installation.ID) + installID := webhook.Installation.ID + + // clear install_id if the installation is deleted + if webhook.Installation.Action == "deleted" { + installID = 0 + } + + err = updateRepoInstallationID(ctx, webhook, r, db, installID) if err != nil { errs = append(errs, err) } diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 9d3e8323d..dea62ac52 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -543,6 +543,7 @@ func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryE }, nil } +// processInstallationEvent is a helper function to process the installation event. func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { h.SetEvent(constants.EventRepository) h.SetEventAction(payload.GetAction()) @@ -563,6 +564,7 @@ func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payl }, nil } +// processInstallationRepositoriesEvent is a helper function to process the installation repositories event. func (c *client) processInstallationRepositoriesEvent(ctx context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { install := new(internal.Installation) From 69e89a1a7d94da1fa8aac2085243b16da7d0729f Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 22 Oct 2024 11:36:42 -0500 Subject: [PATCH 14/56] fix: use deleted action to specify added/removed repos --- api/auth/get_token.go | 9 +++------ scm/github/installation.go | 12 +----------- scm/github/webhook.go | 12 ++++++++++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/api/auth/get_token.go b/api/auth/get_token.go index d123c2716..ccc098c54 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" + "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -66,6 +67,7 @@ import ( func GetAuthToken(c *gin.Context) { // capture middleware values tm := c.MustGet("token-manager").(*token.Manager) + m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) ctx := c.Request.Context() @@ -76,12 +78,7 @@ func GetAuthToken(c *gin.Context) { var err error if c.Request.FormValue("setup_action") == "install" { - // todo: make this better... - // random todos: - // what if a repo is added to the installation before it exists in vela - // then we need to sync repo installID all the time. - // sadly, installID might change if it gets re-installed. - c.Redirect(http.StatusTemporaryRedirect, "https://git.target.com/") + c.Redirect(http.StatusTemporaryRedirect, "https://"+m.Source.Host) return } diff --git a/scm/github/installation.go b/scm/github/installation.go index 3fe0f4185..839e8dfff 100644 --- a/scm/github/installation.go +++ b/scm/github/installation.go @@ -22,9 +22,6 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, errs := []error{} - // if action is "deleted" then the RepositoriesAdded field will indicate the repositories that - // need to have install_id set to zero - // set install_id for repos added to the installation for _, repo := range webhook.Installation.RepositoriesAdded { r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) @@ -37,14 +34,7 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, continue } - installID := webhook.Installation.ID - - // clear install_id if the installation is deleted - if webhook.Installation.Action == "deleted" { - installID = 0 - } - - err = updateRepoInstallationID(ctx, webhook, r, db, installID) + err = updateRepoInstallationID(ctx, webhook, r, db, webhook.Installation.ID) if err != nil { errs = append(errs, err) } diff --git a/scm/github/webhook.go b/scm/github/webhook.go index dea62ac52..c53f14d6a 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -554,8 +554,16 @@ func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payl install.ID = payload.GetInstallation().GetID() install.Org = payload.GetInstallation().GetAccount().GetLogin() - for _, repo := range payload.Repositories { - install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + switch payload.GetAction() { + case "created": + for _, repo := range payload.Repositories { + install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + } + break + case "deleted": + for _, repo := range payload.Repositories { + install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) + } } return &internal.Webhook{ From 9aca5ff0b21db5b48b5c81c5ce1d20346e44b27d Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 23 Oct 2024 14:44:30 -0500 Subject: [PATCH 15/56] feat: webhook handlers, debug code cleanup, installation redirect --- Dockerfile | 11 ++------- api/auth/get_token.go | 25 ++++++++++++++++++--- api/webhook/post.go | 2 +- cmd/vela-server/server.go | 2 +- compiler/native/environment.go | 2 -- docker-compose.yml | 2 -- scm/github/github.go | 2 +- scm/github/installation.go | 41 ++++++++++++++++++++++------------ scm/github/repo.go | 6 ++--- scm/service.go | 3 +++ 10 files changed, 60 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index 071b8e451..48935bc32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,7 @@ FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367eff RUN apk add --update --no-cache ca-certificates -# FROM scratch -FROM golang +FROM scratch COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt @@ -15,10 +14,4 @@ ENV GODEBUG=netdns=go ADD release/vela-server /bin/ -# CMD ["/bin/vela-server"] - -# dlv wrapper - -EXPOSE 4000 -RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest -CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] +CMD ["/bin/vela-server"] \ No newline at end of file diff --git a/api/auth/get_token.go b/api/auth/get_token.go index ccc098c54..013cfa98a 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -5,13 +5,13 @@ package auth import ( "fmt" "net/http" + "strconv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" - "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -67,7 +67,6 @@ import ( func GetAuthToken(c *gin.Context) { // capture middleware values tm := c.MustGet("token-manager").(*token.Manager) - m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) ctx := c.Request.Context() @@ -77,8 +76,28 @@ func GetAuthToken(c *gin.Context) { var err error + // handle scm setup events + // setup_action==install represents the GitHub App installation callback redirect if c.Request.FormValue("setup_action") == "install" { - c.Redirect(http.StatusTemporaryRedirect, "https://"+m.Source.Host) + installID, err := strconv.ParseInt(c.Request.FormValue("installation_id"), 10, 0) + if err != nil { + retErr := fmt.Errorf("unable to parse installation_id: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + r, err := scm.FromContext(c).FinishInstallation(ctx, c.Request, installID) + if err != nil { + retErr := fmt.Errorf("unable to finish installation: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.Redirect(http.StatusTemporaryRedirect, r) return } diff --git a/api/webhook/post.go b/api/webhook/post.go index bd182c723..dd5fb6b89 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -144,7 +144,7 @@ func PostWebhook(c *gin.Context) { return } - c.JSON(http.StatusOK, "handled installation event!") + c.JSON(http.StatusOK, "installation processed successfully") return } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 0c1d5cb4e..7733b9c15 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -132,7 +132,7 @@ func server(c *cli.Context) error { metadata.Vela.OpenIDIssuer = oidcIssuer tm.Issuer = oidcIssuer - jitter := wait.Jitter(0*time.Second, 2.0) + jitter := wait.Jitter(5*time.Second, 2.0) logrus.Infof("retrieving initial platform settings after %v delay", jitter) diff --git a/compiler/native/environment.go b/compiler/native/environment.go index 09d7e5d12..e1d8dd9a5 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -331,13 +331,11 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, n env["VELA_HOST"] = notImplemented env["VELA_NETRC_MACHINE"] = notImplemented env["VELA_NETRC_PASSWORD"] = netrcPassword - logrus.Infof("using netrc password: %s", netrcPassword) env["VELA_NETRC_USERNAME"] = "x-oauth-basic" env["VELA_QUEUE"] = notImplemented env["VELA_RUNTIME"] = notImplemented env["VELA_SOURCE"] = notImplemented env["VELA_VERSION"] = notImplemented - env["VELA_VADER"] = "yes" env["CI"] = "true" // populate environment variables from metadata diff --git a/docker-compose.yml b/docker-compose.yml index 28e235860..a96b4b87f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,6 @@ services: restart: always ports: - '8080:8080' - # dlv - - '4000:4000' depends_on: postgres: condition: service_healthy diff --git a/scm/github/github.go b/scm/github/github.go index 901d42551..c5cd4b411 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -229,7 +229,7 @@ func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) } // helper function to return the GitHub App installation token. -func (c *client) newGithubAppInstallationToken(ctx context.Context, r *api.Repo, repos []string, permissions []string) (string, error) { +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions []string) (string, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient( &http.Client{Transport: c.AppsTransport}). diff --git a/scm/github/installation.go b/scm/github/installation.go index 839e8dfff..6d8b09830 100644 --- a/scm/github/installation.go +++ b/scm/github/installation.go @@ -4,7 +4,9 @@ package github import ( "context" + "fmt" "net/http" + "strings" "time" "github.com/go-vela/server/api/types" @@ -20,14 +22,14 @@ import ( func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, webhook *internal.Webhook, db database.Interface) error { c.Logger.Tracef("processing GitHub App installation") - errs := []error{} + errs := []string{} // set install_id for repos added to the installation for _, repo := range webhook.Installation.RepositoriesAdded { r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { - errs = append(errs, err) + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) } // skip repos that dont exist in vela @@ -36,7 +38,7 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, err = updateRepoInstallationID(ctx, webhook, r, db, webhook.Installation.ID) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) } } @@ -45,7 +47,7 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { - errs = append(errs, err) + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) } // skip repos that dont exist in vela @@ -54,21 +56,13 @@ func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, err = updateRepoInstallationID(ctx, webhook, r, db, 0) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) } } // combine all errors if len(errs) > 0 { - var combined error - for _, e := range errs { - if combined == nil { - combined = e - } else { - combined = errors.Wrap(combined, e.Error()) - } - } - return combined + return errors.New(strings.Join(errs, ", ")) } return nil @@ -142,3 +136,22 @@ func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r return nil } + +// FinishInstallation completes the web flow for a GitHub App installation, returning a redirect to the app installation page. +func (c *client) FinishInstallation(ctx context.Context, request *http.Request, installID int64) (string, error) { + c.Logger.Tracef("finishing GitHub App installation") + + githubAppClient, err := c.newGithubAppClient(ctx) + if err != nil { + return "", err + } + + install, _, err := githubAppClient.Apps.GetInstallation(ctx, installID) + if err != nil { + return "", err + } + + r := install.GetHTMLURL() + + return r, nil +} diff --git a/scm/github/repo.go b/scm/github/repo.go index bab84276d..b205372f1 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -682,7 +682,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App - t, err := c.newGithubAppInstallationToken(ctx, r, []string{}, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, []string{}) if err != nil { return 0, err } @@ -709,7 +709,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { // create client from GitHub App - t, err := c.newGithubAppInstallationToken(ctx, r, []string{}, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, []string{}) if err != nil { return err } @@ -806,7 +806,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo, // todo: pass in THIS repo to only get access to that repo // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation // maybe take an optional list of repos and permission set that is driven by yaml - t, err := c.newGithubAppInstallationToken(ctx, r, repositories, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, repositories, []string{}) if err != nil { logrus.Errorf("unable to get github app installation token: %v", err) } diff --git a/scm/service.go b/scm/service.go index d98d2c535..bf815bdaa 100644 --- a/scm/service.go +++ b/scm/service.go @@ -161,6 +161,9 @@ type Service interface { // ProcessInstallation defines a function that // processes an installation event. ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error + // ProcessInstallation defines a function that + // finishes an installation event and returns a web redirect. + FinishInstallation(context.Context, *http.Request, int64) (string, error) // TODO: Add convert functions to interface? } From 42381904b89020af403d5e9b3b413a3bda27c0fa Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 23 Oct 2024 14:45:21 -0500 Subject: [PATCH 16/56] chore: remove launch.json --- .vscode/launch.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index ecd3df700..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "configurations": [ - { - "name": "Connect to server", - "type": "go", - "request": "attach", - "mode": "remote", - "port": 4000, - "host": "127.0.0.1", - } - ] -} \ No newline at end of file From a08d7e55d8dbdd31bdd453b73e114f00572b950f Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 23 Oct 2024 14:45:57 -0500 Subject: [PATCH 17/56] chore: revert newline --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 48935bc32..1bc26a794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,4 @@ ENV GODEBUG=netdns=go ADD release/vela-server /bin/ -CMD ["/bin/vela-server"] \ No newline at end of file +CMD ["/bin/vela-server"] From c8affa76477e25ec914f50fbd2114dd2a6008564 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 23 Oct 2024 15:55:38 -0500 Subject: [PATCH 18/56] chore: code cleanup, revisions, yaml finalization --- api/auth/get_token.go | 9 +++-- api/step/plan.go | 6 +-- api/step/update.go | 6 +-- cmd/vela-server/scm.go | 4 +- compiler/engine.go | 2 + compiler/native/compile.go | 5 ++- compiler/native/environment.go | 12 +++--- compiler/native/native.go | 2 +- compiler/types/pipeline/git.go | 29 ++++++++++++-- compiler/types/pipeline/git_test.go | 31 +++++++++++++++ compiler/types/yaml/build.go | 10 +---- compiler/types/yaml/git.go | 28 ++++++++++++++ compiler/types/yaml/git_test.go | 60 +++++++++++++++++++++++++++++ scm/flags.go | 4 +- scm/github/github.go | 15 ++++---- scm/github/opts.go | 4 +- scm/github/repo.go | 2 +- scm/service.go | 16 ++++++-- scm/setup.go | 11 +++--- 19 files changed, 200 insertions(+), 56 deletions(-) create mode 100644 compiler/types/pipeline/git_test.go create mode 100644 compiler/types/yaml/git.go create mode 100644 compiler/types/yaml/git_test.go diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 013cfa98a..df1598222 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -52,6 +52,10 @@ import ( // "$ref": "#/definitions/Token" // '307': // description: Redirected for authentication +// '400': +// description: Bad Request +// schema: +// "$ref": "#/definitions/Error" // '401': // description: Unauthorized // schema: @@ -65,17 +69,16 @@ import ( // process a user logging in to Vela from // the API or UI. func GetAuthToken(c *gin.Context) { + var err error + // capture middleware values tm := c.MustGet("token-manager").(*token.Manager) l := c.MustGet("logger").(*logrus.Entry) - ctx := c.Request.Context() // capture the OAuth state if present oAuthState := c.Request.FormValue("state") - var err error - // handle scm setup events // setup_action==install represents the GitHub App installation callback redirect if c.Request.FormValue("setup_action") == "install" { diff --git a/api/step/plan.go b/api/step/plan.go index 4ac86813c..9f3de8655 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -65,12 +65,10 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetCreated(time.Now().UTC().Unix()) if len(c.ReportAs) > 0 { - // todo: is this okay if checks already exist? id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName(), b.GetEvent()) if err != nil { - // todo: need better error-handling - // in a perfect world we warn the user that they need to install the github app to get this to work - logrus.Warnf("unable to create checks for step: %v", err) + // todo: warn the user that they need to install the github app + logrus.Warnf("report_as checks skipped for step: %v", err) } else { s.SetCheckID(id) } diff --git a/api/step/update.go b/api/step/update.go index 7d3ebcddd..818357549 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -159,11 +159,7 @@ func UpdateStep(c *gin.Context) { err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit(), b.GetEvent()) if err != nil { - retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return + l.Warnf("checks skipped for step %s: %v", entry, err) } } diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index 9edc5fcd6..c1b80e010 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -20,14 +20,14 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) { Address: c.String("scm.addr"), ClientID: c.String("scm.client"), ClientSecret: c.String("scm.secret"), + AppID: c.Int64("scm.app.id"), + AppPrivateKey: c.String("scm.app.private_key"), ServerAddress: c.String("server-addr"), ServerWebhookAddress: c.String("scm.webhook.addr"), StatusContext: c.String("scm.context"), WebUIAddress: c.String("webui-addr"), Scopes: c.StringSlice("scm.scopes"), Tracing: tc, - GithubAppID: c.Int64("scm.app.id"), - GithubAppPrivateKey: c.String("scm.app.private_key"), } // setup the scm diff --git a/compiler/engine.go b/compiler/engine.go index f7790c988..2c8e4841d 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -147,6 +147,8 @@ type Engine interface { // WithLabel defines a function that sets // the label(s) in the Engine. WithLabels([]string) Engine + // WithSCM defines a function that sets + // the scm in the Engine. WithSCM(scm.Service) Engine // WithPrivateGitHub defines a function that sets // the private github client in the Engine. diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 05bc21acd..8f153c4f9 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -44,6 +44,9 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * return nil, nil, err } + // set git configurations after parsing them from the yaml configuration + c.WithGit(&p.Git) + // create the API pipeline object from the yaml configuration _pipeline := p.ToPipelineAPI() _pipeline.SetData(data) @@ -306,8 +309,6 @@ func (c *client) compileInline(ctx context.Context, p *yaml.Build, depth int) (* func (c *client) compileSteps(ctx context.Context, p *yaml.Build, _pipeline *api.Pipeline, tmpls map[string]*yaml.Template, r *pipeline.RuleData) (*pipeline.Build, *api.Pipeline, error) { var err error - c.git = &p.Git - // check if the pipeline disabled the clone if p.Metadata.Clone == nil || *p.Metadata.Clone { // inject the clone step diff --git a/compiler/native/environment.go b/compiler/native/environment.go index e1d8dd9a5..c86f89734 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -37,7 +37,7 @@ func (c *client) EnvironmentStage(s *yaml.Stage, globalEnv raw.StringSliceMap) ( // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) + t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -97,7 +97,7 @@ func (c *client) EnvironmentStep(s *yaml.Step, stageEnv raw.StringSliceMap) (*ya // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) + t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -164,7 +164,7 @@ func (c *client) EnvironmentServices(s yaml.ServiceSlice, globalEnv raw.StringSl // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) + t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -210,7 +210,7 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) + t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } @@ -270,11 +270,13 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic return s, nil } +// EnvironmentBuild injects environment variables +// for the build in a yaml configuration. func (c *client) EnvironmentBuild() map[string]string { // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.user, c.repo, c.git.Repositories) + t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) if err != nil { logrus.Errorf("couldnt get netrc password: %v", err) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 4d184e78f..716790866 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -244,7 +244,7 @@ func (c *client) WithSCM(_scm scm.Service) compiler.Engine { return c } -// WithGit sets the git access configurations in the Engine. +// WithGit sets the git configurations in the Engine. func (c *client) WithGit(g *yaml.Git) compiler.Engine { c.git = g diff --git a/compiler/types/pipeline/git.go b/compiler/types/pipeline/git.go index 8799e4d97..6d9784d72 100644 --- a/compiler/types/pipeline/git.go +++ b/compiler/types/pipeline/git.go @@ -2,13 +2,34 @@ package pipeline -// Git is the pipeline representation of the git block for a pipeline. +// Git is the pipeline representation of git configurations for a pipeline. // // swagger:model PipelineGit type Git struct { - Access *Access `json:"access,omitempty" yaml:"access,omitempty"` + Token *Token `json:"token,omitempty" yaml:"token,omitempty"` } -type Access struct { - Repositories []string `json:"repositories,omitempty" yaml:"repositories,omitempty"` +// Token is the pipeline representation of git token access configurations for a pipeline. +// +// swagger:model PipelineGitToken +type Token struct { + Repositories []string `json:"repositories,omitempty" yaml:"repositories,omitempty"` + Permissions map[string]string `json:"permissions,omitempty" yaml:"permissions,omitempty"` +} + +// Empty returns true if the provided struct is empty. +func (g *Git) Empty() bool { + // return true if every field is empty + if g.Token != nil { + if g.Token.Repositories != nil { + return false + } + + if g.Token.Permissions != nil { + return false + } + } + + // return false if any of the fields are provided + return true } diff --git a/compiler/types/pipeline/git_test.go b/compiler/types/pipeline/git_test.go new file mode 100644 index 000000000..aa7b328e5 --- /dev/null +++ b/compiler/types/pipeline/git_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +import "testing" + +func TestPipeline_Git_Empty(t *testing.T) { + // setup tests + tests := []struct { + git *Git + want bool + }{ + { + git: &Git{&Token{Repositories: []string{}}}, + want: false, + }, + { + git: new(Git), + want: true, + }, + } + + // run tests + for _, test := range tests { + got := test.git.Empty() + + if got != test.want { + t.Errorf("Empty is %v, want %t", got, test.want) + } + } +} diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index 9d22d0e97..7988e6905 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -7,18 +7,10 @@ import ( "github.com/go-vela/server/compiler/types/raw" ) -type Git struct { - Access `yaml:"access,omitempty" json:"access,omitempty" jsonschema:"description=Provide the git token specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#access"` -} - -type Access struct { - Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` -} - // Build is the yaml representation of a build for a pipeline. type Build struct { - Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` Version string `yaml:"version,omitempty" json:"version,omitempty" jsonschema:"required,minLength=1,description=Provide syntax version used to evaluate the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/version/"` + Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` Metadata Metadata `yaml:"metadata,omitempty" json:"metadata,omitempty" jsonschema:"description=Pass extra information.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/"` Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide global environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` Worker Worker `yaml:"worker,omitempty" json:"worker,omitempty" jsonschema:"description=Limit the pipeline to certain types of workers.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/"` diff --git a/compiler/types/yaml/git.go b/compiler/types/yaml/git.go new file mode 100644 index 000000000..a71c64f81 --- /dev/null +++ b/compiler/types/yaml/git.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import "github.com/go-vela/server/compiler/types/pipeline" + +// Git is the yaml representation of git configurations for a pipeline. +type Git struct { + Token `yaml:"token,omitempty" json:"token,omitempty" jsonschema:"description=Provide the git token specifications, primarily used for cloning.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#token"` +} + +// Token is the yaml representation of the git token. +// Only applies when using GitHub App installations. +type Token struct { + Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` + Permissions map[string]string `yaml:"permissions,omitempty" json:"permissions,omitempty" jsonschema:"description=Provide a list of repository permissions to apply to the git token.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#permissions"` +} + +// ToPipeline converts the Git type +// to a pipeline Git type. +func (g *Git) ToPipeline() *pipeline.Git { + return &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: g.Repositories, + Permissions: g.Permissions, + }, + } +} diff --git a/compiler/types/yaml/git_test.go b/compiler/types/yaml/git_test.go new file mode 100644 index 000000000..d13d6be48 --- /dev/null +++ b/compiler/types/yaml/git_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Git_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + git *Git + want *pipeline.Git + }{ + { + git: &Git{ + Token: Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{}, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{}, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.git.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/scm/flags.go b/scm/flags.go index fecb944ce..e3745a628 100644 --- a/scm/flags.go +++ b/scm/flags.go @@ -71,12 +71,12 @@ var Flags = []cli.Flag{ EnvVars: []string{"VELA_SCM_APP_ID", "SCM_APP_ID"}, FilePath: "/vela/scm/app_id", Name: "scm.app.id", - Usage: "(optional & experimental) ID for the GitHub App", + Usage: "set ID for the SCM App integration (GitHub App)", }, &cli.StringFlag{ EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY", "SCM_APP_PRIVATE_KEY"}, FilePath: "/vela/scm/app_private_key", Name: "scm.app.private_key", - Usage: "(optional & experimental) path to private key for the GitHub App", + Usage: "set value of base64 encoded SCM App integration (GitHub App) private key", }, } diff --git a/scm/github/github.go b/scm/github/github.go index c5cd4b411..c47a2a93e 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -46,6 +46,10 @@ type config struct { ClientID string // specifies the OAuth client secret from GitHub to use for the GitHub client ClientSecret string + // specifies the ID for the Vela GitHub App + AppID int64 + // specifies the App private key to use for the GitHub client when interacting with App resources + AppPrivateKey string // specifies the Vela server address to use for the GitHub client ServerAddress string // specifies the Vela server address that the scm provider should use to send Vela webhooks @@ -56,9 +60,6 @@ type config struct { WebUIAddress string // specifies the OAuth scopes to use for the GitHub client Scopes []string - // optional and experimental - GithubAppID int64 - GithubAppPrivateKey string } type client struct { @@ -125,11 +126,11 @@ func New(opts ...ClientOpt) (*client, error) { Scopes: githubScopes, } - if c.config.GithubAppID != 0 && len(c.config.GithubAppPrivateKey) > 0 { + if c.config.AppID != 0 && len(c.config.AppPrivateKey) > 0 { // todo: this log isnt accurate, it reads it directly as a string - c.Logger.Infof("sourcing private key from path: %s", c.config.GithubAppPrivateKey) + c.Logger.Infof("sourcing private key from path: %s", c.config.AppPrivateKey) - decodedPEM, err := base64.StdEncoding.DecodeString(c.config.GithubAppPrivateKey) + decodedPEM, err := base64.StdEncoding.DecodeString(c.config.AppPrivateKey) if err != nil { return nil, fmt.Errorf("error decoding base64: %w", err) } @@ -144,7 +145,7 @@ func New(opts ...ClientOpt) (*client, error) { return nil, fmt.Errorf("failed to parse RSA private key: %w", err) } - transport := ghinstallation.NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.GithubAppID, privateKey) + transport := ghinstallation.NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.AppID, privateKey) transport.BaseURL = c.config.API c.AppsTransport = transport diff --git a/scm/github/opts.go b/scm/github/opts.go index 7856a681e..88d5eb2ba 100644 --- a/scm/github/opts.go +++ b/scm/github/opts.go @@ -167,7 +167,7 @@ func WithGithubAppID(id int64) ClientOpt { c.Logger.Trace("configuring ID for GitHub App in github scm client") // set the ID for the GitHub App in the github client - c.config.GithubAppID = id + c.config.AppID = id return nil } @@ -179,7 +179,7 @@ func WithGithubPrivateKey(key string) ClientOpt { c.Logger.Trace("configuring private key for GitHub App in github scm client") // set the private key for the GitHub App in the github client - c.config.GithubAppPrivateKey = key + c.config.AppPrivateKey = key return nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index b205372f1..c3271ceba 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -799,7 +799,7 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, // GetNetrcPassword returns a clone token using the repo's github app installation if it exists. // If not, it defaults to the user OAuth token. -func (c *client) GetNetrcPassword(ctx context.Context, u *api.User, r *api.Repo, repositories []string) (string, error) { +func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, repositories []string) (string, error) { logrus.Infof("getting netrc password") // the app might not be installed diff --git a/scm/service.go b/scm/service.go index bf815bdaa..d7383ae9c 100644 --- a/scm/service.go +++ b/scm/service.go @@ -142,10 +142,9 @@ type Service interface { // GetHTMLURL defines a function that retrieves // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) - - GetNetrcPassword(context.Context, *api.User, *api.Repo, []string) (string, error) - CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) - UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error + // GetNetrc defines a function that returns the netrc + // password injected into build steps. + GetNetrcPassword(context.Context, *api.Repo, *api.User, []string) (string, error) // Webhook SCM Interface Functions @@ -158,6 +157,15 @@ type Service interface { // RedeliverWebhook defines a function that // redelivers the webhook from the SCM. RedeliverWebhook(context.Context, *api.User, *api.Hook) error + + // App Integration SCM Interface Functions + + // CreateChecks defines a function that creates + // a check for a given repo and check id. + CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) + // UpdateChecks defines a function that updates + // a check for a given repo and check id. + UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error // ProcessInstallation defines a function that // processes an installation event. ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error diff --git a/scm/setup.go b/scm/setup.go index 98a672289..9bd292b17 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -27,6 +27,10 @@ type Setup struct { ClientID string // specifies the OAuth client secret from the scm system to use for the scm client ClientSecret string + // specifies App integration id + AppID int64 + // specifies App integration private key + AppPrivateKey string // specifies the Vela server address to use for the scm client ServerAddress string // specifies the Vela server address that the scm provider should use to send Vela webhooks @@ -39,9 +43,6 @@ type Setup struct { Scopes []string // specifies OTel tracing configurations Tracing *tracing.Client - // specifies GitHub App installation configurations - GithubAppID int64 - GithubAppPrivateKey string } // Github creates and returns a Vela service capable of @@ -62,8 +63,8 @@ func (s *Setup) Github() (Service, error) { github.WithWebUIAddress(s.WebUIAddress), github.WithScopes(s.Scopes), github.WithTracing(s.Tracing), - github.WithGithubAppID(s.GithubAppID), - github.WithGithubPrivateKey(s.GithubAppPrivateKey), + github.WithGithubAppID(s.AppID), + github.WithGithubPrivateKey(s.AppPrivateKey), ) } From 8587d2c0afe00ff7e07397dd03c3ad42c9ba1676 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 23 Oct 2024 16:17:28 -0500 Subject: [PATCH 19/56] chore: more cleanup, wip customizable token permission set --- compiler/native/environment.go | 3 ++- internal/webhook.go | 19 +++++++------ scm/github/github.go | 49 +++++++++++++++++++++++++++++++--- scm/github/installation.go | 7 ++--- scm/github/repo.go | 6 ++--- 5 files changed, 66 insertions(+), 18 deletions(-) diff --git a/compiler/native/environment.go b/compiler/native/environment.go index c86f89734..45727262c 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -8,13 +8,14 @@ import ( "os" "strings" + "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/internal" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" ) // EnvironmentStages injects environment variables diff --git a/internal/webhook.go b/internal/webhook.go index 7171d7fce..8b4092e5e 100644 --- a/internal/webhook.go +++ b/internal/webhook.go @@ -22,6 +22,17 @@ type PullRequest struct { Labels []string } +// Installation defines the data pulled from an installation +// while processing a webhook. +// Only applies to GitHub Apps. +type Installation struct { + Action string + ID int64 + Org string + RepositoriesAdded []string + RepositoriesRemoved []string +} + // Webhook defines a struct that is used to return // the required data when processing webhook event // a for a source provider event. @@ -34,14 +45,6 @@ type Webhook struct { Installation *Installation } -type Installation struct { - Action string - ID int64 - Org string - RepositoriesAdded []string - RepositoriesRemoved []string -} - // ShouldSkip uses the build information // associated with the given hook to determine // whether the hook should be skipped. diff --git a/scm/github/github.go b/scm/github/github.go index c47a2a93e..265d05367 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -14,13 +14,13 @@ import ( "strings" "github.com/bradleyfalzon/ghinstallation/v2" - api "github.com/go-vela/server/api/types" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/tracing" ) @@ -230,7 +230,7 @@ func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) } // helper function to return the GitHub App installation token. -func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions []string) (string, error) { +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions map[string]string) (string, error) { // create a github client based off the existing GitHub App configuration client, err := github.NewClient( &http.Client{Transport: c.AppsTransport}). @@ -239,9 +239,27 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R return "", err } + // todo: we want to support passing nothing to get the full permission set + // so move this outside of this function + // make the yaml provide a default when not provided, not the function + + // convert raw permissions to GitHub InstallationPermissions + perms := &github.InstallationPermissions{ + Contents: github.String("read"), + Checks: github.String("write"), + } + + for resource, perm := range permissions { + perms, err = WithGitHubInstallationPermission(perms, resource, perm) + } + + if repos == nil || len(repos) == 0 { + repos = []string{r.GetFullName()} + } + opts := &github.InstallationTokenOptions{ Repositories: repos, - Permissions: &github.InstallationPermissions{}, + Permissions: perms, } // if repo has an install ID, use it to create an installation token @@ -286,3 +304,28 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R return t.GetToken(), nil } + +// WithGitHubInstallationPermission takes permissions and applies a new permission if valid. +func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { + switch strings.ToLower(perm) { + case "read": + case "write": + case "none": + break + default: + return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) + } + + switch strings.ToLower(resource) { + case "contents": + perms.Contents = github.String(resource) + break + case "checks": + perms.Checks = github.String(resource) + break + default: + return perms, fmt.Errorf("invalid permission key given: %s", perm) + } + + return perms, nil +} diff --git a/scm/github/installation.go b/scm/github/installation.go index 6d8b09830..4f62dd47e 100644 --- a/scm/github/installation.go +++ b/scm/github/installation.go @@ -9,13 +9,14 @@ import ( "strings" "time" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "gorm.io/gorm" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "gorm.io/gorm" ) // ProcessInstallation takes a GitHub installation and processes the changes. diff --git a/scm/github/repo.go b/scm/github/repo.go index c3271ceba..5119f6846 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -682,7 +682,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // CreateChecks defines a function that does stuff... func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App - t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) if err != nil { return 0, err } @@ -709,7 +709,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks defines a function that does stuff... func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { // create client from GitHub App - t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) if err != nil { return err } @@ -806,7 +806,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, // todo: pass in THIS repo to only get access to that repo // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation // maybe take an optional list of repos and permission set that is driven by yaml - t, err := c.newGithubAppInstallationRepoToken(ctx, r, repositories, []string{}) + t, err := c.newGithubAppInstallationRepoToken(ctx, r, repositories, map[string]string{}) if err != nil { logrus.Errorf("unable to get github app installation token: %v", err) } From cc81d1b09cb2915c81ec831205715644a3ea77c8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 09:49:04 -0500 Subject: [PATCH 20/56] chore: gut required code from ghinstallation --- go.mod | 3 - go.sum | 6 - .../{installation.go => app_install.go} | 9 +- scm/github/app_transport.go | 282 ++++++++++++++++++ scm/github/github.go | 7 +- 5 files changed, 290 insertions(+), 17 deletions(-) rename scm/github/{installation.go => app_install.go} (98%) create mode 100644 scm/github/app_transport.go diff --git a/go.mod b/go.mod index 7da3f69a8..496fb2d59 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/adhocore/gronx v1.19.1 github.com/alicebob/miniredis/v2 v2.33.0 github.com/aws/aws-sdk-go v1.55.5 - github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/distribution/reference v0.6.0 github.com/drone/envsubst v1.0.3 @@ -88,9 +87,7 @@ require ( github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect - github.com/google/go-github/v62 v62.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/css v1.0.1 // indirect diff --git a/go.sum b/go.sum index d0b72b0ae..1957fb598 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,6 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53AL4ED5MzgEMwGTZag= -github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -108,8 +106,6 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -119,8 +115,6 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= -github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= github.com/google/go-github/v65 v65.0.0 h1:pQ7BmO3DZivvFk92geC0jB0q2m3gyn8vnYPgV7GSLhQ= github.com/google/go-github/v65 v65.0.0/go.mod h1:DvrqWo5hvsdhJvHd4WyVF9ttANN3BniqjP8uTFMNb60= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= diff --git a/scm/github/installation.go b/scm/github/app_install.go similarity index 98% rename from scm/github/installation.go rename to scm/github/app_install.go index 4f62dd47e..858f4aa1d 100644 --- a/scm/github/installation.go +++ b/scm/github/app_install.go @@ -4,19 +4,18 @@ package github import ( "context" + "errors" "fmt" "net/http" "strings" "time" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "gorm.io/gorm" - "github.com/go-vela/server/api/types" - "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + "gorm.io/gorm" ) // ProcessInstallation takes a GitHub installation and processes the changes. diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go new file mode 100644 index 000000000..ee29e4aae --- /dev/null +++ b/scm/github/app_transport.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "bytes" + "context" + "crypto/rsa" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/go-github/v65/github" +) + +const ( + acceptHeader = "application/vnd.github.v3+json" + apiBaseURL = "https://api.github.com" +) + +// AppsTransport provides a http.RoundTripper by wrapping an existing +// http.RoundTripper and provides GitHub Apps authentication as a GitHub App. +// +// Client can also be overwritten, and is useful to change to one which +// provides retry logic if you do experience retryable errors. +// +// See https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/about-authentication-options-for-github-apps/ +type AppsTransport struct { + BaseURL string // BaseURL is the scheme and host for GitHub API, defaults to https://api.github.com + Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport + tr http.RoundTripper // tr is the underlying roundtripper being wrapped + signer Signer // signer signs JWT tokens. + appID int64 // appID is the GitHub App's ID +} + +// NewAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). +func NewAppsTransportFromPrivateKey(tr http.RoundTripper, appID int64, key *rsa.PrivateKey) *AppsTransport { + return &AppsTransport{ + BaseURL: apiBaseURL, + Client: &http.Client{Transport: tr}, + tr: tr, + signer: NewRSASigner(jwt.SigningMethodRS256, key), + appID: appID, + } +} + +// RoundTrip implements http.RoundTripper interface. +func (t *AppsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // GitHub rejects expiry and issue timestamps that are not an integer, + // while the jwt-go library serializes to fractional timestamps. + // Truncate them before passing to jwt-go. + iss := time.Now().Add(-30 * time.Second).Truncate(time.Second) + exp := iss.Add(2 * time.Minute) + claims := &jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(iss), + ExpiresAt: jwt.NewNumericDate(exp), + Issuer: strconv.FormatInt(t.appID, 10), + } + + ss, err := t.signer.Sign(claims) + if err != nil { + return nil, fmt.Errorf("could not sign jwt: %s", err) + } + + req.Header.Set("Authorization", "Bearer "+ss) + req.Header.Add("Accept", acceptHeader) + + resp, err := t.tr.RoundTrip(req) + return resp, err +} + +// Transport provides a http.RoundTripper by wrapping an existing +// http.RoundTripper and provides GitHub Apps authentication as an installation. +// +// Client can also be overwritten, and is useful to change to one which +// provides retry logic if you do experience retryable errors. +// +// See https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/about-authentication-options-for-github-apps/ +type Transport struct { + BaseURL string // BaseURL is the scheme and host for GitHub API, defaults to https://api.github.com + Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport + tr http.RoundTripper // tr is the underlying roundtripper being wrapped + appID int64 // appID is the GitHub App's ID + installationID int64 // installationID is the GitHub App Installation ID + InstallationTokenOptions *github.InstallationTokenOptions // parameters restrict a token's access + appsTransport *AppsTransport + + mu *sync.Mutex + token *accessToken // the installation's access token +} + +// accessToken is an installation access token response from GitHub +type accessToken struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + Permissions github.InstallationPermissions `json:"permissions,omitempty"` + Repositories []github.Repository `json:"repositories,omitempty"` +} + +var _ http.RoundTripper = &Transport{} + +// Client is a HTTP client which sends a http.Request and returns a http.Response +// or an error. +type Client interface { + Do(*http.Request) (*http.Response, error) +} + +// RoundTrip implements http.RoundTripper interface. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + reqBodyClosed := false + if req.Body != nil { + defer func() { + if !reqBodyClosed { + req.Body.Close() + } + }() + } + + token, err := t.Token(req.Context()) + if err != nil { + return nil, err + } + + creq := cloneRequest(req) + creq.Header.Set("Authorization", "token "+token) + + if creq.Header.Get("Accept") == "" { + creq.Header.Add("Accept", acceptHeader) + } + + reqBodyClosed = true + resp, err := t.tr.RoundTrip(creq) + return resp, err +} + +// getRefreshTime returns the time when the token should be refreshed. +func (at *accessToken) getRefreshTime() time.Time { + return at.ExpiresAt.Add(-time.Minute) +} + +// isExpired checks if the access token is expired. +func (at *accessToken) isExpired() bool { + return at == nil || at.getRefreshTime().Before(time.Now()) +} + +// Token checks the active token expiration and renews if necessary. Token returns +// a valid access token. If renewal fails an error is returned. +func (t *Transport) Token(ctx context.Context) (string, error) { + t.mu.Lock() + defer t.mu.Unlock() + if t.token.isExpired() { + // token is not set or expired/nearly expired, so refresh + if err := t.refreshToken(ctx); err != nil { + return "", fmt.Errorf("could not refresh installation id %v's token: %w", t.installationID, err) + } + } + + return t.token.Token, nil +} + +// Expiry returns a transport token's expiration time and refresh time. There is a small grace period +// built in where a token will be refreshed before it expires. expiresAt is the actual token expiry, +// and refreshAt is when a call to Token() will cause it to be refreshed. +func (t *Transport) Expiry() (expiresAt time.Time, refreshAt time.Time, err error) { + if t.token == nil { + return time.Time{}, time.Time{}, errors.New("Expiry() = unknown, err: nil token") + } + return t.token.ExpiresAt, t.token.getRefreshTime(), nil +} + +func (t *Transport) refreshToken(ctx context.Context) error { + // convert InstallationTokenOptions into a ReadWriter to pass as an argument to http.NewRequest + body, err := GetReadWriter(t.InstallationTokenOptions) + if err != nil { + return fmt.Errorf("could not convert installation token parameters into json: %s", err) + } + + requestURL := fmt.Sprintf("%s/app/installations/%v/access_tokens", strings.TrimRight(t.BaseURL, "/"), t.installationID) + req, err := http.NewRequest("POST", requestURL, body) + if err != nil { + return fmt.Errorf("could not create request: %s", err) + } + + // set Content and Accept headers + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + req.Header.Set("Accept", acceptHeader) + + if ctx != nil { + req = req.WithContext(ctx) + } + + t.appsTransport.BaseURL = t.BaseURL + t.appsTransport.Client = t.Client + resp, err := t.appsTransport.RoundTrip(req) + if err != nil { + return fmt.Errorf("could not get access_tokens from GitHub API for installation ID %v: %v", t.installationID, err) + } + + if resp.StatusCode/100 != 2 { + return fmt.Errorf("received non 2xx response status %q when fetching %v", resp.Status, req.URL) + } + + // closing body late, to provide caller a chance to inspect body in an error / non-200 response status situation + defer resp.Body.Close() + + return json.NewDecoder(resp.Body).Decode(&t.token) +} + +// GetReadWriter converts a body interface into an io.ReadWriter object. +func GetReadWriter(i interface{}) (io.ReadWriter, error) { + var buf io.ReadWriter + if i != nil { + buf = new(bytes.Buffer) + enc := json.NewEncoder(buf) + err := enc.Encode(i) + if err != nil { + return nil, err + } + } + return buf, nil +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} + +// Signer is a JWT token signer. This is a wrapper around [jwt.SigningMethod] with predetermined +// key material. +type Signer interface { + // sign the given claims and returns a JWT token string, as specified + // by [jwt.Token.SignedString] + Sign(claims jwt.Claims) (string, error) +} + +// RSASigner signs JWT tokens using RSA keys. +type RSASigner struct { + method *jwt.SigningMethodRSA + key *rsa.PrivateKey +} + +// NewRSASigner creates a new RSASigner with the given RSA key. +func NewRSASigner(method *jwt.SigningMethodRSA, key *rsa.PrivateKey) *RSASigner { + return &RSASigner{ + method: method, + key: key, + } +} + +// Sign signs the JWT claims with the RSA key. +func (s *RSASigner) Sign(claims jwt.Claims) (string, error) { + return jwt.NewWithClaims(s.method, claims).SignedString(s.key) +} + +// AppsTransportOption is a func option for configuring an AppsTransport. +type AppsTransportOption func(*AppsTransport) + +// WithSigner configures the AppsTransport to use the given Signer for generating JWT tokens. +func WithSigner(signer Signer) AppsTransportOption { + return func(at *AppsTransport) { + at.signer = signer + } +} diff --git a/scm/github/github.go b/scm/github/github.go index 265d05367..a42c72487 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -13,7 +13,6 @@ import ( "net/url" "strings" - "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" @@ -67,7 +66,7 @@ type client struct { OAuth *oauth2.Config AuthReq *github.AuthorizationRequest Tracing *tracing.Client - AppsTransport *ghinstallation.AppsTransport + AppsTransport *AppsTransport // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -145,7 +144,8 @@ func New(opts ...ClientOpt) (*client, error) { return nil, fmt.Errorf("failed to parse RSA private key: %w", err) } - transport := ghinstallation.NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.AppID, privateKey) + fmt.Println("using custom round tripper") + transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.AppID, privateKey) transport.BaseURL = c.config.API c.AppsTransport = transport @@ -242,6 +242,7 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R // todo: we want to support passing nothing to get the full permission set // so move this outside of this function // make the yaml provide a default when not provided, not the function + // but also, only if the repo.InstallID is non-empty, for UX on /expand // convert raw permissions to GitHub InstallationPermissions perms := &github.InstallationPermissions{ From bd9cabed2828d60c4ff5a51674c0c5ef88589a31 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 10:04:30 -0500 Subject: [PATCH 21/56] chore: go mod tidy --- .vscode/launch.json | 12 ++++++++++++ Dockerfile | 11 +++++++++-- docker-compose.yml | 2 ++ go.mod | 4 +--- go.sum | 2 ++ 5 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..ecd3df700 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "port": 4000, + "host": "127.0.0.1", + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 1bc26a794..071b8e451 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367eff RUN apk add --update --no-cache ca-certificates -FROM scratch +# FROM scratch +FROM golang COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt @@ -14,4 +15,10 @@ ENV GODEBUG=netdns=go ADD release/vela-server /bin/ -CMD ["/bin/vela-server"] +# CMD ["/bin/vela-server"] + +# dlv wrapper + +EXPOSE 4000 +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest +CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] diff --git a/docker-compose.yml b/docker-compose.yml index a96b4b87f..28e235860 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,8 @@ services: restart: always ports: - '8080:8080' + # dlv + - '4000:4000' depends_on: postgres: condition: service_healthy diff --git a/go.mod b/go.mod index 496fb2d59..a417873a3 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/go-vela/server -go 1.23.2 - -replace github.com/go-vela/types => ../types +go 1.23.1 require ( github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb diff --git a/go.sum b/go.sum index 1957fb598..b1c403b2a 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,8 @@ github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27 github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-vela/types v0.25.0 h1:5jSXgW8uf2ODbhOiWdVmKtbznF/CfNIzkZSYuNQIars= +github.com/go-vela/types v0.25.0/go.mod h1:gyKVRQjNosAJy4AJ164CnEF6jIkwd1y6Cm5pZ6M20ZM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From 43f4ae8d45b07092196aaf924e9319fe4042ed5f Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 10:11:19 -0500 Subject: [PATCH 22/56] chore: revert local debug --- .vscode/launch.json | 12 ------------ Dockerfile | 11 ++--------- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index ecd3df700..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "configurations": [ - { - "name": "Connect to server", - "type": "go", - "request": "attach", - "mode": "remote", - "port": 4000, - "host": "127.0.0.1", - } - ] -} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 071b8e451..1bc26a794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,7 @@ FROM alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367eff RUN apk add --update --no-cache ca-certificates -# FROM scratch -FROM golang +FROM scratch COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt @@ -15,10 +14,4 @@ ENV GODEBUG=netdns=go ADD release/vela-server /bin/ -# CMD ["/bin/vela-server"] - -# dlv wrapper - -EXPOSE 4000 -RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest -CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] +CMD ["/bin/vela-server"] From 96f06a30a109e37bbea9df96a545f8714d1ef401 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 10:29:29 -0500 Subject: [PATCH 23/56] chore: merge with main, fixed --- api/types/report.go | 221 ++++++++++++++++++++++++++++++++++++++ api/types/step.go | 57 ++++++++++ scm/github/app_install.go | 2 +- scm/github/repo.go | 2 +- scm/service.go | 2 +- 5 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 api/types/report.go diff --git a/api/types/report.go b/api/types/report.go new file mode 100644 index 000000000..b24b90c28 --- /dev/null +++ b/api/types/report.go @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +// Report represents the Vela checks report for a build. +type Report struct { + Title *string `json:"title,omitempty"` + Summary *string `json:"summary,omitempty"` + Text *string `json:"text,omitempty"` + AnnotationsCount *int `json:"annotations_count,omitempty"` + AnnotationsURL *string `json:"annotations_url,omitempty"` + Annotations []*Annotation `json:"annotations,omitempty"` +} + +// Annotation represents the Vela annotation for a report. +type Annotation struct { + Path *string `json:"path,omitempty"` + StartLine *int `json:"start_line,omitempty"` + EndLine *int `json:"end_line,omitempty"` + StartColumn *int `json:"start_column,omitempty"` + EndColumn *int `json:"end_column,omitempty"` + AnnotationLevel *string `json:"annotation_level,omitempty"` + Message *string `json:"message,omitempty"` + Title *string `json:"title,omitempty"` + RawDetails *string `json:"raw_details,omitempty"` +} + +// GetTitle returns the Title field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetTitle() string { + // return zero value if Step type or ID field is nil + if r == nil || r.Title == nil { + return "" + } + + return *r.Title +} + +// GetSummary returns the Summary field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetSummary() string { + // return zero value if Step type or ID field is nil + if r == nil || r.Summary == nil { + return "" + } + + return *r.Summary +} + +// GetText returns the Text field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetText() string { + // return zero value if Step type or ID field is nil + if r == nil || r.Text == nil { + return "" + } + + return *r.Text +} + +// GetAnnotationsCount returns the AnnotationsCount field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetAnnotationsCount() int { + // return zero value if Step type or ID field is nil + if r == nil || r.AnnotationsCount == nil { + return 0 + } + + return *r.AnnotationsCount +} + +// GetAnnotationsURL returns the AnnotationsURL field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetAnnotationsURL() string { + // return zero value if Step type or ID field is nil + if r == nil || r.AnnotationsURL == nil { + return "" + } + + return *r.AnnotationsURL +} + +// GetAnnotations returns the Annotations field. +// +// When the provided Report type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Report) GetAnnotations() []*Annotation { + // return zero value if Step type or ID field is nil + if r == nil || r.Annotations == nil { + return []*Annotation{} + } + + return r.Annotations +} + +// GetPath returns the Path field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetPath() string { + // return zero value if Step type or ID field is nil + if a == nil || a.Path == nil { + return "" + } + + return *a.Path +} + +// GetStartLine returns the StartLine field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetStartLine() int { + // return zero value if Step type or ID field is nil + if a == nil || a.StartLine == nil { + return 0 + } + + return *a.StartLine +} + +// GetEndLine returns the EndLine field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetEndLine() int { + // return zero value if Step type or ID field is nil + if a == nil || a.EndLine == nil { + return 0 + } + + return *a.EndLine +} + +// GetStartColumn returns the StartColumn field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetStartColumn() int { + // return zero value if Step type or ID field is nil + if a == nil || a.StartColumn == nil { + return 0 + } + + return *a.StartColumn +} + +// GetEndColumn returns the EndColumn field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetEndColumn() int { + // return zero value if Step type or ID field is nil + if a == nil || a.EndColumn == nil { + return 0 + } + + return *a.EndColumn +} + +// GetAnnotationLevel returns the AnnotationLevel field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetAnnotationLevel() string { + // return zero value if Step type or ID field is nil + if a == nil || a.AnnotationLevel == nil { + return "" + } + + return *a.AnnotationLevel +} + +// GetMessage returns the Message field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetMessage() string { + // return zero value if Step type or ID field is nil + if a == nil || a.Message == nil { + return "" + } + + return *a.Message +} + +// GetTitle returns the Title field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetTitle() string { + // return zero value if Step type or ID field is nil + if a == nil || a.Title == nil { + return "" + } + + return *a.Title +} + +// GetRawDetails returns the RawDetails field. +// +// When the provided Annotation type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (a *Annotation) GetRawDetails() string { + // return zero value if Step type or ID field is nil + if a == nil || a.RawDetails == nil { + return "" + } + + return *a.RawDetails +} diff --git a/api/types/step.go b/api/types/step.go index 52d96f45c..73e44cba5 100644 --- a/api/types/step.go +++ b/api/types/step.go @@ -32,6 +32,8 @@ type Step struct { Runtime *string `json:"runtime,omitempty"` Distribution *string `json:"distribution,omitempty"` ReportAs *string `json:"report_as,omitempty"` + CheckID *int64 `json:"check_id,omitempty"` + Report *Report `json:"report,omitempty"` } // Duration calculates and returns the total amount of @@ -80,6 +82,7 @@ func (s *Step) Environment() map[string]string { "VELA_STEP_STARTED": ToString(s.GetStarted()), "VELA_STEP_STATUS": ToString(s.GetStatus()), "VELA_STEP_REPORT_AS": ToString(s.GetReportAs()), + "VELA_STEP_CHECK_ID": ToString(s.GetCheckID()), } } @@ -304,6 +307,32 @@ func (s *Step) GetReportAs() string { return *s.ReportAs } +// GetCheckID returns the CheckID field. +// +// When the provided Step type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Step) GetCheckID() int64 { + // return zero value if Step type or CheckID field is nil + if s == nil || s.CheckID == nil { + return 0 + } + + return *s.CheckID +} + +// GetReport returns the Report field. +// +// When the provided Step type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Step) GetReport() *Report { + // return zero value if Step type or ReportAs field is nil + if s == nil || s.Report == nil { + return new(Report) + } + + return s.Report +} + // SetID sets the ID field. // // When the provided Step type is nil, it @@ -525,6 +554,32 @@ func (s *Step) SetReportAs(v string) { s.ReportAs = &v } +// SetCheckID sets the CheckID field. +// +// When the provided Step type is nil, it +// will set nothing and immediately return. +func (s *Step) SetCheckID(v int64) { + // return if Step type is nil + if s == nil { + return + } + + s.CheckID = &v +} + +// SetReport sets the Report field. +// +// When the provided Step type is nil, it +// will set nothing and immediately return. +func (s *Step) SetReport(v *Report) { + // return if Step type is nil + if s == nil { + return + } + + s.Report = v +} + // String implements the Stringer interface for the Step type. func (s *Step) String() string { return fmt.Sprintf(`{ @@ -545,6 +600,7 @@ func (s *Step) String() string { Stage: %s, Started: %d, Status: %s, + CheckID: %d, }`, s.GetBuildID(), s.GetCreated(), @@ -563,6 +619,7 @@ func (s *Step) String() string { s.GetStage(), s.GetStarted(), s.GetStatus(), + s.GetCheckID(), ) } diff --git a/scm/github/app_install.go b/scm/github/app_install.go index 858f4aa1d..2ddd06de6 100644 --- a/scm/github/app_install.go +++ b/scm/github/app_install.go @@ -11,9 +11,9 @@ import ( "time" "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" - "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" "gorm.io/gorm" ) diff --git a/scm/github/repo.go b/scm/github/repo.go index a41e5725b..9b7e32c09 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -706,7 +706,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev } // UpdateChecks defines a function that does stuff... -func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *library.Step, commit, event string) error { +func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, commit, event string) error { // create client from GitHub App t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) if err != nil { diff --git a/scm/service.go b/scm/service.go index 98f98ae49..28e41043a 100644 --- a/scm/service.go +++ b/scm/service.go @@ -164,7 +164,7 @@ type Service interface { CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) // UpdateChecks defines a function that updates // a check for a given repo and check id. - UpdateChecks(context.Context, *api.Repo, *library.Step, string, string) error + UpdateChecks(context.Context, *api.Repo, *api.Step, string, string) error // ProcessInstallation defines a function that // processes an installation event. ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error From a21815707ec93a8898b214a912f10bf746082d00 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 10:30:28 -0500 Subject: [PATCH 24/56] fix: imports --- scm/github/app_install.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scm/github/app_install.go b/scm/github/app_install.go index 2ddd06de6..2e2b709e3 100644 --- a/scm/github/app_install.go +++ b/scm/github/app_install.go @@ -10,12 +10,13 @@ import ( "strings" "time" + "github.com/sirupsen/logrus" + "gorm.io/gorm" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" - "github.com/sirupsen/logrus" - "gorm.io/gorm" ) // ProcessInstallation takes a GitHub installation and processes the changes. From 949b3859c78ad77141cf3a7e6b6218b25039d2f3 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 11:06:46 -0500 Subject: [PATCH 25/56] chore: merge with main --- constants/event.go | 3 +++ docker-compose.yml | 2 -- scm/github/app_install.go | 8 +++----- scm/github/repo.go | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/constants/event.go b/constants/event.go index c2ec26aad..ed5cd9224 100644 --- a/constants/event.go +++ b/constants/event.go @@ -28,6 +28,9 @@ const ( // EventTag defines the event type for build and repo tag events. EventTag = "tag" + // EventInstallation defines the event type for scm installation events. + EventInstallation = "installation" + // Alternates for common user inputs that do not match our set constants. // EventPullAlternate defines the alternate event type for build and repo pull_request events. diff --git a/docker-compose.yml b/docker-compose.yml index 28e235860..a96b4b87f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,6 @@ services: restart: always ports: - '8080:8080' - # dlv - - '4000:4000' depends_on: postgres: condition: service_healthy diff --git a/scm/github/app_install.go b/scm/github/app_install.go index 2e2b709e3..58fa3517c 100644 --- a/scm/github/app_install.go +++ b/scm/github/app_install.go @@ -79,7 +79,7 @@ func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r h.SetWebhookID(webhook.Hook.GetWebhookID()) h.SetCreated(webhook.Hook.GetCreated()) h.SetHost(webhook.Hook.GetHost()) - h.SetEvent("installation") + h.SetEvent(constants.EventInstallation) h.SetStatus(webhook.Hook.GetStatus()) r, err := db.UpdateRepo(ctx, r) @@ -140,7 +140,7 @@ func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r // FinishInstallation completes the web flow for a GitHub App installation, returning a redirect to the app installation page. func (c *client) FinishInstallation(ctx context.Context, request *http.Request, installID int64) (string, error) { - c.Logger.Tracef("finishing GitHub App installation") + c.Logger.Tracef("finishing GitHub App installation for ID %d", installID) githubAppClient, err := c.newGithubAppClient(ctx) if err != nil { @@ -152,7 +152,5 @@ func (c *client) FinishInstallation(ctx context.Context, request *http.Request, return "", err } - r := install.GetHTMLURL() - - return r, nil + return install.GetHTMLURL(), nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index 9b7e32c09..d4a16acf2 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -678,7 +678,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str return data.GetName(), data.GetCommit().GetSHA(), nil } -// CreateChecks defines a function that does stuff... +// CreateChecks authenticates with the GitHub App and creates a check run for the repo. func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { // create client from GitHub App t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) @@ -705,7 +705,7 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev return check.GetID(), nil } -// UpdateChecks defines a function that does stuff... +// UpdateChecks authenticates with the GitHub App and updates a check run for the repo. func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, commit, event string) error { // create client from GitHub App t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) From 6efbf902232f03bc88fe5aea26fcf86568f31d60 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 11:44:06 -0500 Subject: [PATCH 26/56] chore: more cleanup --- scm/github/github.go | 2 -- scm/github/repo.go | 1 - 2 files changed, 3 deletions(-) diff --git a/scm/github/github.go b/scm/github/github.go index a42c72487..00f0ef96f 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -144,7 +144,6 @@ func New(opts ...ClientOpt) (*client, error) { return nil, fmt.Errorf("failed to parse RSA private key: %w", err) } - fmt.Println("using custom round tripper") transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.AppID, privateKey) transport.BaseURL = c.config.API @@ -274,7 +273,6 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R return t.GetToken(), nil } - // todo: pagination? // list all installations (a.k.a. orgs) where the GitHub App is installed installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) if err != nil { diff --git a/scm/github/repo.go b/scm/github/repo.go index d4a16acf2..246d34430 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -802,7 +802,6 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, logrus.Infof("getting netrc password") // the app might not be installed - // todo: pass in THIS repo to only get access to that repo // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation // maybe take an optional list of repos and permission set that is driven by yaml t, err := c.newGithubAppInstallationRepoToken(ctx, r, repositories, map[string]string{}) From 9df0c0468d251ef033e8a43d81bdd15c84b6e1b6 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 24 Oct 2024 16:01:23 -0500 Subject: [PATCH 27/56] enhance: code cleanup, moved netrc defaults --- compiler/native/compile.go | 26 +++++- compiler/native/compile_test.go | 106 +++++++++++------------ compiler/native/environment.go | 46 +++------- compiler/native/environment_test.go | 17 ++-- compiler/native/native.go | 19 +++-- compiler/native/script_test.go | 4 +- compiler/native/transform_test.go | 16 ++-- compiler/types/yaml/build.go | 4 - scm/github/github.go | 61 +++++++------- scm/github/repo.go | 126 ++++++++++++++++++++++------ scm/service.go | 14 ++-- 11 files changed, 258 insertions(+), 181 deletions(-) diff --git a/compiler/native/compile.go b/compiler/native/compile.go index d5f83cf91..9250f7a83 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -44,8 +44,30 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * return nil, nil, err } - // set git configurations after parsing them from the yaml configuration - c.WithGit(&p.Git) + // create the netrc using the scm + // this has to occur after Parse because the scm configurations might be set in yaml + // netrc can be provided directly using WithNetrc for situations like local exec + if c.netrc == nil && c.scm != nil { + // ensure restrictive defaults for the netrc for scms that support granular permissions + if p.Git.Repositories == nil { + p.Git.Repositories = []string{c.repo.GetName()} + } + + if p.Git.Permissions == nil { + p.Git.Permissions = map[string]string{ + "contents": "read", + "checks": "write", + } + } + + // get the netrc password from the scm + netrc, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, p.Git.Repositories, p.Git.Permissions) + if err != nil { + return nil, nil, err + } + + c.WithNetrc(netrc) + } // create the API pipeline object from the yaml configuration _pipeline := p.ToPipelineAPI() diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index c9c8c51d5..f47f21f6b 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -54,21 +54,21 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil, "") + initEnv := environment(nil, m, nil, nil, nil) initEnv["HELLO"] = "Hello, Global Environment" - stageEnvInstall := environment(nil, m, nil, nil, "") + stageEnvInstall := environment(nil, m, nil, nil, nil) stageEnvInstall["HELLO"] = "Hello, Global Environment" stageEnvInstall["GRADLE_USER_HOME"] = ".gradle" - stageEnvTest := environment(nil, m, nil, nil, "") + stageEnvTest := environment(nil, m, nil, nil, nil) stageEnvTest["HELLO"] = "Hello, Global Environment" stageEnvTest["GRADLE_USER_HOME"] = "willBeOverwrittenInStep" - cloneEnv := environment(nil, m, nil, nil, "") + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil, "") + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -76,7 +76,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil, "") + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -84,7 +84,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil, "") + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -92,7 +92,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil, "") + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -479,13 +479,13 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil, "") + initEnv := environment(nil, m, nil, nil, nil) initEnv["HELLO"] = "Hello, Global Environment" - cloneEnv := environment(nil, m, nil, nil, "") + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil, "") + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -493,7 +493,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil, "") + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -501,7 +501,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil, "") + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -509,7 +509,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil, "") + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -690,11 +690,11 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil, "") + setupEnv := environment(nil, m, nil, nil, nil) setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil, "") + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -703,7 +703,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil, "") + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -712,7 +712,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil, "") + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -721,14 +721,14 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil, "") + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil, "") + serviceEnv := environment(nil, m, nil, nil, nil) serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -961,11 +961,11 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil, "") + setupEnv := environment(nil, m, nil, nil, nil) setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil, "") + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -974,7 +974,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil, "") + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -983,7 +983,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil, "") + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -992,14 +992,14 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil, "") + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil, "") + serviceEnv := environment(nil, m, nil, nil, nil) serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -1195,9 +1195,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi }, } - setupEnv := environment(nil, m, nil, nil, "") + setupEnv := environment(nil, m, nil, nil, nil) - helloEnv := environment(nil, m, nil, nil, "") + helloEnv := environment(nil, m, nil, nil, nil) helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo sample"}) @@ -1316,9 +1316,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t }, } - setupEnv := environment(nil, m, nil, nil, "") + setupEnv := environment(nil, m, nil, nil, nil) - helloEnv := environment(nil, m, nil, nil, "") + helloEnv := environment(nil, m, nil, nil, nil) helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo inline_templatename"}) @@ -1436,11 +1436,11 @@ func TestNative_Compile_InvalidType(t *testing.T) { }, } - gradleEnv := environment(nil, m, nil, nil, "") + gradleEnv := environment(nil, m, nil, nil, nil) gradleEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" gradleEnv["GRADLE_USER_HOME"] = ".gradle" - dockerEnv := environment(nil, m, nil, nil, "") + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -1493,10 +1493,10 @@ func TestNative_Compile_Clone(t *testing.T) { }, } - fooEnv := environment(nil, m, nil, nil, "") + fooEnv := environment(nil, m, nil, nil, nil) fooEnv["PARAMETER_REGISTRY"] = "foo" - cloneEnv := environment(nil, m, nil, nil, "") + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["PARAMETER_DEPTH"] = "5" wantFalse := &pipeline.Build{ @@ -1512,7 +1512,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1543,7 +1543,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1552,7 +1552,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_clone", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Number: 2, @@ -1583,7 +1583,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1687,10 +1687,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { }, } - defaultFooEnv := environment(nil, m, nil, nil, "") + defaultFooEnv := environment(nil, m, nil, nil, nil) defaultFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultEnv := environment(nil, m, nil, nil, "") + defaultEnv := environment(nil, m, nil, nil, nil) wantDefault := &pipeline.Build{ Version: "1", ID: "__0", @@ -1733,10 +1733,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { goPipelineType := "go" - goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, "") + goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, nil) goFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, "") + defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, nil) wantGo := &pipeline.Build{ Version: "1", ID: "__0", @@ -1779,10 +1779,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { starPipelineType := "starlark" - starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, "") + starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, nil) starlarkFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, "") + defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, nil) wantStarlark := &pipeline.Build{ Version: "1", ID: "__0", @@ -2039,13 +2039,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2072,13 +2072,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil, ""), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2255,7 +2255,7 @@ func convertFileToGithubResponse(file string) (github.RepositoryContent, error) } func generateTestEnv(command string, m *internal.Metadata, pipelineType string) map[string]string { - output := environment(nil, m, nil, nil, "") + output := environment(nil, m, nil, nil, nil) output["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{command}) output["HOME"] = "/root" output["SHELL"] = "/bin/sh" @@ -2312,15 +2312,15 @@ func Test_Compile_Inline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil, "") - testEnv := environment(nil, m, nil, nil, "") + initEnv := environment(nil, m, nil, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["FOO"] = "Hello, foo!" testEnv["HELLO"] = "Hello, Vela!" - stepEnv := environment(nil, m, nil, nil, "") + stepEnv := environment(nil, m, nil, nil, nil) stepEnv["FOO"] = "Hello, foo!" stepEnv["HELLO"] = "Hello, Vela!" stepEnv["PARAMETER_FIRST"] = "foo" - golangEnv := environment(nil, m, nil, nil, "") + golangEnv := environment(nil, m, nil, nil, nil) golangEnv["VELA_REPO_PIPELINE_TYPE"] = "go" type args struct { diff --git a/compiler/native/environment.go b/compiler/native/environment.go index 94a94246d..3f3162137 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -3,13 +3,10 @@ package native import ( - "context" "fmt" "os" "strings" - "github.com/sirupsen/logrus" - api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/compiler/types/yaml" @@ -37,13 +34,8 @@ func (c *client) EnvironmentStage(s *yaml.Stage, globalEnv raw.StringSliceMap) ( // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) - if err != nil { - logrus.Errorf("couldnt get netrc password: %v", err) - } - // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -97,13 +89,8 @@ func (c *client) EnvironmentStep(s *yaml.Step, stageEnv raw.StringSliceMap) (*ya // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) - if err != nil { - logrus.Errorf("couldnt get netrc password: %v", err) - } - // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared stage environment // WARNING: local env can override global + stage @@ -164,13 +151,8 @@ func (c *client) EnvironmentServices(s yaml.ServiceSlice, globalEnv raw.StringSl // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) - if err != nil { - logrus.Errorf("couldnt get netrc password: %v", err) - } - // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -210,13 +192,8 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) - if err != nil { - logrus.Errorf("couldnt get netrc password: %v", err) - } - // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -276,13 +253,8 @@ func (c *client) EnvironmentBuild() map[string]string { // make empty map of environment variables env := make(map[string]string) - t, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, c.git.Repositories) - if err != nil { - logrus.Errorf("couldnt get netrc password: %v", err) - } - // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user, t) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the default environment // variables to the build @@ -316,7 +288,7 @@ func appendMap(originalMap, otherMap map[string]string) map[string]string { } // helper function that creates the standard set of environment variables for a pipeline. -func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, netrcPassword string) map[string]string { +func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, netrc *string) map[string]string { // set default workspace workspace := constants.WorkspaceDefault notImplemented := "TODO" @@ -332,7 +304,7 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, n env["VELA_DISTRIBUTION"] = notImplemented env["VELA_HOST"] = notImplemented env["VELA_NETRC_MACHINE"] = notImplemented - env["VELA_NETRC_PASSWORD"] = netrcPassword + env["VELA_NETRC_PASSWORD"] = notImplemented env["VELA_NETRC_USERNAME"] = "x-oauth-basic" env["VELA_QUEUE"] = notImplemented env["VELA_RUNTIME"] = notImplemented @@ -356,6 +328,10 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, n workspace = fmt.Sprintf("%s/%s/%s/%s", workspace, m.Source.Host, r.GetOrg(), r.GetName()) } + if netrc != nil { + env["VELA_NETRC_PASSWORD"] = *netrc + } + env["VELA_WORKSPACE"] = workspace // populate environment variables from repo api diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index 746f0b59c..a75eeb360 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -42,7 +42,7 @@ func TestNative_EnvironmentStages(t *testing.T) { }, } - env := environment(nil, nil, nil, nil, "") + env := environment(nil, nil, nil, nil, nil) env["HELLO"] = "Hello, Global Message" want := yaml.StageSlice{ @@ -582,12 +582,13 @@ func TestNative_environment(t *testing.T) { target := "production" tests := []struct { - w string - b *api.Build - m *internal.Metadata - r *api.Repo - u *api.User - want map[string]string + w string + b *api.Build + m *internal.Metadata + r *api.Repo + u *api.User + netrc *string + want map[string]string }{ // push { @@ -629,7 +630,7 @@ func TestNative_environment(t *testing.T) { // run test for _, test := range tests { - got := environment(test.b, test.m, test.r, test.u, "") + got := environment(test.b, test.m, test.r, test.u, test.netrc) if diff := cmp.Diff(got, test.want); diff != "" { t.Errorf("environment mismatch (-want +got):\n%s", diff) diff --git a/compiler/native/native.go b/compiler/native/native.go index add83957b..45d4e3ff8 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -15,7 +15,6 @@ import ( "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" - "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/image" "github.com/go-vela/server/scm" @@ -48,7 +47,7 @@ type client struct { user *api.User labels []string scm scm.Service - git *yaml.Git + netrc *string } // FromCLIContext returns a Pipeline implementation that integrates with the supported registries. @@ -237,6 +236,12 @@ func (c *client) WithLabels(labels []string) compiler.Engine { return c } +func (c *client) WithNetrc(n string) compiler.Engine { + c.netrc = &n + + return c +} + // WithSCM sets the scm in the Engine. func (c *client) WithSCM(_scm scm.Service) compiler.Engine { c.scm = _scm @@ -244,9 +249,9 @@ func (c *client) WithSCM(_scm scm.Service) compiler.Engine { return c } -// WithGit sets the git configurations in the Engine. -func (c *client) WithGit(g *yaml.Git) compiler.Engine { - c.git = g +// // WithGit sets the git configurations in the Engine. +// func (c *client) WithGit(g *yaml.Git) compiler.Engine { +// c.git = g - return c -} +// return c +// } diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 12c607da9..410c297c0 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -19,7 +19,7 @@ func TestNative_ScriptStages(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - baseEnv := environment(nil, nil, nil, nil, "") + baseEnv := environment(nil, nil, nil, nil, nil) s := yaml.StageSlice{ &yaml.Stage{ @@ -109,7 +109,7 @@ func TestNative_ScriptSteps(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - emptyEnv := environment(nil, nil, nil, nil, "") + emptyEnv := environment(nil, nil, nil, nil, nil) baseEnv := emptyEnv baseEnv["HOME"] = "/root" diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index f63a310ae..d2906bc8b 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -59,7 +59,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Pull: "always", @@ -72,7 +72,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -138,7 +138,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "__0_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Number: 1, @@ -194,7 +194,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "localOrg_localRepo_1_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Number: 1, @@ -297,14 +297,14 @@ func TestNative_TransformSteps(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Pull: "always", }, &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -365,7 +365,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step___0_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Number: 1, @@ -416,7 +416,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step_localOrg_localRepo_1_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil, ""), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Number: 1, diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index 7988e6905..4705cf083 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -88,10 +88,6 @@ func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { build.Metadata.Environment = []string{"steps", "services", "secrets"} } - if build.Git.Repositories == nil || len(build.Git.Repositories) == 0 { - build.Git.Repositories = []string{} - } - // override the values b.Git = build.Git b.Version = build.Version diff --git a/scm/github/github.go b/scm/github/github.go index 00f0ef96f..979bf0d12 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "encoding/base64" "encoding/pem" + "errors" "fmt" "net/http" "net/http/httptrace" @@ -126,8 +127,7 @@ func New(opts ...ClientOpt) (*client, error) { } if c.config.AppID != 0 && len(c.config.AppPrivateKey) > 0 { - // todo: this log isnt accurate, it reads it directly as a string - c.Logger.Infof("sourcing private key from path: %s", c.config.AppPrivateKey) + c.Logger.Infof("reading github app private key with length %d", len(c.config.AppPrivateKey)) decodedPEM, err := base64.StdEncoding.DecodeString(c.config.AppPrivateKey) if err != nil { @@ -148,6 +148,27 @@ func New(opts ...ClientOpt) (*client, error) { transport.BaseURL = c.config.API c.AppsTransport = transport + + // ensure the github app that was provided is valid + ccc, err := c.newGithubAppClient(context.Background()) + if err != nil { + return nil, fmt.Errorf("error creating github app client: %w", err) + } + + app, _, err := ccc.Apps.Get(context.Background(), "") + if err != nil { + return nil, fmt.Errorf("error getting github app: %w", err) + } + + perms := app.GetPermissions() + + if len(perms.GetContents()) == 0 || (perms.GetContents() != "read" && perms.GetContents() != "write") { + return nil, fmt.Errorf("github app requires contents:read permissions, found: %s", perms.GetContents()) + } + + if len(perms.GetChecks()) == 0 || perms.GetChecks() != "write" { + return nil, fmt.Errorf("github app requires checks:write permissions, found: %s", perms.GetChecks()) + } } return c, nil @@ -229,7 +250,8 @@ func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) } // helper function to return the GitHub App installation token. -func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions map[string]string) (string, error) { +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (string, error) { + // todo: create transport using context to apply tracing // create a github client based off the existing GitHub App configuration client, err := github.NewClient( &http.Client{Transport: c.AppsTransport}). @@ -238,28 +260,9 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R return "", err } - // todo: we want to support passing nothing to get the full permission set - // so move this outside of this function - // make the yaml provide a default when not provided, not the function - // but also, only if the repo.InstallID is non-empty, for UX on /expand - - // convert raw permissions to GitHub InstallationPermissions - perms := &github.InstallationPermissions{ - Contents: github.String("read"), - Checks: github.String("write"), - } - - for resource, perm := range permissions { - perms, err = WithGitHubInstallationPermission(perms, resource, perm) - } - - if repos == nil || len(repos) == 0 { - repos = []string{r.GetFullName()} - } - opts := &github.InstallationTokenOptions{ Repositories: repos, - Permissions: perms, + Permissions: permissions, } // if repo has an install ID, use it to create an installation token @@ -289,10 +292,8 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R } // failsafe in case the repo does not belong to an org where the GitHub App is installed - // todo: should this be an error? - // in reality we should warn them that they should install this app to their org and add this repo if id == 0 { - return "", nil + return "", errors.New("unable to find installation ID for repo") } // create installation token for the repo @@ -306,6 +307,7 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R // WithGitHubInstallationPermission takes permissions and applies a new permission if valid. func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { + // convert permissions from yaml string switch strings.ToLower(perm) { case "read": case "write": @@ -315,13 +317,12 @@ func WithGitHubInstallationPermission(perms *github.InstallationPermissions, res return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) } + // convert resource from yaml string switch strings.ToLower(resource) { case "contents": - perms.Contents = github.String(resource) - break + perms.Contents = github.String(perm) case "checks": - perms.Checks = github.String(resource) - break + perms.Checks = github.String(perm) default: return perms, fmt.Errorf("invalid permission key given: %s", perm) } diff --git a/scm/github/repo.go b/scm/github/repo.go index 246d34430..1b0ed276f 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -678,10 +678,97 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str return data.GetName(), data.GetCommit().GetSHA(), nil } +// GetNetrcPassword returns a clone token using the repo's github app installation if it exists. +// If not, it defaults to the user OAuth token. +func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, repos []string, perms map[string]string) (string, error) { + l := c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }) + + l.Tracef("getting netrc password for %s/%s", r.GetOrg(), r.GetName()) + + var err error + + // repos that the token has access to + // providing no repos, nil, or empty slice will default the token permissions to the list + // of repos added to the installation + // + // the compiler will set restrictive defaults with access to the triggering repo + if repos == nil { + repos = []string{} + } + + // convert repo fullname org/name to just name for usability + for i, repo := range repos { + split := strings.Split(repo, "/") + if len(split) == 2 { + repos[i] = split[1] + } + } + + // permissions that are applied to the token for every repo provided + // providing no permissions, nil, or empty map will default to the permissions + // of the GitHub App installation + // + // the Vela compiler follows a least-privileged-defaults model where + // the list contains only the triggering repo, unless provided in the git yaml block + // + // the default is contents:read and checks:write + ghPerms := &github.InstallationPermissions{ + Contents: github.String("read"), + Checks: github.String("write"), + } + + l.Info("using manual permissions set") + for resource, perm := range perms { + ghPerms, err = WithGitHubInstallationPermission(ghPerms, resource, perm) + if err != nil { + l.Errorf("unable to create github app installation token with permission %s:%s: %v", resource, perm, err) + + // return the legacy token along with no error for backwards compatibility + // todo: return an error based based on app installation requirements + return u.GetToken(), nil + } + } + + // the app might not be installed + // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation + // maybe take an optional list of repos and permission set that is driven by yaml + t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPerms) + if err != nil { + l.Errorf("unable to create github app installation token for repos %v with permissions %v: %v", repos, perms, err) + + // return the legacy token along with no error for backwards compatibility + // todo: return an error based based on app installation requirements + return u.GetToken(), nil + } + + if len(t) != 0 { + l.Tracef("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) + + return t, nil + } + + l.Tracef("using user oauth token for %s/%s", r.GetOrg(), r.GetName()) + + return u.GetToken(), nil +} + // CreateChecks authenticates with the GitHub App and creates a check run for the repo. func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { - // create client from GitHub App - t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) + c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("creating checks for %s/%s@%s", r.GetOrg(), r.GetName(), commit) + + repos := []string{r.GetFullName()} + perms := &github.InstallationPermissions{ + Contents: github.String("read"), + Checks: github.String("write"), + } + + t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, perms) if err != nil { return 0, err } @@ -707,8 +794,18 @@ func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, ev // UpdateChecks authenticates with the GitHub App and updates a check run for the repo. func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, commit, event string) error { - // create client from GitHub App - t, err := c.newGithubAppInstallationRepoToken(ctx, r, []string{}, map[string]string{}) + c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("updating checks for %s/%s@%s", r.GetOrg(), r.GetName(), commit) + + repos := []string{r.GetFullName()} + perms := &github.InstallationPermissions{ + Contents: github.String("read"), + Checks: github.String("write"), + } + + t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, perms) if err != nil { return err } @@ -795,24 +892,3 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, com return nil } - -// GetNetrcPassword returns a clone token using the repo's github app installation if it exists. -// If not, it defaults to the user OAuth token. -func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, repositories []string) (string, error) { - logrus.Infof("getting netrc password") - - // the app might not be installed - // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation - // maybe take an optional list of repos and permission set that is driven by yaml - t, err := c.newGithubAppInstallationRepoToken(ctx, r, repositories, map[string]string{}) - if err != nil { - logrus.Errorf("unable to get github app installation token: %v", err) - } - if len(t) != 0 { - logrus.Infof("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) - return t, nil - } - logrus.Infof("using user oauth token for %s/%s", r.GetOrg(), r.GetName()) - - return u.GetToken(), nil -} diff --git a/scm/service.go b/scm/service.go index 28e41043a..322cd0429 100644 --- a/scm/service.go +++ b/scm/service.go @@ -143,7 +143,13 @@ type Service interface { GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) // GetNetrc defines a function that returns the netrc // password injected into build steps. - GetNetrcPassword(context.Context, *api.Repo, *api.User, []string) (string, error) + GetNetrcPassword(context.Context, *api.Repo, *api.User, []string, map[string]string) (string, error) + // CreateChecks defines a function that creates + // a check for a given repo and check id. + CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) + // UpdateChecks defines a function that updates + // a check for a given repo and check id. + UpdateChecks(context.Context, *api.Repo, *api.Step, string, string) error // Webhook SCM Interface Functions @@ -159,12 +165,6 @@ type Service interface { // App Integration SCM Interface Functions - // CreateChecks defines a function that creates - // a check for a given repo and check id. - CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) - // UpdateChecks defines a function that updates - // a check for a given repo and check id. - UpdateChecks(context.Context, *api.Repo, *api.Step, string, string) error // ProcessInstallation defines a function that // processes an installation event. ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error From 9654dbc4b1bb39cc79f1e1a77c212b8f9475d260 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 25 Oct 2024 09:57:45 -0500 Subject: [PATCH 28/56] enhance: cleanup and organization --- cmd/vela-server/scm.go | 2 +- compiler/native/native.go | 8 +-- scm/github/app_client.go | 136 +++++++++++++++++++++++++++++++++++ scm/github/app_install.go | 4 +- scm/github/driver_test.go | 2 + scm/github/github.go | 144 +++----------------------------------- scm/github/github_test.go | 2 +- scm/github/opts_test.go | 19 ++--- scm/github/repo.go | 26 ++++++- scm/scm.go | 7 +- scm/scm_test.go | 3 +- scm/setup.go | 6 +- scm/setup_test.go | 7 +- 13 files changed, 200 insertions(+), 166 deletions(-) create mode 100644 scm/github/app_client.go diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index c1b80e010..08e416db6 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -33,5 +33,5 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) { // setup the scm // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#New - return scm.New(_setup) + return scm.New(c.Context, _setup) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 45d4e3ff8..6d72e8a8b 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -236,6 +236,7 @@ func (c *client) WithLabels(labels []string) compiler.Engine { return c } +// WithNetrc sets the netrc in the Engine. func (c *client) WithNetrc(n string) compiler.Engine { c.netrc = &n @@ -248,10 +249,3 @@ func (c *client) WithSCM(_scm scm.Service) compiler.Engine { return c } - -// // WithGit sets the git configurations in the Engine. -// func (c *client) WithGit(g *yaml.Git) compiler.Engine { -// c.git = g - -// return c -// } diff --git a/scm/github/app_client.go b/scm/github/app_client.go new file mode 100644 index 000000000..5fca6f35a --- /dev/null +++ b/scm/github/app_client.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/google/go-github/v65/github" + + api "github.com/go-vela/server/api/types" +) + +// NewGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. +func NewGitHubAppTransport(appID int64, privateKey, baseUrl string) (*AppsTransport, error) { + decodedPEM, err := base64.StdEncoding.DecodeString(privateKey) + if err != nil { + return nil, fmt.Errorf("error decoding base64: %w", err) + } + + block, _ := pem.Decode(decodedPEM) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") + } + + _privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse RSA private key: %w", err) + } + + transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, appID, _privateKey) + transport.BaseURL = baseUrl + + return transport, nil +} + +// ValidateGitHubApp ensures the GitHub App configuration is valid. +func (c *client) ValidateGitHubApp(ctx context.Context) error { + client, err := c.newGithubAppClient() + if err != nil { + return fmt.Errorf("error creating github app client: %w", err) + } + + app, _, err := client.Apps.Get(ctx, "") + if err != nil { + return fmt.Errorf("error getting github app: %w", err) + } + + perms := app.GetPermissions() + if len(perms.GetContents()) == 0 || + (perms.GetContents() != "read" && perms.GetContents() != "write") { + return fmt.Errorf("github app requires contents:read permissions, found: %s", perms.GetContents()) + } + + if len(perms.GetChecks()) == 0 || + perms.GetChecks() != "write" { + return fmt.Errorf("github app requires checks:write permissions, found: %s", perms.GetChecks()) + } + + return nil +} + +// newGithubAppClient returns the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. +func (c *client) newGithubAppClient() (*github.Client, error) { + // todo: create transport using context to apply tracing + // create a github client based off the existing GitHub App configuration + client, err := github.NewClient( + &http.Client{ + Transport: c.AppsTransport, + }). + WithEnterpriseURLs(c.config.API, c.config.API) + if err != nil { + return nil, err + } + + return client, nil +} + +// newGithubAppInstallationRepoToken returns the GitHub App installation token. +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (string, error) { + // create a github client based off the existing GitHub App configuration + client, err := c.newGithubAppClient() + if err != nil { + return "", err + } + + opts := &github.InstallationTokenOptions{ + Repositories: repos, + Permissions: permissions, + } + + // if repo has an install ID, use it to create an installation token + if r.GetInstallID() != 0 { + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(ctx, r.GetInstallID(), opts) + if err != nil { + return "", err + } + + return t.GetToken(), nil + } + + // list all installations (a.k.a. orgs) where the GitHub App is installed + installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil { + return "", err + } + + var id int64 + // iterate through the list of installations + for _, install := range installations { + // find the installation that matches the org for the repo + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + id = install.GetID() + } + } + + // failsafe in case the repo does not belong to an org where the GitHub App is installed + if id == 0 { + return "", errors.New("unable to find installation ID for repo") + } + + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(ctx, id, opts) + if err != nil { + return "", err + } + + return t.GetToken(), nil +} diff --git a/scm/github/app_install.go b/scm/github/app_install.go index 58fa3517c..14f3fab79 100644 --- a/scm/github/app_install.go +++ b/scm/github/app_install.go @@ -142,12 +142,12 @@ func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r func (c *client) FinishInstallation(ctx context.Context, request *http.Request, installID int64) (string, error) { c.Logger.Tracef("finishing GitHub App installation for ID %d", installID) - githubAppClient, err := c.newGithubAppClient(ctx) + client, err := c.newGithubAppClient() if err != nil { return "", err } - install, _, err := githubAppClient.Apps.GetInstallation(ctx, installID) + install, _, err := client.Apps.GetInstallation(ctx, installID) if err != nil { return "", err } diff --git a/scm/github/driver_test.go b/scm/github/driver_test.go index 7ea6656d8..9315cb626 100644 --- a/scm/github/driver_test.go +++ b/scm/github/driver_test.go @@ -3,6 +3,7 @@ package github import ( + "context" "reflect" "testing" @@ -14,6 +15,7 @@ func TestGitHub_Driver(t *testing.T) { want := constants.DriverGithub _service, err := New( + context.Background(), WithAddress("https://github.com/"), WithClientID("foo"), WithClientSecret("bar"), diff --git a/scm/github/github.go b/scm/github/github.go index 979bf0d12..9c7c1c048 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -4,15 +4,9 @@ package github import ( "context" - "crypto/x509" - "encoding/base64" - "encoding/pem" - "errors" "fmt" - "net/http" "net/http/httptrace" "net/url" - "strings" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" @@ -20,7 +14,6 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" - api "github.com/go-vela/server/api/types" "github.com/go-vela/server/tracing" ) @@ -76,7 +69,7 @@ type client struct { // a GitHub or a GitHub Enterprise instance. // //nolint:revive // ignore returning unexported client -func New(opts ...ClientOpt) (*client, error) { +func New(ctx context.Context, opts ...ClientOpt) (*client, error) { // create new GitHub client c := new(client) @@ -127,47 +120,18 @@ func New(opts ...ClientOpt) (*client, error) { } if c.config.AppID != 0 && len(c.config.AppPrivateKey) > 0 { - c.Logger.Infof("reading github app private key with length %d", len(c.config.AppPrivateKey)) + c.Logger.Infof("setting up GitHub App integration for App ID %d", c.config.AppID) - decodedPEM, err := base64.StdEncoding.DecodeString(c.config.AppPrivateKey) + transport, err := NewGitHubAppTransport(c.config.AppID, c.config.AppPrivateKey, c.config.API) if err != nil { - return nil, fmt.Errorf("error decoding base64: %w", err) - } - - block, _ := pem.Decode(decodedPEM) - if block == nil { - return nil, fmt.Errorf("failed to parse PEM block containing the key") - } - - privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse RSA private key: %w", err) + return nil, err } - transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, c.config.AppID, privateKey) - - transport.BaseURL = c.config.API c.AppsTransport = transport - // ensure the github app that was provided is valid - ccc, err := c.newGithubAppClient(context.Background()) - if err != nil { - return nil, fmt.Errorf("error creating github app client: %w", err) - } - - app, _, err := ccc.Apps.Get(context.Background(), "") + err = c.ValidateGitHubApp(ctx) if err != nil { - return nil, fmt.Errorf("error getting github app: %w", err) - } - - perms := app.GetPermissions() - - if len(perms.GetContents()) == 0 || (perms.GetContents() != "read" && perms.GetContents() != "write") { - return nil, fmt.Errorf("github app requires contents:read permissions, found: %s", perms.GetContents()) - } - - if len(perms.GetChecks()) == 0 || perms.GetChecks() != "write" { - return nil, fmt.Errorf("github app requires checks:write permissions, found: %s", perms.GetChecks()) + return nil, err } } @@ -190,6 +154,7 @@ func NewTest(urls ...string) (*client, error) { } return New( + context.Background(), WithAddress(address), WithClientID("foo"), WithClientSecret("bar"), @@ -201,7 +166,7 @@ func NewTest(urls ...string) (*client, error) { ) } -// helper function to return the GitHub OAuth client. +// newClientToken returns the GitHub OAuth client. func (c *client) newClientToken(ctx context.Context, token string) *github.Client { // create the token object for the client ts := oauth2.StaticTokenSource( @@ -236,96 +201,3 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien return github } - -// helper function to return the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. -func (c *client) newGithubAppClient(ctx context.Context) (*github.Client, error) { - // todo: create transport using context to apply tracing - // create a github client based off the existing GitHub App configuration - client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) - if err != nil { - return nil, err - } - - return client, nil -} - -// helper function to return the GitHub App installation token. -func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (string, error) { - // todo: create transport using context to apply tracing - // create a github client based off the existing GitHub App configuration - client, err := github.NewClient( - &http.Client{Transport: c.AppsTransport}). - WithEnterpriseURLs(c.config.API, c.config.API) - if err != nil { - return "", err - } - - opts := &github.InstallationTokenOptions{ - Repositories: repos, - Permissions: permissions, - } - - // if repo has an install ID, use it to create an installation token - if r.GetInstallID() != 0 { - // create installation token for the repo - t, _, err := client.Apps.CreateInstallationToken(context.Background(), r.GetInstallID(), opts) - if err != nil { - return "", err - } - - return t.GetToken(), nil - } - - // list all installations (a.k.a. orgs) where the GitHub App is installed - installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) - if err != nil { - return "", err - } - - var id int64 - // iterate through the list of installations - for _, install := range installations { - // find the installation that matches the org for the repo - if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { - id = install.GetID() - } - } - - // failsafe in case the repo does not belong to an org where the GitHub App is installed - if id == 0 { - return "", errors.New("unable to find installation ID for repo") - } - - // create installation token for the repo - t, _, err := client.Apps.CreateInstallationToken(context.Background(), id, opts) - if err != nil { - return "", err - } - - return t.GetToken(), nil -} - -// WithGitHubInstallationPermission takes permissions and applies a new permission if valid. -func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { - // convert permissions from yaml string - switch strings.ToLower(perm) { - case "read": - case "write": - case "none": - break - default: - return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) - } - - // convert resource from yaml string - switch strings.ToLower(resource) { - case "contents": - perms.Contents = github.String(perm) - case "checks": - perms.Checks = github.String(perm) - default: - return perms, fmt.Errorf("invalid permission key given: %s", perm) - } - - return perms, nil -} diff --git a/scm/github/github_test.go b/scm/github/github_test.go index 963aa8076..e497b66ec 100644 --- a/scm/github/github_test.go +++ b/scm/github/github_test.go @@ -32,7 +32,7 @@ func TestGithub_New(t *testing.T) { // run tests for _, test := range tests { - _, err := New( + _, err := New(context.Background(), WithAddress("https://github.com/"), WithClientID(test.id), WithClientSecret("bar"), diff --git a/scm/github/opts_test.go b/scm/github/opts_test.go index 8a6a2617e..bb4769b00 100644 --- a/scm/github/opts_test.go +++ b/scm/github/opts_test.go @@ -3,6 +3,7 @@ package github import ( + "context" "reflect" "testing" @@ -33,7 +34,7 @@ func TestGithub_ClientOpt_WithAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithAddress(test.address), ) @@ -72,7 +73,7 @@ func TestGithub_ClientOpt_WithClientID(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithClientID(test.id), ) @@ -115,7 +116,7 @@ func TestGithub_ClientOpt_WithClientSecret(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithClientSecret(test.secret), ) @@ -158,7 +159,7 @@ func TestGithub_ClientOpt_WithServerAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithServerAddress(test.address), ) @@ -210,7 +211,7 @@ func TestGithub_ClientOpt_WithServerWebhookAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithServerAddress(test.address), WithServerWebhookAddress(test.webhookAddress), ) @@ -254,7 +255,7 @@ func TestGithub_ClientOpt_WithStatusContext(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithStatusContext(test.context), ) @@ -294,7 +295,7 @@ func TestGithub_ClientOpt_WithWebUIAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithWebUIAddress(test.address), ) @@ -329,7 +330,7 @@ func TestGithub_ClientOpt_WithScopes(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithScopes(test.scopes), ) @@ -367,7 +368,7 @@ func TestGithub_ClientOpt_WithTracing(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithTracing(test.tracing), ) diff --git a/scm/github/repo.go b/scm/github/repo.go index 1b0ed276f..84e1f4c00 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -720,7 +720,6 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, Checks: github.String("write"), } - l.Info("using manual permissions set") for resource, perm := range perms { ghPerms, err = WithGitHubInstallationPermission(ghPerms, resource, perm) if err != nil { @@ -892,3 +891,28 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, com return nil } + +// WithGitHubInstallationPermission takes permissions and applies a new permission if valid. +func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { + // convert permissions from yaml string + switch strings.ToLower(perm) { + case "read": + case "write": + case "none": + break + default: + return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) + } + + // convert resource from yaml string + switch strings.ToLower(resource) { + case "contents": + perms.Contents = github.String(perm) + case "checks": + perms.Checks = github.String(perm) + default: + return perms, fmt.Errorf("invalid permission key given: %s", perm) + } + + return perms, nil +} diff --git a/scm/scm.go b/scm/scm.go index b93bde906..037a5d835 100644 --- a/scm/scm.go +++ b/scm/scm.go @@ -3,6 +3,7 @@ package scm import ( + "context" "fmt" "github.com/sirupsen/logrus" @@ -17,7 +18,7 @@ import ( // // * Github // . -func New(s *Setup) (Service, error) { +func New(ctx context.Context, s *Setup) (Service, error) { // validate the setup being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Validate @@ -33,12 +34,12 @@ func New(s *Setup) (Service, error) { // handle the Github scm driver being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Github - return s.Github() + return s.Github(ctx) case constants.DriverGitlab: // handle the Gitlab scm driver being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Gitlab - return s.Gitlab() + return s.Gitlab(ctx) default: // handle an invalid scm driver being provided return nil, fmt.Errorf("invalid scm driver provided: %s", s.Driver) diff --git a/scm/scm_test.go b/scm/scm_test.go index 233108758..e295f0042 100644 --- a/scm/scm_test.go +++ b/scm/scm_test.go @@ -3,6 +3,7 @@ package scm import ( + "context" "testing" ) @@ -72,7 +73,7 @@ func TestSCM_New(t *testing.T) { // run tests for _, test := range tests { - _, err := New(test.setup) + _, err := New(context.Background(), test.setup) if test.failure { if err == nil { diff --git a/scm/setup.go b/scm/setup.go index d6bb79b04..2585b13f4 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -3,6 +3,7 @@ package scm import ( + "context" "fmt" "strings" @@ -47,13 +48,14 @@ type Setup struct { // Github creates and returns a Vela service capable of // integrating with a Github scm system. -func (s *Setup) Github() (Service, error) { +func (s *Setup) Github(ctx context.Context) (Service, error) { logrus.Trace("creating github scm client from setup") // create new Github scm service // // https://pkg.go.dev/github.com/go-vela/server/scm/github?tab=doc#New return github.New( + ctx, github.WithAddress(s.Address), github.WithClientID(s.ClientID), github.WithClientSecret(s.ClientSecret), @@ -70,7 +72,7 @@ func (s *Setup) Github() (Service, error) { // Gitlab creates and returns a Vela service capable of // integrating with a Gitlab scm system. -func (s *Setup) Gitlab() (Service, error) { +func (s *Setup) Gitlab(ctx context.Context) (Service, error) { logrus.Trace("creating gitlab scm client from setup") return nil, fmt.Errorf("unsupported scm driver: %s", constants.DriverGitlab) diff --git a/scm/setup_test.go b/scm/setup_test.go index f3ad759a4..e177a8091 100644 --- a/scm/setup_test.go +++ b/scm/setup_test.go @@ -3,6 +3,7 @@ package scm import ( + "context" "reflect" "testing" ) @@ -21,7 +22,7 @@ func TestSCM_Setup_Github(t *testing.T) { Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, } - _github, err := _setup.Github() + _github, err := _setup.Github(context.Background()) if err != nil { t.Errorf("unable to setup scm: %v", err) } @@ -46,7 +47,7 @@ func TestSCM_Setup_Github(t *testing.T) { // run tests for _, test := range tests { - got, err := test.setup.Github() + got, err := test.setup.Github(context.Background()) if test.failure { if err == nil { @@ -80,7 +81,7 @@ func TestSCM_Setup_Gitlab(t *testing.T) { } // run test - got, err := _setup.Gitlab() + got, err := _setup.Gitlab(context.Background()) if err == nil { t.Errorf("Gitlab should have returned err") } From b60da201a93216873fadf33b064496e07ba39b39 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 25 Oct 2024 11:14:46 -0500 Subject: [PATCH 29/56] enhance: added repo sync to create and repair --- api/repo/create.go | 16 +++++++++++ api/repo/repair.go | 33 +++++++++++++++++++-- scm/github/app_client.go | 2 +- scm/github/repo.go | 62 ++++++++++++++++++++++++++++++++++++++++ scm/service.go | 3 ++ 5 files changed, 112 insertions(+), 4 deletions(-) diff --git a/api/repo/create.go b/api/repo/create.go index 3e35be3f0..097180e28 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -273,6 +273,20 @@ func CreateRepo(c *gin.Context) { } } + // map this repo to an installation if possible + if r.GetInstallID() == 0 { + r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + logrus.Warnf("repo %s has been synced with installation %d", r.GetFullName(), r.GetInstallID()) + // if the repo exists but is inactive if len(dbRepo.GetOrg()) > 0 && !dbRepo.GetActive() { // update the repo owner @@ -281,6 +295,8 @@ func CreateRepo(c *gin.Context) { dbRepo.SetBranch(r.GetBranch()) // activate the repo dbRepo.SetActive(true) + // update the install_id + dbRepo.SetInstallID(r.GetInstallID()) // send API call to update the repo // NOTE: not logging modification out separately diff --git a/api/repo/repair.go b/api/repo/repair.go index 2fd99f9e2..507c280f9 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -163,21 +163,48 @@ func RepairRepo(c *gin.Context) { } } + dirty := false + // if the repo was previously inactive, mark it as active if !r.GetActive() { r.SetActive(true) - // send API call to update the repo + dirty = true + + l.Tracef("repo %s repaired - set to active", r.GetFullName()) + } + + // map this repo to an installation, if possible + if r.GetInstallID() == 0 { + r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // install_id was synced + if r.GetInstallID() != 0 { + dirty = true + + l.Tracef("repo %s repaired - set install_id to %d", r.GetFullName(), r.GetInstallID()) + } + } + + // update the repo in the database, if necessary + if dirty { _, err := database.FromContext(c).UpdateRepo(ctx, r) if err != nil { - retErr := fmt.Errorf("unable to set repo %s to active: %w", r.GetFullName(), err) + retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) return } - l.Infof("repo %s updated - set to active", r.GetFullName()) + l.Infof("repo %s repaired - database updated", r.GetFullName()) } c.JSON(http.StatusOK, fmt.Sprintf("repo %s repaired", r.GetFullName())) diff --git a/scm/github/app_client.go b/scm/github/app_client.go index 5fca6f35a..aa848f052 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -82,7 +82,7 @@ func (c *client) newGithubAppClient() (*github.Client, error) { return client, nil } -// newGithubAppInstallationRepoToken returns the GitHub App installation token. +// newGithubAppInstallationRepoToken returns the GitHub App installation token for a particular repo with granular permissions. func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (string, error) { // create a github client based off the existing GitHub App configuration client, err := c.newGithubAppClient() diff --git a/scm/github/repo.go b/scm/github/repo.go index 84e1f4c00..58cdbb7d7 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -892,6 +892,68 @@ func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, com return nil } +// SyncRepoWithInstallation ensures the repo is synchronized with the scm installation, if it exists. +func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*api.Repo, error) { + c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("syncing app installation for repo %s/%s", r.GetOrg(), r.GetName()) + + client, err := c.newGithubAppClient() + if err != nil { + return r, err + } + + installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil { + return r, err + } + + var installation *github.Installation + for _, install := range installations { + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + installation = install + } + } + + if installation == nil { + return nil, nil + } + + installationCanReadRepo := false + if installation.GetRepositorySelection() != "all" { + client, err := c.newGithubAppClient() + if err != nil { + return r, err + } + + t, _, err := client.Apps.CreateInstallationToken(ctx, installation.GetID(), &github.InstallationTokenOptions{}) + if err != nil { + return r, err + } + + client = c.newClientToken(ctx, t.GetToken()) + + repos, _, err := client.Apps.ListRepos(ctx, &github.ListOptions{}) + if err != nil { + return r, err + } + + for _, repo := range repos.Repositories { + if strings.EqualFold(repo.GetFullName(), r.GetFullName()) { + installationCanReadRepo = true + break + } + } + } + + if installationCanReadRepo { + r.SetInstallID(installation.GetID()) + } + + return r, nil +} + // WithGitHubInstallationPermission takes permissions and applies a new permission if valid. func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { // convert permissions from yaml string diff --git a/scm/service.go b/scm/service.go index 322cd0429..ee57dbe05 100644 --- a/scm/service.go +++ b/scm/service.go @@ -144,6 +144,9 @@ type Service interface { // GetNetrc defines a function that returns the netrc // password injected into build steps. GetNetrcPassword(context.Context, *api.Repo, *api.User, []string, map[string]string) (string, error) + // SyncRepoWithInstallation defines a function that syncs + // a repo with the installation, if it exists. + SyncRepoWithInstallation(context.Context, *api.Repo) (*api.Repo, error) // CreateChecks defines a function that creates // a check for a given repo and check id. CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) From c01442479a22527dab64640402af8daad0f6748b Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 08:50:51 -0500 Subject: [PATCH 30/56] fix: remove checks --- api/repo/create.go | 2 - api/step/plan.go | 10 -- api/step/update.go | 9 -- api/types/report.go | 221 -------------------------------------------- api/types/step.go | 57 ------------ scm/github/repo.go | 139 ---------------------------- scm/service.go | 6 -- 7 files changed, 444 deletions(-) delete mode 100644 api/types/report.go diff --git a/api/repo/create.go b/api/repo/create.go index 097180e28..ae766c4c5 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -285,8 +285,6 @@ func CreateRepo(c *gin.Context) { } } - logrus.Warnf("repo %s has been synced with installation %d", r.GetFullName(), r.GetInstallID()) - // if the repo exists but is inactive if len(dbRepo.GetOrg()) > 0 && !dbRepo.GetActive() { // update the repo owner diff --git a/api/step/plan.go b/api/step/plan.go index 2603da1bb..61c93af86 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -63,16 +63,6 @@ func planStep(ctx context.Context, database database.Interface, scm scm.Service, s.SetReportAs(c.ReportAs) s.SetCreated(time.Now().UTC().Unix()) - if len(c.ReportAs) > 0 { - id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName(), b.GetEvent()) - if err != nil { - // todo: warn the user that they need to install the github app - logrus.Warnf("report_as checks skipped for step: %v", err) - } else { - s.SetCheckID(id) - } - } - // send API call to create the step s, err := database.CreateStep(ctx, s) if err != nil { diff --git a/api/step/update.go b/api/step/update.go index a2b0e890a..d456956d6 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -154,15 +154,6 @@ func UpdateStep(c *gin.Context) { return } - if s.GetCheckID() != 0 { - s.SetReport(input.GetReport()) - - err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit(), b.GetEvent()) - if err != nil { - l.Warnf("checks skipped for step %s: %v", entry, err) - } - } - c.JSON(http.StatusOK, s) // check if the build is in a "final" state diff --git a/api/types/report.go b/api/types/report.go deleted file mode 100644 index b24b90c28..000000000 --- a/api/types/report.go +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package types - -// Report represents the Vela checks report for a build. -type Report struct { - Title *string `json:"title,omitempty"` - Summary *string `json:"summary,omitempty"` - Text *string `json:"text,omitempty"` - AnnotationsCount *int `json:"annotations_count,omitempty"` - AnnotationsURL *string `json:"annotations_url,omitempty"` - Annotations []*Annotation `json:"annotations,omitempty"` -} - -// Annotation represents the Vela annotation for a report. -type Annotation struct { - Path *string `json:"path,omitempty"` - StartLine *int `json:"start_line,omitempty"` - EndLine *int `json:"end_line,omitempty"` - StartColumn *int `json:"start_column,omitempty"` - EndColumn *int `json:"end_column,omitempty"` - AnnotationLevel *string `json:"annotation_level,omitempty"` - Message *string `json:"message,omitempty"` - Title *string `json:"title,omitempty"` - RawDetails *string `json:"raw_details,omitempty"` -} - -// GetTitle returns the Title field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetTitle() string { - // return zero value if Step type or ID field is nil - if r == nil || r.Title == nil { - return "" - } - - return *r.Title -} - -// GetSummary returns the Summary field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetSummary() string { - // return zero value if Step type or ID field is nil - if r == nil || r.Summary == nil { - return "" - } - - return *r.Summary -} - -// GetText returns the Text field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetText() string { - // return zero value if Step type or ID field is nil - if r == nil || r.Text == nil { - return "" - } - - return *r.Text -} - -// GetAnnotationsCount returns the AnnotationsCount field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetAnnotationsCount() int { - // return zero value if Step type or ID field is nil - if r == nil || r.AnnotationsCount == nil { - return 0 - } - - return *r.AnnotationsCount -} - -// GetAnnotationsURL returns the AnnotationsURL field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetAnnotationsURL() string { - // return zero value if Step type or ID field is nil - if r == nil || r.AnnotationsURL == nil { - return "" - } - - return *r.AnnotationsURL -} - -// GetAnnotations returns the Annotations field. -// -// When the provided Report type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (r *Report) GetAnnotations() []*Annotation { - // return zero value if Step type or ID field is nil - if r == nil || r.Annotations == nil { - return []*Annotation{} - } - - return r.Annotations -} - -// GetPath returns the Path field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetPath() string { - // return zero value if Step type or ID field is nil - if a == nil || a.Path == nil { - return "" - } - - return *a.Path -} - -// GetStartLine returns the StartLine field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetStartLine() int { - // return zero value if Step type or ID field is nil - if a == nil || a.StartLine == nil { - return 0 - } - - return *a.StartLine -} - -// GetEndLine returns the EndLine field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetEndLine() int { - // return zero value if Step type or ID field is nil - if a == nil || a.EndLine == nil { - return 0 - } - - return *a.EndLine -} - -// GetStartColumn returns the StartColumn field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetStartColumn() int { - // return zero value if Step type or ID field is nil - if a == nil || a.StartColumn == nil { - return 0 - } - - return *a.StartColumn -} - -// GetEndColumn returns the EndColumn field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetEndColumn() int { - // return zero value if Step type or ID field is nil - if a == nil || a.EndColumn == nil { - return 0 - } - - return *a.EndColumn -} - -// GetAnnotationLevel returns the AnnotationLevel field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetAnnotationLevel() string { - // return zero value if Step type or ID field is nil - if a == nil || a.AnnotationLevel == nil { - return "" - } - - return *a.AnnotationLevel -} - -// GetMessage returns the Message field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetMessage() string { - // return zero value if Step type or ID field is nil - if a == nil || a.Message == nil { - return "" - } - - return *a.Message -} - -// GetTitle returns the Title field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetTitle() string { - // return zero value if Step type or ID field is nil - if a == nil || a.Title == nil { - return "" - } - - return *a.Title -} - -// GetRawDetails returns the RawDetails field. -// -// When the provided Annotation type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (a *Annotation) GetRawDetails() string { - // return zero value if Step type or ID field is nil - if a == nil || a.RawDetails == nil { - return "" - } - - return *a.RawDetails -} diff --git a/api/types/step.go b/api/types/step.go index 73e44cba5..52d96f45c 100644 --- a/api/types/step.go +++ b/api/types/step.go @@ -32,8 +32,6 @@ type Step struct { Runtime *string `json:"runtime,omitempty"` Distribution *string `json:"distribution,omitempty"` ReportAs *string `json:"report_as,omitempty"` - CheckID *int64 `json:"check_id,omitempty"` - Report *Report `json:"report,omitempty"` } // Duration calculates and returns the total amount of @@ -82,7 +80,6 @@ func (s *Step) Environment() map[string]string { "VELA_STEP_STARTED": ToString(s.GetStarted()), "VELA_STEP_STATUS": ToString(s.GetStatus()), "VELA_STEP_REPORT_AS": ToString(s.GetReportAs()), - "VELA_STEP_CHECK_ID": ToString(s.GetCheckID()), } } @@ -307,32 +304,6 @@ func (s *Step) GetReportAs() string { return *s.ReportAs } -// GetCheckID returns the CheckID field. -// -// When the provided Step type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Step) GetCheckID() int64 { - // return zero value if Step type or CheckID field is nil - if s == nil || s.CheckID == nil { - return 0 - } - - return *s.CheckID -} - -// GetReport returns the Report field. -// -// When the provided Step type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Step) GetReport() *Report { - // return zero value if Step type or ReportAs field is nil - if s == nil || s.Report == nil { - return new(Report) - } - - return s.Report -} - // SetID sets the ID field. // // When the provided Step type is nil, it @@ -554,32 +525,6 @@ func (s *Step) SetReportAs(v string) { s.ReportAs = &v } -// SetCheckID sets the CheckID field. -// -// When the provided Step type is nil, it -// will set nothing and immediately return. -func (s *Step) SetCheckID(v int64) { - // return if Step type is nil - if s == nil { - return - } - - s.CheckID = &v -} - -// SetReport sets the Report field. -// -// When the provided Step type is nil, it -// will set nothing and immediately return. -func (s *Step) SetReport(v *Report) { - // return if Step type is nil - if s == nil { - return - } - - s.Report = v -} - // String implements the Stringer interface for the Step type. func (s *Step) String() string { return fmt.Sprintf(`{ @@ -600,7 +545,6 @@ func (s *Step) String() string { Stage: %s, Started: %d, Status: %s, - CheckID: %d, }`, s.GetBuildID(), s.GetCreated(), @@ -619,7 +563,6 @@ func (s *Step) String() string { s.GetStage(), s.GetStarted(), s.GetStatus(), - s.GetCheckID(), ) } diff --git a/scm/github/repo.go b/scm/github/repo.go index 58cdbb7d7..0f1462db0 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -4,7 +4,6 @@ package github import ( "context" - "errors" "fmt" "net/http" "strconv" @@ -754,144 +753,6 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, return u.GetToken(), nil } -// CreateChecks authenticates with the GitHub App and creates a check run for the repo. -func (c *client) CreateChecks(ctx context.Context, r *api.Repo, commit, step, event string) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("creating checks for %s/%s@%s", r.GetOrg(), r.GetName(), commit) - - repos := []string{r.GetFullName()} - perms := &github.InstallationPermissions{ - Contents: github.String("read"), - Checks: github.String("write"), - } - - t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, perms) - if err != nil { - return 0, err - } - - if len(t) == 0 { - return 0, errors.New("unable to get github app installation token") - } - - client := c.newClientToken(ctx, t) - - opts := github.CreateCheckRunOptions{ - Name: fmt.Sprintf("vela-%s-%s", event, step), - HeadSHA: commit, - } - - check, _, err := client.Checks.CreateCheckRun(ctx, r.GetOrg(), r.GetName(), opts) - if err != nil { - return 0, err - } - - return check.GetID(), nil -} - -// UpdateChecks authenticates with the GitHub App and updates a check run for the repo. -func (c *client) UpdateChecks(ctx context.Context, r *api.Repo, s *api.Step, commit, event string) error { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("updating checks for %s/%s@%s", r.GetOrg(), r.GetName(), commit) - - repos := []string{r.GetFullName()} - perms := &github.InstallationPermissions{ - Contents: github.String("read"), - Checks: github.String("write"), - } - - t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, perms) - if err != nil { - return err - } - - if len(t) == 0 { - return errors.New("unable to get github app installation token") - } - - client := c.newClientToken(ctx, t) - - var ( - conclusion string - status string - ) - // set the conclusion and status for the step check depending on what the status of the step is - switch s.GetStatus() { - case constants.StatusPending: - conclusion = "neutral" - status = "queued" - case constants.StatusPendingApproval: - conclusion = "action_required" - status = "queued" - case constants.StatusRunning: - conclusion = "neutral" - status = "in_progress" - case constants.StatusSuccess: - conclusion = "success" - status = "completed" - case constants.StatusFailure: - conclusion = "failure" - status = "completed" - case constants.StatusCanceled: - conclusion = "cancelled" - status = "completed" - case constants.StatusKilled: - conclusion = "cancelled" - status = "completed" - case constants.StatusSkipped: - conclusion = "skipped" - status = "completed" - default: - conclusion = "neutral" - status = "completed" - } - - var annotations []*github.CheckRunAnnotation - - for _, reportAnnotation := range s.GetReport().GetAnnotations() { - annotation := &github.CheckRunAnnotation{ - Path: github.String(reportAnnotation.GetPath()), - StartLine: github.Int(reportAnnotation.GetStartLine()), - EndLine: github.Int(reportAnnotation.GetEndLine()), - StartColumn: github.Int(reportAnnotation.GetStartColumn()), - EndColumn: github.Int(reportAnnotation.GetEndColumn()), - AnnotationLevel: github.String(reportAnnotation.GetAnnotationLevel()), - Message: github.String(reportAnnotation.GetMessage()), - Title: github.String(reportAnnotation.GetTitle()), - RawDetails: github.String(reportAnnotation.GetRawDetails()), - } - - annotations = append(annotations, annotation) - } - - output := &github.CheckRunOutput{ - Title: github.String(s.GetReport().GetTitle()), - Summary: github.String(s.GetReport().GetSummary()), - Text: github.String(s.GetReport().GetText()), - AnnotationsCount: github.Int(s.GetReport().GetAnnotationsCount()), - AnnotationsURL: github.String(s.GetReport().GetAnnotationsURL()), - Annotations: annotations, - } - - opts := github.UpdateCheckRunOptions{ - Name: fmt.Sprintf("vela-%s-%s", event, s.GetName()), - Conclusion: github.String(conclusion), - Status: github.String(status), - Output: output, - } - - _, _, err = client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) - if err != nil { - return err - } - - return nil -} - // SyncRepoWithInstallation ensures the repo is synchronized with the scm installation, if it exists. func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*api.Repo, error) { c.Logger.WithFields(logrus.Fields{ diff --git a/scm/service.go b/scm/service.go index ee57dbe05..9e8ff96d9 100644 --- a/scm/service.go +++ b/scm/service.go @@ -147,12 +147,6 @@ type Service interface { // SyncRepoWithInstallation defines a function that syncs // a repo with the installation, if it exists. SyncRepoWithInstallation(context.Context, *api.Repo) (*api.Repo, error) - // CreateChecks defines a function that creates - // a check for a given repo and check id. - CreateChecks(context.Context, *api.Repo, string, string, string) (int64, error) - // UpdateChecks defines a function that updates - // a check for a given repo and check id. - UpdateChecks(context.Context, *api.Repo, *api.Step, string, string) error // Webhook SCM Interface Functions From 771d1a6925eeb2c57c11a179cbcfc824dc60f1e3 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 09:02:00 -0500 Subject: [PATCH 31/56] chore: cleanup and linting --- compiler/native/compile.go | 4 ++-- constants/app_install.go | 16 ++++++++++++++++ scm/github/app_client.go | 5 +++-- scm/github/app_transport.go | 6 +++++- scm/github/repo.go | 6 ++++-- 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 constants/app_install.go diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 5dc2d7008..2e5ab3c5c 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -55,8 +55,8 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * if p.Git.Permissions == nil { p.Git.Permissions = map[string]string{ - "contents": "read", - "checks": "write", + constants.AppInstallResourceContents: constants.AppInstallPermissionRead, + constants.AppInstallResourceChecks: constants.AppInstallPermissionWrite, } } diff --git a/constants/app_install.go b/constants/app_install.go new file mode 100644 index 000000000..fdde64bd8 --- /dev/null +++ b/constants/app_install.go @@ -0,0 +1,16 @@ +package constants + +// see: https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps?apiVersion=2022-11-28 +const ( + // The string value for GitHub App install read permissions. + AppInstallPermissionRead = "read" + // The string value for GitHub App install write permissions. + AppInstallPermissionWrite = "write" +) + +const ( + // The string value for GitHub App install contents resource. + AppInstallResourceContents = "contents" + // The string value for GitHub App install checks resource. + AppInstallResourceChecks = "checks" +) diff --git a/scm/github/app_client.go b/scm/github/app_client.go index aa848f052..27a8fa9e5 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-github/v65/github" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" ) // NewGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. @@ -54,12 +55,12 @@ func (c *client) ValidateGitHubApp(ctx context.Context) error { perms := app.GetPermissions() if len(perms.GetContents()) == 0 || - (perms.GetContents() != "read" && perms.GetContents() != "write") { + (perms.GetContents() != constants.AppInstallPermissionRead && perms.GetContents() != constants.AppInstallPermissionWrite) { return fmt.Errorf("github app requires contents:read permissions, found: %s", perms.GetContents()) } if len(perms.GetChecks()) == 0 || - perms.GetChecks() != "write" { + perms.GetChecks() != constants.AppInstallPermissionWrite { return fmt.Errorf("github app requires checks:write permissions, found: %s", perms.GetChecks()) } diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index ee29e4aae..deab34ecf 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -96,7 +96,7 @@ type Transport struct { token *accessToken // the installation's access token } -// accessToken is an installation access token response from GitHub +// accessToken is an installation access token response from GitHub. type accessToken struct { Token string `json:"token"` ExpiresAt time.Time `json:"expires_at"` @@ -115,6 +115,7 @@ type Client interface { // RoundTrip implements http.RoundTripper interface. func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { reqBodyClosed := false + if req.Body != nil { defer func() { if !reqBodyClosed { @@ -154,7 +155,9 @@ func (at *accessToken) isExpired() bool { // a valid access token. If renewal fails an error is returned. func (t *Transport) Token(ctx context.Context) (string, error) { t.mu.Lock() + defer t.mu.Unlock() + if t.token.isExpired() { // token is not set or expired/nearly expired, so refresh if err := t.refreshToken(ctx); err != nil { @@ -201,6 +204,7 @@ func (t *Transport) refreshToken(ctx context.Context) error { t.appsTransport.BaseURL = t.BaseURL t.appsTransport.Client = t.Client + resp, err := t.appsTransport.RoundTrip(req) if err != nil { return fmt.Errorf("could not get access_tokens from GitHub API for installation ID %v: %v", t.installationID, err) diff --git a/scm/github/repo.go b/scm/github/repo.go index 0f1462db0..cb45c37b7 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -715,8 +715,8 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, // // the default is contents:read and checks:write ghPerms := &github.InstallationPermissions{ - Contents: github.String("read"), - Checks: github.String("write"), + Contents: github.String(constants.AppInstallPermissionRead), + Checks: github.String(constants.AppInstallPermissionWrite), } for resource, perm := range perms { @@ -771,6 +771,7 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap } var installation *github.Installation + for _, install := range installations { if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { installation = install @@ -782,6 +783,7 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap } installationCanReadRepo := false + if installation.GetRepositorySelection() != "all" { client, err := c.newGithubAppClient() if err != nil { From 51960e23db88227bfecab35cdd12a42024fc1e47 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 09:06:38 -0500 Subject: [PATCH 32/56] chore: cleanup and linting --- api/step/plan.go | 6 +++--- constants/app_install.go | 3 +++ scm/github/app_transport.go | 26 +++++++++++++++++--------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/api/step/plan.go b/api/step/plan.go index 61c93af86..ff2f42996 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -28,7 +28,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service // iterate through all steps for each pipeline stage for _, step := range stage.Steps { // create the step object - s, err := planStep(ctx, database, scm, b, r, step, stage.Name) + s, err := planStep(ctx, database, scm, b, step, stage.Name) if err != nil { return steps, err } @@ -39,7 +39,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service // iterate through all pipeline steps for _, step := range p.Steps { - s, err := planStep(ctx, database, scm, b, r, step, "") + s, err := planStep(ctx, database, scm, b, step, "") if err != nil { return steps, err } @@ -50,7 +50,7 @@ func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service return steps, nil } -func planStep(ctx context.Context, database database.Interface, scm scm.Service, b *types.Build, r *types.Repo, c *pipeline.Container, stage string) (*types.Step, error) { +func planStep(ctx context.Context, database database.Interface, scm scm.Service, b *types.Build, c *pipeline.Container, stage string) (*types.Step, error) { // create the step object s := new(types.Step) s.SetBuildID(b.GetID()) diff --git a/constants/app_install.go b/constants/app_install.go index fdde64bd8..2a2d53cb3 100644 --- a/constants/app_install.go +++ b/constants/app_install.go @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 + +// App Install vars. package constants // see: https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps?apiVersion=2022-11-28 diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index deab34ecf..5038fe59a 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -72,8 +72,7 @@ func (t *AppsTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Set("Authorization", "Bearer "+ss) req.Header.Add("Accept", acceptHeader) - resp, err := t.tr.RoundTrip(req) - return resp, err + return t.tr.RoundTrip(req) } // Transport provides a http.RoundTripper by wrapping an existing @@ -182,13 +181,14 @@ func (t *Transport) refreshToken(ctx context.Context) error { // convert InstallationTokenOptions into a ReadWriter to pass as an argument to http.NewRequest body, err := GetReadWriter(t.InstallationTokenOptions) if err != nil { - return fmt.Errorf("could not convert installation token parameters into json: %s", err) + return fmt.Errorf("could not convert installation token parameters into json: %w", err) } requestURL := fmt.Sprintf("%s/app/installations/%v/access_tokens", strings.TrimRight(t.BaseURL, "/"), t.installationID) + req, err := http.NewRequest("POST", requestURL, body) if err != nil { - return fmt.Errorf("could not create request: %s", err) + return fmt.Errorf("could not create request: %w", err) } // set Content and Accept headers @@ -223,14 +223,18 @@ func (t *Transport) refreshToken(ctx context.Context) error { // GetReadWriter converts a body interface into an io.ReadWriter object. func GetReadWriter(i interface{}) (io.ReadWriter, error) { var buf io.ReadWriter + if i != nil { buf = new(bytes.Buffer) + enc := json.NewEncoder(buf) + err := enc.Encode(i) if err != nil { return nil, err } } + return buf, nil } @@ -238,14 +242,18 @@ func GetReadWriter(i interface{}) (io.ReadWriter, error) { // The clone is a shallow copy of the struct and its Header map. func cloneRequest(r *http.Request) *http.Request { // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r + _r := new(http.Request) + + *_r = *r + // deep copy of the Header - r2.Header = make(http.Header, len(r.Header)) + _r.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { - r2.Header[k] = append([]string(nil), s...) + _r.Header[k] = append([]string(nil), s...) } - return r2 + + return _r } // Signer is a JWT token signer. This is a wrapper around [jwt.SigningMethod] with predetermined From 6ec7bbcdcba01217d3f982d5ef8858fcc01d3ad0 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 09:22:13 -0500 Subject: [PATCH 33/56] chore: cleanup and linting --- api/build/plan.go | 2 +- api/step/plan.go | 2 +- scm/github/app_client.go | 4 ++-- scm/github/app_transport.go | 6 ++++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/api/build/plan.go b/api/build/plan.go index 7fa62def0..dc6f26c18 100644 --- a/api/build/plan.go +++ b/api/build/plan.go @@ -57,7 +57,7 @@ func PlanBuild(ctx context.Context, database database.Interface, scm scm.Service } // plan all steps for the build - steps, err := step.PlanSteps(ctx, database, scm, p, b, r) + steps, err := step.PlanSteps(ctx, database, scm, p, b) if err != nil { // clean up the objects from the pipeline in the database CleanBuild(ctx, database, b, services, steps, err) diff --git a/api/step/plan.go b/api/step/plan.go index ff2f42996..cb85d1c35 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -19,7 +19,7 @@ import ( // PlanSteps is a helper function to plan all steps // in the build for execution. This creates the steps // for the build. -func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build, r *types.Repo) ([]*types.Step, error) { +func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build) ([]*types.Step, error) { // variable to store planned steps steps := []*types.Step{} diff --git a/scm/github/app_client.go b/scm/github/app_client.go index 27a8fa9e5..57e5e4041 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -19,7 +19,7 @@ import ( ) // NewGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. -func NewGitHubAppTransport(appID int64, privateKey, baseUrl string) (*AppsTransport, error) { +func NewGitHubAppTransport(appID int64, privateKey, baseURL string) (*AppsTransport, error) { decodedPEM, err := base64.StdEncoding.DecodeString(privateKey) if err != nil { return nil, fmt.Errorf("error decoding base64: %w", err) @@ -36,7 +36,7 @@ func NewGitHubAppTransport(appID int64, privateKey, baseUrl string) (*AppsTransp } transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, appID, _privateKey) - transport.BaseURL = baseUrl + transport.BaseURL = baseURL return transport, nil } diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index 5038fe59a..070fc4ad8 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -37,7 +37,8 @@ type AppsTransport struct { Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport tr http.RoundTripper // tr is the underlying roundtripper being wrapped signer Signer // signer signs JWT tokens. - appID int64 // appID is the GitHub App's ID + //nolint:unused // ignore false positive + appID int64 // appID is the GitHub App's ID } // NewAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). @@ -174,6 +175,7 @@ func (t *Transport) Expiry() (expiresAt time.Time, refreshAt time.Time, err erro if t.token == nil { return time.Time{}, time.Time{}, errors.New("Expiry() = unknown, err: nil token") } + return t.token.ExpiresAt, t.token.getRefreshTime(), nil } @@ -207,7 +209,7 @@ func (t *Transport) refreshToken(ctx context.Context) error { resp, err := t.appsTransport.RoundTrip(req) if err != nil { - return fmt.Errorf("could not get access_tokens from GitHub API for installation ID %v: %v", t.installationID, err) + return fmt.Errorf("could not get access_tokens from GitHub API for installation ID %v: %w", t.installationID, err) } if resp.StatusCode/100 != 2 { From a3d0821f6453056487ac1bc4e0fa2ae79e28462c Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 11:12:03 -0500 Subject: [PATCH 34/56] chore: more linting --- compiler/types/yaml/build.go | 2 +- compiler/types/yaml/git.go | 2 +- scm/github/app_transport.go | 10 ++++------ scm/github/webhook.go | 5 ++--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index 07e0eae6e..b2a6e872b 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -18,7 +18,7 @@ type Build struct { Stages StageSlice `yaml:"stages,omitempty" json:"stages,omitempty" jsonschema:"oneof_required=stages,description=Provide parallel execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/"` Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"oneof_required=steps,description=Provide sequential execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/"` Templates TemplateSlice `yaml:"templates,omitempty" json:"templates,omitempty" jsonschema:"description=Provide the name of templates to expand.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/"` - Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` + Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` } // ToPipelineAPI converts the Build type to an API Pipeline type. diff --git a/compiler/types/yaml/git.go b/compiler/types/yaml/git.go index a71c64f81..7391714e5 100644 --- a/compiler/types/yaml/git.go +++ b/compiler/types/yaml/git.go @@ -13,7 +13,7 @@ type Git struct { // Only applies when using GitHub App installations. type Token struct { Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` - Permissions map[string]string `yaml:"permissions,omitempty" json:"permissions,omitempty" jsonschema:"description=Provide a list of repository permissions to apply to the git token.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#permissions"` + Permissions map[string]string `yaml:"permissions,omitempty" json:"permissions,omitempty" jsonschema:"description=Provide a list of repository permissions to apply to the git token.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#permissions"` } // ToPipeline converts the Git type diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index 070fc4ad8..9857f81fc 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -37,8 +37,7 @@ type AppsTransport struct { Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport tr http.RoundTripper // tr is the underlying roundtripper being wrapped signer Signer // signer signs JWT tokens. - //nolint:unused // ignore false positive - appID int64 // appID is the GitHub App's ID + appID int64 // appID is the GitHub App's ID } // NewAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). @@ -67,7 +66,7 @@ func (t *AppsTransport) RoundTrip(req *http.Request) (*http.Response, error) { ss, err := t.signer.Sign(claims) if err != nil { - return nil, fmt.Errorf("could not sign jwt: %s", err) + return nil, fmt.Errorf("could not sign jwt: %w", err) } req.Header.Set("Authorization", "Bearer "+ss) @@ -87,7 +86,6 @@ type Transport struct { BaseURL string // BaseURL is the scheme and host for GitHub API, defaults to https://api.github.com Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport tr http.RoundTripper // tr is the underlying roundtripper being wrapped - appID int64 // appID is the GitHub App's ID installationID int64 // installationID is the GitHub App Installation ID InstallationTokenOptions *github.InstallationTokenOptions // parameters restrict a token's access appsTransport *AppsTransport @@ -137,8 +135,8 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { } reqBodyClosed = true - resp, err := t.tr.RoundTrip(creq) - return resp, err + + return t.tr.RoundTrip(creq) } // getRefreshTime returns the time when the token should be refreshed. diff --git a/scm/github/webhook.go b/scm/github/webhook.go index a3bed07e1..54ed1dee5 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -544,7 +544,7 @@ func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryE } // processInstallationEvent is a helper function to process the installation event. -func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { +func (c *client) processInstallationEvent(_ context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { h.SetEvent(constants.EventRepository) h.SetEventAction(payload.GetAction()) @@ -559,7 +559,6 @@ func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payl for _, repo := range payload.Repositories { install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) } - break case "deleted": for _, repo := range payload.Repositories { install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) @@ -573,7 +572,7 @@ func (c *client) processInstallationEvent(ctx context.Context, h *api.Hook, payl } // processInstallationRepositoriesEvent is a helper function to process the installation repositories event. -func (c *client) processInstallationRepositoriesEvent(ctx context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { +func (c *client) processInstallationRepositoriesEvent(_ context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { install := new(internal.Installation) install.Action = payload.GetAction() From 47e29b4fcdfbae6c7e640708e870594454250524 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 11:13:49 -0500 Subject: [PATCH 35/56] chore: more linting --- scm/github/app_install.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scm/github/app_install.go b/scm/github/app_install.go index 14f3fab79..16c03924c 100644 --- a/scm/github/app_install.go +++ b/scm/github/app_install.go @@ -20,7 +20,7 @@ import ( ) // ProcessInstallation takes a GitHub installation and processes the changes. -func (c *client) ProcessInstallation(ctx context.Context, request *http.Request, webhook *internal.Webhook, db database.Interface) error { +func (c *client) ProcessInstallation(ctx context.Context, _ *http.Request, webhook *internal.Webhook, db database.Interface) error { c.Logger.Tracef("processing GitHub App installation") errs := []string{} @@ -139,7 +139,7 @@ func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r } // FinishInstallation completes the web flow for a GitHub App installation, returning a redirect to the app installation page. -func (c *client) FinishInstallation(ctx context.Context, request *http.Request, installID int64) (string, error) { +func (c *client) FinishInstallation(ctx context.Context, _ *http.Request, installID int64) (string, error) { c.Logger.Tracef("finishing GitHub App installation for ID %d", installID) client, err := c.newGithubAppClient() From 03c4232ce1918c84e9b6ada37cffb278120501c2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 11:14:29 -0500 Subject: [PATCH 36/56] chore: more linting --- compiler/types/pipeline/git.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/types/pipeline/git.go b/compiler/types/pipeline/git.go index 6d9784d72..eccf6f19c 100644 --- a/compiler/types/pipeline/git.go +++ b/compiler/types/pipeline/git.go @@ -14,7 +14,7 @@ type Git struct { // swagger:model PipelineGitToken type Token struct { Repositories []string `json:"repositories,omitempty" yaml:"repositories,omitempty"` - Permissions map[string]string `json:"permissions,omitempty" yaml:"permissions,omitempty"` + Permissions map[string]string `json:"permissions,omitempty" yaml:"permissions,omitempty"` } // Empty returns true if the provided struct is empty. From 836d5cffcf393dcf17ce09fd9acf090e9f57c3e9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 11:57:57 -0500 Subject: [PATCH 37/56] chore: constants, cleanup, permissions helper --- api/auth/get_token.go | 3 +- compiler/native/compile.go | 2 +- compiler/types/pipeline/git.go | 4 +-- compiler/types/yaml/build.go | 2 +- constants/app_install.go | 26 +++++++++++--- scm/github/app_client.go | 64 ++++++++++++++++++++++++++++++---- scm/github/repo.go | 12 +++---- scm/github/webhook.go | 4 +-- 8 files changed, 93 insertions(+), 24 deletions(-) diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 197a04b90..e82da4be6 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" @@ -80,7 +81,7 @@ func GetAuthToken(c *gin.Context) { // handle scm setup events // setup_action==install represents the GitHub App installation callback redirect - if c.Request.FormValue("setup_action") == "install" { + if c.Request.FormValue("setup_action") == constants.AppInstallSetupActionInstall { installID, err := strconv.ParseInt(c.Request.FormValue("installation_id"), 10, 0) if err != nil { retErr := fmt.Errorf("unable to parse installation_id: %w", err) diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 2e5ab3c5c..c751d9f9b 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -61,7 +61,7 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * } // get the netrc password from the scm - netrc, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, p.Git.Repositories, p.Git.Permissions) + netrc, err := c.scm.GetNetrcPassword(ctx, c.repo, c.user, p.Git.Repositories, p.Git.Permissions) if err != nil { return nil, nil, err } diff --git a/compiler/types/pipeline/git.go b/compiler/types/pipeline/git.go index eccf6f19c..a7628abf9 100644 --- a/compiler/types/pipeline/git.go +++ b/compiler/types/pipeline/git.go @@ -19,7 +19,7 @@ type Token struct { // Empty returns true if the provided struct is empty. func (g *Git) Empty() bool { - // return true if every field is empty + // return false if any of the fields are provided if g.Token != nil { if g.Token.Repositories != nil { return false @@ -30,6 +30,6 @@ func (g *Git) Empty() bool { } } - // return false if any of the fields are provided + // return true if all fields are empty return true } diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index b2a6e872b..008546fdb 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -65,7 +65,6 @@ func (b *Build) ToPipelineAPI() *api.Pipeline { func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { // build we try unmarshalling to build := new(struct { - Git Git Version string Metadata Metadata Environment raw.StringSliceMap @@ -75,6 +74,7 @@ func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { Stages StageSlice Steps StepSlice Templates TemplateSlice + Git Git }) // attempt to unmarshal as a build type diff --git a/constants/app_install.go b/constants/app_install.go index 2a2d53cb3..d9acf5f9a 100644 --- a/constants/app_install.go +++ b/constants/app_install.go @@ -5,15 +5,33 @@ package constants // see: https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps?apiVersion=2022-11-28 const ( - // The string value for GitHub App install read permissions. + // GitHub App install permission 'none'. + AppInstallPermissionNone = "none" + // GitHub App install permission 'read'. AppInstallPermissionRead = "read" - // The string value for GitHub App install write permissions. + // GitHub App install permission 'write'. AppInstallPermissionWrite = "write" ) const ( - // The string value for GitHub App install contents resource. + // GitHub App install contents resource. AppInstallResourceContents = "contents" - // The string value for GitHub App install checks resource. + // GitHub App install checks resource. AppInstallResourceChecks = "checks" ) + +const ( + // GitHub App install repositories selection when "all" repositories are selected. + AppInstallRepositoriesSelectionAll = "all" + // GitHub App install repositories selection when a subset of repositories are selected. + AppInstallRepositoriesSelectionSelected = "selected" +) + +const ( + // GitHub App install setup_action type 'install'. + AppInstallSetupActionInstall = "install" + // GitHub App install event type 'created'. + AppInstallCreated = "created" + // GitHub App install event type 'deleted'. + AppInstallDeleted = "deleted" +) diff --git a/scm/github/app_client.go b/scm/github/app_client.go index 57e5e4041..f91ba4605 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -54,14 +54,65 @@ func (c *client) ValidateGitHubApp(ctx context.Context) error { } perms := app.GetPermissions() - if len(perms.GetContents()) == 0 || - (perms.GetContents() != constants.AppInstallPermissionRead && perms.GetContents() != constants.AppInstallPermissionWrite) { - return fmt.Errorf("github app requires contents:read permissions, found: %s", perms.GetContents()) + + type perm struct { + resource string + requiredPermission string + actualPermission string + } + + // GitHub App installation requires the following permissions + // - contents:read + // - checks:write + requiredPermissions := []perm{ + { + resource: constants.AppInstallResourceContents, + requiredPermission: constants.AppInstallPermissionRead, + actualPermission: perms.GetContents(), + }, + { + resource: constants.AppInstallResourceChecks, + requiredPermission: constants.AppInstallPermissionWrite, + actualPermission: perms.GetChecks(), + }, + } + + for _, p := range requiredPermissions { + err := hasPermission(p.resource, p.requiredPermission, p.actualPermission) + if err != nil { + return err + } + } + + return nil +} + +// hasPermission takes a resource:perm pair and checks if the actual permission matches the expected permission or is supersceded by a higher permission. +func hasPermission(resource, requiredPerm, actualPerm string) error { + if len(actualPerm) == 0 { + return fmt.Errorf("github app missing permission %s:%s", resource, requiredPerm) + } + + permitted := false + + switch requiredPerm { + case constants.AppInstallPermissionNone: + permitted = true + case constants.AppInstallPermissionRead: + if actualPerm == constants.AppInstallPermissionRead || + actualPerm == constants.AppInstallPermissionWrite { + permitted = true + } + case constants.AppInstallPermissionWrite: + if actualPerm == constants.AppInstallPermissionWrite { + permitted = true + } + default: + return fmt.Errorf("invalid required permission type: %s", requiredPerm) } - if len(perms.GetChecks()) == 0 || - perms.GetChecks() != constants.AppInstallPermissionWrite { - return fmt.Errorf("github app requires checks:write permissions, found: %s", perms.GetChecks()) + if !permitted { + return fmt.Errorf("github app requires permission %s:%s, found: %s", constants.AppInstallResourceContents, constants.AppInstallPermissionRead, actualPerm) } return nil @@ -69,7 +120,6 @@ func (c *client) ValidateGitHubApp(ctx context.Context) error { // newGithubAppClient returns the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. func (c *client) newGithubAppClient() (*github.Client, error) { - // todo: create transport using context to apply tracing // create a github client based off the existing GitHub App configuration client, err := github.NewClient( &http.Client{ diff --git a/scm/github/repo.go b/scm/github/repo.go index cb45c37b7..ba10ca059 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -784,7 +784,7 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap installationCanReadRepo := false - if installation.GetRepositorySelection() != "all" { + if installation.GetRepositorySelection() != constants.AppInstallRepositoriesSelectionAll { client, err := c.newGithubAppClient() if err != nil { return r, err @@ -821,9 +821,9 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { // convert permissions from yaml string switch strings.ToLower(perm) { - case "read": - case "write": - case "none": + case constants.AppInstallPermissionNone: + case constants.AppInstallPermissionRead: + case constants.AppInstallPermissionWrite: break default: return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) @@ -831,9 +831,9 @@ func WithGitHubInstallationPermission(perms *github.InstallationPermissions, res // convert resource from yaml string switch strings.ToLower(resource) { - case "contents": + case constants.AppInstallResourceContents: perms.Contents = github.String(perm) - case "checks": + case constants.AppInstallResourceChecks: perms.Checks = github.String(perm) default: return perms, fmt.Errorf("invalid permission key given: %s", perm) diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 54ed1dee5..31c83fb42 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -555,11 +555,11 @@ func (c *client) processInstallationEvent(_ context.Context, h *api.Hook, payloa install.Org = payload.GetInstallation().GetAccount().GetLogin() switch payload.GetAction() { - case "created": + case constants.AppInstallCreated: for _, repo := range payload.Repositories { install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) } - case "deleted": + case constants.AppInstallDeleted: for _, repo := range payload.Repositories { install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) } From 858da17afa757c629e7a78c6df173628db789450 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 12:14:05 -0500 Subject: [PATCH 38/56] enhance: constants, helper funcs, reorganize client code, apply tracing to app transports --- compiler/registry/github/github.go | 6 +- compiler/registry/github/template.go | 2 +- scm/github/access.go | 10 +- scm/github/app_client.go | 142 ++++++++------------------- scm/github/app_transport.go | 43 +++++++- scm/github/authentication.go | 2 +- scm/github/changeset.go | 4 +- scm/github/deployment.go | 8 +- scm/github/github.go | 120 ++++++++++++++-------- scm/github/github_test.go | 2 +- scm/github/org.go | 2 +- scm/github/repo.go | 26 ++--- scm/github/user.go | 2 +- scm/github/webhook.go | 2 +- scm/github/webhook_test.go | 2 +- 15 files changed, 193 insertions(+), 180 deletions(-) diff --git a/compiler/registry/github/github.go b/compiler/registry/github/github.go index 09924393c..41be1dfcb 100644 --- a/compiler/registry/github/github.go +++ b/compiler/registry/github/github.go @@ -50,7 +50,7 @@ func New(ctx context.Context, address, token string) (*client, error) { if len(token) > 0 { // create GitHub OAuth client with user's token - gitClient = c.newClientToken(ctx, token) + gitClient = c.newOAuthTokenClient(ctx, token) } // overwrite the github client @@ -59,8 +59,8 @@ func New(ctx context.Context, address, token string) (*client, error) { return c, nil } -// newClientToken is a helper function to return the GitHub oauth2 client. -func (c *client) newClientToken(ctx context.Context, token string) *github.Client { +// newOAuthTokenClient is a helper function to return the GitHub oauth2 client. +func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github.Client { // create the token object for the client ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, diff --git a/compiler/registry/github/template.go b/compiler/registry/github/template.go index cd3dfc75f..8e0add5cc 100644 --- a/compiler/registry/github/template.go +++ b/compiler/registry/github/template.go @@ -19,7 +19,7 @@ func (c *client) Template(ctx context.Context, u *api.User, s *registry.Source) cli := c.Github if u != nil { // create GitHub OAuth client with user's token - cli = c.newClientToken(ctx, u.GetToken()) + cli = c.newOAuthTokenClient(ctx, u.GetToken()) } // create the options to pass diff --git a/scm/github/access.go b/scm/github/access.go index a1e7f5d4d..1bd4dd2e3 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -31,7 +31,7 @@ func (c *client) OrgAccess(ctx context.Context, u *api.User, org string) (string } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture org access level for user membership, _, err := client.Organizations.GetOrgMembership(ctx, *u.Name, org) @@ -67,7 +67,7 @@ func (c *client) RepoAccess(ctx context.Context, name, token, org, repo string) } // create github oauth client with the given token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture repo access level for user perm, _, err := client.Repositories.GetPermissionLevel(ctx, org, repo, name) @@ -98,7 +98,7 @@ func (c *client) TeamAccess(ctx context.Context, u *api.User, org, team string) } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -148,7 +148,7 @@ func (c *client) ListUsersTeamsForOrg(ctx context.Context, u *api.User, org stri }).Tracef("capturing %s team membership for org %s", u.GetName(), org) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -193,7 +193,7 @@ func (c *client) RepoContributor(ctx context.Context, owner *api.User, sender, o }).Tracef("capturing %s contributor status for repo %s/%s", sender, org, repo) // create GitHub OAuth client with repo owner's token - client := c.newClientToken(ctx, owner.GetToken()) + client := c.newOAuthTokenClient(ctx, owner.GetToken()) // set the max per page for the options to capture the list of repos opts := github.ListContributorsOptions{ diff --git a/scm/github/app_client.go b/scm/github/app_client.go index f91ba4605..de2243514 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -4,118 +4,54 @@ package github import ( "context" - "crypto/x509" - "encoding/base64" - "encoding/pem" "errors" - "fmt" "net/http" + "net/http/httptrace" + "net/url" "strings" "github.com/google/go-github/v65/github" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "golang.org/x/oauth2" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/constants" ) -// NewGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. -func NewGitHubAppTransport(appID int64, privateKey, baseURL string) (*AppsTransport, error) { - decodedPEM, err := base64.StdEncoding.DecodeString(privateKey) - if err != nil { - return nil, fmt.Errorf("error decoding base64: %w", err) - } - - block, _ := pem.Decode(decodedPEM) - if block == nil { - return nil, fmt.Errorf("failed to parse PEM block containing the key") - } - - _privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse RSA private key: %w", err) - } - - transport := NewAppsTransportFromPrivateKey(http.DefaultTransport, appID, _privateKey) - transport.BaseURL = baseURL - - return transport, nil -} - -// ValidateGitHubApp ensures the GitHub App configuration is valid. -func (c *client) ValidateGitHubApp(ctx context.Context) error { - client, err := c.newGithubAppClient() - if err != nil { - return fmt.Errorf("error creating github app client: %w", err) - } - - app, _, err := client.Apps.Get(ctx, "") - if err != nil { - return fmt.Errorf("error getting github app: %w", err) - } - - perms := app.GetPermissions() - - type perm struct { - resource string - requiredPermission string - actualPermission string - } - - // GitHub App installation requires the following permissions - // - contents:read - // - checks:write - requiredPermissions := []perm{ - { - resource: constants.AppInstallResourceContents, - requiredPermission: constants.AppInstallPermissionRead, - actualPermission: perms.GetContents(), - }, - { - resource: constants.AppInstallResourceChecks, - requiredPermission: constants.AppInstallPermissionWrite, - actualPermission: perms.GetChecks(), - }, - } - - for _, p := range requiredPermissions { - err := hasPermission(p.resource, p.requiredPermission, p.actualPermission) - if err != nil { - return err - } - } - - return nil -} - -// hasPermission takes a resource:perm pair and checks if the actual permission matches the expected permission or is supersceded by a higher permission. -func hasPermission(resource, requiredPerm, actualPerm string) error { - if len(actualPerm) == 0 { - return fmt.Errorf("github app missing permission %s:%s", resource, requiredPerm) - } - - permitted := false - - switch requiredPerm { - case constants.AppInstallPermissionNone: - permitted = true - case constants.AppInstallPermissionRead: - if actualPerm == constants.AppInstallPermissionRead || - actualPerm == constants.AppInstallPermissionWrite { - permitted = true - } - case constants.AppInstallPermissionWrite: - if actualPerm == constants.AppInstallPermissionWrite { - permitted = true - } - default: - return fmt.Errorf("invalid required permission type: %s", requiredPerm) - } - - if !permitted { - return fmt.Errorf("github app requires permission %s:%s, found: %s", constants.AppInstallResourceContents, constants.AppInstallPermissionRead, actualPerm) - } - - return nil +// newOAuthTokenClient returns the GitHub OAuth client. +func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github.Client { + // create the token object for the client + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + + // create the OAuth client + tc := oauth2.NewClient(ctx, ts) + // if c.SkipVerify { + // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ + // Proxy: http.ProxyFromEnvironment, + // TLSClientConfig: &tls.Config{ + // InsecureSkipVerify: true, + // }, + // } + // } + + if c.Tracing.Config.EnableTracing { + tc.Transport = otelhttp.NewTransport( + tc.Transport, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + }), + ) + } + + // create the GitHub client from the OAuth client + github := github.NewClient(tc) + + // ensure the proper URL is set in the GitHub client + github.BaseURL, _ = url.Parse(c.config.API) + + return github } // newGithubAppClient returns the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index 9857f81fc..c7c68a3b7 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -6,11 +6,15 @@ import ( "bytes" "context" "crypto/rsa" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "errors" "fmt" "io" "net/http" + "net/http/httptrace" "strconv" "strings" "sync" @@ -18,6 +22,8 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/google/go-github/v65/github" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) const ( @@ -40,8 +46,41 @@ type AppsTransport struct { appID int64 // appID is the GitHub App's ID } -// NewAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). -func NewAppsTransportFromPrivateKey(tr http.RoundTripper, appID int64, key *rsa.PrivateKey) *AppsTransport { +// newGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. +func (c *client) newGitHubAppTransport(appID int64, privateKey, baseURL string) (*AppsTransport, error) { + decodedPEM, err := base64.StdEncoding.DecodeString(privateKey) + if err != nil { + return nil, fmt.Errorf("error decoding base64: %w", err) + } + + block, _ := pem.Decode(decodedPEM) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") + } + + _privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse RSA private key: %w", err) + } + + transport := c.newAppsTransportFromPrivateKey(http.DefaultTransport, appID, _privateKey) + transport.BaseURL = baseURL + + // apply tracing to the transport + if c.Tracing.Config.EnableTracing { + transport.tr = otelhttp.NewTransport( + transport.tr, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + }), + ) + } + + return transport, nil +} + +// newAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). +func (c *client) newAppsTransportFromPrivateKey(tr http.RoundTripper, appID int64, key *rsa.PrivateKey) *AppsTransport { return &AppsTransport{ BaseURL: apiBaseURL, Client: &http.Client{Transport: tr}, diff --git a/scm/github/authentication.go b/scm/github/authentication.go index f7e86bb89..2d9c7f95c 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -21,7 +21,7 @@ func (c *client) Authorize(ctx context.Context, token string) (string, error) { c.Logger.Trace("authorizing user with token") // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture the current user making the call u, _, err := client.Users.Get(ctx, "") diff --git a/scm/github/changeset.go b/scm/github/changeset.go index 2aa07a445..7a9732fc4 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -21,7 +21,7 @@ func (c *client) Changeset(ctx context.Context, r *api.Repo, sha string) ([]stri }).Tracef("capturing commit changeset for %s/commit/%s", r.GetFullName(), sha) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) s := []string{} // set the max per page for the options to capture the commit @@ -50,7 +50,7 @@ func (c *client) ChangesetPR(ctx context.Context, r *api.Repo, number int) ([]st }).Tracef("capturing pull request changeset for %s/pull/%d", r.GetFullName(), number) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) s := []string{} f := []*github.CommitFile{} diff --git a/scm/github/deployment.go b/scm/github/deployment.go index e7f8ed0b4..f1c32db88 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -22,7 +22,7 @@ func (c *client) GetDeployment(ctx context.Context, u *api.User, r *api.Repo, id }).Tracef("capturing deployment %d for repo %s", id, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture the deployment deployment, _, err := client.Repositories.GetDeployment(ctx, r.GetOrg(), r.GetName(), id) @@ -63,7 +63,7 @@ func (c *client) GetDeploymentCount(ctx context.Context, u *api.User, r *api.Rep }).Tracef("counting deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // create variable to track the deployments deployments := []*github.Deployment{} @@ -105,7 +105,7 @@ func (c *client) GetDeploymentList(ctx context.Context, u *api.User, r *api.Repo }).Tracef("listing deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // set pagination options for listing deployments opts := &github.DeploymentsListOptions{ @@ -164,7 +164,7 @@ func (c *client) CreateDeployment(ctx context.Context, u *api.User, r *api.Repo, }).Tracef("creating deployment for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) var payload interface{} if d.Payload == nil { diff --git a/scm/github/github.go b/scm/github/github.go index 9c7c1c048..539a539a7 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -5,15 +5,12 @@ package github import ( "context" "fmt" - "net/http/httptrace" - "net/url" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" + "github.com/go-vela/server/constants" "github.com/go-vela/server/tracing" ) @@ -122,7 +119,7 @@ func New(ctx context.Context, opts ...ClientOpt) (*client, error) { if c.config.AppID != 0 && len(c.config.AppPrivateKey) > 0 { c.Logger.Infof("setting up GitHub App integration for App ID %d", c.config.AppID) - transport, err := NewGitHubAppTransport(c.config.AppID, c.config.AppPrivateKey, c.config.API) + transport, err := c.newGitHubAppTransport(c.config.AppID, c.config.AppPrivateKey, c.config.API) if err != nil { return nil, err } @@ -138,6 +135,83 @@ func New(ctx context.Context, opts ...ClientOpt) (*client, error) { return c, nil } +// ValidateGitHubApp ensures the GitHub App configuration is valid. +func (c *client) ValidateGitHubApp(ctx context.Context) error { + client, err := c.newGithubAppClient() + if err != nil { + return fmt.Errorf("error creating github app client: %w", err) + } + + app, _, err := client.Apps.Get(ctx, "") + if err != nil { + return fmt.Errorf("error getting github app: %w", err) + } + + perms := app.GetPermissions() + + type perm struct { + resource string + requiredPermission string + actualPermission string + } + + // GitHub App installation requires the following permissions + // - contents:read + // - checks:write + requiredPermissions := []perm{ + { + resource: constants.AppInstallResourceContents, + requiredPermission: constants.AppInstallPermissionRead, + actualPermission: perms.GetContents(), + }, + { + resource: constants.AppInstallResourceChecks, + requiredPermission: constants.AppInstallPermissionWrite, + actualPermission: perms.GetChecks(), + }, + } + + for _, p := range requiredPermissions { + err := hasPermission(p.resource, p.requiredPermission, p.actualPermission) + if err != nil { + return err + } + } + + return nil +} + +// hasPermission takes a resource:perm pair and checks if the actual permission matches the expected permission or is supersceded by a higher permission. +func hasPermission(resource, requiredPerm, actualPerm string) error { + if len(actualPerm) == 0 { + return fmt.Errorf("github app missing permission %s:%s", resource, requiredPerm) + } + + permitted := false + + switch requiredPerm { + case constants.AppInstallPermissionNone: + permitted = true + case constants.AppInstallPermissionRead: + if actualPerm == constants.AppInstallPermissionRead || + actualPerm == constants.AppInstallPermissionWrite { + permitted = true + } + case constants.AppInstallPermissionWrite: + if actualPerm == constants.AppInstallPermissionWrite { + permitted = true + } + default: + return fmt.Errorf("invalid required permission type: %s", requiredPerm) + } + + if !permitted { + return fmt.Errorf("github app requires permission %s:%s, found: %s", constants.AppInstallResourceContents, constants.AppInstallPermissionRead, actualPerm) + } + + return nil +} + // NewTest returns a SCM implementation that integrates with the provided // mock server. Only the url from the mock server is required. // @@ -165,39 +239,3 @@ func NewTest(urls ...string) (*client, error) { WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) } - -// newClientToken returns the GitHub OAuth client. -func (c *client) newClientToken(ctx context.Context, token string) *github.Client { - // create the token object for the client - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - - // create the OAuth client - tc := oauth2.NewClient(ctx, ts) - // if c.SkipVerify { - // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ - // Proxy: http.ProxyFromEnvironment, - // TLSClientConfig: &tls.Config{ - // InsecureSkipVerify: true, - // }, - // } - // } - - if c.Tracing.Config.EnableTracing { - tc.Transport = otelhttp.NewTransport( - tc.Transport, - otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { - return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) - }), - ) - } - - // create the GitHub client from the OAuth client - github := github.NewClient(tc) - - // ensure the proper URL is set in the GitHub client - github.BaseURL, _ = url.Parse(c.config.API) - - return github -} diff --git a/scm/github/github_test.go b/scm/github/github_test.go index e497b66ec..8cfae3844 100644 --- a/scm/github/github_test.go +++ b/scm/github/github_test.go @@ -72,7 +72,7 @@ func TestGithub_newClientToken(t *testing.T) { client, _ := NewTest(s.URL) // run test - got := client.newClientToken(context.Background(), "foobar") + got := client.newOAuthTokenClient(context.Background(), "foobar") //nolint:staticcheck // ignore false positive if got == nil { diff --git a/scm/github/org.go b/scm/github/org.go index 8c9e95986..ec1f8e314 100644 --- a/scm/github/org.go +++ b/scm/github/org.go @@ -19,7 +19,7 @@ func (c *client) GetOrgName(ctx context.Context, u *api.User, o string) (string, }).Tracef("retrieving org information for %s", o) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the org info orgInfo, resp, err := client.Organizations.Get(ctx, o) diff --git a/scm/github/repo.go b/scm/github/repo.go index ba10ca059..f6ad559b2 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -55,7 +55,7 @@ func (c *client) Config(ctx context.Context, u *api.User, r *api.Repo, ref strin }).Tracef("capturing configuration file for %s/commit/%s", r.GetFullName(), ref) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // default pipeline file names files := []string{".vela.yml", ".vela.yaml"} @@ -107,7 +107,7 @@ func (c *client) DestroyWebhook(ctx context.Context, u *api.User, org, name stri }).Tracef("deleting repository webhooks for %s/%s", org, name) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture the hooks for the repo hooks, _, err := client.Repositories.ListHooks(ctx, org, name, nil) @@ -167,7 +167,7 @@ func (c *client) CreateWebhook(ctx context.Context, u *api.User, r *api.Repo, h }).Tracef("creating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -241,7 +241,7 @@ func (c *client) Update(ctx context.Context, u *api.User, r *api.Repo, hookID in }).Tracef("updating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -305,7 +305,7 @@ func (c *client) Status(ctx context.Context, u *api.User, b *api.Build, org, nam } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) context := fmt.Sprintf("%s/%s", c.config.StatusContext, b.GetEvent()) url := fmt.Sprintf("%s/%s/%s/%d", c.config.WebUIAddress, org, name, b.GetNumber()) @@ -424,7 +424,7 @@ func (c *client) StepStatus(ctx context.Context, u *api.User, b *api.Build, s *a } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) context := fmt.Sprintf("%s/%s/%s", c.config.StatusContext, b.GetEvent(), s.GetReportAs()) url := fmt.Sprintf("%s/%s/%s/%d#%d", c.config.WebUIAddress, org, name, b.GetNumber(), s.GetNumber()) @@ -487,7 +487,7 @@ func (c *client) GetRepo(ctx context.Context, u *api.User, r *api.Repo) (*api.Re }).Tracef("retrieving repository information for %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the repo info repo, resp, err := client.Repositories.Get(ctx, r.GetOrg(), r.GetName()) @@ -507,7 +507,7 @@ func (c *client) GetOrgAndRepoName(ctx context.Context, u *api.User, o string, r }).Tracef("retrieving repository information for %s/%s", o, r) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the repo info repo, _, err := client.Repositories.Get(ctx, o, r) @@ -525,7 +525,7 @@ func (c *client) ListUserRepos(ctx context.Context, u *api.User) ([]*api.Repo, e }).Tracef("listing source repositories for %s", u.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) r := []*github.Repository{} f := []*api.Repo{} @@ -605,7 +605,7 @@ func (c *client) GetPullRequest(ctx context.Context, r *api.Repo, number int) (s }).Tracef("retrieving pull request %d for repo %s", number, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) pull, _, err := client.PullRequests.Get(ctx, r.GetOrg(), r.GetName(), number) if err != nil { @@ -629,7 +629,7 @@ func (c *client) GetHTMLURL(ctx context.Context, u *api.User, org, repo, name, r }).Tracef("capturing html_url for %s/%s/%s@%s", org, repo, name, ref) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // set the reference for the options to capture the repository contents opts := &github.RepositoryContentGetOptions{ @@ -665,7 +665,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str }).Tracef("retrieving branch %s for repo %s", branch, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) maxRedirects := 3 @@ -795,7 +795,7 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap return r, err } - client = c.newClientToken(ctx, t.GetToken()) + client = c.newOAuthTokenClient(ctx, t.GetToken()) repos, _, err := client.Apps.ListRepos(ctx, &github.ListOptions{}) if err != nil { diff --git a/scm/github/user.go b/scm/github/user.go index d014256e4..3ceaf75ac 100644 --- a/scm/github/user.go +++ b/scm/github/user.go @@ -16,7 +16,7 @@ func (c *client) GetUserID(ctx context.Context, name string, token string) (stri }).Tracef("capturing SCM user id for %s", name) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture user user, _, err := client.Users.Get(ctx, name) diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 31c83fb42..51fa7638c 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -103,7 +103,7 @@ func (c *client) VerifyWebhook(ctx context.Context, request *http.Request, r *ap // RedeliverWebhook redelivers webhooks from GitHub. func (c *client) RedeliverWebhook(ctx context.Context, u *api.User, h *api.Hook) error { // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // capture the delivery ID of the hook using GitHub API deliveryID, err := c.getDeliveryID(ctx, client, h) diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index 1ef1685fe..c228a5491 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -1542,7 +1542,7 @@ func TestGithub_GetDeliveryID(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") - ghClient := client.newClientToken(context.Background(), *u.Token) + ghClient := client.newOAuthTokenClient(context.Background(), *u.Token) // run test got, err := client.getDeliveryID(context.TODO(), ghClient, _hook) From 249099d16e7a7057bd47d965f5f7c2fa2d7309b8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 12:19:02 -0500 Subject: [PATCH 39/56] fix: unexport func --- scm/github/app_transport.go | 7 +++---- scm/github/repo.go | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index c7c68a3b7..d005fb420 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -28,7 +28,6 @@ import ( const ( acceptHeader = "application/vnd.github.v3+json" - apiBaseURL = "https://api.github.com" ) // AppsTransport provides a http.RoundTripper by wrapping an existing @@ -58,12 +57,12 @@ func (c *client) newGitHubAppTransport(appID int64, privateKey, baseURL string) return nil, fmt.Errorf("failed to parse PEM block containing the key") } - _privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + parsedPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { return nil, fmt.Errorf("failed to parse RSA private key: %w", err) } - transport := c.newAppsTransportFromPrivateKey(http.DefaultTransport, appID, _privateKey) + transport := c.newAppsTransportFromPrivateKey(http.DefaultTransport, appID, parsedPrivateKey) transport.BaseURL = baseURL // apply tracing to the transport @@ -82,7 +81,7 @@ func (c *client) newGitHubAppTransport(appID int64, privateKey, baseURL string) // newAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). func (c *client) newAppsTransportFromPrivateKey(tr http.RoundTripper, appID int64, key *rsa.PrivateKey) *AppsTransport { return &AppsTransport{ - BaseURL: apiBaseURL, + BaseURL: defaultAPI, Client: &http.Client{Transport: tr}, tr: tr, signer: NewRSASigner(jwt.SigningMethodRS256, key), diff --git a/scm/github/repo.go b/scm/github/repo.go index f6ad559b2..f6b84f1fd 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -720,7 +720,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, } for resource, perm := range perms { - ghPerms, err = WithGitHubInstallationPermission(ghPerms, resource, perm) + ghPerms, err = applyGitHubInstallationPermission(ghPerms, resource, perm) if err != nil { l.Errorf("unable to create github app installation token with permission %s:%s: %v", resource, perm, err) @@ -817,8 +817,8 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap return r, nil } -// WithGitHubInstallationPermission takes permissions and applies a new permission if valid. -func WithGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { +// applyGitHubInstallationPermission takes permissions and applies a new permission if valid. +func applyGitHubInstallationPermission(perms *github.InstallationPermissions, resource, perm string) (*github.InstallationPermissions, error) { // convert permissions from yaml string switch strings.ToLower(perm) { case constants.AppInstallPermissionNone: From 941019145f9a25ddd6300ccdb1ce00ef8bf574bd Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 13:55:49 -0500 Subject: [PATCH 40/56] fix: repo.install_id test updates --- api/types/repo.go | 6 +- api/types/repo_test.go | 4 +- compiler/native/environment_test.go | 78 ++++++++++++------- database/build/clean_test.go | 1 + database/build/count_deployment_test.go | 1 + database/build/count_org_test.go | 2 + database/build/get_repo_test.go | 1 + database/build/get_test.go | 1 + database/build/last_repo_test.go | 1 + database/build/list_org_test.go | 2 + .../build/list_pending_running_repo_test.go | 2 + database/build/list_repo_test.go | 1 + database/build/list_test.go | 1 + database/deployment/get_repo_test.go | 1 + database/deployment/get_test.go | 1 + database/deployment/list_repo_test.go | 2 + database/deployment/list_test.go | 1 + database/hook/get_repo_test.go | 1 + database/hook/get_test.go | 1 + database/hook/get_webhook_test.go | 1 + database/hook/last_repo_test.go | 1 + database/hook/list_repo_test.go | 1 + database/hook/list_test.go | 1 + database/pipeline/get_repo_test.go | 1 + database/pipeline/get_test.go | 1 + database/pipeline/list_repo_test.go | 1 + database/pipeline/list_test.go | 2 + database/repo/create_test.go | 7 +- database/repo/get_org_test.go | 1 + database/repo/get_test.go | 1 + database/repo/list_org_test.go | 2 + database/repo/list_test.go | 2 + database/repo/list_user_test.go | 2 + database/repo/update_test.go | 7 +- database/schedule/count_active_test.go | 1 + database/schedule/count_repo_test.go | 1 + database/schedule/count_test.go | 1 + database/schedule/create_test.go | 1 + database/schedule/delete_test.go | 1 + database/schedule/get_repo_test.go | 1 + database/schedule/get_test.go | 1 + database/schedule/list_active_test.go | 1 + database/schedule/list_repo_test.go | 1 + database/schedule/list_test.go | 1 + database/schedule/update_test.go | 2 + database/types/repo_test.go | 49 ++++++------ database/types/schedule_test.go | 6 +- mock/server/repo.go | 11 ++- router/middleware/build/build_test.go | 1 + router/middleware/hook/hook_test.go | 1 + router/middleware/pipeline/pipeline_test.go | 1 + router/middleware/repo/repo_test.go | 1 + 52 files changed, 155 insertions(+), 66 deletions(-) diff --git a/api/types/repo.go b/api/types/repo.go index 345240173..b81a061e9 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -657,7 +657,6 @@ func (r *Repo) String() string { Counter: %d, FullName: %s, ID: %d, - InstallID: %d, Link: %s, Name: %s, Org: %s, @@ -668,7 +667,8 @@ func (r *Repo) String() string { Timeout: %d, Topics: %s, Trusted: %t, - Visibility: %s + Visibility: %s, + InstallID: %d }`, r.GetActive(), r.GetAllowEvents().List(), @@ -679,7 +679,6 @@ func (r *Repo) String() string { r.GetCounter(), r.GetFullName(), r.GetID(), - r.GetInstallID(), r.GetLink(), r.GetName(), r.GetOrg(), @@ -691,5 +690,6 @@ func (r *Repo) String() string { r.GetTopics(), r.GetTrusted(), r.GetVisibility(), + r.GetInstallID(), ) } diff --git a/api/types/repo_test.go b/api/types/repo_test.go index c7e20407d..fa8fb2970 100644 --- a/api/types/repo_test.go +++ b/api/types/repo_test.go @@ -303,7 +303,8 @@ func TestTypes_Repo_String(t *testing.T) { Timeout: %d, Topics: %s, Trusted: %t, - Visibility: %s + Visibility: %s, + InstallID: %d }`, r.GetActive(), r.GetAllowEvents().List(), @@ -325,6 +326,7 @@ func TestTypes_Repo_String(t *testing.T) { r.GetTopics(), r.GetTrusted(), r.GetVisibility(), + r.GetInstallID(), ) // run test diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index a75eeb360..f988a1213 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -174,7 +174,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -351,7 +351,7 @@ func TestNative_EnvironmentServices(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -510,7 +510,7 @@ func TestNative_EnvironmentSecrets(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -580,6 +580,8 @@ func TestNative_environment(t *testing.T) { // deployment deploy := "deployment" target := "production" + // netrc + netrc := "foo" tests := []struct { w string @@ -592,39 +594,53 @@ func TestNative_environment(t *testing.T) { }{ // push { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &push, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &str, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &push, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &str, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // tag { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // pull_request { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &pull, EventAction: &pullact, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &pull, EventAction: &pullact, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // deployment { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + }, + // todo: netrc + { + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, } @@ -695,12 +711,15 @@ func Test_client_EnvironmentBuild(t *testing.T) { // deployment deploy := "deployment" target := "production" + // netrc + netrc := "foo" type fields struct { build *api.Build metadata *internal.Metadata repo *api.Repo user *api.User + netrc *string } tests := []struct { @@ -713,12 +732,14 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}}, {"tag", fields{ build: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, {"pull_request", fields{ @@ -726,6 +747,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, {"deployment", fields{ @@ -733,6 +755,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, } @@ -743,6 +766,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: tt.fields.metadata, repo: tt.fields.repo, user: tt.fields.user, + netrc: tt.fields.netrc, } got := c.EnvironmentBuild() if diff := cmp.Diff(got, tt.want); diff != "" { diff --git a/database/build/clean_test.go b/database/build/clean_test.go index 179dd2fb8..b263dfe27 100644 --- a/database/build/clean_test.go +++ b/database/build/clean_test.go @@ -24,6 +24,7 @@ func TestBuild_Engine_CleanBuilds(t *testing.T) { _repo.SetVisibility("public") _repo.SetPipelineType("yaml") _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repo.SetAllowEvents(api.NewEventsFromMask(1)) _owner := testutils.APIUser() diff --git a/database/build/count_deployment_test.go b/database/build/count_deployment_test.go index a956dbba5..dc4ca89a8 100644 --- a/database/build/count_deployment_test.go +++ b/database/build/count_deployment_test.go @@ -24,6 +24,7 @@ func TestBuild_Engine_CountBuildsForDeployment(t *testing.T) { _repo.SetVisibility("public") _repo.SetPipelineType("yaml") _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repo.SetAllowEvents(api.NewEventsFromMask(1)) _owner := testutils.APIUser() diff --git a/database/build/count_org_test.go b/database/build/count_org_test.go index 5454b82dd..3b894459a 100644 --- a/database/build/count_org_test.go +++ b/database/build/count_org_test.go @@ -31,6 +31,7 @@ func TestBuild_Engine_CountBuildsForOrg(t *testing.T) { _repoOne.SetVisibility("public") _repoOne.SetPipelineType("yaml") _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -42,6 +43,7 @@ func TestBuild_Engine_CountBuildsForOrg(t *testing.T) { _repoTwo.SetVisibility("public") _repoTwo.SetPipelineType("yaml") _repoTwo.SetTopics([]string{}) + _repoTwo.SetInstallID(0) _buildOne := testutils.APIBuild() _buildOne.SetID(1) diff --git a/database/build/get_repo_test.go b/database/build/get_repo_test.go index 2be48a249..d3f365c6a 100644 --- a/database/build/get_repo_test.go +++ b/database/build/get_repo_test.go @@ -32,6 +32,7 @@ func TestBuild_Engine_GetBuildForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _build := testutils.APIBuild() _build.SetID(1) diff --git a/database/build/get_test.go b/database/build/get_test.go index 3dc2e95e8..a00c26902 100644 --- a/database/build/get_test.go +++ b/database/build/get_test.go @@ -33,6 +33,7 @@ func TestBuild_Engine_GetBuild(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _build := testutils.APIBuild() _build.SetID(1) diff --git a/database/build/last_repo_test.go b/database/build/last_repo_test.go index 55b74e5b6..28da019cd 100644 --- a/database/build/last_repo_test.go +++ b/database/build/last_repo_test.go @@ -32,6 +32,7 @@ func TestBuild_Engine_LastBuildForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _build := testutils.APIBuild() _build.SetID(1) diff --git a/database/build/list_org_test.go b/database/build/list_org_test.go index 965955891..f02b881e2 100644 --- a/database/build/list_org_test.go +++ b/database/build/list_org_test.go @@ -33,6 +33,7 @@ func TestBuild_Engine_ListBuildsForOrg(t *testing.T) { _repoOne.SetPipelineType("yaml") _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -45,6 +46,7 @@ func TestBuild_Engine_ListBuildsForOrg(t *testing.T) { _repoTwo.SetPipelineType("yaml") _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) _repoTwo.SetTopics([]string{}) + _repoTwo.SetInstallID(0) _buildOne := testutils.APIBuild() _buildOne.SetID(1) diff --git a/database/build/list_pending_running_repo_test.go b/database/build/list_pending_running_repo_test.go index e8a9b323b..8d7bb9348 100644 --- a/database/build/list_pending_running_repo_test.go +++ b/database/build/list_pending_running_repo_test.go @@ -31,6 +31,7 @@ func TestBuild_Engine_ListPendingAndRunningBuildsForRepo(t *testing.T) { _repoOne.SetPipelineType("yaml") _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -43,6 +44,7 @@ func TestBuild_Engine_ListPendingAndRunningBuildsForRepo(t *testing.T) { _repoTwo.SetPipelineType("yaml") _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) _repoTwo.SetTopics([]string{}) + _repoTwo.SetInstallID(0) _buildOne := testutils.APIBuild() _buildOne.SetID(1) diff --git a/database/build/list_repo_test.go b/database/build/list_repo_test.go index 8071e3a67..68bdaa250 100644 --- a/database/build/list_repo_test.go +++ b/database/build/list_repo_test.go @@ -33,6 +33,7 @@ func TestBuild_Engine_ListBuildsForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _buildOne := testutils.APIBuild() _buildOne.SetID(1) diff --git a/database/build/list_test.go b/database/build/list_test.go index e440a162c..5d71c8041 100644 --- a/database/build/list_test.go +++ b/database/build/list_test.go @@ -33,6 +33,7 @@ func TestBuild_Engine_ListBuilds(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _buildOne := testutils.APIBuild() _buildOne.SetID(1) diff --git a/database/deployment/get_repo_test.go b/database/deployment/get_repo_test.go index f4931b3b3..4ef0a3827 100644 --- a/database/deployment/get_repo_test.go +++ b/database/deployment/get_repo_test.go @@ -32,6 +32,7 @@ func TestDeployment_Engine_GetDeploymentForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/deployment/get_test.go b/database/deployment/get_test.go index c1124b08d..aa28732a4 100644 --- a/database/deployment/get_test.go +++ b/database/deployment/get_test.go @@ -32,6 +32,7 @@ func TestDeployment_Engine_GetDeployment(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _deploymentOne := testutils.APIDeployment() _deploymentOne.SetID(1) diff --git a/database/deployment/list_repo_test.go b/database/deployment/list_repo_test.go index 50b20760d..7461074e0 100644 --- a/database/deployment/list_repo_test.go +++ b/database/deployment/list_repo_test.go @@ -32,6 +32,7 @@ func TestDeployment_Engine_ListDeploymentsForRepo(t *testing.T) { _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) _repoOne.SetPipelineType(constants.PipelineTypeYAML) _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -44,6 +45,7 @@ func TestDeployment_Engine_ListDeploymentsForRepo(t *testing.T) { _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) _repoTwo.SetPipelineType(constants.PipelineTypeYAML) _repoTwo.SetTopics([]string{}) + _repoTwo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/deployment/list_test.go b/database/deployment/list_test.go index 14e9577ea..96506d540 100644 --- a/database/deployment/list_test.go +++ b/database/deployment/list_test.go @@ -32,6 +32,7 @@ func TestDeployment_Engine_ListDeployments(t *testing.T) { _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) _repoOne.SetPipelineType(constants.PipelineTypeYAML) _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/get_repo_test.go b/database/hook/get_repo_test.go index bc1eb0fb9..103f2e866 100644 --- a/database/hook/get_repo_test.go +++ b/database/hook/get_repo_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_GetHookForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/get_test.go b/database/hook/get_test.go index 9e6664784..a9f8b6207 100644 --- a/database/hook/get_test.go +++ b/database/hook/get_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_GetHook(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/get_webhook_test.go b/database/hook/get_webhook_test.go index 37ab712ff..d6173b007 100644 --- a/database/hook/get_webhook_test.go +++ b/database/hook/get_webhook_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_GetHookByWebhookID(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/last_repo_test.go b/database/hook/last_repo_test.go index 70af398ec..327a14b09 100644 --- a/database/hook/last_repo_test.go +++ b/database/hook/last_repo_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_LastHookForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/list_repo_test.go b/database/hook/list_repo_test.go index ea3301f2c..4f9489bec 100644 --- a/database/hook/list_repo_test.go +++ b/database/hook/list_repo_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_ListHooksForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/hook/list_test.go b/database/hook/list_test.go index 5d8aa4fe9..60edeb82f 100644 --- a/database/hook/list_test.go +++ b/database/hook/list_test.go @@ -32,6 +32,7 @@ func TestHook_Engine_ListHooks(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repoBuild := new(api.Repo) _repoBuild.SetID(1) diff --git a/database/pipeline/get_repo_test.go b/database/pipeline/get_repo_test.go index f7de4f72b..700c9fc15 100644 --- a/database/pipeline/get_repo_test.go +++ b/database/pipeline/get_repo_test.go @@ -32,6 +32,7 @@ func TestPipeline_Engine_GetPipelineForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _pipeline := testutils.APIPipeline() _pipeline.SetID(1) diff --git a/database/pipeline/get_test.go b/database/pipeline/get_test.go index 8022eb410..30df9fd7f 100644 --- a/database/pipeline/get_test.go +++ b/database/pipeline/get_test.go @@ -32,6 +32,7 @@ func TestPipeline_Engine_GetPipeline(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _pipeline := testutils.APIPipeline() _pipeline.SetID(1) diff --git a/database/pipeline/list_repo_test.go b/database/pipeline/list_repo_test.go index 77696e05d..dcaa0d065 100644 --- a/database/pipeline/list_repo_test.go +++ b/database/pipeline/list_repo_test.go @@ -32,6 +32,7 @@ func TestPipeline_Engine_ListPipelinesForRepo(t *testing.T) { _repo.SetAllowEvents(api.NewEventsFromMask(1)) _repo.SetPipelineType(constants.PipelineTypeYAML) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _pipelineOne := testutils.APIPipeline() _pipelineOne.SetID(1) diff --git a/database/pipeline/list_test.go b/database/pipeline/list_test.go index 4dc624a55..a86c3e284 100644 --- a/database/pipeline/list_test.go +++ b/database/pipeline/list_test.go @@ -32,6 +32,7 @@ func TestPipeline_Engine_ListPipelines(t *testing.T) { _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) _repoOne.SetPipelineType(constants.PipelineTypeYAML) _repoOne.SetTopics([]string{}) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -44,6 +45,7 @@ func TestPipeline_Engine_ListPipelines(t *testing.T) { _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) _repoTwo.SetPipelineType(constants.PipelineTypeYAML) _repoTwo.SetTopics([]string{}) + _repoTwo.SetInstallID(0) _pipelineOne := testutils.APIPipeline() _pipelineOne.SetID(1) diff --git a/database/repo/create_test.go b/database/repo/create_test.go index 07b8204ab..a06e8dfb2 100644 --- a/database/repo/create_test.go +++ b/database/repo/create_test.go @@ -25,6 +25,7 @@ func TestRepo_Engine_CreateRepo(t *testing.T) { _repo.SetPipelineType("yaml") _repo.SetPreviousName("oldName") _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -34,9 +35,9 @@ func TestRepo_Engine_CreateRepo(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "repos" -("user_id","hash","org","name","full_name","link","clone","branch","topics","build_limit","timeout","counter","visibility","private","trusted","active","allow_events","pipeline_type","previous_name","approve_build","id") -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21) RETURNING "id"`). - WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, nil, "yaml", "oldName", nil, 1). +("user_id","hash","org","name","full_name","link","clone","branch","topics","build_limit","timeout","counter","visibility","private","trusted","active","allow_events","pipeline_type","previous_name","approve_build","install_id","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22) RETURNING "id"`). + WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, nil, "yaml", "oldName", nil, 0, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/repo/get_org_test.go b/database/repo/get_org_test.go index ed912ca1f..3e4c2ea0f 100644 --- a/database/repo/get_org_test.go +++ b/database/repo/get_org_test.go @@ -26,6 +26,7 @@ func TestRepo_Engine_GetRepoForOrg(t *testing.T) { _repo.SetVisibility("public") _repo.SetPipelineType("yaml") _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repo.SetAllowEvents(api.NewEventsFromMask(1)) _owner := testutils.APIUser().Crop() diff --git a/database/repo/get_test.go b/database/repo/get_test.go index 720f9bf07..264d4b585 100644 --- a/database/repo/get_test.go +++ b/database/repo/get_test.go @@ -26,6 +26,7 @@ func TestRepo_Engine_GetRepo(t *testing.T) { _repo.SetVisibility("public") _repo.SetPipelineType("yaml") _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repo.SetAllowEvents(api.NewEventsFromMask(1)) _owner := testutils.APIUser().Crop() diff --git a/database/repo/list_org_test.go b/database/repo/list_org_test.go index 9aad55864..328f6749b 100644 --- a/database/repo/list_org_test.go +++ b/database/repo/list_org_test.go @@ -34,6 +34,7 @@ func TestRepo_Engine_ListReposForOrg(t *testing.T) { _repoOne.SetPipelineType("yaml") _repoOne.SetTopics([]string{}) _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -46,6 +47,7 @@ func TestRepo_Engine_ListReposForOrg(t *testing.T) { _repoTwo.SetPipelineType("yaml") _repoTwo.SetTopics([]string{}) _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) + _repoTwo.SetInstallID(0) _buildOne := new(api.Build) _buildOne.SetID(1) diff --git a/database/repo/list_test.go b/database/repo/list_test.go index 176d70369..56d12ab74 100644 --- a/database/repo/list_test.go +++ b/database/repo/list_test.go @@ -33,6 +33,7 @@ func TestRepo_Engine_ListRepos(t *testing.T) { _repoOne.SetPipelineType("yaml") _repoOne.SetTopics([]string{}) _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -45,6 +46,7 @@ func TestRepo_Engine_ListRepos(t *testing.T) { _repoTwo.SetPipelineType("yaml") _repoTwo.SetTopics([]string{}) _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) + _repoTwo.SetInstallID(0) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() diff --git a/database/repo/list_user_test.go b/database/repo/list_user_test.go index 5b9fda28a..ccb650215 100644 --- a/database/repo/list_user_test.go +++ b/database/repo/list_user_test.go @@ -34,6 +34,7 @@ func TestRepo_Engine_ListReposForUser(t *testing.T) { _repoOne.SetPipelineType("yaml") _repoOne.SetTopics([]string{}) _repoOne.SetAllowEvents(api.NewEventsFromMask(1)) + _repoOne.SetInstallID(0) _repoTwo := testutils.APIRepo() _repoTwo.SetID(2) @@ -46,6 +47,7 @@ func TestRepo_Engine_ListReposForUser(t *testing.T) { _repoTwo.SetPipelineType("yaml") _repoTwo.SetTopics([]string{}) _repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) + _repoTwo.SetInstallID(0) _buildOne := new(api.Build) _buildOne.SetID(1) diff --git a/database/repo/update_test.go b/database/repo/update_test.go index a447d6973..5ced61f0c 100644 --- a/database/repo/update_test.go +++ b/database/repo/update_test.go @@ -28,6 +28,7 @@ func TestRepo_Engine_UpdateRepo(t *testing.T) { _repo.SetPreviousName("oldName") _repo.SetApproveBuild(constants.ApproveForkAlways) _repo.SetTopics([]string{}) + _repo.SetInstallID(0) _repo.SetAllowEvents(api.NewEventsFromMask(1)) _postgres, _mock := testPostgres(t) @@ -35,9 +36,9 @@ func TestRepo_Engine_UpdateRepo(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "repos" -SET "user_id"=$1,"hash"=$2,"org"=$3,"name"=$4,"full_name"=$5,"link"=$6,"clone"=$7,"branch"=$8,"topics"=$9,"build_limit"=$10,"timeout"=$11,"counter"=$12,"visibility"=$13,"private"=$14,"trusted"=$15,"active"=$16,"allow_events"=$17,"pipeline_type"=$18,"previous_name"=$19,"approve_build"=$20 -WHERE "id" = $21`). - WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, 1, "yaml", "oldName", constants.ApproveForkAlways, 1). +SET "user_id"=$1,"hash"=$2,"org"=$3,"name"=$4,"full_name"=$5,"link"=$6,"clone"=$7,"branch"=$8,"topics"=$9,"build_limit"=$10,"timeout"=$11,"counter"=$12,"visibility"=$13,"private"=$14,"trusted"=$15,"active"=$16,"allow_events"=$17,"pipeline_type"=$18,"previous_name"=$19,"approve_build"=$20,"install_id"=$21 +WHERE "id" = $22`). + WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, 1, "yaml", "oldName", constants.ApproveForkAlways, 0, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/schedule/count_active_test.go b/database/schedule/count_active_test.go index 5cb6efb4f..f8c83d0fe 100644 --- a/database/schedule/count_active_test.go +++ b/database/schedule/count_active_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_CountActiveSchedules(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/count_repo_test.go b/database/schedule/count_repo_test.go index 847b341cc..1c38876d1 100644 --- a/database/schedule/count_repo_test.go +++ b/database/schedule/count_repo_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/count_test.go b/database/schedule/count_test.go index df58cdd45..23c058ff1 100644 --- a/database/schedule/count_test.go +++ b/database/schedule/count_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_CountSchedules(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/create_test.go b/database/schedule/create_test.go index cb29cd8aa..122b36468 100644 --- a/database/schedule/create_test.go +++ b/database/schedule/create_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_CreateSchedule(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/delete_test.go b/database/schedule/delete_test.go index ca7351863..3f976e784 100644 --- a/database/schedule/delete_test.go +++ b/database/schedule/delete_test.go @@ -49,6 +49,7 @@ func TestSchedule_Engine_DeleteSchedule(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/get_repo_test.go b/database/schedule/get_repo_test.go index 07f49bcf3..3e97a0dbe 100644 --- a/database/schedule/get_repo_test.go +++ b/database/schedule/get_repo_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/get_test.go b/database/schedule/get_test.go index 88ad9df3c..5e831fd51 100644 --- a/database/schedule/get_test.go +++ b/database/schedule/get_test.go @@ -51,6 +51,7 @@ func TestSchedule_Engine_GetSchedule(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/list_active_test.go b/database/schedule/list_active_test.go index d015f0e38..d3932d1b7 100644 --- a/database/schedule/list_active_test.go +++ b/database/schedule/list_active_test.go @@ -51,6 +51,7 @@ func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/list_repo_test.go b/database/schedule/list_repo_test.go index a1b8f7131..45b4c67d1 100644 --- a/database/schedule/list_repo_test.go +++ b/database/schedule/list_repo_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/list_test.go b/database/schedule/list_test.go index 5d8b5d738..c5e7b8b36 100644 --- a/database/schedule/list_test.go +++ b/database/schedule/list_test.go @@ -51,6 +51,7 @@ func TestSchedule_Engine_ListSchedules(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index bcf675edb..bd291396f 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -50,6 +50,7 @@ func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) @@ -164,6 +165,7 @@ func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { _repo.SetPipelineType("") _repo.SetPreviousName("") _repo.SetApproveBuild(constants.ApproveNever) + _repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) diff --git a/database/types/repo_test.go b/database/types/repo_test.go index 949985490..8bc3f2b68 100644 --- a/database/types/repo_test.go +++ b/database/types/repo_test.go @@ -193,6 +193,7 @@ func TestTypes_Repo_ToAPI(t *testing.T) { want.SetPipelineType("yaml") want.SetPreviousName("oldName") want.SetApproveBuild(constants.ApproveNever) + want.SetInstallID(0) // run test got := testRepo().ToAPI() @@ -320,37 +321,38 @@ func TestTypes_Repo_Validate(t *testing.T) { func TestTypes_RepoFromAPI(t *testing.T) { // setup types - r := new(api.Repo) + repo := new(api.Repo) owner := testutils.APIUser() owner.SetID(1) - r.SetID(1) - r.SetOwner(owner) - r.SetHash("superSecretHash") - r.SetOrg("github") - r.SetName("octocat") - r.SetFullName("github/octocat") - r.SetLink("https://github.com/github/octocat") - r.SetClone("https://github.com/github/octocat.git") - r.SetBranch("main") - r.SetTopics([]string{"cloud", "security"}) - r.SetBuildLimit(10) - r.SetTimeout(30) - r.SetCounter(0) - r.SetVisibility("public") - r.SetPrivate(false) - r.SetTrusted(false) - r.SetActive(true) - r.SetAllowEvents(api.NewEventsFromMask(1)) - r.SetPipelineType("yaml") - r.SetPreviousName("oldName") - r.SetApproveBuild(constants.ApproveNever) + repo.SetID(1) + repo.SetOwner(owner) + repo.SetHash("superSecretHash") + repo.SetOrg("github") + repo.SetName("octocat") + repo.SetFullName("github/octocat") + repo.SetLink("https://github.com/github/octocat") + repo.SetClone("https://github.com/github/octocat.git") + repo.SetBranch("main") + repo.SetTopics([]string{"cloud", "security"}) + repo.SetBuildLimit(10) + repo.SetTimeout(30) + repo.SetCounter(0) + repo.SetVisibility("public") + repo.SetPrivate(false) + repo.SetTrusted(false) + repo.SetActive(true) + repo.SetAllowEvents(api.NewEventsFromMask(1)) + repo.SetPipelineType("yaml") + repo.SetPreviousName("oldName") + repo.SetApproveBuild(constants.ApproveNever) + repo.SetInstallID(0) want := testRepo() want.Owner = User{} // run test - got := RepoFromAPI(r) + got := RepoFromAPI(repo) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("RepoFromAPI() mismatch (-want +got):\n%s", diff) @@ -382,6 +384,7 @@ func testRepo() *Repo { PipelineType: sql.NullString{String: "yaml", Valid: true}, PreviousName: sql.NullString{String: "oldName", Valid: true}, ApproveBuild: sql.NullString{String: constants.ApproveNever, Valid: true}, + InstallID: sql.NullInt64{Int64: 0, Valid: true}, Owner: *testUser(), } diff --git a/database/types/schedule_test.go b/database/types/schedule_test.go index 10b4da233..1f9542ed7 100644 --- a/database/types/schedule_test.go +++ b/database/types/schedule_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/adhocore/gronx" + "github.com/google/go-cmp/cmp" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" @@ -94,6 +95,7 @@ func TestTypes_Schedule_ToAPI(t *testing.T) { repo.SetPipelineType("yaml") repo.SetPreviousName("oldName") repo.SetApproveBuild(constants.ApproveNever) + repo.SetInstallID(0) currTime := time.Now().UTC() nextTime, _ := gronx.NextTickAfter("0 0 * * *", currTime, false) @@ -116,8 +118,8 @@ func TestTypes_Schedule_ToAPI(t *testing.T) { // run test got := testSchedule().ToAPI() - if !reflect.DeepEqual(got, want) { - t.Errorf("ToAPI is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("RepoFromAPI() mismatch (-want +got):\n%s", diff) } } diff --git a/mock/server/repo.go b/mock/server/repo.go index 9b156669e..801f71a1d 100644 --- a/mock/server/repo.go +++ b/mock/server/repo.go @@ -59,8 +59,9 @@ const ( } }, "approve_build": "fork-always", - "previous_name": "" -}` + "previous_name": "", + "install_id": 0 + }` // ReposResp represents a JSON return for one to many repos. ReposResp = `[ @@ -78,7 +79,8 @@ const ( "visibility": "public", "private": false, "trusted": true, - "active": true + "active": true, + "install_id": 0 }, { "id": 2, @@ -94,7 +96,8 @@ const ( "visibility": "public", "private": false, "trusted": true, - "active": true + "active": true, + "install_id": 0 } ]` ) diff --git a/router/middleware/build/build_test.go b/router/middleware/build/build_test.go index aacf14949..68f10deff 100644 --- a/router/middleware/build/build_test.go +++ b/router/middleware/build/build_test.go @@ -53,6 +53,7 @@ func TestBuild_Establish(t *testing.T) { r.SetName("bar") r.SetFullName("foo/bar") r.SetVisibility("public") + r.SetInstallID(0) want := new(api.Build) want.SetID(1) diff --git a/router/middleware/hook/hook_test.go b/router/middleware/hook/hook_test.go index dc4316215..0c582e553 100644 --- a/router/middleware/hook/hook_test.go +++ b/router/middleware/hook/hook_test.go @@ -52,6 +52,7 @@ func TestHook_Establish(t *testing.T) { r.SetName("bar") r.SetFullName("foo/bar") r.SetVisibility("public") + r.SetInstallID(0) want := new(api.Hook) want.SetID(1) diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index a12dd544b..16daf5a95 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -82,6 +82,7 @@ func TestPipeline_Establish(t *testing.T) { r.SetName("bar") r.SetFullName("foo/bar") r.SetVisibility("public") + r.SetInstallID(0) want := new(api.Pipeline) want.SetID(1) diff --git a/router/middleware/repo/repo_test.go b/router/middleware/repo/repo_test.go index c20067c62..a331b5ae8 100644 --- a/router/middleware/repo/repo_test.go +++ b/router/middleware/repo/repo_test.go @@ -66,6 +66,7 @@ func TestRepo_Establish(t *testing.T) { want.SetPipelineType("yaml") want.SetPreviousName("") want.SetApproveBuild("") + want.SetInstallID(0) got := new(api.Repo) From e239480977bf462b6bd083b75056e633023fe82d Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 14:03:30 -0500 Subject: [PATCH 41/56] chore: cleanup --- compiler/registry/github/github.go | 8 -------- mock/server/repo.go | 4 ++-- scm/github/app_client.go | 8 -------- scm/github/app_transport.go | 4 ++-- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/compiler/registry/github/github.go b/compiler/registry/github/github.go index 41be1dfcb..f5027a5b5 100644 --- a/compiler/registry/github/github.go +++ b/compiler/registry/github/github.go @@ -68,14 +68,6 @@ func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github. // create the OAuth client tc := oauth2.NewClient(ctx, ts) - // if c.SkipVerify { - // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ - // Proxy: http.ProxyFromEnvironment, - // TLSClientConfig: &tls.Config{ - // InsecureSkipVerify: true, - // }, - // } - // } // create the GitHub client from the OAuth client github := github.NewClient(tc) diff --git a/mock/server/repo.go b/mock/server/repo.go index 801f71a1d..1db1fc733 100644 --- a/mock/server/repo.go +++ b/mock/server/repo.go @@ -80,7 +80,7 @@ const ( "private": false, "trusted": true, "active": true, - "install_id": 0 + "install_id": 0 }, { "id": 2, @@ -97,7 +97,7 @@ const ( "private": false, "trusted": true, "active": true, - "install_id": 0 + "install_id": 0 } ]` ) diff --git a/scm/github/app_client.go b/scm/github/app_client.go index de2243514..64067de09 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -27,14 +27,6 @@ func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github. // create the OAuth client tc := oauth2.NewClient(ctx, ts) - // if c.SkipVerify { - // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ - // Proxy: http.ProxyFromEnvironment, - // TLSClientConfig: &tls.Config{ - // InsecureSkipVerify: true, - // }, - // } - // } if c.Tracing.Config.EnableTracing { tc.Transport = otelhttp.NewTransport( diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index d005fb420..21d3c85f3 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -92,8 +92,8 @@ func (c *client) newAppsTransportFromPrivateKey(tr http.RoundTripper, appID int6 // RoundTrip implements http.RoundTripper interface. func (t *AppsTransport) RoundTrip(req *http.Request) (*http.Response, error) { // GitHub rejects expiry and issue timestamps that are not an integer, - // while the jwt-go library serializes to fractional timestamps. - // Truncate them before passing to jwt-go. + // while the jwt-go library serializes to fractional timestamps + // then truncate them before passing to jwt-go. iss := time.Now().Add(-30 * time.Second).Truncate(time.Second) exp := iss.Add(2 * time.Minute) claims := &jwt.RegisteredClaims{ From b19e50e0dc43be107de848796bc17107e802e1c6 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 14:16:02 -0500 Subject: [PATCH 42/56] fix: tests --- database/types/schedule_test.go | 2 +- scm/github/webhook_test.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/database/types/schedule_test.go b/database/types/schedule_test.go index 1f9542ed7..fe712f2fd 100644 --- a/database/types/schedule_test.go +++ b/database/types/schedule_test.go @@ -119,7 +119,7 @@ func TestTypes_Schedule_ToAPI(t *testing.T) { got := testSchedule().ToAPI() if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("RepoFromAPI() mismatch (-want +got):\n%s", diff) + t.Errorf("ScheduleToAPI() mismatch (-want +got):\n%s", diff) } } diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index c228a5491..8f7f5bf06 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -1197,8 +1197,8 @@ func TestGitHub_ProcessWebhook_RepositoryRename(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1260,8 +1260,8 @@ func TestGitHub_ProcessWebhook_RepositoryTransfer(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1323,8 +1323,8 @@ func TestGitHub_ProcessWebhook_RepositoryArchived(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1386,8 +1386,8 @@ func TestGitHub_ProcessWebhook_RepositoryEdited(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1449,8 +1449,8 @@ func TestGitHub_ProcessWebhook_Repository(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } From 9d62db7f8f5d2da908bd8c82fa1295b9d165ebbb Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 14:21:55 -0500 Subject: [PATCH 43/56] fix: tests --- database/integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database/integration_test.go b/database/integration_test.go index 89eda666a..2aa338eac 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -2491,6 +2491,7 @@ func newResources() *Resources { repoOne.SetPreviousName("") repoOne.SetApproveBuild(constants.ApproveNever) repoOne.SetAllowEvents(api.NewEventsFromMask(1)) + repoOne.SetInstallID(0) repoTwo := new(api.Repo) repoTwo.SetID(2) @@ -2514,6 +2515,7 @@ func newResources() *Resources { repoTwo.SetPreviousName("") repoTwo.SetApproveBuild(constants.ApproveForkAlways) repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) + repoTwo.SetInstallID(0) buildOne := new(api.Build) buildOne.SetID(1) From 618152dca2a5fabcc036011c0c86b4835de86dc6 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 14:42:57 -0500 Subject: [PATCH 44/56] fix: move netrc defaults into github implementation --- compiler/native/compile.go | 14 +------------- scm/github/repo.go | 19 +++++++++++++------ scm/service.go | 3 ++- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/compiler/native/compile.go b/compiler/native/compile.go index c751d9f9b..5e0f7a857 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -48,20 +48,8 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * // this has to occur after Parse because the scm configurations might be set in yaml // netrc can be provided directly using WithNetrc for situations like local exec if c.netrc == nil && c.scm != nil { - // ensure restrictive defaults for the netrc for scms that support granular permissions - if p.Git.Repositories == nil { - p.Git.Repositories = []string{c.repo.GetName()} - } - - if p.Git.Permissions == nil { - p.Git.Permissions = map[string]string{ - constants.AppInstallResourceContents: constants.AppInstallPermissionRead, - constants.AppInstallResourceChecks: constants.AppInstallPermissionWrite, - } - } - // get the netrc password from the scm - netrc, err := c.scm.GetNetrcPassword(ctx, c.repo, c.user, p.Git.Repositories, p.Git.Permissions) + netrc, err := c.scm.GetNetrcPassword(ctx, c.repo, c.user, p.Git) if err != nil { return nil, nil, err } diff --git a/scm/github/repo.go b/scm/github/repo.go index f6b84f1fd..91f0c5e21 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -14,6 +14,7 @@ import ( "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/constants" ) @@ -679,7 +680,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // GetNetrcPassword returns a clone token using the repo's github app installation if it exists. // If not, it defaults to the user OAuth token. -func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, repos []string, perms map[string]string) (string, error) { +func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, g yaml.Git) (string, error) { l := c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -692,10 +693,11 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, // repos that the token has access to // providing no repos, nil, or empty slice will default the token permissions to the list // of repos added to the installation - // - // the compiler will set restrictive defaults with access to the triggering repo + repos := g.Repositories + + // use triggering repo as a restrictive default if repos == nil { - repos = []string{} + repos = []string{r.GetName()} } // convert repo fullname org/name to just name for usability @@ -719,7 +721,12 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, Checks: github.String(constants.AppInstallPermissionWrite), } - for resource, perm := range perms { + permissions := g.Permissions + if permissions == nil { + permissions = map[string]string{} + } + + for resource, perm := range permissions { ghPerms, err = applyGitHubInstallationPermission(ghPerms, resource, perm) if err != nil { l.Errorf("unable to create github app installation token with permission %s:%s: %v", resource, perm, err) @@ -735,7 +742,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, // maybe take an optional list of repos and permission set that is driven by yaml t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPerms) if err != nil { - l.Errorf("unable to create github app installation token for repos %v with permissions %v: %v", repos, perms, err) + l.Errorf("unable to create github app installation token for repos %v with permissions %v: %v", repos, permissions, err) // return the legacy token along with no error for backwards compatibility // todo: return an error based based on app installation requirements diff --git a/scm/service.go b/scm/service.go index 9e8ff96d9..beb5eec0b 100644 --- a/scm/service.go +++ b/scm/service.go @@ -7,6 +7,7 @@ import ( "net/http" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" ) @@ -143,7 +144,7 @@ type Service interface { GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) // GetNetrc defines a function that returns the netrc // password injected into build steps. - GetNetrcPassword(context.Context, *api.Repo, *api.User, []string, map[string]string) (string, error) + GetNetrcPassword(context.Context, *api.Repo, *api.User, yaml.Git) (string, error) // SyncRepoWithInstallation defines a function that syncs // a repo with the installation, if it exists. SyncRepoWithInstallation(context.Context, *api.Repo) (*api.Repo, error) From 744cb00cdc8902fca21583aebf910fb35b2ba89e Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 15:36:35 -0500 Subject: [PATCH 45/56] enhance: new webhook tests --- compiler/native/environment_test.go | 6 +- constants/app_install.go | 4 + constants/event.go | 3 + database/step/table.go | 2 - database/types/repo_test.go | 48 ++--- .../testdata/hooks/installation_created.json | 100 +++++++++++ .../testdata/hooks/installation_deleted.json | 100 +++++++++++ .../installation_repositories_added.json | 103 +++++++++++ .../installation_repositories_removed.json | 103 +++++++++++ scm/github/webhook.go | 5 +- scm/github/webhook_test.go | 170 ++++++++++++++++++ 11 files changed, 614 insertions(+), 30 deletions(-) create mode 100644 scm/github/testdata/hooks/installation_created.json create mode 100644 scm/github/testdata/hooks/installation_deleted.json create mode 100644 scm/github/testdata/hooks/installation_repositories_added.json create mode 100644 scm/github/testdata/hooks/installation_repositories_removed.json diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index f988a1213..fc7717888 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -632,15 +632,15 @@ func TestNative_environment(t *testing.T) { netrc: &netrc, want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, - // todo: netrc + // netrc { w: workspace, b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - netrc: &netrc, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + netrc: nil, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, } diff --git a/constants/app_install.go b/constants/app_install.go index d9acf5f9a..34f146aad 100644 --- a/constants/app_install.go +++ b/constants/app_install.go @@ -25,6 +25,10 @@ const ( AppInstallRepositoriesSelectionAll = "all" // GitHub App install repositories selection when a subset of repositories are selected. AppInstallRepositoriesSelectionSelected = "selected" + // GitHub App install event type 'added'. + AppInstallRepositoriesAdded = "added" + // GitHub App install event type 'removed'. + AppInstallRepositoriesRemoved = "removed" ) const ( diff --git a/constants/event.go b/constants/event.go index ed5cd9224..a2ab76e3a 100644 --- a/constants/event.go +++ b/constants/event.go @@ -31,6 +31,9 @@ const ( // EventInstallation defines the event type for scm installation events. EventInstallation = "installation" + // EventInstallationRepositories defines the event type for scm installation_repositories events. + EventInstallationRepositories = "installation_repositories" + // Alternates for common user inputs that do not match our set constants. // EventPullAlternate defines the alternate event type for build and repo pull_request events. diff --git a/database/step/table.go b/database/step/table.go index 80ddf30ce..af7431c70 100644 --- a/database/step/table.go +++ b/database/step/table.go @@ -30,7 +30,6 @@ steps ( host VARCHAR(250), runtime VARCHAR(250), distribution VARCHAR(250), - check_id INTEGER, report_as VARCHAR(250), UNIQUE(build_id, number) ); @@ -57,7 +56,6 @@ steps ( host TEXT, runtime TEXT, distribution TEXT, - check_id INTEGER, report_as TEXT, UNIQUE(build_id, number) ); diff --git a/database/types/repo_test.go b/database/types/repo_test.go index 8bc3f2b68..f66d0b15f 100644 --- a/database/types/repo_test.go +++ b/database/types/repo_test.go @@ -321,38 +321,38 @@ func TestTypes_Repo_Validate(t *testing.T) { func TestTypes_RepoFromAPI(t *testing.T) { // setup types - repo := new(api.Repo) + r := new(api.Repo) owner := testutils.APIUser() owner.SetID(1) - repo.SetID(1) - repo.SetOwner(owner) - repo.SetHash("superSecretHash") - repo.SetOrg("github") - repo.SetName("octocat") - repo.SetFullName("github/octocat") - repo.SetLink("https://github.com/github/octocat") - repo.SetClone("https://github.com/github/octocat.git") - repo.SetBranch("main") - repo.SetTopics([]string{"cloud", "security"}) - repo.SetBuildLimit(10) - repo.SetTimeout(30) - repo.SetCounter(0) - repo.SetVisibility("public") - repo.SetPrivate(false) - repo.SetTrusted(false) - repo.SetActive(true) - repo.SetAllowEvents(api.NewEventsFromMask(1)) - repo.SetPipelineType("yaml") - repo.SetPreviousName("oldName") - repo.SetApproveBuild(constants.ApproveNever) - repo.SetInstallID(0) + r.SetID(1) + r.SetOwner(owner) + r.SetHash("superSecretHash") + r.SetOrg("github") + r.SetName("octocat") + r.SetFullName("github/octocat") + r.SetLink("https://github.com/github/octocat") + r.SetClone("https://github.com/github/octocat.git") + r.SetBranch("main") + r.SetTopics([]string{"cloud", "security"}) + r.SetBuildLimit(10) + r.SetTimeout(30) + r.SetCounter(0) + r.SetVisibility("public") + r.SetPrivate(false) + r.SetTrusted(false) + r.SetActive(true) + r.SetAllowEvents(api.NewEventsFromMask(1)) + r.SetPipelineType("yaml") + r.SetPreviousName("oldName") + r.SetApproveBuild(constants.ApproveNever) + r.SetInstallID(0) want := testRepo() want.Owner = User{} // run test - got := RepoFromAPI(repo) + got := RepoFromAPI(r) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("RepoFromAPI() mismatch (-want +got):\n%s", diff) diff --git a/scm/github/testdata/hooks/installation_created.json b/scm/github/testdata/hooks/installation_created.json new file mode 100644 index 000000000..c3904b286 --- /dev/null +++ b/scm/github/testdata/hooks/installation_created.json @@ -0,0 +1,100 @@ +{ + "action": "created", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_deleted.json b/scm/github/testdata/hooks/installation_deleted.json new file mode 100644 index 000000000..9972e0cf9 --- /dev/null +++ b/scm/github/testdata/hooks/installation_deleted.json @@ -0,0 +1,100 @@ +{ + "action": "deleted", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_repositories_added.json b/scm/github/testdata/hooks/installation_repositories_added.json new file mode 100644 index 000000000..f75fedcd1 --- /dev/null +++ b/scm/github/testdata/hooks/installation_repositories_added.json @@ -0,0 +1,103 @@ +{ + "action": "added", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories_added": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "repositories_removed": [ + + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } + } \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_repositories_removed.json b/scm/github/testdata/hooks/installation_repositories_removed.json new file mode 100644 index 000000000..476193185 --- /dev/null +++ b/scm/github/testdata/hooks/installation_repositories_removed.json @@ -0,0 +1,103 @@ +{ + "action": "removed", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories_added": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "repositories_removed": [ + + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } + } \ No newline at end of file diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 51fa7638c..fee557e45 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -545,7 +545,7 @@ func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryE // processInstallationEvent is a helper function to process the installation event. func (c *client) processInstallationEvent(_ context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { - h.SetEvent(constants.EventRepository) + h.SetEvent(constants.EventInstallation) h.SetEventAction(payload.GetAction()) install := new(internal.Installation) @@ -573,6 +573,9 @@ func (c *client) processInstallationEvent(_ context.Context, h *api.Hook, payloa // processInstallationRepositoriesEvent is a helper function to process the installation repositories event. func (c *client) processInstallationRepositoriesEvent(_ context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { + h.SetEvent(constants.EventInstallationRepositories) + h.SetEventAction(payload.GetAction()) + install := new(internal.Installation) install.Action = payload.GetAction() diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index 8f7f5bf06..3867e1a86 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -1555,3 +1555,173 @@ func TestGithub_GetDeliveryID(t *testing.T) { t.Errorf("getDeliveryID returned: %v; want: %v", got, want) } } + +func TestGitHub_ProcessWebhook_Installation(t *testing.T) { + // setup tests + var createdHook api.Hook + createdHook.SetNumber(1) + createdHook.SetSourceID("7bd477e4-4415-11e9-9359-0d41fdf9567e") + createdHook.SetWebhookID(123456) + createdHook.SetCreated(time.Now().UTC().Unix()) + createdHook.SetHost("github.com") + createdHook.SetEvent(constants.EventInstallation) + createdHook.SetEventAction(constants.AppInstallCreated) + createdHook.SetStatus(constants.StatusSuccess) + + deletedHook := createdHook + deletedHook.SetEventAction(constants.AppInstallDeleted) + + tests := []struct { + name string + file string + wantHook *api.Hook + wantInstall *internal.Installation + }{ + { + name: "installation created", + file: "testdata/hooks/installation_created.json", + wantHook: &createdHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallCreated, + ID: 1, + RepositoriesAdded: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + { + name: "installation deleted", + file: "testdata/hooks/installation_deleted.json", + wantHook: &deletedHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallDeleted, + ID: 1, + RepositoriesRemoved: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + } + + // setup router + s := httptest.NewServer(http.NotFoundHandler()) + defer s.Close() + + for _, tt := range tests { + // setup request + body, err := os.Open(tt.file) + if err != nil { + t.Errorf("unable to open file: %v", err) + } + + defer body.Close() + + request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/test", body) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("User-Agent", "GitHub-Hookshot/a22606a") + request.Header.Set("X-GitHub-Delivery", "7bd477e4-4415-11e9-9359-0d41fdf9567e") + request.Header.Set("X-GitHub-Hook-ID", "123456") + request.Header.Set("X-GitHub-Event", "installation") + + // setup client + client, _ := NewTest(s.URL) + + want := &internal.Webhook{ + Hook: tt.wantHook, + Installation: tt.wantInstall, + } + + got, err := client.ProcessWebhook(context.TODO(), request) + + if err != nil { + t.Errorf("ProcessWebhook returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) + } + } +} + +func TestGitHub_ProcessWebhook_InstallationRepositories(t *testing.T) { + // setup tests + var reposAddedHook api.Hook + reposAddedHook.SetNumber(1) + reposAddedHook.SetSourceID("7bd477e4-4415-11e9-9359-0d41fdf9567e") + reposAddedHook.SetWebhookID(123456) + reposAddedHook.SetCreated(time.Now().UTC().Unix()) + reposAddedHook.SetHost("github.com") + reposAddedHook.SetEvent(constants.EventInstallationRepositories) + reposAddedHook.SetEventAction(constants.AppInstallRepositoriesAdded) + reposAddedHook.SetStatus(constants.StatusSuccess) + + reposRemovedHook := reposAddedHook + reposRemovedHook.SetEventAction(constants.AppInstallRepositoriesRemoved) + + tests := []struct { + name string + file string + wantHook *api.Hook + wantInstall *internal.Installation + }{ + { + name: "installation_repositories repos added", + file: "testdata/hooks/installation_repositories_added.json", + wantHook: &reposAddedHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallRepositoriesAdded, + ID: 1, + RepositoriesAdded: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + { + name: "installation_repositories repos removed", + file: "testdata/hooks/installation_repositories_removed.json", + wantHook: &reposRemovedHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallRepositoriesRemoved, + ID: 1, + RepositoriesRemoved: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + } + + // setup router + s := httptest.NewServer(http.NotFoundHandler()) + defer s.Close() + + for _, tt := range tests { + // setup request + body, err := os.Open(tt.file) + if err != nil { + t.Errorf("unable to open file: %v", err) + } + + defer body.Close() + + request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/test", body) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("User-Agent", "GitHub-Hookshot/a22606a") + request.Header.Set("X-GitHub-Delivery", "7bd477e4-4415-11e9-9359-0d41fdf9567e") + request.Header.Set("X-GitHub-Hook-ID", "123456") + request.Header.Set("X-GitHub-Event", "installation_repositories") + + // setup client + client, _ := NewTest(s.URL) + + want := &internal.Webhook{ + Hook: tt.wantHook, + Installation: tt.wantInstall, + } + + got, err := client.ProcessWebhook(context.TODO(), request) + + if err != nil { + t.Errorf("ProcessWebhook returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) + } + } +} From 9e0c98f3c1d7a5662a427d6aa1dd2be80852b543 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 29 Oct 2024 17:12:20 -0500 Subject: [PATCH 46/56] enhance: new scm tests --- scm/github/app_client.go | 8 + scm/github/app_transport.go | 16 + scm/github/github.go | 2 +- scm/github/repo.go | 14 +- scm/github/repo_test.go | 287 ++++++++++++++++++ .../testdata/installation_repositories.json | 123 ++++++++ scm/github/testdata/installations.json | 52 ++++ .../testdata/installations_access_tokens.json | 134 ++++++++ 8 files changed, 626 insertions(+), 10 deletions(-) create mode 100644 scm/github/testdata/installation_repositories.json create mode 100644 scm/github/testdata/installations.json create mode 100644 scm/github/testdata/installations_access_tokens.json diff --git a/scm/github/app_client.go b/scm/github/app_client.go index 64067de09..b33004227 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -48,6 +48,10 @@ func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github. // newGithubAppClient returns the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. func (c *client) newGithubAppClient() (*github.Client, error) { + if c.AppsTransport == nil { + return nil, errors.New("unable to create github app client: no AppsTransport configured") + } + // create a github client based off the existing GitHub App configuration client, err := github.NewClient( &http.Client{ @@ -75,6 +79,8 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R } // if repo has an install ID, use it to create an installation token + // todo: fix this: its fine but return the installation token object and + // update the repo when you can find a token for it... if r.GetInstallID() != 0 { // create installation token for the repo t, _, err := client.Apps.CreateInstallationToken(ctx, r.GetInstallID(), opts) @@ -96,6 +102,8 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R for _, install := range installations { // find the installation that matches the org for the repo if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + // todo: right here... what if we're using this function to generate a netrc and the + // installation doesnt have access to that particular repo id = install.GetID() } } diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index 21d3c85f3..ca0234535 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -5,6 +5,7 @@ package github import ( "bytes" "context" + "crypto/rand" "crypto/rsa" "crypto/x509" "encoding/base64" @@ -330,3 +331,18 @@ func WithSigner(signer Signer) AppsTransportOption { at.signer = signer } } + +// NewTestAppsTransport creates a new AppsTransport for testing purposes. +func NewTestAppsTransport(baseURL string) *AppsTransport { + pk, _ := rsa.GenerateKey(rand.Reader, 2048) + return &AppsTransport{ + BaseURL: baseURL, + Client: &http.Client{Transport: http.DefaultTransport}, + tr: http.DefaultTransport, + signer: &RSASigner{ + method: jwt.SigningMethodRS256, + key: pk, + }, + appID: 1, + } +} diff --git a/scm/github/github.go b/scm/github/github.go index 539a539a7..fd52421f9 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -117,7 +117,7 @@ func New(ctx context.Context, opts ...ClientOpt) (*client, error) { } if c.config.AppID != 0 && len(c.config.AppPrivateKey) > 0 { - c.Logger.Infof("setting up GitHub App integration for App ID %d", c.config.AppID) + c.Logger.Infof("configurating github app integration for app_id %d", c.config.AppID) transport, err := c.newGitHubAppTransport(c.config.AppID, c.config.AppPrivateKey, c.config.API) if err != nil { diff --git a/scm/github/repo.go b/scm/github/repo.go index 91f0c5e21..2d3289784 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -729,11 +729,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, for resource, perm := range permissions { ghPerms, err = applyGitHubInstallationPermission(ghPerms, resource, perm) if err != nil { - l.Errorf("unable to create github app installation token with permission %s:%s: %v", resource, perm, err) - - // return the legacy token along with no error for backwards compatibility - // todo: return an error based based on app installation requirements - return u.GetToken(), nil + return u.GetToken(), err } } @@ -742,10 +738,10 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, // maybe take an optional list of repos and permission set that is driven by yaml t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPerms) if err != nil { - l.Errorf("unable to create github app installation token for repos %v with permissions %v: %v", repos, permissions, err) - // return the legacy token along with no error for backwards compatibility // todo: return an error based based on app installation requirements + l.Tracef("unable to create github app installation token for repos %v with permissions %v: %v", repos, permissions, err) + return u.GetToken(), nil } @@ -833,7 +829,7 @@ func applyGitHubInstallationPermission(perms *github.InstallationPermissions, re case constants.AppInstallPermissionWrite: break default: - return perms, fmt.Errorf("invalid permission value given for %s: %s", resource, perm) + return perms, fmt.Errorf("invalid permission level given for : in %s:%s", resource, perm) } // convert resource from yaml string @@ -843,7 +839,7 @@ func applyGitHubInstallationPermission(perms *github.InstallationPermissions, re case constants.AppInstallResourceChecks: perms.Checks = github.String(perm) default: - return perms, fmt.Errorf("invalid permission key given: %s", perm) + return perms, fmt.Errorf("invalid permission resource given for : in %s:%s", resource, perm) } return perms, nil diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 502c1cc7c..f10907ab7 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -13,8 +13,11 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/google/go-cmp/cmp" + "github.com/google/go-github/v65/github" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/constants" ) @@ -1621,3 +1624,287 @@ func TestGithub_GetBranch(t *testing.T) { t.Errorf("Commit is %v, want %v", gotCommit, wantCommit) } } + +func TestGithub_SyncRepoWithInstallation(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/app/installations", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations.json") + }) + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + engine.GET("/api/v3/installation/repositories", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installation_repositories.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + tests := []struct { + name string + org string + repo string + wantInstallID int64 + wantStatusCode int + }{ + { + name: "match", + org: "octocat", + repo: "Hello-World", + wantInstallID: 1, + wantStatusCode: http.StatusOK, + }, + { + name: "no match", + repo: "octocat/Hello-World2", + wantInstallID: 0, + wantStatusCode: http.StatusOK, + }, + } + for _, test := range tests { + // setup types + r := new(api.Repo) + r.SetOrg(test.org) + r.SetName(test.repo) + r.SetFullName(fmt.Sprintf("%s/%s", test.org, test.repo)) + + client, _ := NewTest(s.URL) + client.AppsTransport = NewTestAppsTransport(s.URL) + + // run test + got, err := client.SyncRepoWithInstallation(context.TODO(), r) + + if resp.Code != test.wantStatusCode { + t.Errorf("SyncRepoWithInstallation %s returned %v, want %v", test.name, resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("SyncRepoWithInstallation %s returned err: %v", test.name, err) + } + + if got.GetInstallID() != test.wantInstallID { + t.Errorf("SyncRepoWithInstallation %s returned %v, want %v", test.name, got.GetInstallID(), test.wantInstallID) + } + } +} + +func TestGithub_applyGitHubInstallationPermission(t *testing.T) { + tests := []struct { + name string + perms *github.InstallationPermissions + resource string + perm string + wantPerms *github.InstallationPermissions + wantErr bool + }{ + { + name: "valid read permission for contents", + perms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionNone), + }, + resource: constants.AppInstallResourceContents, + perm: constants.AppInstallPermissionRead, + wantPerms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionRead), + }, + wantErr: false, + }, + { + name: "valid write permission for checks", + perms: &github.InstallationPermissions{ + Checks: github.String(constants.AppInstallPermissionNone), + }, + resource: constants.AppInstallResourceChecks, + perm: constants.AppInstallPermissionWrite, + wantPerms: &github.InstallationPermissions{ + Checks: github.String(constants.AppInstallPermissionWrite), + }, + wantErr: false, + }, + { + name: "invalid permission value", + perms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionNone), + }, + resource: constants.AppInstallResourceContents, + perm: "invalid", + wantPerms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionNone), + }, + wantErr: true, + }, + { + name: "invalid permission key", + perms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionNone), + }, + resource: "invalid", + perm: constants.AppInstallPermissionRead, + wantPerms: &github.InstallationPermissions{ + Contents: github.String(constants.AppInstallPermissionNone), + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := applyGitHubInstallationPermission(tt.perms, tt.resource, tt.perm) + if (err != nil) != tt.wantErr { + t.Errorf("applyGitHubInstallationPermission() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.wantPerms, got); diff != "" { + t.Errorf("applyGitHubInstallationPermission() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestGithub_GetNetrcPassword(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/app/installations", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations.json") + }) + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + installedRepo := new(api.Repo) + installedRepo.SetOrg("octocat") + installedRepo.SetName("Hello-World") + installedRepo.SetInstallID(1) + + oauthRepo := new(api.Repo) + oauthRepo.SetOrg("octocat") + oauthRepo.SetName("Hello-World2") + oauthRepo.SetInstallID(0) + + u := new(api.User) + u.SetName("foo") + u.SetToken("bar") + + tests := []struct { + name string + repo *api.Repo + user *api.User + git yaml.Git + appTransport bool + wantToken string + wantErr bool + }{ + { + name: "installation token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appTransport: true, + wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + wantErr: false, + }, + { + name: "no app configured returns user oauth token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appTransport: false, + wantToken: "bar", + wantErr: false, + }, + { + name: "repo not installed returns user oauth token", + repo: oauthRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appTransport: true, + wantToken: "bar", + wantErr: false, + }, + { + name: "invalid permission resource", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"invalid": "read"}, + }, + }, + appTransport: true, + wantToken: "bar", + wantErr: true, + }, + { + name: "invalid permission level", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "invalid"}, + }, + }, + appTransport: true, + wantToken: "bar", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, _ := NewTest(s.URL) + if test.appTransport { + client.AppsTransport = NewTestAppsTransport(s.URL) + } + + got, err := client.GetNetrcPassword(context.TODO(), test.repo, test.user, test.git) + if (err != nil) != test.wantErr { + t.Errorf("GetNetrcPassword() error = %v, wantErr %v", err, test.wantErr) + return + } + if got != test.wantToken { + t.Errorf("GetNetrcPassword() = %v, want %v", got, test.wantToken) + } + }) + } +} diff --git a/scm/github/testdata/installation_repositories.json b/scm/github/testdata/installation_repositories.json new file mode 100644 index 000000000..9eb501cb5 --- /dev/null +++ b/scm/github/testdata/installation_repositories.json @@ -0,0 +1,123 @@ +{ + "total_count": 1, + "repositories": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "clone_url": "https://github.com/octocat/Hello-World.git", + "mirror_url": "git:git.example.com/octocat/Hello-World", + "hooks_url": "https://api.github.com/repos/octocat/Hello-World/hooks", + "svn_url": "https://svn.github.com/octocat/Hello-World", + "homepage": "https://github.com", + "language": null, + "forks_count": 9, + "stargazers_count": 80, + "watchers_count": 80, + "size": 108, + "default_branch": "master", + "open_issues_count": 0, + "is_template": true, + "topics": [ + "octocat", + "atom", + "electron", + "api" + ], + "has_issues": true, + "has_projects": true, + "has_wiki": true, + "has_pages": false, + "has_downloads": true, + "archived": false, + "disabled": false, + "visibility": "public", + "pushed_at": "2011-01-26T19:06:43Z", + "created_at": "2011-01-26T19:01:12Z", + "updated_at": "2011-01-26T19:14:43Z", + "allow_rebase_merge": true, + "template_repository": null, + "temp_clone_token": "ABTLWHOULUVAXGTRYU7OC2876QJ2O", + "allow_squash_merge": true, + "allow_auto_merge": false, + "delete_branch_on_merge": true, + "allow_merge_commit": true, + "subscribers_count": 42, + "network_count": 0, + "license": { + "key": "mit", + "name": "MIT License", + "url": "https://api.github.com/licenses/mit", + "spdx_id": "MIT", + "node_id": "MDc6TGljZW5zZW1pdA==", + "html_url": "https://github.com/licenses/mit" + }, + "forks": 1, + "open_issues": 1, + "watchers": 1 + } + ] + } \ No newline at end of file diff --git a/scm/github/testdata/installations.json b/scm/github/testdata/installations.json new file mode 100644 index 000000000..736849702 --- /dev/null +++ b/scm/github/testdata/installations.json @@ -0,0 +1,52 @@ +[ + { + "id": 1, + "account": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "access_tokens_url": "https://api.github.com/app/installations/1/access_tokens", + "repositories_url": "https://api.github.com/installation/repositories", + "html_url": "https://github.com/organizations/github/settings/installations/1", + "app_id": 1, + "target_id": 1, + "target_type": "Organization", + "permissions": { + "checks": "write", + "metadata": "read", + "contents": "read" + }, + "events": [ + "push", + "pull_request" + ], + "single_file_name": "config.yaml", + "has_multiple_single_files": true, + "single_file_paths": [ + "config.yml", + ".github/issue_TEMPLATE.md" + ], + "repository_selection": "selected", + "created_at": "2017-07-08T16:18:44-04:00", + "updated_at": "2017-07-08T16:18:44-04:00", + "app_slug": "github-actions", + "suspended_at": null, + "suspended_by": null + } +] \ No newline at end of file diff --git a/scm/github/testdata/installations_access_tokens.json b/scm/github/testdata/installations_access_tokens.json new file mode 100644 index 000000000..86b705880 --- /dev/null +++ b/scm/github/testdata/installations_access_tokens.json @@ -0,0 +1,134 @@ +{ + "token": "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + "expires_at": "2016-07-11T22:14:10Z", + "permissions": { + "issues": "write", + "contents": "read" + }, + "repository_selection": "selected", + "repositories": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "clone_url": "https://github.com/octocat/Hello-World.git", + "mirror_url": "git:git.example.com/octocat/Hello-World", + "hooks_url": "https://api.github.com/repos/octocat/Hello-World/hooks", + "svn_url": "https://svn.github.com/octocat/Hello-World", + "homepage": "https://github.com", + "language": null, + "forks_count": 9, + "stargazers_count": 80, + "watchers_count": 80, + "size": 108, + "default_branch": "master", + "open_issues_count": 0, + "is_template": true, + "topics": [ + "octocat", + "atom", + "electron", + "api" + ], + "has_issues": true, + "has_projects": true, + "has_wiki": true, + "has_pages": false, + "has_downloads": true, + "archived": false, + "disabled": false, + "visibility": "public", + "pushed_at": "2011-01-26T19:06:43Z", + "created_at": "2011-01-26T19:01:12Z", + "updated_at": "2011-01-26T19:14:43Z", + "permissions": { + "admin": false, + "push": false, + "pull": true + }, + "allow_rebase_merge": true, + "template_repository": null, + "temp_clone_token": "ABTLWHOULUVAXGTRYU7OC2876QJ2O", + "allow_squash_merge": true, + "allow_auto_merge": false, + "delete_branch_on_merge": true, + "allow_merge_commit": true, + "subscribers_count": 42, + "network_count": 0, + "license": { + "key": "mit", + "name": "MIT License", + "url": "https://api.github.com/licenses/mit", + "spdx_id": "MIT", + "node_id": "MDc6TGljZW5zZW1pdA==", + "html_url": "https://github.com/licenses/mit" + }, + "forks": 1, + "open_issues": 1, + "watchers": 1 + } + ] + } \ No newline at end of file From ebf1786321d6cc667a95dbc24d314bfd84fb4f32 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 09:24:35 -0500 Subject: [PATCH 47/56] enhance: helper for installationCanReadRepo --- scm/github/app_client.go | 91 +++++++++++++++++++++++++++------------- scm/github/repo.go | 48 ++++++++------------- 2 files changed, 80 insertions(+), 59 deletions(-) diff --git a/scm/github/app_client.go b/scm/github/app_client.go index b33004227..27a793d81 100644 --- a/scm/github/app_client.go +++ b/scm/github/app_client.go @@ -5,6 +5,7 @@ package github import ( "context" "errors" + "fmt" "net/http" "net/http/httptrace" "net/url" @@ -16,6 +17,7 @@ import ( "golang.org/x/oauth2" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" ) // newOAuthTokenClient returns the GitHub OAuth client. @@ -66,11 +68,11 @@ func (c *client) newGithubAppClient() (*github.Client, error) { } // newGithubAppInstallationRepoToken returns the GitHub App installation token for a particular repo with granular permissions. -func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (string, error) { +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (*github.InstallationToken, int64, error) { // create a github client based off the existing GitHub App configuration client, err := c.newGithubAppClient() if err != nil { - return "", err + return nil, 0, err } opts := &github.InstallationTokenOptions{ @@ -78,46 +80,79 @@ func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.R Permissions: permissions, } + id := r.GetInstallID() + // if repo has an install ID, use it to create an installation token - // todo: fix this: its fine but return the installation token object and - // update the repo when you can find a token for it... - if r.GetInstallID() != 0 { - // create installation token for the repo - t, _, err := client.Apps.CreateInstallationToken(ctx, r.GetInstallID(), opts) + if id == 0 { + // list all installations (a.k.a. orgs) where the GitHub App is installed + installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) if err != nil { - return "", err + return nil, 0, err } - return t.GetToken(), nil - } - - // list all installations (a.k.a. orgs) where the GitHub App is installed - installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) - if err != nil { - return "", err - } - - var id int64 - // iterate through the list of installations - for _, install := range installations { - // find the installation that matches the org for the repo - if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { - // todo: right here... what if we're using this function to generate a netrc and the - // installation doesnt have access to that particular repo - id = install.GetID() + // iterate through the list of installations + for _, install := range installations { + // find the installation that matches the org for the repo + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + if install.GetRepositorySelection() == constants.AppInstallRepositoriesSelectionSelected { + installationCanReadRepo, err := c.installationCanReadRepo(ctx, r, install) + if err != nil { + return nil, 0, fmt.Errorf("installation for org %s exists but unable to check if it can read repo %s: %w", install.GetAccount().GetLogin(), r.GetFullName(), err) + } + + if !installationCanReadRepo { + return nil, 0, fmt.Errorf("installation for org %s exists but does not have access to repo %s", install.GetAccount().GetLogin(), r.GetFullName()) + } + } + + id = install.GetID() + } } } // failsafe in case the repo does not belong to an org where the GitHub App is installed if id == 0 { - return "", errors.New("unable to find installation ID for repo") + return nil, 0, errors.New("unable to find installation ID for repo") } // create installation token for the repo t, _, err := client.Apps.CreateInstallationToken(ctx, id, opts) if err != nil { - return "", err + return nil, 0, err + } + + return t, id, nil +} + +// installationCanReadRepo checks if the installation can read the repo. +func (c *client) installationCanReadRepo(ctx context.Context, r *api.Repo, installation *github.Installation) (bool, error) { + installationCanReadRepo := false + + if installation.GetRepositorySelection() == constants.AppInstallRepositoriesSelectionSelected { + client, err := c.newGithubAppClient() + if err != nil { + return false, err + } + + t, _, err := client.Apps.CreateInstallationToken(ctx, installation.GetID(), &github.InstallationTokenOptions{}) + if err != nil { + return false, err + } + + client = c.newOAuthTokenClient(ctx, t.GetToken()) + + repos, _, err := client.Apps.ListRepos(ctx, &github.ListOptions{}) + if err != nil { + return false, err + } + + for _, repo := range repos.Repositories { + if strings.EqualFold(repo.GetFullName(), r.GetFullName()) { + installationCanReadRepo = true + break + } + } } - return t.GetToken(), nil + return installationCanReadRepo, nil } diff --git a/scm/github/repo.go b/scm/github/repo.go index 2d3289784..f5af49fad 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -16,6 +16,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/constants" + "github.com/go-vela/server/database" ) // ConfigBackoff is a wrapper for Config that will retry five times if the function @@ -733,10 +734,10 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, } } - // the app might not be installed + // the app might not be installedm therefore we retain backwords compatibility via the user oauth token // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation - // maybe take an optional list of repos and permission set that is driven by yaml - t, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPerms) + // the optional list of repos and permissions are driven by yaml + installToken, installID, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPerms) if err != nil { // return the legacy token along with no error for backwards compatibility // todo: return an error based based on app installation requirements @@ -745,10 +746,18 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, return u.GetToken(), nil } - if len(t) != 0 { + if installToken != nil && len(installToken.GetToken()) != 0 { l.Tracef("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) - return t, nil + // sync the install ID with the repo + r.SetInstallID(installID) + + _, err = database.FromContext(ctx).UpdateRepo(ctx, r) + if err != nil { + c.Logger.Tracef("unable to update repo with install ID %d: %v", installID, err) + } + + return installToken.GetToken(), nil } l.Tracef("using user oauth token for %s/%s", r.GetOrg(), r.GetName()) @@ -785,32 +794,9 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap return nil, nil } - installationCanReadRepo := false - - if installation.GetRepositorySelection() != constants.AppInstallRepositoriesSelectionAll { - client, err := c.newGithubAppClient() - if err != nil { - return r, err - } - - t, _, err := client.Apps.CreateInstallationToken(ctx, installation.GetID(), &github.InstallationTokenOptions{}) - if err != nil { - return r, err - } - - client = c.newOAuthTokenClient(ctx, t.GetToken()) - - repos, _, err := client.Apps.ListRepos(ctx, &github.ListOptions{}) - if err != nil { - return r, err - } - - for _, repo := range repos.Repositories { - if strings.EqualFold(repo.GetFullName(), r.GetFullName()) { - installationCanReadRepo = true - break - } - } + installationCanReadRepo, err := c.installationCanReadRepo(ctx, r, installation) + if err != nil { + return r, err } if installationCanReadRepo { From 6be732d8818c1f894fd816db4043dcab3c033ae2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 09:40:51 -0500 Subject: [PATCH 48/56] chore: more tests (installationCanReadRepo) --- scm/github/app_transport.go | 1 + .../{app_client.go => github_client.go} | 0 scm/github/github_client_test.go | 125 ++++++++++++++++++ scm/github/repo_test.go | 46 +++---- 4 files changed, 149 insertions(+), 23 deletions(-) rename scm/github/{app_client.go => github_client.go} (100%) create mode 100644 scm/github/github_client_test.go diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go index ca0234535..34ebb0742 100644 --- a/scm/github/app_transport.go +++ b/scm/github/app_transport.go @@ -335,6 +335,7 @@ func WithSigner(signer Signer) AppsTransportOption { // NewTestAppsTransport creates a new AppsTransport for testing purposes. func NewTestAppsTransport(baseURL string) *AppsTransport { pk, _ := rsa.GenerateKey(rand.Reader, 2048) + return &AppsTransport{ BaseURL: baseURL, Client: &http.Client{Transport: http.DefaultTransport}, diff --git a/scm/github/app_client.go b/scm/github/github_client.go similarity index 100% rename from scm/github/app_client.go rename to scm/github/github_client.go diff --git a/scm/github/github_client_test.go b/scm/github/github_client_test.go new file mode 100644 index 000000000..5d289f874 --- /dev/null +++ b/scm/github/github_client_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/google/go-github/v65/github" +) + +func TestClient_installationCanReadRepo(t *testing.T) { + // setup types + accessibleRepo := new(api.Repo) + accessibleRepo.SetOrg("octocat") + accessibleRepo.SetName("Hello-World") + accessibleRepo.SetFullName("octocat/Hello-World") + accessibleRepo.SetInstallID(0) + + inaccessibleRepo := new(api.Repo) + inaccessibleRepo.SetOrg("octocat") + inaccessibleRepo.SetName("Hello-World") + inaccessibleRepo.SetFullName("octocat/Hello-World2") + inaccessibleRepo.SetInstallID(4) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + engine.GET("/api/v3/installation/repositories", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installation_repositories.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + oauthClient, _ := NewTest(s.URL) + + appsClient, err := NewTest(s.URL) + if err != nil { + t.Errorf("unable to create GitHub App client: %v", err) + } + + appsClient.AppsTransport = NewTestAppsTransport("") + + // setup tests + tests := []struct { + name string + client *client + repo *api.Repo + installation *github.Installation + appsTransport bool + want bool + wantErr bool + }{ + { + name: "installation can read repo", + client: appsClient, + repo: accessibleRepo, + installation: &github.Installation{ + ID: github.Int64(1), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: true, + wantErr: false, + }, + { + name: "installation cannot read repo", + client: appsClient, + repo: inaccessibleRepo, + installation: &github.Installation{ + ID: github.Int64(2), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: false, + wantErr: false, + }, + { + name: "no GitHub App client", + client: oauthClient, + repo: accessibleRepo, + installation: &github.Installation{ + ID: github.Int64(1), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: false, + wantErr: true, + }, + } + + // run tests + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.client.installationCanReadRepo(context.Background(), tt.repo, tt.installation) + if (err != nil) != tt.wantErr { + t.Errorf("installationCanReadRepo() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("installationCanReadRepo() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index f10907ab7..898f072ac 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -1810,13 +1810,13 @@ func TestGithub_GetNetrcPassword(t *testing.T) { u.SetToken("bar") tests := []struct { - name string - repo *api.Repo - user *api.User - git yaml.Git - appTransport bool - wantToken string - wantErr bool + name string + repo *api.Repo + user *api.User + git yaml.Git + appsTransport bool + wantToken string + wantErr bool }{ { name: "installation token", @@ -1828,9 +1828,9 @@ func TestGithub_GetNetrcPassword(t *testing.T) { Permissions: map[string]string{"contents": "read"}, }, }, - appTransport: true, - wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", - wantErr: false, + appsTransport: true, + wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + wantErr: false, }, { name: "no app configured returns user oauth token", @@ -1842,9 +1842,9 @@ func TestGithub_GetNetrcPassword(t *testing.T) { Permissions: map[string]string{"contents": "read"}, }, }, - appTransport: false, - wantToken: "bar", - wantErr: false, + appsTransport: false, + wantToken: "bar", + wantErr: false, }, { name: "repo not installed returns user oauth token", @@ -1856,9 +1856,9 @@ func TestGithub_GetNetrcPassword(t *testing.T) { Permissions: map[string]string{"contents": "read"}, }, }, - appTransport: true, - wantToken: "bar", - wantErr: false, + appsTransport: true, + wantToken: "bar", + wantErr: false, }, { name: "invalid permission resource", @@ -1870,9 +1870,9 @@ func TestGithub_GetNetrcPassword(t *testing.T) { Permissions: map[string]string{"invalid": "read"}, }, }, - appTransport: true, - wantToken: "bar", - wantErr: true, + appsTransport: true, + wantToken: "bar", + wantErr: true, }, { name: "invalid permission level", @@ -1884,16 +1884,16 @@ func TestGithub_GetNetrcPassword(t *testing.T) { Permissions: map[string]string{"contents": "invalid"}, }, }, - appTransport: true, - wantToken: "bar", - wantErr: true, + appsTransport: true, + wantToken: "bar", + wantErr: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { client, _ := NewTest(s.URL) - if test.appTransport { + if test.appsTransport { client.AppsTransport = NewTestAppsTransport(s.URL) } From ffae8698f6ca8ab30151cc45c3e933cda581ba1d Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 10:09:32 -0500 Subject: [PATCH 49/56] chore: more tests (app transport) --- scm/github/app_transport_test.go | 169 +++++++++++++++++++ scm/github/github_client_test.go | 2 + scm/github/repo_test.go | 272 +++++++++++++++---------------- 3 files changed, 307 insertions(+), 136 deletions(-) create mode 100644 scm/github/app_transport_test.go diff --git a/scm/github/app_transport_test.go b/scm/github/app_transport_test.go new file mode 100644 index 000000000..3bf27443c --- /dev/null +++ b/scm/github/app_transport_test.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/go-cmp/cmp" +) + +func TestGitHub_cloneRequest(t *testing.T) { + tests := []struct { + name string + request *http.Request + }{ + { + name: "basic request", + request: &http.Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + }, + }, + { + name: "request with body", + request: &http.Request{ + Method: "POST", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{"key":"value"}`)), + }, + }, + { + name: "request with multiple headers", + request: &http.Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Accept": []string{"application/json"}, + "Authorization": []string{"Bearer token"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clonedReq := cloneRequest(tt.request) + + if clonedReq == tt.request { + t.Errorf("cloneRequest() = %v, want different instance", clonedReq) + } + + if diff := cmp.Diff(clonedReq.Header, tt.request.Header); diff != "" { + t.Errorf("cloneRequest() headers mismatch (-want +got):\n%s", diff) + } + + if clonedReq.Method != tt.request.Method { + t.Errorf("cloneRequest() method = %v, want %v", clonedReq.Method, tt.request.Method) + } + + if clonedReq.URL.String() != tt.request.URL.String() { + t.Errorf("cloneRequest() URL = %v, want %v", clonedReq.URL, tt.request.URL) + } + }) + } +} + +func TestAppsTransport_RoundTrip(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + + s := httptest.NewServer(engine) + defer s.Close() + + _url, _ := url.Parse(s.URL) + + tests := []struct { + name string + transport *AppsTransport + request *http.Request + wantHeader string + wantErr bool + }{ + { + name: "valid GET request", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "GET", + URL: _url, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + }, + wantHeader: "Bearer ", + wantErr: false, + }, + { + name: "valid POST request", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "POST", + URL: _url, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{"key":"value"}`)), + }, + wantHeader: "Bearer ", + wantErr: false, + }, + { + name: "request with invalid URL", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "GET", + URL: &url.URL{Path: "://invalid-url"}, + Header: http.Header{}, + }, + wantHeader: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := tt.transport.RoundTrip(tt.request) + if (err != nil) != tt.wantErr { + t.Errorf("RoundTrip() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if got := tt.request.Header.Get("Authorization"); !strings.HasPrefix(got, tt.wantHeader) { + t.Errorf("RoundTrip() Authorization header = %v, want prefix %v", got, tt.wantHeader) + } + } + if resp != nil { + resp.Body.Close() + } + }) + } +} diff --git a/scm/github/github_client_test.go b/scm/github/github_client_test.go index 5d289f874..dc045ce4c 100644 --- a/scm/github/github_client_test.go +++ b/scm/github/github_client_test.go @@ -28,6 +28,8 @@ func TestClient_installationCanReadRepo(t *testing.T) { inaccessibleRepo.SetFullName("octocat/Hello-World2") inaccessibleRepo.SetInstallID(4) + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() _, engine := gin.CreateTestContext(resp) diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 898f072ac..b8516c795 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -1625,6 +1625,142 @@ func TestGithub_GetBranch(t *testing.T) { } } +func TestGithub_GetNetrcPassword(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/app/installations", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations.json") + }) + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + installedRepo := new(api.Repo) + installedRepo.SetOrg("octocat") + installedRepo.SetName("Hello-World") + installedRepo.SetInstallID(1) + + oauthRepo := new(api.Repo) + oauthRepo.SetOrg("octocat") + oauthRepo.SetName("Hello-World2") + oauthRepo.SetInstallID(0) + + u := new(api.User) + u.SetName("foo") + u.SetToken("bar") + + tests := []struct { + name string + repo *api.Repo + user *api.User + git yaml.Git + appsTransport bool + wantToken string + wantErr bool + }{ + { + name: "installation token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: true, + wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + wantErr: false, + }, + { + name: "no app configured returns user oauth token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: false, + wantToken: "bar", + wantErr: false, + }, + { + name: "repo not installed returns user oauth token", + repo: oauthRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: false, + }, + { + name: "invalid permission resource", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"invalid": "read"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: true, + }, + { + name: "invalid permission level", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "invalid"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, _ := NewTest(s.URL) + if test.appsTransport { + client.AppsTransport = NewTestAppsTransport(s.URL) + } + + got, err := client.GetNetrcPassword(context.TODO(), test.repo, test.user, test.git) + if (err != nil) != test.wantErr { + t.Errorf("GetNetrcPassword() error = %v, wantErr %v", err, test.wantErr) + return + } + if got != test.wantToken { + t.Errorf("GetNetrcPassword() = %v, want %v", got, test.wantToken) + } + }) + } +} + func TestGithub_SyncRepoWithInstallation(t *testing.T) { // setup context gin.SetMode(gin.TestMode) @@ -1772,139 +1908,3 @@ func TestGithub_applyGitHubInstallationPermission(t *testing.T) { }) } } - -func TestGithub_GetNetrcPassword(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) - - resp := httptest.NewRecorder() - _, engine := gin.CreateTestContext(resp) - - // setup mock server - engine.GET("/api/v3/app/installations", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/installations.json") - }) - engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/installations_access_tokens.json") - }) - - s := httptest.NewServer(engine) - defer s.Close() - - installedRepo := new(api.Repo) - installedRepo.SetOrg("octocat") - installedRepo.SetName("Hello-World") - installedRepo.SetInstallID(1) - - oauthRepo := new(api.Repo) - oauthRepo.SetOrg("octocat") - oauthRepo.SetName("Hello-World2") - oauthRepo.SetInstallID(0) - - u := new(api.User) - u.SetName("foo") - u.SetToken("bar") - - tests := []struct { - name string - repo *api.Repo - user *api.User - git yaml.Git - appsTransport bool - wantToken string - wantErr bool - }{ - { - name: "installation token", - repo: installedRepo, - user: u, - git: yaml.Git{ - Token: yaml.Token{ - Repositories: []string{"Hello-World"}, - Permissions: map[string]string{"contents": "read"}, - }, - }, - appsTransport: true, - wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", - wantErr: false, - }, - { - name: "no app configured returns user oauth token", - repo: installedRepo, - user: u, - git: yaml.Git{ - Token: yaml.Token{ - Repositories: []string{"Hello-World"}, - Permissions: map[string]string{"contents": "read"}, - }, - }, - appsTransport: false, - wantToken: "bar", - wantErr: false, - }, - { - name: "repo not installed returns user oauth token", - repo: oauthRepo, - user: u, - git: yaml.Git{ - Token: yaml.Token{ - Repositories: []string{"Hello-World"}, - Permissions: map[string]string{"contents": "read"}, - }, - }, - appsTransport: true, - wantToken: "bar", - wantErr: false, - }, - { - name: "invalid permission resource", - repo: installedRepo, - user: u, - git: yaml.Git{ - Token: yaml.Token{ - Repositories: []string{"Hello-World"}, - Permissions: map[string]string{"invalid": "read"}, - }, - }, - appsTransport: true, - wantToken: "bar", - wantErr: true, - }, - { - name: "invalid permission level", - repo: installedRepo, - user: u, - git: yaml.Git{ - Token: yaml.Token{ - Repositories: []string{"Hello-World"}, - Permissions: map[string]string{"contents": "invalid"}, - }, - }, - appsTransport: true, - wantToken: "bar", - wantErr: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, _ := NewTest(s.URL) - if test.appsTransport { - client.AppsTransport = NewTestAppsTransport(s.URL) - } - - got, err := client.GetNetrcPassword(context.TODO(), test.repo, test.user, test.git) - if (err != nil) != test.wantErr { - t.Errorf("GetNetrcPassword() error = %v, wantErr %v", err, test.wantErr) - return - } - if got != test.wantToken { - t.Errorf("GetNetrcPassword() = %v, want %v", got, test.wantToken) - } - }) - } -} From 8668615a1bd74054a4244c0cec5e70a868b8d2bb Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 10:10:23 -0500 Subject: [PATCH 50/56] fix: gci --- scm/github/github_client_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scm/github/github_client_test.go b/scm/github/github_client_test.go index dc045ce4c..57de6f7d7 100644 --- a/scm/github/github_client_test.go +++ b/scm/github/github_client_test.go @@ -9,9 +9,10 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/google/go-github/v65/github" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" - "github.com/google/go-github/v65/github" ) func TestClient_installationCanReadRepo(t *testing.T) { From 2246b3d12dc6e1d23b0ebb6b7dedd4499789ae1f Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 10:38:45 -0500 Subject: [PATCH 51/56] fix: set db in the compiler to support repo sync --- api/build/compile_publish.go | 1 + compiler/engine.go | 4 ++++ compiler/native/compile.go | 2 +- compiler/native/native.go | 9 +++++++++ scm/github/repo.go | 14 ++++++++------ scm/github/repo_test.go | 2 +- scm/service.go | 2 +- 7 files changed, 25 insertions(+), 9 deletions(-) diff --git a/api/build/compile_publish.go b/api/build/compile_publish.go index 40a385780..4a0f2d659 100644 --- a/api/build/compile_publish.go +++ b/api/build/compile_publish.go @@ -269,6 +269,7 @@ func CompileAndPublish( WithUser(u). WithLabels(cfg.Labels). WithSCM(scm). + WithDatabase(database). Compile(ctx, pipelineFile) if err != nil { // format the error message with extra information diff --git a/compiler/engine.go b/compiler/engine.go index 3f5010978..ab4555296 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -10,6 +10,7 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" "github.com/go-vela/server/scm" ) @@ -150,6 +151,9 @@ type Engine interface { // WithSCM defines a function that sets // the scm in the Engine. WithSCM(scm.Service) Engine + // WithDatabase defines a function that sets + // the database in the Engine. + WithDatabase(database.Interface) Engine // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(context.Context, string, string) Engine diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 5e0f7a857..4cba9384d 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -49,7 +49,7 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * // netrc can be provided directly using WithNetrc for situations like local exec if c.netrc == nil && c.scm != nil { // get the netrc password from the scm - netrc, err := c.scm.GetNetrcPassword(ctx, c.repo, c.user, p.Git) + netrc, err := c.scm.GetNetrcPassword(ctx, c.db, c.repo, c.user, p.Git) if err != nil { return nil, nil, err } diff --git a/compiler/native/native.go b/compiler/native/native.go index 6d72e8a8b..d7e170c11 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/image" "github.com/go-vela/server/scm" @@ -46,6 +47,7 @@ type client struct { repo *api.Repo user *api.User labels []string + db database.Interface scm scm.Service netrc *string } @@ -249,3 +251,10 @@ func (c *client) WithSCM(_scm scm.Service) compiler.Engine { return c } + +// WithDatabase sets the database in the Engine. +func (c *client) WithDatabase(db database.Interface) compiler.Engine { + c.db = db + + return c +} diff --git a/scm/github/repo.go b/scm/github/repo.go index f5af49fad..55ffa3113 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -681,7 +681,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str // GetNetrcPassword returns a clone token using the repo's github app installation if it exists. // If not, it defaults to the user OAuth token. -func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, g yaml.Git) (string, error) { +func (c *client) GetNetrcPassword(ctx context.Context, db database.Interface, r *api.Repo, u *api.User, g yaml.Git) (string, error) { l := c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -749,12 +749,14 @@ func (c *client) GetNetrcPassword(ctx context.Context, r *api.Repo, u *api.User, if installToken != nil && len(installToken.GetToken()) != 0 { l.Tracef("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) - // sync the install ID with the repo - r.SetInstallID(installID) + // (optional) sync the install ID with the repo + if db != nil { + r.SetInstallID(installID) - _, err = database.FromContext(ctx).UpdateRepo(ctx, r) - if err != nil { - c.Logger.Tracef("unable to update repo with install ID %d: %v", installID, err) + _, err = db.UpdateRepo(ctx, r) + if err != nil { + c.Logger.Tracef("unable to update repo with install ID %d: %v", installID, err) + } } return installToken.GetToken(), nil diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index b8516c795..938f1660b 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -1749,7 +1749,7 @@ func TestGithub_GetNetrcPassword(t *testing.T) { client.AppsTransport = NewTestAppsTransport(s.URL) } - got, err := client.GetNetrcPassword(context.TODO(), test.repo, test.user, test.git) + got, err := client.GetNetrcPassword(context.TODO(), nil, test.repo, test.user, test.git) if (err != nil) != test.wantErr { t.Errorf("GetNetrcPassword() error = %v, wantErr %v", err, test.wantErr) return diff --git a/scm/service.go b/scm/service.go index beb5eec0b..7dd6e75bf 100644 --- a/scm/service.go +++ b/scm/service.go @@ -144,7 +144,7 @@ type Service interface { GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) // GetNetrc defines a function that returns the netrc // password injected into build steps. - GetNetrcPassword(context.Context, *api.Repo, *api.User, yaml.Git) (string, error) + GetNetrcPassword(context.Context, database.Interface, *api.Repo, *api.User, yaml.Git) (string, error) // SyncRepoWithInstallation defines a function that syncs // a repo with the installation, if it exists. SyncRepoWithInstallation(context.Context, *api.Repo) (*api.Repo, error) From ac86e63d6cf2969e0a8f4091c419eba621a7d76a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 30 Oct 2024 10:50:55 -0500 Subject: [PATCH 52/56] fix: only update when necessary, fix test --- scm/github/repo.go | 2 +- .../testdata/hooks/installation_repositories_removed.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scm/github/repo.go b/scm/github/repo.go index 55ffa3113..abef37151 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -750,7 +750,7 @@ func (c *client) GetNetrcPassword(ctx context.Context, db database.Interface, r l.Tracef("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) // (optional) sync the install ID with the repo - if db != nil { + if db != nil && r.GetInstallID() != installID { r.SetInstallID(installID) _, err = db.UpdateRepo(ctx, r) diff --git a/scm/github/testdata/hooks/installation_repositories_removed.json b/scm/github/testdata/hooks/installation_repositories_removed.json index 476193185..b7a82ec5b 100644 --- a/scm/github/testdata/hooks/installation_repositories_removed.json +++ b/scm/github/testdata/hooks/installation_repositories_removed.json @@ -49,6 +49,9 @@ "suspended_at": null }, "repositories_added": [ + + ], + "repositories_removed": [ { "id": 1, "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", @@ -63,9 +66,6 @@ "full_name": "Codertocat/Hello-World2", "private": false } - ], - "repositories_removed": [ - ], "requester": null, "enterprise": { From 3916cbeebba6d447cfa834b9fe0005c04f128954 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 31 Oct 2024 10:36:08 -0500 Subject: [PATCH 53/56] fix: return repo --- scm/github/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scm/github/repo.go b/scm/github/repo.go index abef37151..d5ca50ad5 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -793,7 +793,7 @@ func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*ap } if installation == nil { - return nil, nil + return r, nil } installationCanReadRepo, err := c.installationCanReadRepo(ctx, r, installation) From f7d472344004bb24fc98166d7e1c940834fd951f Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 5 Nov 2024 09:45:35 -0600 Subject: [PATCH 54/56] chore: merge with main --- api/webhook/post.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/webhook/post.go b/api/webhook/post.go index 16f96b42e..45889285d 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -160,8 +160,6 @@ func PostWebhook(c *gin.Context) { l.Debugf("hook generated from SCM: %v", h) l.Debugf("repo generated from SCM: %v", r) - db := database.FromContext(c) - // if event is repository event, handle separately and return if strings.EqualFold(h.GetEvent(), constants.EventRepository) { r, err = handleRepositoryEvent(ctx, l, db, m, h, r) From fae9c1f0d487cd2c288c4b2cdd1d1e577dd01162 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 5 Nov 2024 09:48:10 -0600 Subject: [PATCH 55/56] chore: merge with main --- scm/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scm/setup.go b/scm/setup.go index 2585b13f4..a2a0224a4 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -72,7 +72,7 @@ func (s *Setup) Github(ctx context.Context) (Service, error) { // Gitlab creates and returns a Vela service capable of // integrating with a Gitlab scm system. -func (s *Setup) Gitlab(ctx context.Context) (Service, error) { +func (s *Setup) Gitlab(_ context.Context) (Service, error) { logrus.Trace("creating gitlab scm client from setup") return nil, fmt.Errorf("unsupported scm driver: %s", constants.DriverGitlab) From 8e0fe7f9dfd87ebe09abc1b31c2d51de4105ab02 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 5 Nov 2024 09:58:42 -0600 Subject: [PATCH 56/56] chore: lint --- api/repo/repair.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/repo/repair.go b/api/repo/repair.go index f4db0d74a..bb15e2734 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -62,6 +62,8 @@ import ( // RepairRepo represents the API handler to remove // and then create a webhook for a repo. +// +//nolint:funlen // ignore statement count func RepairRepo(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*internal.Metadata)