feat: opt-in gh app integration

api/auth/get_token.go

@@ -5,6 +5,7 @@ package auth
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
@@ -36,6 +37,10 @@ import (
 //   name: redirect_uri
 //   description: The URL where the user will be sent after authorization
 //   type: string
+// - in: query
+//   name: setup_action
+//   description: The specific setup action callback identifier
+//   type: string
 // responses:
 //   '200':
 //     description: Successfully authenticated
@@ -46,6 +51,10 @@ import (
 //       "$ref": "#/definitions/Token"
 //   '307':
 //     description: Redirected for authentication
+//   '400':
+//     description: Bad Request
+//     schema:
+//       "$ref": "#/definitions/Error"
 //   '401':
 //     description: Unauthorized
 //     schema:
@@ -69,6 +78,32 @@ func GetAuthToken(c *gin.Context) {
 	// capture the OAuth state if present
 	oAuthState := c.Request.FormValue("state")
 
+	// handle scm setup events
+	// setup_action==install represents the GitHub App installation callback redirect
+	if c.Request.FormValue("setup_action") == "install" {
+		installID, err := strconv.ParseInt(c.Request.FormValue("installation_id"), 10, 0)
+		if err != nil {
+			retErr := fmt.Errorf("unable to parse installation_id: %w", err)
+
+			util.HandleError(c, http.StatusBadRequest, retErr)
+
+			return
+		}
+
+		r, err := scm.FromContext(c).FinishInstallation(ctx, c.Request, installID)
+		if err != nil {
+			retErr := fmt.Errorf("unable to finish installation: %w", err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+
+		c.Redirect(http.StatusTemporaryRedirect, r)
+
+		return
+	}
+
 	// capture the OAuth code if present
 	code := c.Request.FormValue("code")
 	if len(code) == 0 {

api/build/compile_publish.go

@@ -268,6 +268,7 @@ func CompileAndPublish(
 			WithRepo(repo).
 			WithUser(u).
 			WithLabels(cfg.Labels).
+			WithSCM(scm).
 			Compile(ctx, pipelineFile)
 		if err != nil {
 			// format the error message with extra information

api/build/plan.go

@@ -57,7 +57,7 @@ func PlanBuild(ctx context.Context, database database.Interface, scm scm.Service
 	}
 
 	// plan all steps for the build
-	steps, err := step.PlanSteps(ctx, database, scm, p, b)
+	steps, err := step.PlanSteps(ctx, database, scm, p, b, r)
 	if err != nil {
 		// clean up the objects from the pipeline in the database
 		CleanBuild(ctx, database, b, services, steps, err)

api/repo/create.go

@@ -273,6 +273,18 @@ func CreateRepo(c *gin.Context) {
 		}
 	}
 
+	// map this repo to an installation if possible
+	if r.GetInstallID() == 0 {
+		r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r)
+		if err != nil {
+			retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+	}
+
 	// if the repo exists but is inactive
 	if len(dbRepo.GetOrg()) > 0 && !dbRepo.GetActive() {
 		// update the repo owner
@@ -281,6 +293,8 @@ func CreateRepo(c *gin.Context) {
 		dbRepo.SetBranch(r.GetBranch())
 		// activate the repo
 		dbRepo.SetActive(true)
+		// update the install_id
+		dbRepo.SetInstallID(r.GetInstallID())
 
 		// send API call to update the repo
 		// NOTE: not logging modification out separately

api/repo/repair.go

@@ -62,7 +62,7 @@
 
 // RepairRepo represents the API handler to remove
 // and then create a webhook for a repo.
 func RepairRepo(c *gin.Context) {
 	// capture middleware values
 	m := c.MustGet("metadata").(*internal.Metadata)
 	l := c.MustGet("logger").(*logrus.Entry)
@@ -163,21 +163,48 @@
 		}
 	}
 
+	dirty := false
+
 	// if the repo was previously inactive, mark it as active
 	if !r.GetActive() {
 		r.SetActive(true)
 
-		// send API call to update the repo
+		dirty = true
+
+		l.Tracef("repo %s repaired - set to active", r.GetFullName())
+	}
+
+	// map this repo to an installation, if possible
+	if r.GetInstallID() == 0 {
+		r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r)
+		if err != nil {
+			retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err)
+
+			util.HandleError(c, http.StatusInternalServerError, retErr)
+
+			return
+		}
+
+		// install_id was synced
+		if r.GetInstallID() != 0 {
+			dirty = true
+
+			l.Tracef("repo %s repaired - set install_id to %d", r.GetFullName(), r.GetInstallID())
+		}
+	}
+
+	// update the repo in the database, if necessary
+	if dirty {
 		_, err := database.FromContext(c).UpdateRepo(ctx, r)
 		if err != nil {
-			retErr := fmt.Errorf("unable to set repo %s to active: %w", r.GetFullName(), err)
+			retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err)
 
 			util.HandleError(c, http.StatusInternalServerError, retErr)
 
 			return
 		}
 
-		l.Infof("repo %s updated - set to active", r.GetFullName())
+		l.Infof("repo %s repaired - database updated", r.GetFullName())
 	}
 
 	c.JSON(http.StatusOK, fmt.Sprintf("repo %s repaired", r.GetFullName()))

api/step/plan.go

@@ -19,7 +19,7 @@
 // PlanSteps is a helper function to plan all steps
 // in the build for execution. This creates the steps
 // for the build.
-func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build) ([]*types.Step, error) {
+func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *types.Build, r *types.Repo) ([]*types.Step, error) {
 	// variable to store planned steps
 	steps := []*types.Step{}
 

api/types/repo.go

@@ -32,6 +32,7 @@ type Repo struct {
 	PipelineType *string   `json:"pipeline_type,omitempty"`
 	PreviousName *string   `json:"previous_name,omitempty"`
 	ApproveBuild *string   `json:"approve_build,omitempty"`
+	InstallID    *int64    `json:"install_id,omitempty"`
 }
 
 // Environment returns a list of environment variables
@@ -345,6 +346,19 @@ func (r *Repo) GetApproveBuild() string {
 	return *r.ApproveBuild
 }
 
+// GetInstallID returns the InstallID field.
+//
+// When the provided Repo type is nil, or the field within
+// the type is nil, it returns the zero value for the field.
+func (r *Repo) GetInstallID() int64 {
+	// return zero value if Repo type or InstallID field is nil
+	if r == nil || r.InstallID == nil {
+		return 0
+	}
+
+	return *r.InstallID
+}
+
 // SetID sets the ID field.
 //
 // When the provided Repo type is nil, it
@@ -618,6 +632,19 @@ func (r *Repo) SetApproveBuild(v string) {
 	r.ApproveBuild = &v
 }
 
+// SetInstallID sets the InstallID field.
+//
+// When the provided Repo type is nil, it
+// will set nothing and immediately return.
+func (r *Repo) SetInstallID(v int64) {
+	// return if Repo type is nil
+	if r == nil {
+		return
+	}
+
+	r.InstallID = &v
+}
+
 // String implements the Stringer interface for the Repo type.
 func (r *Repo) String() string {
 	return fmt.Sprintf(`{
@@ -630,6 +657,7 @@ func (r *Repo) String() string {
   Counter: %d,
   FullName: %s,
   ID: %d,
+  InstallID: %d,
   Link: %s,
   Name: %s,
   Org: %s,
@@ -651,6 +679,7 @@ func (r *Repo) String() string {
 		r.GetCounter(),
 		r.GetFullName(),
 		r.GetID(),
+		r.GetInstallID(),
 		r.GetLink(),
 		r.GetName(),
 		r.GetOrg(),

api/webhook/post.go

@@ -84,6 +84,7 @@
 	// capture middleware values
 	m := c.MustGet("metadata").(*internal.Metadata)
 	l := c.MustGet("logger").(*logrus.Entry)
+	db := database.FromContext(c)
 	ctx := c.Request.Context()
 
 	l.Debug("webhook received")
@@ -133,6 +134,20 @@
 		return
 	}
 
+	if webhook.Installation != nil {
+		err = scm.FromContext(c).ProcessInstallation(ctx, c.Request, webhook, db)
+		if err != nil {
+			retErr := fmt.Errorf("unable to process installation: %w", err)
+			util.HandleError(c, http.StatusBadRequest, retErr)
+
+			return
+		}
+
+		c.JSON(http.StatusOK, "installation processed successfully")
+
+		return
+	}
+
 	// check if the hook should be skipped
 	if skip, skipReason := webhook.ShouldSkip(); skip {
 		c.JSON(http.StatusOK, fmt.Sprintf("skipping build: %s", skipReason))
@@ -189,7 +204,7 @@
 
 	defer func() {
 		// send API call to update the webhook
 		_, err = database.FromContext(c).UpdateHook(ctx, h)
 		if err != nil {
 			l.Errorf("unable to update webhook %s/%d: %v", r.GetFullName(), h.GetNumber(), err)
 		}
@@ -658,7 +673,7 @@
 	case "archived", "unarchived", constants.ActionEdited:
 		l.Debugf("repository action %s for %s", h.GetEventAction(), r.GetFullName())
 		// send call to get repository from database
 		dbRepo, err := database.FromContext(c).GetRepoForOrg(ctx, r.GetOrg(), r.GetName())
 		if err != nil {
 			retErr := fmt.Errorf("%s: failed to get repo %s: %w", baseErr, r.GetFullName(), err)
 
@@ -669,7 +684,7 @@
 		}
 
 		// send API call to capture the last hook for the repo
 		lastHook, err := database.FromContext(c).LastHookForRepo(ctx, dbRepo)
 		if err != nil {
 			retErr := fmt.Errorf("unable to get last hook for repo %s: %w", r.GetFullName(), err)
 

cmd/vela-server/scm.go

@@ -20,6 +20,8 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) {
 		Address:              c.String("scm.addr"),
 		ClientID:             c.String("scm.client"),
 		ClientSecret:         c.String("scm.secret"),
+		AppID:                c.Int64("scm.app.id"),
+		AppPrivateKey:        c.String("scm.app.private_key"),
 		ServerAddress:        c.String("server-addr"),
 		ServerWebhookAddress: c.String("scm.webhook.addr"),
 		StatusContext:        c.String("scm.context"),
@@ -31,5 +33,5 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) {
 	// setup the scm
 	//
 	// https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#New
-	return scm.New(_setup)
+	return scm.New(c.Context, _setup)
 }

compiler/engine.go

@@ -11,6 +11,7 @@ import (
 	"github.com/go-vela/server/compiler/types/raw"
 	"github.com/go-vela/server/compiler/types/yaml"
 	"github.com/go-vela/server/internal"
+	"github.com/go-vela/server/scm"
 )
 
 // Engine represents an interface for converting a yaml
@@ -146,6 +147,9 @@ type Engine interface {
 	// WithLabel defines a function that sets
 	// the label(s) in the Engine.
 	WithLabels([]string) Engine
+	// WithSCM defines a function that sets
+	// the scm in the Engine.
+	WithSCM(scm.Service) Engine
 	// WithPrivateGitHub defines a function that sets
 	// the private github client in the Engine.
 	WithPrivateGitHub(context.Context, string, string) Engine

compiler/native/compile.go

@@ -44,6 +44,31 @@
 		return nil, nil, err
 	}
 
+	// create the netrc using the scm
+	// this has to occur after Parse because the scm configurations might be set in yaml
+	// netrc can be provided directly using WithNetrc for situations like local exec
+	if c.netrc == nil && c.scm != nil {
+		// ensure restrictive defaults for the netrc for scms that support granular permissions
+		if p.Git.Repositories == nil {
+			p.Git.Repositories = []string{c.repo.GetName()}
+		}
+
+		if p.Git.Permissions == nil {
+			p.Git.Permissions = map[string]string{
+				constants.AppInstallResourceContents: constants.AppInstallPermissionRead,
+				constants.AppInstallResourceChecks:   constants.AppInstallPermissionWrite,
+			}
+		}
+
+		// get the netrc password from the scm
+		netrc, err := c.scm.GetNetrcPassword(context.Background(), c.repo, c.user, p.Git.Repositories, p.Git.Permissions)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		c.WithNetrc(netrc)
+	}
+
 	// create the API pipeline object from the yaml configuration
 	_pipeline := p.ToPipelineAPI()
 	_pipeline.SetData(data)
@@ -330,7 +355,7 @@
 
 	if c.ModificationService.Endpoint != "" {
 		// send config to external endpoint for modification
 		p, err = c.modifyConfig(p, c.build, c.repo)
 		if err != nil {
 			return nil, _pipeline, err
 		}
@@ -425,7 +450,7 @@
 
 	if c.ModificationService.Endpoint != "" {
 		// send config to external endpoint for modification
 		p, err = c.modifyConfig(p, c.build, c.repo)
 		if err != nil {
 			return nil, _pipeline, err
 		}