Merge branch 'main' into enhance/live-queue-update

.github/workflows/integration-test.yml

@@ -15,7 +15,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:15-alpine
+        image: postgres:15-alpine@sha256:8bc3c893342c766481df5fde58fab6f1a1115b94eb56778126163305243e9709
         env:
           POSTGRES_DB: vela
           POSTGRES_PASSWORD: notARealPassword12345

Dockerfile

@@ -2,7 +2,7 @@
 #
 # Use of this source code is governed by the LICENSE file in this repository.
 
-FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70 as certs
+FROM alpine:3.18.3@sha256:c5c5fda71656f28e49ac9c5416b3643eaa6a108a8093151d6d1afc9463be8e33 as certs
 
 RUN apk add --update --no-cache ca-certificates
 

Dockerfile-alpine

@@ -2,7 +2,7 @@
 #
 # Use of this source code is governed by the LICENSE file in this repository.
 
-FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70
+FROM alpine:3.18.3@sha256:c5c5fda71656f28e49ac9c5416b3643eaa6a108a8093151d6d1afc9463be8e33
 
 RUN apk add --update --no-cache ca-certificates
 

api/build/executable.go

@@ -82,6 +82,7 @@ func GetBuildExecutable(c *gin.Context) {
 		"subject": cl.Subject,
 	}).Infof("reading build executable %s/%d", r.GetFullName(), b.GetNumber())
 
+	// send database call to pop the requested build executable from the table
 	bExecutable, err := database.FromContext(c).PopBuildExecutable(ctx, b.GetID())
 	if err != nil {
 		retErr := fmt.Errorf("unable to pop build executable: %w", err)

api/build/publish.go

@@ -17,9 +17,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// PublishToQueue is a helper function that creates
-// a build item and publishes it to the queue.
+// PublishToQueue is a helper function that pushes the build executable to the database
+// and publishes a queue item (build, repo, user) to the queue.
 func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) {
+	// marshal pipeline build into byte data to add to the build executable object
 	byteExecutable, err := json.Marshal(p)
 	if err != nil {
 		logrus.Errorf("Failed to marshal build executable %d for %s: %v", b.GetNumber(), r.GetFullName(), err)
@@ -30,10 +31,12 @@ func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interf
 		return
 	}
 
+	// create build executable to push to database
 	bExecutable := new(library.BuildExecutable)
 	bExecutable.SetBuildID(b.GetID())
 	bExecutable.SetData(byteExecutable)
 
+	// send database call to create a build executable
 	err = db.CreateBuildExecutable(ctx, bExecutable)
 	if err != nil {
 		logrus.Errorf("Failed to publish build executable to database %d for %s: %v", b.GetNumber(), r.GetFullName(), err)
@@ -44,6 +47,7 @@ func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interf
 		return
 	}
 
+	// convert build, repo, and user into queue item
 	item := types.ToItem(b, r, u)
 
 	logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName())
@@ -60,6 +64,7 @@ func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interf
 
 	logrus.Infof("Establishing route for build %d for %s", b.GetNumber(), r.GetFullName())
 
+	// determine the route on which to publish the queue item
 	route, err := queue.Route(&p.Worker)
 	if err != nil {
 		logrus.Errorf("unable to set route for build %d for %s: %v", b.GetNumber(), r.GetFullName(), err)
@@ -72,6 +77,7 @@ func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interf
 
 	logrus.Infof("Publishing item for build %d for %s to queue %s", b.GetNumber(), r.GetFullName(), route)
 
+	// push item on to the queue
 	err = queue.Push(context.Background(), route, byteItem)
 	if err != nil {
 		logrus.Errorf("Retrying; Failed to publish build %d for %s: %v", b.GetNumber(), r.GetFullName(), err)

api/webhook/post.go

@@ -113,7 +113,8 @@ func PostWebhook(c *gin.Context) {
 	// -------------------- End of TODO: --------------------
 
 	// process the webhook from the source control provider
-	// comment, number, h, r, b
+	//
+	// populate build, hook, repo resources as well as PR Number / PR Comment if necessary
 	webhook, err := scm.FromContext(c).ProcessWebhook(c.Request)
 	if err != nil {
 		retErr := fmt.Errorf("unable to parse webhook: %w", err)
@@ -142,7 +143,7 @@ func PostWebhook(c *gin.Context) {
 			return
 		}
 
-		// if there were actual changes to the repo, return the repo object
+		// if there were actual changes to the repo (database call populated ID field), return the repo object
 		if r.GetID() != 0 {
 			c.JSON(http.StatusOK, r)
 			return
@@ -345,7 +346,8 @@ func PostWebhook(c *gin.Context) {
 	logrus.Debug("updating status to pending")
 	b.SetStatus(constants.StatusPending)
 
-	// if this is a comment on a pull_request event
+	// if the event is issue_comment and the issue is a pull request,
+	// call SCM for more data not provided in webhook payload
 	if strings.EqualFold(b.GetEvent(), constants.EventComment) && webhook.PRNumber > 0 {
 		commit, branch, baseref, headref, err := scm.FromContext(c).GetPullRequest(u, repo, webhook.PRNumber)
 		if err != nil {
@@ -366,6 +368,7 @@ func PostWebhook(c *gin.Context) {
 
 	// variable to store changeset files
 	var files []string
+
 	// check if the build event is not issue_comment or pull_request
 	if !strings.EqualFold(b.GetEvent(), constants.EventComment) &&
 		!strings.EqualFold(b.GetEvent(), constants.EventPull) {
@@ -423,7 +426,7 @@ func PostWebhook(c *gin.Context) {
 			time.Sleep(time.Duration(i) * time.Second)
 		}
 
-		// send API call to attempt to capture the pipeline
+		// send database call to attempt to capture the pipeline if we already processed it before
 		pipeline, err = database.FromContext(c).GetPipelineForRepo(ctx, b.GetCommit(), repo)
 		if err != nil { // assume the pipeline doesn't exist in the database yet
 			// send API call to capture the pipeline configuration file
@@ -463,7 +466,7 @@ func PostWebhook(c *gin.Context) {
 			return
 		}
 
-		// update repo fields with any changes from SCM process
+		// update DB record of repo (repo) with any changes captured from webhook payload (r)
 		repo.SetTopics(r.GetTopics())
 		repo.SetBranch(r.GetBranch())
 
@@ -525,7 +528,7 @@ func PostWebhook(c *gin.Context) {
 		// before compiling. After we're done compiling, we reset the pipeline type.
 		repo.SetPipelineType(pipelineType)
 
-		// skip the build if only the init or clone steps are found
+		// skip the build if pipeline compiled to only the init and clone steps
 		skip := build.SkipEmptyBuild(p)
 		if skip != "" {
 			// set build to successful status
@@ -678,6 +681,8 @@ func PostWebhook(c *gin.Context) {
 	)
 }
 
+// handleRepositoryEvent is a helper function that processes repository events from the SCM and updates
+// the database resources with any relevant changes resulting from the event, such as name changes, transfers, etc.
 func handleRepositoryEvent(ctx context.Context, c *gin.Context, m *types.Metadata, h *library.Hook, r *library.Repo) (*library.Repo, error) {
 	logrus.Debugf("webhook is repository event, making necessary updates to repo %s", r.GetFullName())
 
@@ -690,7 +695,7 @@ func handleRepositoryEvent(ctx context.Context, c *gin.Context, m *types.Metadat
 	}()
 
 	switch h.GetEventAction() {
-	// if action is rename, go through rename routine
+	// if action is renamed or transferred, go through rename routine
 	case constants.ActionRenamed, constants.ActionTransferred:
 		r, err := renameRepository(ctx, h, r, c, m)
 		if err != nil {

api/worker/create.go

@@ -89,7 +89,7 @@ func CreateWorker(c *gin.Context) {
 		"worker": input.GetHostname(),
 	}).Infof("creating new worker %s", input.GetHostname())
 
-	err = database.FromContext(c).CreateWorker(input)
+	_, err = database.FromContext(c).CreateWorker(input)
 	if err != nil {
 		retErr := fmt.Errorf("unable to create worker: %w", err)
 

api/worker/refresh.go

@@ -79,7 +79,7 @@ func Refresh(c *gin.Context) {
 	w.SetLastCheckedIn(time.Now().Unix())
 
 	// send API call to update the worker
-	err := database.FromContext(c).UpdateWorker(w)
+	_, err := database.FromContext(c).UpdateWorker(w)
 	if err != nil {
 		retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err)
 

api/worker/update.go

@@ -124,7 +124,7 @@ func UpdateWorker(c *gin.Context) {
 	}
 
 	// send API call to update the worker
-	err = database.FromContext(c).UpdateWorker(w)
+	w, err = database.FromContext(c).UpdateWorker(w)
 	if err != nil {
 		retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err)
 
@@ -133,8 +133,5 @@ func UpdateWorker(c *gin.Context) {
 		return
 	}
 
-	// send API call to capture the updated worker
-	w, _ = database.FromContext(c).GetWorkerForHostname(w.GetHostname())
-
 	c.JSON(http.StatusOK, w)
 }

compiler/native/compile_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/go-vela/types/constants"
 	"github.com/go-vela/types/raw"
 
-	"github.com/google/go-github/v53/github"
+	"github.com/google/go-github/v54/github"
 
 	"testing"
 	"time"

compiler/registry/github/github.go

@@ -9,7 +9,7 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/google/go-github/v53/github"
+	"github.com/google/go-github/v54/github"
 	"golang.org/x/oauth2"
 )
 

compiler/registry/github/github_test.go

@@ -12,7 +12,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/google/go-github/v53/github"
+	"github.com/google/go-github/v54/github"
 	"golang.org/x/oauth2"
 )
 

compiler/registry/github/template.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/go-vela/types/library"
 
-	"github.com/google/go-github/v53/github"
+	"github.com/google/go-github/v54/github"
 )
 
 // Template captures the templated pipeline configuration from the GitHub repo.

database/integration_test.go

@@ -1760,7 +1760,7 @@ func testWorkers(t *testing.T, db Interface, resources *Resources) {
 
 	// create the workers
 	for _, worker := range resources.Workers {
-		err := db.CreateWorker(worker)
+		_, err := db.CreateWorker(worker)
 		if err != nil {
 			t.Errorf("unable to create worker %d: %v", worker.GetID(), err)
 		}
@@ -1802,16 +1802,11 @@ func testWorkers(t *testing.T, db Interface, resources *Resources) {
 	// update the workers
 	for _, worker := range resources.Workers {
 		worker.SetActive(false)
-		err = db.UpdateWorker(worker)
+		got, err := db.UpdateWorker(worker)
 		if err != nil {
 			t.Errorf("unable to update worker %d: %v", worker.GetID(), err)
 		}
 
-		// lookup the worker by ID
-		got, err := db.GetWorker(worker.GetID())
-		if err != nil {
-			t.Errorf("unable to get worker %d by ID: %v", worker.GetID(), err)
-		}
 		if !reflect.DeepEqual(got, worker) {
 			t.Errorf("GetWorker() is %v, want %v", got, worker)
 		}

database/worker/count_test.go

@@ -37,12 +37,12 @@ func TestWorker_Engine_CountWorkers(t *testing.T) {
 	_sqlite := testSqlite(t)
 	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
 
-	err := _sqlite.CreateWorker(_workerOne)
+	_, err := _sqlite.CreateWorker(_workerOne)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}
 
-	err = _sqlite.CreateWorker(_workerTwo)
+	_, err = _sqlite.CreateWorker(_workerTwo)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}

database/worker/create.go

@@ -12,7 +12,7 @@ import (
 )
 
 // CreateWorker creates a new worker in the database.
-func (e *engine) CreateWorker(w *library.Worker) error {
+func (e *engine) CreateWorker(w *library.Worker) (*library.Worker, error) {
 	e.logger.WithFields(logrus.Fields{
 		"worker": w.GetHostname(),
 	}).Tracef("creating worker %s in the database", w.GetHostname())
@@ -27,12 +27,11 @@ func (e *engine) CreateWorker(w *library.Worker) error {
 	// https://pkg.go.dev/github.com/go-vela/types/database#Worker.Validate
 	err := worker.Validate()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	// send query to the database
-	return e.client.
-		Table(constants.TableWorker).
-		Create(worker).
-		Error
+	result := e.client.Table(constants.TableWorker).Create(worker)
+
+	return worker.ToLibrary(), result.Error
 }

database/worker/create_test.go

@@ -5,6 +5,7 @@
 package worker
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
@@ -55,7 +56,7 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`).
 	// run tests
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			err := test.database.CreateWorker(_worker)
+			got, err := test.database.CreateWorker(_worker)
 
 			if test.failure {
 				if err == nil {
@@ -68,6 +69,10 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`).
 			if err != nil {
 				t.Errorf("CreateWorker for %s returned err: %v", test.name, err)
 			}
+
+			if !reflect.DeepEqual(got, _worker) {
+				t.Errorf("CreateWorker for %s returned %s, want %s", test.name, got, _worker)
+			}
 		})
 	}
 }

database/worker/delete_test.go

@@ -29,7 +29,7 @@ func TestWorker_Engine_DeleteWorker(t *testing.T) {
 	_sqlite := testSqlite(t)
 	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
 
-	err := _sqlite.CreateWorker(_worker)
+	_, err := _sqlite.CreateWorker(_worker)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}

database/worker/get_hostname_test.go

@@ -34,7 +34,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) {
 	_sqlite := testSqlite(t)
 	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
 
-	err := _sqlite.CreateWorker(_worker)
+	_, err := _sqlite.CreateWorker(_worker)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}

database/worker/get_test.go

@@ -34,7 +34,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) {
 	_sqlite := testSqlite(t)
 	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
 
-	err := _sqlite.CreateWorker(_worker)
+	_, err := _sqlite.CreateWorker(_worker)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}

database/worker/interface.go

@@ -29,7 +29,7 @@ type WorkerInterface interface {
 	// CountWorkers defines a function that gets the count of all workers.
 	CountWorkers() (int64, error)
 	// CreateWorker defines a function that creates a new worker.
-	CreateWorker(*library.Worker) error
+	CreateWorker(*library.Worker) (*library.Worker, error)
 	// DeleteWorker defines a function that deletes an existing worker.
 	DeleteWorker(*library.Worker) error
 	// GetWorker defines a function that gets a worker by ID.
@@ -39,5 +39,5 @@ type WorkerInterface interface {
 	// ListWorkers defines a function that gets a list of all workers.
 	ListWorkers() ([]*library.Worker, error)
 	// UpdateWorker defines a function that updates an existing worker.
-	UpdateWorker(*library.Worker) error
+	UpdateWorker(*library.Worker) (*library.Worker, error)
 }

database/worker/list_test.go

@@ -47,12 +47,12 @@ func TestWorker_Engine_ListWorkers(t *testing.T) {
 	_sqlite := testSqlite(t)
 	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
 
-	err := _sqlite.CreateWorker(_workerOne)
+	_, err := _sqlite.CreateWorker(_workerOne)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}
 
-	err = _sqlite.CreateWorker(_workerTwo)
+	_, err = _sqlite.CreateWorker(_workerTwo)
 	if err != nil {
 		t.Errorf("unable to create test worker for sqlite: %v", err)
 	}