feat(queuing)!: build executable server-side implementation (#927)

* init commit

* adding db tests

* compiled to build itinerary

* add mock

* itinerary -> executable

* fix mock

* update some extra db configs

* pull in updated types

* encrypt and decrypt

* Update database/executable/interface.go

Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com>

* pesky linter

* add integration test for executables

* try this

* with encryption resource opt

* Update database/executable/opts.go

Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com>

---------

Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com>
Co-authored-by: David May <1301201+wass3r@users.noreply.github.com>

api/build/executable.go

@@ -0,0 +1,93 @@
+// Copyright (c) 2023 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package build
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/go-vela/server/database"
+	"github.com/go-vela/server/router/middleware/build"
+	"github.com/go-vela/server/router/middleware/claims"
+	"github.com/go-vela/server/router/middleware/org"
+	"github.com/go-vela/server/router/middleware/repo"
+	"github.com/go-vela/server/util"
+	"github.com/sirupsen/logrus"
+)
+
+// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/executable builds GetBuildExecutable
+//
+// Get a build executable in the configured backend
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: path
+//   name: org
+//   description: Name of the org
+//   required: true
+//   type: string
+// - in: path
+//   name: repo
+//   description: Name of the repo
+//   required: true
+//   type: string
+// - in: path
+//   name: build
+//   description: Build number to retrieve
+//   required: true
+//   type: integer
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully retrieved the build executable
+//     type: json
+//     schema:
+//       "$ref": "#/definitions/Build"
+//   '400':
+//     description: Bad request
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '401':
+//     description: Unauthorized
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Could not retrieve build executable
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// GetBuildExecutable represents the API handler to capture
+// a build executable for a repo from the configured backend.
+func GetBuildExecutable(c *gin.Context) {
+	// capture middleware values
+	b := build.Retrieve(c)
+	o := org.Retrieve(c)
+	r := repo.Retrieve(c)
+	cl := claims.Retrieve(c)
+
+	// update engine logger with API metadata
+	//
+	// https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields
+	logrus.WithFields(logrus.Fields{
+		"build":   b.GetNumber(),
+		"org":     o,
+		"repo":    r.GetName(),
+		"subject": cl.Subject,
+	}).Infof("reading build executable %s/%d", r.GetFullName(), b.GetNumber())
+
+	bExecutable, err := database.FromContext(c).PopBuildExecutable(b.GetID())
+	if err != nil {
+		retErr := fmt.Errorf("unable to pop build executable: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusOK, bExecutable)
+}

api/build/publish.go

@@ -20,7 +20,31 @@ import (
 // PublishToQueue is a helper function that creates
 // a build item and publishes it to the queue.
 func PublishToQueue(ctx context.Context, queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) {
-	item := types.ToItem(p, b, r, u)
+	byteExecutable, err := json.Marshal(p)
+	if err != nil {
+		logrus.Errorf("Failed to marshal build executable %d for %s: %v", b.GetNumber(), r.GetFullName(), err)
+
+		// error out the build
+		CleanBuild(ctx, db, b, nil, nil, err)
+
+		return
+	}
+
+	bExecutable := new(library.BuildExecutable)
+	bExecutable.SetBuildID(b.GetID())
+	bExecutable.SetData(byteExecutable)
+
+	err = db.CreateBuildExecutable(bExecutable)
+	if err != nil {
+		logrus.Errorf("Failed to publish build executable to database %d for %s: %v", b.GetNumber(), r.GetFullName(), err)
+
+		// error out the build
+		CleanBuild(ctx, db, b, nil, nil, err)
+
+		return
+	}
+
+	item := types.ToItem(b, r, u)
 
 	logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName())
 

database/database.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/go-vela/server/database/build"
+	"github.com/go-vela/server/database/executable"
 	"github.com/go-vela/server/database/hook"
 	"github.com/go-vela/server/database/log"
 	"github.com/go-vela/server/database/pipeline"
@@ -61,6 +62,7 @@ type (
 		logger *logrus.Entry
 
 		build.BuildInterface
+		executable.BuildExecutableInterface
 		hook.HookInterface
 		log.LogInterface
 		pipeline.PipelineInterface

database/executable/create.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2023 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package executable
+
+import (
+	"fmt"
+
+	"github.com/go-vela/types/constants"
+	"github.com/go-vela/types/database"
+	"github.com/go-vela/types/library"
+	"github.com/sirupsen/logrus"
+)
+
+// CreateBuildExecutable creates a new build executable in the database.
+func (e *engine) CreateBuildExecutable(b *library.BuildExecutable) error {
+	e.logger.WithFields(logrus.Fields{
+		"build": b.GetBuildID(),
+	}).Tracef("creating build executable for build %d in the database", b.GetBuildID())
+
+	// cast the library type to database type
+	//
+	// https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutableFromLibrary
+	executable := database.BuildExecutableFromLibrary(b)
+
+	// validate the necessary fields are populated
+	//
+	// https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Validate
+	err := executable.Validate()
+	if err != nil {
+		return err
+	}
+
+	// compress data for the build executable
+	//
+	// https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Compress
+	err = executable.Compress(e.config.CompressionLevel)
+	if err != nil {
+		return err
+	}
+
+	// encrypt the data field for the build executable
+	//
+	// https://pkg.go.dev/github.com/go-vela/types/database#BuildExecutable.Encrypt
+	err = executable.Encrypt(e.config.EncryptionKey)
+	if err != nil {
+		return fmt.Errorf("unable to encrypt build executable for build %d: %w", b.GetBuildID(), err)
+	}
+
+	// send query to the database
+	return e.client.
+		Table(constants.TableBuildExecutable).
+		Create(executable).
+		Error
+}

database/executable/create_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2023 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package executable
+
+import (
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+)
+
+func TestExecutable_Engine_CreateBuildExecutable(t *testing.T) {
+	// setup types
+	_bExecutable := testBuildExecutable()
+	_bExecutable.SetID(1)
+	_bExecutable.SetBuildID(1)
+
+	_postgres, _mock := testPostgres(t)
+	defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }()
+
+	// create expected result in mock
+	_rows := sqlmock.NewRows([]string{"id"}).AddRow(1)
+
+	// ensure the mock expects the query
+	_mock.ExpectQuery(`INSERT INTO "build_executables"
+("build_id","data","id")
+VALUES ($1,$2,$3) RETURNING "id"`).
+		WithArgs(1, AnyArgument{}, 1).
+		WillReturnRows(_rows)
+
+	_sqlite := testSqlite(t)
+	defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()
+
+	// setup tests
+	tests := []struct {
+		failure  bool
+		name     string
+		database *engine
+	}{
+		{
+			failure:  false,
+			name:     "postgres",
+			database: _postgres,
+		},
+		{
+			failure:  false,
+			name:     "sqlite3",
+			database: _sqlite,
+		},
+	}
+
+	// run tests
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := test.database.CreateBuildExecutable(_bExecutable)
+
+			if test.failure {
+				if err == nil {
+					t.Errorf("CreateBuildExecutable for %s should have returned err", test.name)
+				}
+
+				return
+			}
+
+			if err != nil {
+				t.Errorf("CreateBuildExecutable for %s returned err: %v", test.name, err)
+			}
+		})
+	}
+}

database/executable/executable.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2023 Target Brands, Inc. All rights reserved.
+//
+// Use of this source code is governed by the LICENSE file in this repository.
+
+package executable
+
+import (
+	"fmt"
+
+	"github.com/go-vela/types/constants"
+	"github.com/sirupsen/logrus"
+
+	"gorm.io/gorm"
+)
+
+type (
+	// config represents the settings required to create the engine that implements the BuildExecutableService interface.
+	config struct {
+		// specifies the level of compression to use for the BuildExecutable engine
+		CompressionLevel int
+		// specifies the encryption key to use for the BuildExecutable engine
+		EncryptionKey string
+		// specifies to skip creating tables and indexes for the BuildExecutable engine
+		SkipCreation bool
+		// specifies the driver for proper popping query
+		Driver string
+	}
+
+	// engine represents the build executable functionality that implements the BuildExecutableService interface.
+	engine struct {
+		// engine configuration settings used in build executable functions
+		config *config
+
+		// gorm.io/gorm database client used in build executable functions
+		//
+		// https://pkg.go.dev/gorm.io/gorm#DB
+		client *gorm.DB
+
+		// sirupsen/logrus logger used in build executable functions
+		//
+		// https://pkg.go.dev/github.com/sirupsen/logrus#Entry
+		logger *logrus.Entry
+	}
+)
+
+// New creates and returns a Vela service for integrating with build executables in the database.
+//
+//nolint:revive // ignore returning unexported engine
+func New(opts ...EngineOpt) (*engine, error) {
+	// create new BuildExecutable engine
+	e := new(engine)
+
+	// create new fields
+	e.client = new(gorm.DB)
+	e.config = new(config)
+	e.logger = new(logrus.Entry)
+
+	// apply all provided configuration options
+	for _, opt := range opts {
+		err := opt(e)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// check if we should skip creating build executable database objects
+	if e.config.SkipCreation {
+		e.logger.Warning("skipping creation of build executables table and indexes in the database")
+
+		return e, nil
+	}
+
+	// create the build executables table
+	err := e.CreateBuildExecutableTable(e.client.Config.Dialector.Name())
+	if err != nil {
+		return nil, fmt.Errorf("unable to create %s table: %w", constants.TableBuildExecutable, err)
+	}
+
+	return e, nil
+}