fix issues with teams, improved e2e-tests and add missing tests for teams and sm, clean up unused code

.github/workflows/docker-image.yml → .github/workflows/build-image.yml

@@ -1,4 +1,4 @@
-name: ci
+name: Build Image
 
 on:
   workflow_dispatch:

.github/workflows/test.yml

@@ -0,0 +1,55 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - '**.go'
+      - '.github/workflows/test.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - '**.go'
+      - '.github/workflows/test.yml'
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+
+    # This assumes that the configuration file connects to Redis at localhost:6379
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.20.0'
+
+      - name: Create config directory
+        run: sudo mkdir -p /etc/deploy && sudo chown $USER /etc/deploy
+
+      - name: Create config file from secret
+        run: |
+          echo "${{ vars.CONFIG_FILE }}" | sudo tee /etc/deploy/config.yaml > /dev/null
+
+      - name: Export configuration file path
+        run: echo "DEPLOY_CONFIG_FILE=/etc/deploy/config.yaml" >> $GITHUB_ENV
+
+      - name: Start main program
+        run: |
+          go build -o go-deploy .     # Build go-deploy
+          ./go-deploy --test-mode &   # Starts in the background
+          sleep 15                    # Wait a bit to ensure it has started
+
+      - name: Run e2e tests
+        run: go test ./test/e2e/...

models/config/helpers.go

@@ -1,10 +1,10 @@
 package config
 
 import (
-	roleModel "go-deploy/models/sys/role"
+	roleModels "go-deploy/models/sys/role"
 )
 
-func (e *ConfigType) GetRole(roleName string) *roleModel.Role {
+func (e *ConfigType) GetRole(roleName string) *roleModels.Role {
 	for _, role := range e.Roles {
 		if role.Name == roleName {
 			return &role
@@ -14,8 +14,8 @@ func (e *ConfigType) GetRole(roleName string) *roleModel.Role {
 	return nil
 }
 
-func (e *ConfigType) GetRolesByIamGroups(iamGroups []string) []roleModel.Role {
-	var roles []roleModel.Role
+func (e *ConfigType) GetRolesByIamGroups(iamGroups []string) []roleModels.Role {
+	var roles []roleModels.Role
 
 	for _, role := range e.Roles {
 		for _, iamGroup := range iamGroups {

models/dto/body/job.go

@@ -15,5 +15,5 @@ type JobRead struct {
 }
 
 type JobUpdate struct {
-	Status *string `json:"status" binding:"omitempty,oneof=pending running failed terminated finished"`
+	Status *string `json:"status" binding:"omitempty,oneof=pending running failed terminated finished completed"`
 }

models/dto/body/notification.go

@@ -4,6 +4,7 @@ import "time"
 
 type NotificationRead struct {
 	ID        string                 `json:"id"`
+	UserID    string                 `json:"userId"`
 	Type      string                 `json:"type"`
 	Content   map[string]interface{} `json:"content"`
 	CreatedAt time.Time              `json:"createdAt"`

models/dto/body/team.go

@@ -52,7 +52,7 @@ type TeamRead struct {
 	ID          string         `json:"id"`
 	Name        string         `json:"name"`
 	OwnerID     string         `json:"ownerId"`
-	Description *string        `json:"description,omitempty"`
+	Description string         `json:"description,omitempty"`
 	Resources   []TeamResource `json:"resources"`
 	Members     []TeamMember   `json:"members"`
 	CreatedAt   time.Time      `json:"createdAt"`

models/dto/query/job.go

@@ -4,7 +4,7 @@ type JobList struct {
 	Pagination
 
 	All    bool    `form:"all" binding:"omitempty,boolean"`
-	Status *string `form:"status" binding:"omitempty,oneof=pending running failed terminated finished"`
+	Status *string `form:"status" binding:"omitempty,oneof=pending running failed terminated finished completed"`
 	Type   *string `form:"type" binding:"omitempty,ascii"`
 	UserID *string `form:"userId" binding:"omitempty,uuid4"`
 }

models/sys/deployment/client.go

@@ -7,6 +7,7 @@ import (
 	"go-deploy/models/sys/base/resource"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
+	"time"
 )
 
 type Client struct {
@@ -102,6 +103,25 @@ func (client *Client) WithNoActivities() *Client {
 
 func (client *Client) WithGitHubWebhookID(id int64) *Client {
 	filter := bson.D{{"subsystems.github.webhook.id", id}}
+
+	client.ResourceClient.AddExtraFilter(filter)
+	client.ActivityResourceClient.AddExtraFilter(filter)
+
+	return client
+}
+
+func (client *Client) WithNameRegex(name string) *Client {
+	filter := bson.D{{"name", bson.D{{"$regex", name}}}}
+
+	client.ResourceClient.AddExtraFilter(filter)
+	client.ActivityResourceClient.AddExtraFilter(filter)
+
+	return client
+}
+
+func (client *Client) OlderThan(timestamp time.Time) *Client {
+	filter := bson.D{{"createdAt", bson.D{{"$lt", timestamp}}}}
+
 	client.ResourceClient.AddExtraFilter(filter)
 	client.ActivityResourceClient.AddExtraFilter(filter)
 

models/sys/job/client.go

@@ -47,12 +47,24 @@ func (client *Client) IncludeTypes(types ...string) *Client {
 	return client
 }
 
+func (client *Client) IncludeStatus(status ...string) *Client {
+	client.AddExtraFilter(bson.D{{"status", bson.D{{"$in", status}}}})
+
+	return client
+}
+
 func (client *Client) ExcludeStatus(status ...string) *Client {
 	client.AddExtraFilter(bson.D{{"status", bson.D{{"$nin", status}}}})
 
 	return client
 }
 
+func (client *Client) ExcludeTypes(types ...string) *Client {
+	client.AddExtraFilter(bson.D{{"type", bson.D{{"$nin", types}}}})
+
+	return client
+}
+
 func (client *Client) FilterArgs(argName string, filter interface{}) *Client {
 	client.AddExtraFilter(bson.D{{"args." + argName, filter}})
 

models/sys/notification/dto.go

@@ -5,6 +5,7 @@ import "go-deploy/models/dto/body"
 func (notification *Notification) ToDTO() body.NotificationRead {
 	return body.NotificationRead{
 		ID:        notification.ID,
+		UserID:    notification.UserID,
 		Type:      notification.Type,
 		Content:   notification.Content,
 		CreatedAt: notification.CreatedAt,

models/sys/team/client.go

@@ -5,6 +5,7 @@ import (
 	"go-deploy/models/sys/base"
 	"go-deploy/models/sys/base/resource"
 	"go.mongodb.org/mongo-driver/bson"
+	"time"
 )
 
 type Client struct {
@@ -49,3 +50,17 @@ func (client *Client) WithResourceID(resourceID string) *Client {
 
 	return client
 }
+
+func (client *Client) WithNameRegex(name string) *Client {
+	filter := bson.D{{"name", bson.D{{"$regex", name}}}}
+	client.ResourceClient.AddExtraFilter(filter)
+
+	return client
+}
+
+func (client *Client) OlderThan(timestamp time.Time) *Client {
+	filter := bson.D{{"createdAt", bson.D{{"$lt", timestamp}}}}
+	client.ResourceClient.AddExtraFilter(filter)
+
+	return client
+}

models/sys/team/db.go

@@ -14,7 +14,7 @@ func (client *Client) Create(id, ownerID string, params *CreateParams) (*Team, e
 	team := &Team{
 		ID:          id,
 		Name:        params.Name,
-		Description: &params.Description,
+		Description: params.Description,
 		OwnerID:     ownerID,
 		CreatedAt:   time.Now(),
 		ResourceMap: params.ResourceMap,
@@ -59,7 +59,9 @@ func (client *Client) ListByUserID(userID string) ([]Team, error) {
 }
 
 func (client *Client) UpdateWithParams(id string, params *UpdateParams) error {
-	updateData := bson.D{}
+	updateData := bson.D{
+		{"updatedAt", time.Now()},
+	}
 
 	models.AddIfNotNil(&updateData, "name", params.Name)
 	models.AddIfNotNil(&updateData, "description", params.Description)
@@ -74,7 +76,9 @@ func (client *Client) UpdateWithParams(id string, params *UpdateParams) error {
 }
 
 func (client *Client) UpdateMember(id string, memberID string, member *Member) error {
-	updateData := bson.D{}
+	updateData := bson.D{
+		{"updatedAt", time.Now()},
+	}
 
 	models.AddIfNotNil(&updateData, "memberMap."+memberID, member)
 
@@ -84,7 +88,3 @@ func (client *Client) UpdateMember(id string, memberID string, member *Member) e
 
 	return client.SetWithBsonByID(id, updateData)
 }
-
-func (client *Client) MarkUpdated(id string) error {
-	return client.SetWithBsonByID(id, bson.D{{"updatedAt", time.Now()}})
-}

models/sys/team/dto.go

@@ -52,6 +52,7 @@ func (params *CreateParams) FromDTO(teamCreateDTO *body.TeamCreate, ownerID stri
 	params.Name = teamCreateDTO.Name
 	params.MemberMap = make(map[string]Member)
 	params.Description = teamCreateDTO.Description
+	params.ResourceMap = make(map[string]Resource)
 
 	for _, resourceDTO := range teamCreateDTO.Resources {
 		if resource := getResourceFunc(resourceDTO); resource != nil {

models/sys/team/team.go

@@ -32,7 +32,7 @@ type Resource struct {
 type Team struct {
 	ID          string    `bson:"id"`
 	Name        string    `bson:"name"`
-	Description *string   `bson:"description"`
+	Description string    `bson:"description"`
 	OwnerID     string    `bson:"ownerId"`
 	CreatedAt   time.Time `bson:"createdAt"`
 	UpdatedAt   time.Time `bson:"updatedAt"`

models/sys/user/dto.go

@@ -2,11 +2,11 @@ package user
 
 import (
 	"go-deploy/models/dto/body"
-	roleModel "go-deploy/models/sys/role"
+	roleModels "go-deploy/models/sys/role"
 	"log"
 )
 
-func (user *User) ToDTO(effectiveRole *roleModel.Role, usage *Usage, storageURL *string) body.UserRead {
+func (user *User) ToDTO(effectiveRole *roleModels.Role, usage *Usage, storageURL *string) body.UserRead {
 	publicKeys := make([]body.PublicKey, len(user.PublicKeys))
 	for i, key := range user.PublicKeys {
 		publicKeys[i] = body.PublicKey{
@@ -21,7 +21,7 @@ func (user *User) ToDTO(effectiveRole *roleModel.Role, usage *Usage, storageURL
 
 	if effectiveRole == nil {
 		log.Println("effective role is nil when creating user read for user", user.Username)
-		effectiveRole = &roleModel.Role{
+		effectiveRole = &roleModels.Role{
 			Name:        "unknown",
 			Description: "unknown",
 		}

models/sys/vm/client.go

@@ -7,6 +7,7 @@ import (
 	"go-deploy/models/sys/base/resource"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
+	"time"
 )
 
 type Client struct {
@@ -108,3 +109,21 @@ func (client *Client) WithCustomFilter(filter bson.D) *Client {
 
 	return client
 }
+
+func (client *Client) WithNameRegex(name string) *Client {
+	filter := bson.D{{"name", bson.D{{"$regex", name}}}}
+
+	client.ResourceClient.AddExtraFilter(filter)
+	client.ActivityResourceClient.AddExtraFilter(filter)
+
+	return client
+}
+
+func (client *Client) OlderThan(timestamp time.Time) *Client {
+	filter := bson.D{{"createdAt", bson.D{{"$lt", timestamp}}}}
+
+	client.ResourceClient.AddExtraFilter(filter)
+	client.ActivityResourceClient.AddExtraFilter(filter)
+
+	return client
+}

models/sys/vm/vm.go

@@ -2,7 +2,7 @@ package vm
 
 import (
 	"go-deploy/models/sys/activity"
-	gpuModel "go-deploy/models/sys/gpu"
+	gpuModels "go-deploy/models/sys/gpu"
 	"time"
 )
 
@@ -56,7 +56,7 @@ func (vm *VM) DoingActivity(activity string) bool {
 }
 
 func (vm *VM) HasGPU() bool {
-	exists, err := gpuModel.New().WithVM(vm.ID).ExistsAny()
+	exists, err := gpuModels.New().WithVM(vm.ID).ExistsAny()
 	if err != nil {
 		return false
 	}
@@ -65,16 +65,16 @@ func (vm *VM) HasGPU() bool {
 }
 
 func (vm *VM) GetGpuID() *string {
-	idStruct, err := gpuModel.New().WithVM(vm.ID).GetID()
+	idStruct, err := gpuModels.New().WithVM(vm.ID).GetID()
 	if err != nil || idStruct == nil {
 		return nil
 	}
 
 	return &idStruct.ID
 }
 
-func (vm *VM) GetGpu() *gpuModel.GPU {
-	gpu, err := gpuModel.New().WithVM(vm.ID).Get()
+func (vm *VM) GetGpu() *gpuModels.GPU {
+	gpu, err := gpuModels.New().WithVM(vm.ID).Get()
 	if err != nil || gpu == nil {
 		return nil
 	}

pkg/app/server.go

@@ -34,11 +34,11 @@ func shutdown() {
 	db.Shutdown()
 }
 
-func Create(options *Options) *App {
+func Create(opts *Options) *App {
 	config.SetupEnvironment()
 	metrics.Setup()
 
-	if options.TestMode {
+	if opts.TestMode {
 		config.Config.TestMode = true
 		config.Config.MongoDB.Name = config.Config.MongoDB.Name + "-test"
 	}
@@ -56,7 +56,7 @@ func Create(options *Options) *App {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	for _, flag := range options.Flags {
+	for _, flag := range opts.Flags {
 		// handle api worker separately
 		if flag.Name == "api" {
 			continue
@@ -69,7 +69,7 @@ func Create(options *Options) *App {
 
 	var httpServer *http.Server
 
-	if options.Flags.GetPassedValue("api").(bool) {
+	if opts.Flags.GetPassedValue("api").(bool) {
 		ginMode, exists := os.LookupEnv("GIN_MODE")
 		if exists {
 			gin.SetMode(ginMode)