From 8703e8613688d293450898b06b3a9bb1e277cdae Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 11:50:46 +0200 Subject: [PATCH 01/14] Increase e2e timeout for test from 30m to 90m --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0330ee52..6543e9d4 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -51,5 +51,5 @@ jobs: done - name: Run e2e tests - run: go test -timeout 30m ./test/e2e/... + run: go test -timeout 90m ./test/e2e/... \ No newline at end of file From ad4949e796bf32f2313f54e0928751130db89968 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:39:37 +0200 Subject: [PATCH 02/14] Split e2e tests in GitHub actions to multiple steps --- .github/workflows/run-tests.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6543e9d4..8bb2cbcd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -50,6 +50,20 @@ jobs: sleep 1 done - - name: Run e2e tests - run: go test -timeout 90m ./test/e2e/... - \ No newline at end of file + - name: Run e2e tests (VMs) + run: go test -timeout 90m ./test/e2e/v2/gpu_groups ./test/e2e/v2/gpu_leases + + - name: Run e2e tests (Deployments) + run: go test -timeout 90m ./test/e2e/v2/deployments + + - name: Run e2e tests (SMs) + run: go test -timeout 90m ./test/e2e/v2/sms + + - name: Run e2e tests (Jobs) + run: go test -timeout 90m ./test/e2e/v2/jobs ./test/e2e/v2/resource_migrations + + - name: Run e2e tests (Users) + run: go test -timeout 90m ./test/e2e/v2/users ./test/e2e/v2/teams ./test/e2e/v2/notifications + + - name: Run e2e tests (System) + run: go test -timeout 90m ./test/e2e/v2/zones \ No newline at end of file From 6163b65a38a8db4a113907d4c0cdbd96aee95dea Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:39:58 +0200 Subject: [PATCH 03/14] Fix bug where GPU lease would dissappear after attaching to VM --- service/v2/vms/gpu_leases/gpu_lease_service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/v2/vms/gpu_leases/gpu_lease_service.go b/service/v2/vms/gpu_leases/gpu_lease_service.go index 27ae6fe1..341bbb1e 100644 --- a/service/v2/vms/gpu_leases/gpu_lease_service.go +++ b/service/v2/vms/gpu_leases/gpu_lease_service.go @@ -41,8 +41,8 @@ func (c *Client) Get(id string, opts ...opts.GetGpuLeaseOpts) (*model.GpuLease, return nil, makeError(err) } - if leaseByUserID == nil { - return nil, nil + if leaseByUserID != nil { + return leaseByUserID, nil } // 3. User has access to the parent VM through a team From 0cf729486ffaedaea70b9fe0336d907c029b2063 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:41:18 +0200 Subject: [PATCH 04/14] Fix bug where status updater could panic --- pkg/subsystems/k8s/status.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/subsystems/k8s/status.go b/pkg/subsystems/k8s/status.go index 46874179..6219cd74 100644 --- a/pkg/subsystems/k8s/status.go +++ b/pkg/subsystems/k8s/status.go @@ -90,7 +90,9 @@ func (client *Client) deploymentStatusWatcher(ctx context.Context, handler func( } } case <-recreateInterval: - watcher.Stop() + if watcher != nil { + watcher.Stop() + } watcher, err = setupDeploymentWatcher(client.Namespace) if err != nil { log.Println("Failed to restart Deployment status watcher, sleeping for 10 seconds before retrying") @@ -141,7 +143,9 @@ func (client *Client) vmStatusWatcher(ctx context.Context, handler func(string, handler(vmStatus.Name, vmStatus) } case <-recreateInterval: - watcher.Stop() + if watcher != nil { + watcher.Stop() + } watcher, err = setupVmWatcher(client.Namespace) if err != nil { log.Println("Failed to restart VM status watcher, sleeping for 10 seconds before retrying") @@ -192,7 +196,9 @@ func (client *Client) vmiStatusWatcher(ctx context.Context, handler func(string, handler(vmiStatus.Name, vmiStatus) } case <-recreateInterval: - watcher.Stop() + if watcher != nil { + watcher.Stop() + } watcher, err = setupVmWatcher(client.Namespace) if err != nil { log.Println("Failed to restart VM instance status watcher, sleeping for 10 seconds before retrying") @@ -340,7 +346,9 @@ func (client *Client) eventWatcher(ctx context.Context, handler func(string, int }(e) } case <-recreateInterval: - watcher.Stop() + if watcher != nil { + watcher.Stop() + } watcher, err = setupEventWatcher(client.Namespace) if err != nil { log.Println("Failed to restart Event status watcher, sleeping for 10 seconds before retrying") From 00fbc5bcd25984ae74f36acfcb892c54031d8242 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:41:49 +0200 Subject: [PATCH 05/14] Fix more structured disable/enabled of VM test --- test/e2e/common.go | 4 ++-- test/e2e/v2/gpu_groups/routes_test.go | 2 +- test/e2e/v2/gpu_leases/routes_test.go | 2 +- test/e2e/v2/vms/routes_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index 4a97bb40..33432388 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -25,8 +25,8 @@ const ( CheckInterval = 1 * time.Second MaxChecks = 900 // 900 * CheckInterval (1) seconds = 15 minutes - // V2TestsEnabled flag is used temporarily to enable V2 tests until V2 zone is fully operational - V2TestsEnabled = true + // VmTestsEnabled flag is used temporarily to enable V2 tests until V2 zone is fully operational + VmTestsEnabled = true ) func GetUserID(user string) string { diff --git a/test/e2e/v2/gpu_groups/routes_test.go b/test/e2e/v2/gpu_groups/routes_test.go index 1fbac046..586f7bdf 100644 --- a/test/e2e/v2/gpu_groups/routes_test.go +++ b/test/e2e/v2/gpu_groups/routes_test.go @@ -8,7 +8,7 @@ import ( ) func TestMain(m *testing.M) { - if e2e.V2TestsEnabled { + if e2e.VmTestsEnabled { e2e.Setup() code := m.Run() e2e.Shutdown() diff --git a/test/e2e/v2/gpu_leases/routes_test.go b/test/e2e/v2/gpu_leases/routes_test.go index 52a10228..b6895da6 100644 --- a/test/e2e/v2/gpu_leases/routes_test.go +++ b/test/e2e/v2/gpu_leases/routes_test.go @@ -10,7 +10,7 @@ import ( ) func TestMain(m *testing.M) { - if e2e.V2TestsEnabled { + if e2e.VmTestsEnabled { e2e.Setup() code := m.Run() e2e.Shutdown() diff --git a/test/e2e/v2/vms/routes_test.go b/test/e2e/v2/vms/routes_test.go index 96bf168c..5edf2cd6 100644 --- a/test/e2e/v2/vms/routes_test.go +++ b/test/e2e/v2/vms/routes_test.go @@ -15,7 +15,7 @@ import ( ) func TestMain(m *testing.M) { - if e2e.V2TestsEnabled { + if e2e.VmTestsEnabled { e2e.Setup() code := m.Run() e2e.Shutdown() From 61f068b1c6527b17c555523b97cf9c906b5e286f Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:42:01 +0200 Subject: [PATCH 06/14] Add mock support for GPU groups --- models/config/config.go | 11 +------ pkg/services/synchronize/fetch_gpu.go | 47 +++++++++++++++++++++++++-- scripts/local/config.yml.tmpl | 8 +---- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/models/config/config.go b/models/config/config.go index 3f82bb8e..20d820bf 100644 --- a/models/config/config.go +++ b/models/config/config.go @@ -67,6 +67,7 @@ type ConfigType struct { PrivilegedGPUs []string `yaml:"privilegedGpus"` ExcludedHosts []string `yaml:"excludedHosts"` ExcludedGPUs []string `yaml:"excludedGpus"` + AddMock bool `yaml:"addMock"` } `yaml:"gpu"` Registry struct { @@ -101,16 +102,6 @@ type ConfigType struct { Password string `yaml:"password"` } - SysApi struct { - URL string `yaml:"url"` - User string `yaml:"user"` - Password string `yaml:"password"` - ClientID string `yaml:"clientId"` - // UseMock is a flag that indicates whether the sys-api should be mocked - // This is useful for testing purposes, since the sys-api cannot be run locally - UseMock bool `yaml:"useMock"` - } `yaml:"sys-api"` - Harbor struct { URL string `yaml:"url"` User string `yaml:"user"` diff --git a/pkg/services/synchronize/fetch_gpu.go b/pkg/services/synchronize/fetch_gpu.go index a6829ca2..7e3c1db1 100644 --- a/pkg/services/synchronize/fetch_gpu.go +++ b/pkg/services/synchronize/fetch_gpu.go @@ -157,11 +157,54 @@ func listLatestGPUs() (*body.SystemGpuInfo, error) { return nil, makeError(err) } + var result *body.SystemGpuInfo if len(systemGpuInfo) > 0 { - return &systemGpuInfo[0].GpuInfo, nil + result = &systemGpuInfo[0].GpuInfo } - return nil, nil + if config.Config.GPU.AddMock { + // Add one mock GPUs in each zone + for _, zone := range config.Config.EnabledZones() { + if result == nil { + result = &body.SystemGpuInfo{} + } + + result.HostGpuInfo = append(result.HostGpuInfo, body.HostGpuInfo{ + HostBase: body.HostBase{ + Name: "Mock Host 1", + DisplayName: "Mock Host 1", + Zone: zone.Name, + }, + GPUs: []host_api.GpuInfo{{ + Name: "Mock GPU 1", + Vendor: "NVIDIA", + VendorID: "10de", + DeviceID: "1eb0", + }, { + Name: "Mock GPU 2", + Vendor: "NVIDIA", + VendorID: "10de", + DeviceID: "2230", + }}, + }) + + result.HostGpuInfo = append(result.HostGpuInfo, body.HostGpuInfo{ + HostBase: body.HostBase{ + Name: "Mock Host 2", + DisplayName: "Mock Host 2", + Zone: zone.Name, + }, + GPUs: []host_api.GpuInfo{{ + Name: "Mock GPU 1", + Vendor: "NVIDIA", + VendorID: "10de", + DeviceID: "1eb0", + }}, + }) + } + } + + return result, nil } func createGpuGroupName(gpu *host_api.GpuInfo) *string { diff --git a/scripts/local/config.yml.tmpl b/scripts/local/config.yml.tmpl index 49aa5af5..e7b65532 100644 --- a/scripts/local/config.yml.tmpl +++ b/scripts/local/config.yml.tmpl @@ -70,6 +70,7 @@ gpu: privilegedGpus: excludedHosts: excludedGpus: + addMock: true deployment: defaultZone: local @@ -165,13 +166,6 @@ redis: url: $redis_url password: $redis_password -sys-api: - url: - user: - password: - clientId: - useMock: true - harbor: url: $harbor_url user: $harbor_user From 2d325d9f50133bc402370039ec101383b95a828d Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 15:47:34 +0200 Subject: [PATCH 07/14] Add support for VM status ErrorUnschedulable --- pkg/app/status_codes/code.go | 2 ++ pkg/services/status_update/vm_status_listener.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/app/status_codes/code.go b/pkg/app/status_codes/code.go index 099f1e63..2dcba4ce 100644 --- a/pkg/app/status_codes/code.go +++ b/pkg/app/status_codes/code.go @@ -21,6 +21,7 @@ const ( ResourceMountFailed = 10040 ResourceImagePullFailed = 10041 ResourceDisabled = 10042 + ResourceUnschedulable = 10043 JobPending = 10140 JobFinished = 10141 @@ -56,6 +57,7 @@ var MsgFlags = map[int]string{ ResourceMountFailed: "resourceMountFailed", ResourceImagePullFailed: "resourceImagePullFailed", ResourceDisabled: "resourceDisabled", + ResourceUnschedulable: "resourceUnschedulable", JobPending: "pending", JobRunning: "running", diff --git a/pkg/services/status_update/vm_status_listener.go b/pkg/services/status_update/vm_status_listener.go index f999063b..bdaf73bb 100644 --- a/pkg/services/status_update/vm_status_listener.go +++ b/pkg/services/status_update/vm_status_listener.go @@ -86,6 +86,8 @@ func parseVmStatus(status *model.VmStatus) string { statusCode = status_codes.ResourceStopping case "Terminating": statusCode = status_codes.ResourceDeleting + case "ErrorUnschedulable": + statusCode = status_codes.ResourceUnschedulable case "CrashLoopBackOff", "Unknown", "Unschedulable", "ErrImagePull", "ImagePullBackOff", "PvcNotFound", "DataVolumeError": statusCode = status_codes.ResourceError default: From 0b1849d1417184d647bc80f68690bcf5c3310d80 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 16:23:10 +0200 Subject: [PATCH 08/14] Fix missing e2e tests in GitHub actions --- .github/workflows/run-tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 8bb2cbcd..9a7c56af 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -51,7 +51,7 @@ jobs: done - name: Run e2e tests (VMs) - run: go test -timeout 90m ./test/e2e/v2/gpu_groups ./test/e2e/v2/gpu_leases + run: go test -timeout 90m ./test/e2e/v2/gpu_groups ./test/e2e/v2/gpu_leases ./test/e2e/v2/vms - name: Run e2e tests (Deployments) run: go test -timeout 90m ./test/e2e/v2/deployments @@ -60,7 +60,10 @@ jobs: run: go test -timeout 90m ./test/e2e/v2/sms - name: Run e2e tests (Jobs) - run: go test -timeout 90m ./test/e2e/v2/jobs ./test/e2e/v2/resource_migrations + run: go test -timeout 90m ./test/e2e/v2/jobs + + - name: Run e2e tests (Resource Migrations) + run: go test -timeout 90m ./test/e2e/v2/resource_migrations - name: Run e2e tests (Users) run: go test -timeout 90m ./test/e2e/v2/users ./test/e2e/v2/teams ./test/e2e/v2/notifications From a09ab7c7cea24d9ec45ae9a5fdcaee0f22345e97 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 16:23:34 +0200 Subject: [PATCH 09/14] Fix inconsistencies with URL in read with VM http proxy --- dto/v2/body/vm_port.go | 4 ++-- models/model/deployment.go | 18 ------------------ models/model/deployment_convert.go | 6 +++++- models/model/vm.go | 22 ++++++++++++++++++++++ models/model/vm_convert.go | 15 ++++++++++++--- routers/api/v2/vm.go | 26 ++++++++++++++++++++++++-- 6 files changed, 65 insertions(+), 26 deletions(-) diff --git a/dto/v2/body/vm_port.go b/dto/v2/body/vm_port.go index 0616818a..242203a1 100644 --- a/dto/v2/body/vm_port.go +++ b/dto/v2/body/vm_port.go @@ -30,8 +30,8 @@ type CustomDomainRead struct { } type HttpProxyRead struct { - Name string `json:"name" bson:"name,omitempty" binding:"required,rfc1035,min=3,max=30"` - URL *string `json:"url,omitempty,omitempty"` + Name string `json:"name" bson:"name"` + URL *string `json:"url,omitempty" bson:"url,omitempty"` CustomDomain *CustomDomainRead `json:"customDomain,omitempty" bson:"customDomain,omitempty"` } diff --git a/models/model/deployment.go b/models/model/deployment.go index 644b48ce..fc1f1f22 100644 --- a/models/model/deployment.go +++ b/models/model/deployment.go @@ -76,24 +76,6 @@ func (deployment *Deployment) GetURL(externalPort *int) *string { return nil } -// GetCustomDomainURL returns the custom domain URL of the deployment. -// If the app does not have a custom domain, it will return nil. -// This method does not check whether the custom domain is active, and does -// not check if the ingress exists. -func (deployment *Deployment) GetCustomDomainURL() *string { - app := deployment.GetMainApp() - if app == nil { - return nil - } - - if app.CustomDomain != nil && len(app.CustomDomain.Domain) > 0 { - url := fmt.Sprintf("https://%s", app.CustomDomain.Domain) - return &url - } - - return nil -} - // Ready returns true if the deployment is not being created or deleted. func (deployment *Deployment) Ready() bool { return !deployment.DoingActivity(ActivityBeingCreated) && !deployment.DoingActivity(ActivityBeingDeleted) diff --git a/models/model/deployment_convert.go b/models/model/deployment_convert.go index 2f392793..6945086d 100644 --- a/models/model/deployment_convert.go +++ b/models/model/deployment_convert.go @@ -87,9 +87,13 @@ func (deployment *Deployment) ToDTO(smURL *string, externalPort *int, teams []st var customDomain *body.CustomDomainRead if app.CustomDomain != nil { + extPortStr := "" + if externalPort != nil && *externalPort != 443 { + extPortStr = fmt.Sprintf(":%d", *externalPort) + } customDomain = &body.CustomDomainRead{ Domain: app.CustomDomain.Domain, - URL: fmt.Sprintf("https://%s", app.CustomDomain.Domain), + URL: fmt.Sprintf("https://%s%s", app.CustomDomain.Domain, extPortStr), Status: app.CustomDomain.Status, Secret: app.CustomDomain.Secret, } diff --git a/models/model/vm.go b/models/model/vm.go index 723aa9a1..8f61d31d 100644 --- a/models/model/vm.go +++ b/models/model/vm.go @@ -69,6 +69,28 @@ func (vm *VM) BeingDeleted() bool { return vm.DoingActivity(ActivityBeingDeleted) } +// GetHttpProxyURL returns the URL of a VM's HTTP proxy. +// If the K8s ingress does not exist, it will return nil, or if the ingress does not have a host, it will return nil. +func (vm *VM) GetHttpProxyURL(name string, externalPort *int) *string { + ingress := vm.Subsystems.K8s.GetIngress(fmt.Sprintf("%s-%s", vm.Name, name)) + if ingress == nil || !ingress.Created() { + return nil + } + + if len(ingress.Hosts) > 0 && len(ingress.Hosts[0]) > 0 { + url := fmt.Sprintf("https://%s", ingress.Hosts[0]) + + // If we have a custom port, we need to append it to the URL + if externalPort != nil && *externalPort != 443 { + url = fmt.Sprintf("%s:%d", url, *externalPort) + } + + return &url + } + + return nil +} + func (vm *VM) GetExternalPort(privatePort int, protocol string) *int { pfrName := fmt.Sprintf("priv-%d-prot-%s", privatePort, protocol) service := vm.Subsystems.K8s.GetService(fmt.Sprintf("%s-%s", vm.Name, pfrName)) diff --git a/models/model/vm_convert.go b/models/model/vm_convert.go index b7932310..abab2e73 100644 --- a/models/model/vm_convert.go +++ b/models/model/vm_convert.go @@ -9,7 +9,7 @@ import ( ) // ToDTOv2 converts a VM to a body.VmRead. -func (vm *VM) ToDTOv2(gpuLease *GpuLease, teams []string, sshConnectionString *string) body.VmRead { +func (vm *VM) ToDTOv2(gpuLease *GpuLease, teams []string, externalPort *int, sshConnectionString *string) body.VmRead { var host *string if vm.Host != nil { host = &vm.Host.Name @@ -42,17 +42,26 @@ func (vm *VM) ToDTOv2(gpuLease *GpuLease, teams []string, sshConnectionString *s var httpProxy *body.HttpProxyRead if port.HttpProxy != nil { + extPortStr := "" + if externalPort != nil && *externalPort != 443 { + extPortStr = fmt.Sprintf(":%d", *externalPort) + } + var customDomain *body.CustomDomainRead if port.HttpProxy.CustomDomain != nil { customDomain = &body.CustomDomainRead{ Domain: port.HttpProxy.CustomDomain.Domain, - URL: fmt.Sprintf("https://%s", port.HttpProxy.CustomDomain.Domain), + URL: fmt.Sprintf("https://%s%s", port.HttpProxy.CustomDomain.Domain, extPortStr), Secret: port.HttpProxy.CustomDomain.Secret, Status: port.HttpProxy.CustomDomain.Status, } } - httpProxy = &body.HttpProxyRead{Name: port.HttpProxy.Name, CustomDomain: customDomain} + httpProxy = &body.HttpProxyRead{ + Name: port.HttpProxy.Name, + URL: vm.GetHttpProxyURL(port.HttpProxy.Name, externalPort), + CustomDomain: customDomain, + } } ports = append(ports, body.PortRead{ diff --git a/routers/api/v2/vm.go b/routers/api/v2/vm.go index 1cca1cad..38169fd9 100644 --- a/routers/api/v2/vm.go +++ b/routers/api/v2/vm.go @@ -10,12 +10,15 @@ import ( configModels "go-deploy/models/config" "go-deploy/models/model" "go-deploy/models/version" + "go-deploy/pkg/config" "go-deploy/pkg/sys" "go-deploy/service" sErrors "go-deploy/service/errors" teamOpts "go-deploy/service/v2/teams/opts" v2Utils "go-deploy/service/v2/utils" "go-deploy/service/v2/vms/opts" + "strconv" + "strings" ) // GetVM @@ -75,7 +78,7 @@ func GetVM(c *gin.Context) { sshConnectionString, _ := deployV2.VMs().SshConnectionString(vm.ID) lease, _ := deployV2.VMs().GpuLeases().GetByVmID(vm.ID) - context.Ok(vm.ToDTOv2(lease, teamIDs, sshConnectionString)) + context.Ok(vm.ToDTOv2(lease, teamIDs, getVmAppExternalPort(vm.Zone), sshConnectionString)) } // ListVMs @@ -139,7 +142,7 @@ func ListVMs(c *gin.Context) { teamIDs, _ := deployV2.Teams().ListIDs(teamOpts.ListOpts{ResourceID: vm.ID}) sshConnectionString, _ := deployV2.VMs().SshConnectionString(vm.ID) lease, _ := deployV2.VMs().GpuLeases().GetByVmID(vm.ID) - dtoVMs[i] = vm.ToDTOv2(lease, teamIDs, sshConnectionString) + dtoVMs[i] = vm.ToDTOv2(lease, teamIDs, getVmAppExternalPort(vm.Zone), sshConnectionString) } context.Ok(dtoVMs) @@ -408,3 +411,22 @@ func UpdateVM(c *gin.Context) { JobID: &jobID, }) } + +func getVmAppExternalPort(zoneName string) *int { + zone := config.Config.GetZone(zoneName) + if zone == nil { + return nil + } + + split := strings.Split(zone.Domains.ParentVmApp, ":") + if len(split) > 1 { + port, err := strconv.Atoi(split[1]) + if err != nil { + return nil + } + + return &port + } + + return nil +} From dadda8aa619ae730d258a0c1ab2d0bed1807bbc3 Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 20:45:51 +0200 Subject: [PATCH 10/14] Increase fetch timeout in tests --- test/e2e/common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index 33432388..94c4e3e7 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -23,7 +23,7 @@ const ( TestDomain = "test-deploy.saffronbun.com" CheckInterval = 1 * time.Second - MaxChecks = 900 // 900 * CheckInterval (1) seconds = 15 minutes + MaxChecks = 3600 // 1 hour // VmTestsEnabled flag is used temporarily to enable V2 tests until V2 zone is fully operational VmTestsEnabled = true From 0af49953e4d9897fbde387ee8debcc26ea9109ad Mon Sep 17 00:00:00 2001 From: saffronjam Date: Sun, 2 Jun 2024 23:52:10 +0200 Subject: [PATCH 11/14] Remove t.Parallel() for VM tests --- test/e2e/v2/vms/routes_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/e2e/v2/vms/routes_test.go b/test/e2e/v2/vms/routes_test.go index 5edf2cd6..177eedc8 100644 --- a/test/e2e/v2/vms/routes_test.go +++ b/test/e2e/v2/vms/routes_test.go @@ -24,7 +24,7 @@ func TestMain(m *testing.M) { } func TestList(t *testing.T) { - t.Parallel() + //t.Parallel() queries := []string{ "?page=1&pageSize=10", @@ -38,7 +38,7 @@ func TestList(t *testing.T) { } func TestCreate(t *testing.T) { - t.Parallel() + //t.Parallel() requestBody := body.VmCreate{ Name: e2e.GenName(), @@ -60,7 +60,7 @@ func TestCreate(t *testing.T) { } func TestCreateWithInvalidBody(t *testing.T) { - t.Parallel() + //t.Parallel() longName := body.VmCreate{ Name: "e2e-", @@ -199,7 +199,7 @@ func TestCreateWithInvalidBody(t *testing.T) { } func TestUpdate(t *testing.T) { - t.Parallel() + //t.Parallel() vm := v2.WithDefaultVM(t) @@ -246,7 +246,7 @@ func TestUpdate(t *testing.T) { } func TestCreateShared(t *testing.T) { - t.Parallel() + //t.Parallel() vm := v2.WithDefaultVM(t) team := v2.WithTeam(t, body.TeamCreate{ @@ -273,7 +273,7 @@ func TestCreateShared(t *testing.T) { } func TestAction(t *testing.T) { - t.Parallel() + //t.Parallel() actions := []string{"stop", "start", "restart"} vm := v2.WithDefaultVM(t) @@ -285,7 +285,7 @@ func TestAction(t *testing.T) { } func TestInvalidAction(t *testing.T) { - t.Parallel() + //t.Parallel() actions := []string{"some command", "invalid"} From 5ccf9f50fe05e1b751ab4d8b74d9e8245d8f546c Mon Sep 17 00:00:00 2001 From: saffronjam Date: Mon, 3 Jun 2024 16:41:27 +0200 Subject: [PATCH 12/14] Disable VM tests --- test/acc/common.go | 4 ++++ test/acc/subsystems/k8s/vm_test.go | 9 +++++++++ test/e2e/common.go | 3 +-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/test/acc/common.go b/test/acc/common.go index 2a391531..0a486777 100644 --- a/test/acc/common.go +++ b/test/acc/common.go @@ -11,6 +11,10 @@ import ( "github.com/google/uuid" ) +const ( + VmTestsEnabled = false +) + func Setup() { err := log.SetupLogger(mode.Test) if err != nil { diff --git a/test/acc/subsystems/k8s/vm_test.go b/test/acc/subsystems/k8s/vm_test.go index 7a4cf748..43e7bb45 100644 --- a/test/acc/subsystems/k8s/vm_test.go +++ b/test/acc/subsystems/k8s/vm_test.go @@ -3,10 +3,15 @@ package k8s import ( "github.com/stretchr/testify/assert" "go-deploy/test" + "go-deploy/test/acc" "testing" ) func TestCreateVM(t *testing.T) { + if !acc.VmTestsEnabled { + t.Skip("vm tests are disabled") + } + t.Parallel() c, _ := withContext(t) @@ -14,6 +19,10 @@ func TestCreateVM(t *testing.T) { } func TestUpdateVM(t *testing.T) { + if !acc.VmTestsEnabled { + t.Skip("vm tests are disabled") + } + t.Parallel() c, _ := withContext(t) diff --git a/test/e2e/common.go b/test/e2e/common.go index 94c4e3e7..d381ff02 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -25,8 +25,7 @@ const ( CheckInterval = 1 * time.Second MaxChecks = 3600 // 1 hour - // VmTestsEnabled flag is used temporarily to enable V2 tests until V2 zone is fully operational - VmTestsEnabled = true + VmTestsEnabled = false ) func GetUserID(user string) string { From cbe309b79a9d592b391760428b990b9e18be401f Mon Sep 17 00:00:00 2001 From: saffronjam Date: Mon, 3 Jun 2024 17:01:58 +0200 Subject: [PATCH 13/14] Fix better logging for when setting up logger --- pkg/db/key_value/client.go | 2 +- pkg/services/logger/pod_event_listener.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/db/key_value/client.go b/pkg/db/key_value/client.go index 21646266..18ffbcde 100644 --- a/pkg/db/key_value/client.go +++ b/pkg/db/key_value/client.go @@ -107,7 +107,7 @@ func (client *Client) SetUpExpirationListener(ctx context.Context, pattern strin err := handler(msg.Payload) if err != nil { utils.PrettyPrintError(fmt.Errorf("failed to handle expired key event for key %s. details: %w", msg.Payload, err)) - return + continue } } } diff --git a/pkg/services/logger/pod_event_listener.go b/pkg/services/logger/pod_event_listener.go index 2983cb0f..c474bf9b 100644 --- a/pkg/services/logger/pod_event_listener.go +++ b/pkg/services/logger/pod_event_listener.go @@ -44,6 +44,7 @@ func PodEventListener(ctx context.Context) error { if !exists { // Clean up the keys + log.Printf("Pod %s not longer exists. Cleaning up keys", podName) _ = kvc.Del(LogKey(podName)) _ = kvc.Del(LastLogKey(podName)) _ = kvc.Del(OwnerLogKey(podName)) From 54e10c4a8c1ef303c8894e989832104627fed78b Mon Sep 17 00:00:00 2001 From: saffronjam Date: Mon, 3 Jun 2024 17:02:29 +0200 Subject: [PATCH 14/14] Fix broken e2e test after adding support for custom port for custom domain in DeploymentRead --- test/e2e/v2/deployment_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/v2/deployment_utils.go b/test/e2e/v2/deployment_utils.go index 9bf4e597..462cb6ea 100644 --- a/test/e2e/v2/deployment_utils.go +++ b/test/e2e/v2/deployment_utils.go @@ -197,7 +197,7 @@ func WithDeployment(t *testing.T, requestBody body.DeploymentCreate, user ...str if requestBody.CustomDomain != nil { punyEncoded, err := idna.New().ToASCII("https://" + *requestBody.CustomDomain) assert.NoError(t, err, "custom domain was not puny encoded") - assert.Equal(t, punyEncoded, deploymentRead.CustomDomain.URL) + assert.Contains(t, deploymentRead.CustomDomain.URL, punyEncoded, "custom domain was not set") } return deploymentRead, deploymentCreated.JobID