diff --git a/models/sys/job/job.go b/models/sys/job/job.go index 4f10947b..6b00b722 100644 --- a/models/sys/job/job.go +++ b/models/sys/job/job.go @@ -36,17 +36,16 @@ const ( ) type Job struct { - ID string `bson:"id" json:"id"` - UserID string `bson:"userId" json:"userId"` - Type string `bson:"type" json:"type"` - Args map[string]interface{} `bson:"args" json:"args"` - ResourceIDs []string `bson:"resourceIds" json:"resourceIds"` - CreatedAt time.Time `bson:"createdAt" json:"createdAt"` - LastRunAt time.Time `bson:"lastRunAt" json:"lastRunAt"` - FinishedAt time.Time `bson:"finishedAt" json:"finishedAt"` - RunAfter time.Time `bson:"runAfter" json:"runAfter"` - Status string `bson:"status" json:"status"` - ErrorLogs []string `bson:"errorLogs" json:"errorLogs"` + ID string `bson:"id" json:"id"` + UserID string `bson:"userId" json:"userId"` + Type string `bson:"type" json:"type"` + Args map[string]interface{} `bson:"args" json:"args"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + LastRunAt time.Time `bson:"lastRunAt" json:"lastRunAt"` + FinishedAt time.Time `bson:"finishedAt" json:"finishedAt"` + RunAfter time.Time `bson:"runAfter" json:"runAfter"` + Status string `bson:"status" json:"status"` + ErrorLogs []string `bson:"errorLogs" json:"errorLogs"` } type UpdateParams struct { diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 13b26058..cff31f2b 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -67,7 +67,7 @@ func DeleteVM(job *jobModel.Job) error { defer cancel() go func() { - err = waitForJobs(ctx, relatedJobs, []string{jobModel.StatusCompleted, jobModel.StatusFailed, jobModel.StatusTerminated}) + err = waitForJobs(ctx, relatedJobs, []string{jobModel.StatusCompleted, jobModel.StatusTerminated}) if err != nil { log.Println("failed to wait for related jobs", id, ". details:", err) } @@ -241,7 +241,7 @@ func DeleteDeployment(job *jobModel.Job) error { defer cancel() go func() { - err = waitForJobs(ctx, relatedJobs, []string{jobModel.StatusCompleted, jobModel.StatusFailed, jobModel.StatusTerminated}) + err = waitForJobs(ctx, relatedJobs, []string{jobModel.StatusCompleted, jobModel.StatusTerminated}) if err != nil { log.Println("failed to wait for related jobs", id, ". details:", err) } diff --git a/pkg/jobs/runner.go b/pkg/jobs/runner.go index 3bc9a575..c6130b1a 100644 --- a/pkg/jobs/runner.go +++ b/pkg/jobs/runner.go @@ -17,11 +17,12 @@ func NewRunner(job *jobModel.Job) *Runner { } func (runner *Runner) Run() { - if jobDef := GetJobDef(runner.Job); jobDef != nil { if jobDef.TerminateFunc != nil { shouldTerminate, err := jobDef.TerminateFunc(runner.Job) if err != nil { + utils.PrettyPrintError(fmt.Errorf("error executing job (%s) terminate function, terminating the job instead. details: %w", runner.Job.Type, err)) + err = jobModel.New().MarkTerminated(runner.Job.ID, err.Error()) if err != nil { utils.PrettyPrintError(fmt.Errorf("error marking job as terminated. details: %w", err)) @@ -32,6 +33,7 @@ func (runner *Runner) Run() { if shouldTerminate { err = jobModel.New().MarkTerminated(runner.Job.ID, "gracefully terminated by system") + utils.PrettyPrintError(fmt.Errorf("job (%s) gracefully terminated by system", runner.Job.Type)) if err != nil { utils.PrettyPrintError(fmt.Errorf("error marking job as terminated. details: %w", err)) return diff --git a/pkg/workers/confirm/workers.go b/pkg/workers/confirm/workers.go index a5f4d429..7a8aea2c 100644 --- a/pkg/workers/confirm/workers.go +++ b/pkg/workers/confirm/workers.go @@ -34,7 +34,7 @@ func deploymentConfirmer(ctx context.Context) { continue } - relatedJobs, err := jobModel.New().GetByArgs(map[string]interface{}{ + relatedJobs, err := jobModel.New().ExcludeScheduled().GetByArgs(map[string]interface{}{ "id": deployment.ID, }) @@ -45,7 +45,6 @@ func deploymentConfirmer(ctx context.Context) { allFinished := slices.IndexFunc(relatedJobs, func(j jobModel.Job) bool { return j.Status != jobModel.StatusCompleted && - j.Status != jobModel.StatusFailed && j.Status != jobModel.StatusTerminated }) == -1 @@ -91,7 +90,7 @@ func vmConfirmer(ctx context.Context) { continue } - relatedJobs, err := jobModel.New().GetByArgs(map[string]interface{}{ + relatedJobs, err := jobModel.New().ExcludeScheduled().GetByArgs(map[string]interface{}{ "id": vm.ID, }) @@ -102,7 +101,6 @@ func vmConfirmer(ctx context.Context) { allFinished := slices.IndexFunc(relatedJobs, func(j jobModel.Job) bool { return j.Status != jobModel.StatusCompleted && - j.Status != jobModel.StatusFailed && j.Status != jobModel.StatusTerminated }) == -1