From c46e7116857793c190a158f255766609f66a3892 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 4 Jan 2024 16:38:59 -0600 Subject: [PATCH 01/98] api: new /jobs/statuses endpoint example usage: ``` $ cat jobs.json { "jobs": [ {"id": "clients", "namespace": "default"}, {"id": "fail"} ] } $ cat jobs.json | nomad operator api -X POST /v1/jobs/statuses | jq . ``` --- api/jobs.go | 7 +++ command/agent/http.go | 1 + command/agent/job_endpoint.go | 30 ++++++++++++ nomad/jobs_endpoint.go | 89 +++++++++++++++++++++++++++++++++++ nomad/server.go | 1 + nomad/structs/job.go | 29 ++++++++++++ 6 files changed, 157 insertions(+) create mode 100644 nomad/jobs_endpoint.go diff --git a/api/jobs.go b/api/jobs.go index 3bd60c4ef4c..6aa2ad9f399 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -1544,3 +1544,10 @@ func (j *Jobs) ActionExec(ctx context.Context, return s.run(ctx) } + +type JobsStatusesRequest struct { + Jobs []struct { // TODO: proper type + ID string `json:"id"` + Namespace string `json:"namespace"` + } `json:"jobs"` // TODO: unkeyed? +} diff --git a/command/agent/http.go b/command/agent/http.go index c08d4fdf510..679948b591e 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -382,6 +382,7 @@ func (s *HTTPServer) ResolveToken(req *http.Request) (*acl.ACL, error) { func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest)) s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) + s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobsStatusesRequest)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 05f1f04a162..a00810d2b49 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2108,3 +2108,33 @@ func validateEvalPriorityOpt(priority int) HTTPCodedError { } return nil } + +func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != http.MethodPost { + return nil, CodedError(405, ErrInvalidMethod) + } + var in api.JobsStatusesRequest + if err := decodeBody(req, &in); err != nil { + return nil, err + } + + args := structs.JobsStatusesRequest{} + for _, j := range in.Jobs { + if j.Namespace == "" { + j.Namespace = "default" + } + args.Jobs = append(args.Jobs, structs.NamespacedID{ + ID: j.ID, + // note: can't just use QueryOptions.Namespace, because each job may have a different NS + Namespace: j.Namespace, + }) + } + + var out structs.JobsStatusesResponse + if err := s.agent.RPC("Jobs.Statuses", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + return out, nil +} diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go new file mode 100644 index 00000000000..4523f1d82f0 --- /dev/null +++ b/nomad/jobs_endpoint.go @@ -0,0 +1,89 @@ +package nomad + +import ( + "fmt" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/structs" +) + +func NewJobsEndpoint(s *Server, ctx *RPCContext) *Jobs { + return &Jobs{ + srv: s, + ctx: ctx, + logger: s.logger.Named("jobs"), + } +} + +type Jobs struct { + srv *Server + ctx *RPCContext + logger hclog.Logger +} + +func (j *Jobs) Statuses( + args *structs.JobsStatusesRequest, + reply *structs.JobsStatusesResponse) error { + // TODO: auth, rate limiting, etc... + + if reply.Jobs == nil { + reply.Jobs = make(map[string]structs.JobStatus) + } + + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + // TODO: make this block properly + var idx uint64 + + for _, j := range args.Jobs { + ns := j.Namespace + js := structs.JobStatus{ID: j.ID, Namespace: j.Namespace} + + allocs, err := state.AllocsByJob(ws, ns, j.ID, false) + if err != nil { + return err + } + for _, a := range allocs { + alloc := structs.JobStatusAlloc{ + ID: a.ID, + Group: a.TaskGroup, + ClientStatus: a.ClientStatus, + } + if a.DeploymentStatus != nil { + alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary + alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy + } + js.Allocs = append(js.Allocs, alloc) + if a.ModifyIndex > idx { + idx = a.ModifyIndex + } + } + + deploys, err := state.DeploymentsByJobID(ws, ns, j.ID, false) + if err != nil { + return err + } + for _, d := range deploys { + if d.Active() { + js.DeploymentID = d.ID + break + } + if d.ModifyIndex > idx { + idx = d.ModifyIndex + } + } + + nsid := fmt.Sprintf("%s@%s", j.ID, j.Namespace) + reply.Jobs[nsid] = js + } + reply.Index = idx + j.srv.setQueryMeta(&reply.QueryMeta) + return nil + + }} + return j.srv.blockingRPC(&opts) +} diff --git a/nomad/server.go b/nomad/server.go index a488481d997..08f5d775848 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1341,6 +1341,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { _ = server.Register(NewDeploymentEndpoint(s, ctx)) _ = server.Register(NewEvalEndpoint(s, ctx)) _ = server.Register(NewJobEndpoints(s, ctx)) + _ = server.Register(NewJobsEndpoint(s, ctx)) _ = server.Register(NewKeyringEndpoint(s, ctx, s.encrypter)) _ = server.Register(NewNamespaceEndpoint(s, ctx)) _ = server.Register(NewNodeEndpoint(s, ctx)) diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 8eb69b917e4..4df1f5f3ef6 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -16,6 +16,35 @@ const ( JobServiceRegistrationsRPCMethod = "Job.GetServiceRegistrations" ) +type JobsStatusesRequest struct { + Jobs []NamespacedID + QueryOptions +} + +type JobsStatusesResponse struct { + Jobs map[string]JobStatus + QueryMeta +} + +type JobStatus struct { + ID string + Namespace string + Allocs []JobStatusAlloc + DeploymentID string +} + +type JobStatusAlloc struct { + ID string + Group string + ClientStatus string + DeploymentStatus JobStatusDeployment +} + +type JobStatusDeployment struct { + Canary bool + Healthy bool +} + // JobServiceRegistrationsRequest is the request object used to list all // service registrations belonging to the specified Job.ID. type JobServiceRegistrationsRequest struct { From 30f82b733e54bf167dabf8fc1606a517e3630375 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 4 Jan 2024 16:59:10 -0600 Subject: [PATCH 02/98] make blocking-query block with ?index= query param e.g. ``` $ cat jobs.json | nomad operator api -X POST /v1/jobs/statuses?index=1681 | jq . ``` --- command/agent/job_endpoint.go | 7 ++++++- nomad/jobs_endpoint.go | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index a00810d2b49..53db591b2da 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2113,12 +2113,17 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req if req.Method != http.MethodPost { return nil, CodedError(405, ErrInvalidMethod) } + + args := structs.JobsStatusesRequest{} + if parseWait(resp, req, &args.QueryOptions) { + return nil, nil // seems whack. + } + var in api.JobsStatusesRequest if err := decodeBody(req, &in); err != nil { return nil, err } - args := structs.JobsStatusesRequest{} for _, j := range in.Jobs { if j.Namespace == "" { j.Namespace = "default" diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 4523f1d82f0..caabd07d5e6 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -36,7 +36,6 @@ func (j *Jobs) Statuses( queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, run: func(ws memdb.WatchSet, state *state.StateStore) error { - // TODO: make this block properly var idx uint64 for _, j := range args.Jobs { From c32d019c9d4f3d8c8ad8ad9786d0f997ac03288c Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 8 Jan 2024 11:26:58 -0600 Subject: [PATCH 03/98] add JobSummary.Children.Desired --- nomad/state/state_store.go | 9 +++++++++ nomad/structs/structs.go | 1 + 2 files changed, 10 insertions(+) diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index af538fa8888..1ba97a143b6 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -5517,7 +5517,9 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, hasSummaryChanged = true } + var totalCount int64 for _, tg := range job.TaskGroups { + totalCount += int64(tg.Count) if _, ok := summary.Summary[tg.Name]; !ok { newSummary := structs.TaskGroupSummary{ Complete: 0, @@ -5529,6 +5531,7 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, hasSummaryChanged = true } } + summary.Children.Desired = totalCount // The job summary has changed, so update the modify index. if hasSummaryChanged { @@ -5887,6 +5890,12 @@ func (s *StateStore) updateSummaryWithAlloc(index uint64, alloc *structs.Allocat return nil } + var totalCount int64 + for _, tg := range alloc.Job.TaskGroups { + totalCount += int64(tg.Count) + } + jobSummary.Children.Desired = totalCount + tgSummary, ok := jobSummary.Summary[alloc.TaskGroup] if !ok { return fmt.Errorf("unable to find task group in the job summary: %v", alloc.TaskGroup) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 5611647090f..8c1431d321e 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5112,6 +5112,7 @@ type JobChildrenSummary struct { Pending int64 Running int64 Dead int64 + Desired int64 } // Copy returns a new copy of a JobChildrenSummary From 8ba63f52471b22c0099ea3c000ace03f2b63e35b Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 10 Jan 2024 13:06:23 -0600 Subject: [PATCH 04/98] GroupCountSum in new /statuses --- nomad/jobs_endpoint.go | 21 +++++++++++++++++++-- nomad/structs/job.go | 10 ++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index caabd07d5e6..7eed1e835eb 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -40,7 +40,22 @@ func (j *Jobs) Statuses( for _, j := range args.Jobs { ns := j.Namespace - js := structs.JobStatus{ID: j.ID, Namespace: j.Namespace} + job, err := state.JobByID(ws, ns, j.ID) + if err != nil { + return err + } + if job == nil { + continue + } + + js := structs.JobStatus{ + ID: j.ID, + Namespace: j.Namespace, + } + js.Type = job.Type + for _, tg := range job.TaskGroups { + js.GroupCountSum += tg.Count + } allocs, err := state.AllocsByJob(ws, ns, j.ID, false) if err != nil { @@ -54,7 +69,9 @@ func (j *Jobs) Statuses( } if a.DeploymentStatus != nil { alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary - alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy + if a.DeploymentStatus.Healthy != nil { + alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy + } } js.Allocs = append(js.Allocs, alloc) if a.ModifyIndex > idx { diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 4df1f5f3ef6..c134a60c144 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -27,10 +27,12 @@ type JobsStatusesResponse struct { } type JobStatus struct { - ID string - Namespace string - Allocs []JobStatusAlloc - DeploymentID string + ID string + Namespace string + Type string + Allocs []JobStatusAlloc + GroupCountSum int + DeploymentID string } type JobStatusAlloc struct { From 4653b7ca14fd3222687cdbde135f5bcc84a267ca Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 10 Jan 2024 17:12:08 -0600 Subject: [PATCH 05/98] /jobs/statuses2 endpoint for the whole /ui/jobs index table --- command/agent/http.go | 1 + command/agent/job_endpoint.go | 19 ++++ nomad/jobs_endpoint.go | 170 ++++++++++++++++++++++++++++++++++ nomad/structs/job.go | 21 +++++ 4 files changed, 211 insertions(+) diff --git a/command/agent/http.go b/command/agent/http.go index 679948b591e..c4b4a91b507 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -383,6 +383,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest)) s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobsStatusesRequest)) + s.mux.HandleFunc("/v1/jobs/statuses2", s.wrap(s.JobsStatuses2Request)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 53db591b2da..a6e56d2d801 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2143,3 +2143,22 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req setMeta(resp, &out.QueryMeta) return out, nil } + +func (s *HTTPServer) JobsStatuses2Request(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != http.MethodGet { + return nil, CodedError(405, ErrInvalidMethod) + } + + args := structs.JobsStatuses2Request{} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil // seems whack + } + + var out structs.JobsStatuses2Response + if err := s.agent.RPC("Jobs.Statuses2", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + return out, nil +} diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 7eed1e835eb..c50000f1fdb 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -2,10 +2,15 @@ package nomad import ( "fmt" + "net/http" + "time" + "github.com/armon/go-metrics" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/state/paginator" "github.com/hashicorp/nomad/nomad/structs" ) @@ -103,3 +108,168 @@ func (j *Jobs) Statuses( }} return j.srv.blockingRPC(&opts) } + +func (j *Jobs) Statuses2( + args *structs.JobsStatuses2Request, + reply *structs.JobsStatuses2Response) error { + + // totally lifted from Job.List + authErr := j.srv.Authenticate(j.ctx, args) + if done, err := j.srv.forward("Jobs.Statuses2", args, args, reply); done { + return err + } + j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) + if authErr != nil { + return structs.ErrPermissionDenied + } + defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) + + namespace := args.RequestNamespace() + + // Check for list-job permissions + aclObj, err := j.srv.ResolveACL(args) + if err != nil { + return err + } + if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { + return structs.ErrPermissionDenied + } + allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + // Capture all the jobs + var err error + var iter memdb.ResultIterator + + // Get the namespaces the user is allowed to access. + allowableNamespaces, err := allowedNSes(aclObj, state, allow) + if err == structs.ErrPermissionDenied { + // return empty jobs if token isn't authorized for any + // namespace, matching other endpoints + reply.Jobs = make(map[string]structs.UIJob) + } else if err != nil { + return err + } else { + if prefix := args.QueryOptions.Prefix; prefix != "" { + iter, err = state.JobsByIDPrefix(ws, namespace, prefix) + } else if namespace != structs.AllNamespacesSentinel { + iter, err = state.JobsByNamespace(ws, namespace) + } else { + iter, err = state.Jobs(ws) + } + if err != nil { + return err + } + + tokenizer := paginator.NewStructsTokenizer( + iter, + paginator.StructsTokenizerOptions{ + WithNamespace: true, + WithID: true, + }, + ) + filters := []paginator.Filter{ + paginator.NamespaceFilter{ + AllowableNamespaces: allowableNamespaces, + }, + } + + jobs := make(map[string]structs.UIJob) + pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, + func(raw interface{}) error { + job := raw.(*structs.Job) + //summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) + //if err != nil || summary == nil { + // return fmt.Errorf("unable to look up summary for job: %v", job.ID) + //} + uiJob := structs.UIJob{ + NamespacedID: structs.NamespacedID{ + ID: job.ID, + Namespace: job.Namespace, + }, + Name: job.Name, + Type: job.Type, + NodePool: job.NodePool, + Priority: job.Priority, + Version: job.Version, + // included here for completeness, populated below. + Allocs: nil, + GroupCountSum: 0, + DeploymentID: "", + } + + for _, tg := range job.TaskGroups { + uiJob.GroupCountSum += tg.Count + } + + allocs, err := state.AllocsByJob(ws, namespace, job.ID, false) + if err != nil { + return err + } + for _, a := range allocs { + alloc := structs.JobStatusAlloc{ + ID: a.ID, + Group: a.TaskGroup, + ClientStatus: a.ClientStatus, + } + if a.DeploymentStatus != nil { + alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary + if a.DeploymentStatus.Healthy != nil { + alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy + } + } + uiJob.Allocs = append(uiJob.Allocs, alloc) + } + + deploys, err := state.DeploymentsByJobID(ws, namespace, job.ID, false) + if err != nil { + return err + } + for _, d := range deploys { + if d.Active() { + uiJob.DeploymentID = d.ID + break + } + } + + nsID := fmt.Sprintf("%s@%s", job.ID, job.Namespace) + jobs[nsID] = uiJob + return nil + }) + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to create result paginator: %v", err) + } + + nextToken, err := pager.Page() + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to read result page: %v", err) + } + + reply.QueryMeta.NextToken = nextToken + reply.Jobs = jobs + } + + var idx uint64 + for _, table := range []string{"jobs, allocs", "deployment"} { + i, err := state.Index(table) + if err != nil { + return err + } + if i > idx { + idx = i + } + } + reply.Index = idx + + // Set the query response + j.srv.setQueryMeta(&reply.QueryMeta) + return nil + }} + return j.srv.blockingRPC(&opts) +} diff --git a/nomad/structs/job.go b/nomad/structs/job.go index c134a60c144..555394d613b 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -16,6 +16,27 @@ const ( JobServiceRegistrationsRPCMethod = "Job.GetServiceRegistrations" ) +type JobsStatuses2Request struct { + QueryOptions +} + +type JobsStatuses2Response struct { + Jobs map[string]UIJob + QueryMeta +} + +type UIJob struct { + NamespacedID + Name string + Type string + NodePool string + Priority int + Allocs []JobStatusAlloc + GroupCountSum int + DeploymentID string + Version uint64 +} + type JobsStatusesRequest struct { Jobs []NamespacedID QueryOptions From 209643c0b4dd1ac9cd67acbf6338ba200a25b320 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 10 Jan 2024 18:04:39 -0600 Subject: [PATCH 06/98] fix a really silly bug for blocking queries --- nomad/jobs_endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index c50000f1fdb..7f9fd072b13 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -256,7 +256,7 @@ func (j *Jobs) Statuses2( } var idx uint64 - for _, table := range []string{"jobs, allocs", "deployment"} { + for _, table := range []string{"jobs", "allocs", "deployment"} { i, err := state.Index(table) if err != nil { return err From ef1f8ee0a05c097ed3e27ea652e1b684286395e5 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 11 Jan 2024 11:10:07 -0600 Subject: [PATCH 07/98] add Datacenters --- nomad/jobs_endpoint.go | 11 ++++++----- nomad/structs/job.go | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 7f9fd072b13..64ec7f4d943 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -191,11 +191,12 @@ func (j *Jobs) Statuses2( ID: job.ID, Namespace: job.Namespace, }, - Name: job.Name, - Type: job.Type, - NodePool: job.NodePool, - Priority: job.Priority, - Version: job.Version, + Name: job.Name, + Type: job.Type, + NodePool: job.NodePool, + Datacenters: job.Datacenters, + Priority: job.Priority, + Version: job.Version, // included here for completeness, populated below. Allocs: nil, GroupCountSum: 0, diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 555394d613b..9eddb945403 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -30,6 +30,7 @@ type UIJob struct { Name string Type string NodePool string + Datacenters []string Priority int Allocs []JobStatusAlloc GroupCountSum int From c98a90e5f1da3c68afe886b562083dba15f2eb2b Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 11 Jan 2024 11:22:50 -0600 Subject: [PATCH 08/98] flat list of jobs will need to read response headers for index, next_token --- command/agent/job_endpoint.go | 2 +- nomad/jobs_endpoint.go | 7 +++---- nomad/structs/job.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index a6e56d2d801..72c7d8e9f78 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2160,5 +2160,5 @@ func (s *HTTPServer) JobsStatuses2Request(resp http.ResponseWriter, req *http.Re } setMeta(resp, &out.QueryMeta) - return out, nil + return out.Jobs, nil } diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 64ec7f4d943..1741368b797 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -150,7 +150,7 @@ func (j *Jobs) Statuses2( if err == structs.ErrPermissionDenied { // return empty jobs if token isn't authorized for any // namespace, matching other endpoints - reply.Jobs = make(map[string]structs.UIJob) + reply.Jobs = make([]structs.UIJob, 0) } else if err != nil { return err } else { @@ -178,7 +178,7 @@ func (j *Jobs) Statuses2( }, } - jobs := make(map[string]structs.UIJob) + var jobs []structs.UIJob pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) @@ -237,8 +237,7 @@ func (j *Jobs) Statuses2( } } - nsID := fmt.Sprintf("%s@%s", job.ID, job.Namespace) - jobs[nsID] = uiJob + jobs = append(jobs, uiJob) return nil }) if err != nil { diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 9eddb945403..68dc67cf558 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -21,7 +21,7 @@ type JobsStatuses2Request struct { } type JobsStatuses2Response struct { - Jobs map[string]UIJob + Jobs []UIJob QueryMeta } From a85fe9b2ef782302f083e1416f686d2646a01201 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 12 Jan 2024 13:36:40 -0600 Subject: [PATCH 09/98] make /statuses the same shape as /statuses2 --- command/agent/job_endpoint.go | 2 +- nomad/jobs_endpoint.go | 158 ++++++++++++++-------------------- nomad/structs/job.go | 18 ++-- 3 files changed, 73 insertions(+), 105 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 72c7d8e9f78..6432f912c8b 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2141,7 +2141,7 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req } setMeta(resp, &out.QueryMeta) - return out, nil + return out.Jobs, nil } func (s *HTTPServer) JobsStatuses2Request(resp http.ResponseWriter, req *http.Request) (interface{}, error) { diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 1741368b797..4fed8cd6cd7 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -1,7 +1,6 @@ package nomad import ( - "fmt" "net/http" "time" @@ -33,16 +32,10 @@ func (j *Jobs) Statuses( reply *structs.JobsStatusesResponse) error { // TODO: auth, rate limiting, etc... - if reply.Jobs == nil { - reply.Jobs = make(map[string]structs.JobStatus) - } - opts := blockingOptions{ queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, run: func(ws memdb.WatchSet, state *state.StateStore) error { - var idx uint64 - for _, j := range args.Jobs { ns := j.Namespace job, err := state.JobByID(ws, ns, j.ID) @@ -52,59 +45,26 @@ func (j *Jobs) Statuses( if job == nil { continue } - - js := structs.JobStatus{ - ID: j.ID, - Namespace: j.Namespace, - } - js.Type = job.Type - for _, tg := range job.TaskGroups { - js.GroupCountSum += tg.Count - } - - allocs, err := state.AllocsByJob(ws, ns, j.ID, false) + uiJob, err := UIJobFromJob(ws, state, job) if err != nil { return err } - for _, a := range allocs { - alloc := structs.JobStatusAlloc{ - ID: a.ID, - Group: a.TaskGroup, - ClientStatus: a.ClientStatus, - } - if a.DeploymentStatus != nil { - alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary - if a.DeploymentStatus.Healthy != nil { - alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy - } - } - js.Allocs = append(js.Allocs, alloc) - if a.ModifyIndex > idx { - idx = a.ModifyIndex - } - } + reply.Jobs = append(reply.Jobs, uiJob) + } - deploys, err := state.DeploymentsByJobID(ws, ns, j.ID, false) + var idx uint64 + for _, table := range []string{"jobs", "allocs", "deployment"} { + i, err := state.Index(table) if err != nil { return err } - for _, d := range deploys { - if d.Active() { - js.DeploymentID = d.ID - break - } - if d.ModifyIndex > idx { - idx = d.ModifyIndex - } + if i > idx { + idx = i } - - nsid := fmt.Sprintf("%s@%s", j.ID, j.Namespace) - reply.Jobs[nsid] = js } reply.Index = idx j.srv.setQueryMeta(&reply.QueryMeta) return nil - }} return j.srv.blockingRPC(&opts) } @@ -186,56 +146,10 @@ func (j *Jobs) Statuses2( //if err != nil || summary == nil { // return fmt.Errorf("unable to look up summary for job: %v", job.ID) //} - uiJob := structs.UIJob{ - NamespacedID: structs.NamespacedID{ - ID: job.ID, - Namespace: job.Namespace, - }, - Name: job.Name, - Type: job.Type, - NodePool: job.NodePool, - Datacenters: job.Datacenters, - Priority: job.Priority, - Version: job.Version, - // included here for completeness, populated below. - Allocs: nil, - GroupCountSum: 0, - DeploymentID: "", - } - - for _, tg := range job.TaskGroups { - uiJob.GroupCountSum += tg.Count - } - - allocs, err := state.AllocsByJob(ws, namespace, job.ID, false) - if err != nil { - return err - } - for _, a := range allocs { - alloc := structs.JobStatusAlloc{ - ID: a.ID, - Group: a.TaskGroup, - ClientStatus: a.ClientStatus, - } - if a.DeploymentStatus != nil { - alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary - if a.DeploymentStatus.Healthy != nil { - alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy - } - } - uiJob.Allocs = append(uiJob.Allocs, alloc) - } - - deploys, err := state.DeploymentsByJobID(ws, namespace, job.ID, false) + uiJob, err := UIJobFromJob(ws, state, job) if err != nil { return err } - for _, d := range deploys { - if d.Active() { - uiJob.DeploymentID = d.ID - break - } - } jobs = append(jobs, uiJob) return nil @@ -273,3 +187,57 @@ func (j *Jobs) Statuses2( }} return j.srv.blockingRPC(&opts) } + +func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) (structs.UIJob, error) { + uiJob := structs.UIJob{ + NamespacedID: structs.NamespacedID{ + ID: job.ID, + Namespace: job.Namespace, + }, + Name: job.Name, + Type: job.Type, + NodePool: job.NodePool, + Datacenters: job.Datacenters, + Priority: job.Priority, + Version: job.Version, + // included here for completeness, populated below. + Allocs: nil, + GroupCountSum: 0, + DeploymentID: "", + } + for _, tg := range job.TaskGroups { + uiJob.GroupCountSum += tg.Count + } + + allocs, err := state.AllocsByJob(ws, job.Namespace, job.ID, false) + if err != nil { + return uiJob, err + } + for _, a := range allocs { + alloc := structs.JobStatusAlloc{ + ID: a.ID, + Group: a.TaskGroup, + ClientStatus: a.ClientStatus, + } + // TODO: use methods instead of fields directly? + if a.DeploymentStatus != nil { + alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary + if a.DeploymentStatus.Healthy != nil { + alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy + } + } + uiJob.Allocs = append(uiJob.Allocs, alloc) + } + + deploys, err := state.DeploymentsByJobID(ws, job.Namespace, job.ID, false) + if err != nil { + return uiJob, err + } + for _, d := range deploys { + if d.Active() { + uiJob.DeploymentID = d.ID + break + } + } + return uiJob, nil +} diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 68dc67cf558..22441d2ff55 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -44,18 +44,18 @@ type JobsStatusesRequest struct { } type JobsStatusesResponse struct { - Jobs map[string]JobStatus + Jobs []UIJob QueryMeta } -type JobStatus struct { - ID string - Namespace string - Type string - Allocs []JobStatusAlloc - GroupCountSum int - DeploymentID string -} +//type JobStatus struct { +// ID string +// Namespace string +// Type string +// Allocs []JobStatusAlloc +// GroupCountSum int +// DeploymentID string +//} type JobStatusAlloc struct { ID string From 259de6ecc9f49b26f4c77c4b34e50e5cd7ef5736 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 12 Jan 2024 14:09:49 -0600 Subject: [PATCH 10/98] /statuses only unblock on changes to the jobs being watched --- nomad/jobs_endpoint.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 4fed8cd6cd7..b087a16c9a7 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -36,6 +36,7 @@ func (j *Jobs) Statuses( queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, run: func(ws memdb.WatchSet, state *state.StateStore) error { + var jobs []structs.UIJob for _, j := range args.Jobs { ns := j.Namespace job, err := state.JobByID(ws, ns, j.ID) @@ -45,24 +46,16 @@ func (j *Jobs) Statuses( if job == nil { continue } - uiJob, err := UIJobFromJob(ws, state, job) + uiJob, idx, err := UIJobFromJob(ws, state, job) if err != nil { return err } - reply.Jobs = append(reply.Jobs, uiJob) - } - - var idx uint64 - for _, table := range []string{"jobs", "allocs", "deployment"} { - i, err := state.Index(table) - if err != nil { - return err - } - if i > idx { - idx = i + jobs = append(jobs, uiJob) + if idx > reply.Index { + reply.Index = idx } } - reply.Index = idx + reply.Jobs = jobs j.srv.setQueryMeta(&reply.QueryMeta) return nil }} @@ -146,7 +139,7 @@ func (j *Jobs) Statuses2( //if err != nil || summary == nil { // return fmt.Errorf("unable to look up summary for job: %v", job.ID) //} - uiJob, err := UIJobFromJob(ws, state, job) + uiJob, _, err := UIJobFromJob(ws, state, job) if err != nil { return err } @@ -188,7 +181,9 @@ func (j *Jobs) Statuses2( return j.srv.blockingRPC(&opts) } -func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) (structs.UIJob, error) { +func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) (structs.UIJob, uint64, error) { + idx := job.ModifyIndex + uiJob := structs.UIJob{ NamespacedID: structs.NamespacedID{ ID: job.ID, @@ -211,7 +206,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) allocs, err := state.AllocsByJob(ws, job.Namespace, job.ID, false) if err != nil { - return uiJob, err + return uiJob, idx, err } for _, a := range allocs { alloc := structs.JobStatusAlloc{ @@ -227,17 +222,23 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) } } uiJob.Allocs = append(uiJob.Allocs, alloc) + if a.ModifyIndex > idx { + idx = a.ModifyIndex + } } deploys, err := state.DeploymentsByJobID(ws, job.Namespace, job.ID, false) if err != nil { - return uiJob, err + return uiJob, idx, err } for _, d := range deploys { + if d.ModifyIndex > idx { + idx = d.ModifyIndex + } if d.Active() { uiJob.DeploymentID = d.ID break } } - return uiJob, nil + return uiJob, idx, nil } From be7583d36f91407a1e8056c0fbfe20e096360a19 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 12 Jan 2024 14:32:18 -0600 Subject: [PATCH 11/98] add JobVersion to allocs --- nomad/jobs_endpoint.go | 1 + nomad/structs/job.go | 1 + 2 files changed, 2 insertions(+) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index b087a16c9a7..e6cda55a721 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -213,6 +213,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) ID: a.ID, Group: a.TaskGroup, ClientStatus: a.ClientStatus, + JobVersion: a.Job.Version, } // TODO: use methods instead of fields directly? if a.DeploymentStatus != nil { diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 22441d2ff55..72f93ffdf0c 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -62,6 +62,7 @@ type JobStatusAlloc struct { Group string ClientStatus string DeploymentStatus JobStatusDeployment + JobVersion uint64 } type JobStatusDeployment struct { From f366cd6199504010ce3e773ad666498cbd4a972e Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 12 Jan 2024 15:07:11 -0600 Subject: [PATCH 12/98] /statuses2 use new index logic too --- nomad/jobs_endpoint.go | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index e6cda55a721..db579682925 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -139,12 +139,15 @@ func (j *Jobs) Statuses2( //if err != nil || summary == nil { // return fmt.Errorf("unable to look up summary for job: %v", job.ID) //} - uiJob, _, err := UIJobFromJob(ws, state, job) + uiJob, idx, err := UIJobFromJob(ws, state, job) if err != nil { return err } jobs = append(jobs, uiJob) + if idx > reply.Index { + reply.Index = idx + } return nil }) if err != nil { @@ -162,18 +165,6 @@ func (j *Jobs) Statuses2( reply.Jobs = jobs } - var idx uint64 - for _, table := range []string{"jobs", "allocs", "deployment"} { - i, err := state.Index(table) - if err != nil { - return err - } - if i > idx { - idx = i - } - } - reply.Index = idx - // Set the query response j.srv.setQueryMeta(&reply.QueryMeta) return nil From 39b29d48513d088a94d9eafc3d260658de5e13a9 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 16 Jan 2024 15:54:28 -0600 Subject: [PATCH 13/98] unblock if job goes away (gc) --- nomad/jobs_endpoint.go | 45 +++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index db579682925..cad52164677 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -7,6 +7,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-set/v2" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/state/paginator" @@ -32,14 +33,18 @@ func (j *Jobs) Statuses( reply *structs.JobsStatusesResponse) error { // TODO: auth, rate limiting, etc... + // compare between state unblocks to see if the RPC should unblock (namely, if any jobs have gone away) + prevJobs := set.New[structs.NamespacedID](0) + opts := blockingOptions{ queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, run: func(ws memdb.WatchSet, state *state.StateStore) error { - var jobs []structs.UIJob + var err error + jobs := make([]structs.UIJob, 0) + newJobs := set.New[structs.NamespacedID](0) for _, j := range args.Jobs { - ns := j.Namespace - job, err := state.JobByID(ws, ns, j.ID) + job, err := state.JobByID(ws, j.Namespace, j.ID) if err != nil { return err } @@ -51,10 +56,19 @@ func (j *Jobs) Statuses( return err } jobs = append(jobs, uiJob) + newJobs.Insert(job.NamespacedID()) if idx > reply.Index { reply.Index = idx } } + // mainly for if a job goes away + if !newJobs.Equal(prevJobs) { + reply.Index, err = state.Index("jobs") + if err != nil { + return err + } + } + prevJobs = newJobs reply.Jobs = jobs j.srv.setQueryMeta(&reply.QueryMeta) return nil @@ -89,6 +103,9 @@ func (j *Jobs) Statuses2( } allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) + // compare between state unblocks to see if the RPC should unblock (namely, if any jobs have gone away) + prevJobs := set.New[structs.NamespacedID](0) + // Setup the blocking query opts := blockingOptions{ queryOpts: &args.QueryOptions, @@ -131,7 +148,8 @@ func (j *Jobs) Statuses2( }, } - var jobs []structs.UIJob + jobs := make([]structs.UIJob, 0) + newJobs := set.New[structs.NamespacedID](0) pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) @@ -145,6 +163,8 @@ func (j *Jobs) Statuses2( } jobs = append(jobs, uiJob) + newJobs.Insert(job.NamespacedID()) + if idx > reply.Index { reply.Index = idx } @@ -161,6 +181,14 @@ func (j *Jobs) Statuses2( http.StatusBadRequest, "failed to read result page: %v", err) } + if !newJobs.Equal(prevJobs) { + reply.Index, err = state.Index("jobs") + if err != nil { + return err + } + } + prevJobs = newJobs + reply.QueryMeta.NextToken = nextToken reply.Jobs = jobs } @@ -214,7 +242,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) } } uiJob.Allocs = append(uiJob.Allocs, alloc) - if a.ModifyIndex > idx { + if a.ModifyIndex > idx { // TODO: unblock if an alloc goes away (like GC) idx = a.ModifyIndex } } @@ -224,12 +252,11 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) return uiJob, idx, err } for _, d := range deploys { - if d.ModifyIndex > idx { - idx = d.ModifyIndex - } if d.Active() { uiJob.DeploymentID = d.ID - break + } + if d.ModifyIndex > idx { + idx = d.ModifyIndex } } return uiJob, idx, nil From 3f882f9a6fbb987fb2980f60ce287eb194330946 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 16 Jan 2024 15:56:54 -0600 Subject: [PATCH 14/98] one /jobs/statuses3 to rule them all with iterator experiments --- command/agent/http.go | 1 + command/agent/job_endpoint.go | 40 +++++++++ nomad/jobs_endpoint.go | 130 +++++++++++++++++++++++++++ nomad/state/state_store_multi_job.go | 89 ++++++++++++++++++ 4 files changed, 260 insertions(+) create mode 100644 nomad/state/state_store_multi_job.go diff --git a/command/agent/http.go b/command/agent/http.go index c4b4a91b507..3cd47c321b0 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -384,6 +384,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobsStatusesRequest)) s.mux.HandleFunc("/v1/jobs/statuses2", s.wrap(s.JobsStatuses2Request)) + s.mux.HandleFunc("/v1/jobs/statuses3", s.wrap(s.JobsStatuses3Request)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 6432f912c8b..beb0733f056 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2162,3 +2162,43 @@ func (s *HTTPServer) JobsStatuses2Request(resp http.ResponseWriter, req *http.Re setMeta(resp, &out.QueryMeta) return out.Jobs, nil } + +func (s *HTTPServer) JobsStatuses3Request(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.JobsStatusesRequest{} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil // seems whack + } + switch req.Method { + case http.MethodGet: + // GET requests will be treated as "get all jobs" but also with filtering and pagination and such + case http.MethodPost: + // POST requests expect a list of Jobs in the request body, which will then be filtered/paginated, etc. + var in api.JobsStatusesRequest + if err := decodeBody(req, &in); err != nil { + return nil, err + } + if len(in.Jobs) == 0 { + return nil, CodedError(http.StatusBadRequest, "no jobs in request") + } + for _, j := range in.Jobs { + if j.Namespace == "" { + j.Namespace = "default" + } + args.Jobs = append(args.Jobs, structs.NamespacedID{ + ID: j.ID, + // note: can't just use QueryOptions.Namespace, because each job may have a different NS + Namespace: j.Namespace, + }) + } + default: + return nil, CodedError(405, ErrInvalidMethod) + } + + var out structs.JobsStatusesResponse + if err := s.agent.RPC("Jobs.Statuses3", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + return out.Jobs, nil +} diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index cad52164677..cd772039a02 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -261,3 +261,133 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) } return uiJob, idx, nil } + +func (j *Jobs) Statuses3( + args *structs.JobsStatusesRequest, + reply *structs.JobsStatusesResponse) error { + + // totally lifted from Job.List + authErr := j.srv.Authenticate(j.ctx, args) + if done, err := j.srv.forward("Jobs.Statuses3", args, args, reply); done { + return err + } + j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) + if authErr != nil { + return structs.ErrPermissionDenied + } + defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) + + namespace := args.RequestNamespace() + + // Check for list-job permissions + aclObj, err := j.srv.ResolveACL(args) + if err != nil { + return err + } + if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { + return structs.ErrPermissionDenied + } + allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) + + // compare between state run() unblocks to see if the RPC should unblock. + // i.e. if new job(s) shift the page, or when job(s) go away. + prevJobs := set.New[structs.NamespacedID](0) + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + var err error + var iter memdb.ResultIterator + + // Get the namespaces the user is allowed to access. + allowableNamespaces, err := allowedNSes(aclObj, state, allow) + if err == structs.ErrPermissionDenied { + // return empty jobs if token isn't authorized for any + // namespace, matching other endpoints + reply.Jobs = make([]structs.UIJob, 0) + } else if err != nil { + return err + } else { + if prefix := args.QueryOptions.Prefix; prefix != "" { + iter, err = state.JobsByIDPrefix(ws, namespace, prefix) + + } else if len(args.Jobs) > 0 { + // new experiment: only fetch specific jobs if requested + iter, err = state.JobsByIDs2(ws, args.Jobs) + + } else if namespace != structs.AllNamespacesSentinel { + iter, err = state.JobsByNamespace(ws, namespace) + } else { + iter, err = state.Jobs(ws) + } + if err != nil { + return err + } + + tokenizer := paginator.NewStructsTokenizer( + iter, + paginator.StructsTokenizerOptions{ + WithNamespace: true, + WithID: true, + }, + ) + filters := []paginator.Filter{ + paginator.NamespaceFilter{ + AllowableNamespaces: allowableNamespaces, + }, + } + + jobs := make([]structs.UIJob, 0) + newJobs := set.New[structs.NamespacedID](0) + pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, + func(raw interface{}) error { + job := raw.(*structs.Job) + if job == nil { + return nil + } + + uiJob, idx, err := UIJobFromJob(ws, state, job) + if err != nil { + return err + } + + jobs = append(jobs, uiJob) + newJobs.Insert(job.NamespacedID()) + + if idx > reply.Index { + reply.Index = idx + } + return nil + }) + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to create result paginator: %v", err) + } + + nextToken, err := pager.Page() + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to read result page: %v", err) + } + + // if the page has updated, or a job has gone away, bump the index to latest jobs entry. + if !newJobs.Equal(prevJobs) { + reply.Index, err = state.Index("jobs") + if err != nil { + return err + } + } + prevJobs = newJobs + + reply.QueryMeta.NextToken = nextToken + reply.Jobs = jobs + } + + // Set the query response + j.srv.setQueryMeta(&reply.QueryMeta) + return nil + }} + return j.srv.blockingRPC(&opts) +} diff --git a/nomad/state/state_store_multi_job.go b/nomad/state/state_store_multi_job.go new file mode 100644 index 00000000000..7851e66556a --- /dev/null +++ b/nomad/state/state_store_multi_job.go @@ -0,0 +1,89 @@ +package state + +import ( + "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-set/v2" + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *StateStore) JobsByIDs(ws memdb.WatchSet, nsIDs []structs.NamespacedID) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + // this thing reads through all jobs, which seems pretty inefficient... + iterAll, err := txn.Get("jobs", "id") + if err != nil { + return nil, err + } + idSet := set.From[structs.NamespacedID](nsIDs) + iter := &JobsIterator{ + SuperIter: iterAll, + Filter: func(j *structs.Job) bool { + return idSet.Contains(j.NamespacedID()) + }, + } + ws.Add(iter.WatchCh()) + return iter, nil +} + +var _ memdb.ResultIterator = &JobsIterator{} + +type JobsIterator struct { + SuperIter memdb.ResultIterator + Filter func(*structs.Job) bool +} + +func (j *JobsIterator) WatchCh() <-chan struct{} { + return j.SuperIter.WatchCh() +} + +// Next will always return a *structs.Job, or nil when there are none left. +func (j *JobsIterator) Next() interface{} { + for { + next := j.SuperIter.Next() + // The Santa Clause 3: The Escape Clause + if next == nil { + return nil + } + job := next.(*structs.Job) + if j.Filter(job) { + return job + } + } +} + +func (s *StateStore) JobsByIDs2(ws memdb.WatchSet, nsIDs []structs.NamespacedID) (memdb.ResultIterator, error) { + return &JobsIterator2{ + Jobs: nsIDs, + state: s, + ws: ws, + }, nil +} + +var _ memdb.ResultIterator = &JobsIterator2{} + +type JobsIterator2 struct { + // these states are not protected from concurrent access... + Jobs []structs.NamespacedID + idx int + + // this is feelin pretty wild... + state *StateStore + ws memdb.WatchSet + watchCh <-chan struct{} +} + +func (j *JobsIterator2) WatchCh() <-chan struct{} { + return j.watchCh +} + +func (j *JobsIterator2) Next() interface{} { + if len(j.Jobs) < j.idx+1 { + return nil + } + nsID := j.Jobs[j.idx] + j.idx++ + job, err := j.state.JobByIDTxn(j.ws, nsID.Namespace, nsID.ID, j.state.db.ReadTxn()) // state and ws here feel wrong... + if err != nil { + return nil // hmm, losing the error... + } + return job +} From 927ec874a5f046fe921a84cd09e989b1e25dc643 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 16 Jan 2024 16:29:32 -0600 Subject: [PATCH 15/98] use filters feature of paginator instead of bespoke ResultIterator experiments --- nomad/jobs_endpoint.go | 15 +++-- nomad/state/state_store_multi_job.go | 89 ---------------------------- 2 files changed, 10 insertions(+), 94 deletions(-) delete mode 100644 nomad/state/state_store_multi_job.go diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index cd772039a02..11f564e073e 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -312,11 +312,6 @@ func (j *Jobs) Statuses3( } else { if prefix := args.QueryOptions.Prefix; prefix != "" { iter, err = state.JobsByIDPrefix(ws, namespace, prefix) - - } else if len(args.Jobs) > 0 { - // new experiment: only fetch specific jobs if requested - iter, err = state.JobsByIDs2(ws, args.Jobs) - } else if namespace != structs.AllNamespacesSentinel { iter, err = state.JobsByNamespace(ws, namespace) } else { @@ -338,6 +333,16 @@ func (j *Jobs) Statuses3( AllowableNamespaces: allowableNamespaces, }, } + // only provide specific jobs if requested. + if len(args.Jobs) > 0 { + jobSet := set.From[structs.NamespacedID](args.Jobs) + filters = append(filters, paginator.GenericFilter{ + Allow: func(i interface{}) (bool, error) { + job := i.(*structs.Job) + return jobSet.Contains(job.NamespacedID()), nil + }, + }) + } jobs := make([]structs.UIJob, 0) newJobs := set.New[structs.NamespacedID](0) diff --git a/nomad/state/state_store_multi_job.go b/nomad/state/state_store_multi_job.go deleted file mode 100644 index 7851e66556a..00000000000 --- a/nomad/state/state_store_multi_job.go +++ /dev/null @@ -1,89 +0,0 @@ -package state - -import ( - "github.com/hashicorp/go-memdb" - "github.com/hashicorp/go-set/v2" - "github.com/hashicorp/nomad/nomad/structs" -) - -func (s *StateStore) JobsByIDs(ws memdb.WatchSet, nsIDs []structs.NamespacedID) (memdb.ResultIterator, error) { - txn := s.db.ReadTxn() - // this thing reads through all jobs, which seems pretty inefficient... - iterAll, err := txn.Get("jobs", "id") - if err != nil { - return nil, err - } - idSet := set.From[structs.NamespacedID](nsIDs) - iter := &JobsIterator{ - SuperIter: iterAll, - Filter: func(j *structs.Job) bool { - return idSet.Contains(j.NamespacedID()) - }, - } - ws.Add(iter.WatchCh()) - return iter, nil -} - -var _ memdb.ResultIterator = &JobsIterator{} - -type JobsIterator struct { - SuperIter memdb.ResultIterator - Filter func(*structs.Job) bool -} - -func (j *JobsIterator) WatchCh() <-chan struct{} { - return j.SuperIter.WatchCh() -} - -// Next will always return a *structs.Job, or nil when there are none left. -func (j *JobsIterator) Next() interface{} { - for { - next := j.SuperIter.Next() - // The Santa Clause 3: The Escape Clause - if next == nil { - return nil - } - job := next.(*structs.Job) - if j.Filter(job) { - return job - } - } -} - -func (s *StateStore) JobsByIDs2(ws memdb.WatchSet, nsIDs []structs.NamespacedID) (memdb.ResultIterator, error) { - return &JobsIterator2{ - Jobs: nsIDs, - state: s, - ws: ws, - }, nil -} - -var _ memdb.ResultIterator = &JobsIterator2{} - -type JobsIterator2 struct { - // these states are not protected from concurrent access... - Jobs []structs.NamespacedID - idx int - - // this is feelin pretty wild... - state *StateStore - ws memdb.WatchSet - watchCh <-chan struct{} -} - -func (j *JobsIterator2) WatchCh() <-chan struct{} { - return j.watchCh -} - -func (j *JobsIterator2) Next() interface{} { - if len(j.Jobs) < j.idx+1 { - return nil - } - nsID := j.Jobs[j.idx] - j.idx++ - job, err := j.state.JobByIDTxn(j.ws, nsID.Namespace, nsID.ID, j.state.db.ReadTxn()) // state and ws here feel wrong... - if err != nil { - return nil // hmm, losing the error... - } - return job -} From c6579f127900ac6140186490b0a93a22b0218eaa Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 19 Jan 2024 16:31:20 -0600 Subject: [PATCH 16/98] exclude child jobs, add ChildStatuses --- nomad/jobs_endpoint.go | 27 +++++++++++++++++++++++++++ nomad/structs/job.go | 1 + 2 files changed, 28 insertions(+) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 11f564e073e..c5fe5cfa3cf 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -216,6 +216,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) Version: job.Version, // included here for completeness, populated below. Allocs: nil, + ChildStatuses: nil, GroupCountSum: 0, DeploymentID: "", } @@ -223,6 +224,27 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) uiJob.GroupCountSum += tg.Count } + if job.IsParameterized() || job.IsPeriodic() { + children, err := state.JobsByIDPrefix(ws, job.Namespace, job.ID) + if err != nil { + return uiJob, idx, err + } + for { + child := children.Next() + if child == nil { + break + } + j := child.(*structs.Job) + if j.ParentID != job.ID { + continue + } + if j.ModifyIndex > idx { + idx = j.ModifyIndex + } + uiJob.ChildStatuses = append(uiJob.ChildStatuses, j.Status) + } + } + allocs, err := state.AllocsByJob(ws, job.Namespace, job.ID, false) if err != nil { return uiJob, idx, err @@ -332,6 +354,11 @@ func (j *Jobs) Statuses3( paginator.NamespaceFilter{ AllowableNamespaces: allowableNamespaces, }, + // don't include child jobs; we'll look them up later, per parent. + paginator.GenericFilter{Allow: func(i interface{}) (bool, error) { + job := i.(*structs.Job) + return job.ParentID == "", nil + }}, } // only provide specific jobs if requested. if len(args.Jobs) > 0 { diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 72f93ffdf0c..762312b5bb8 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -33,6 +33,7 @@ type UIJob struct { Datacenters []string Priority int Allocs []JobStatusAlloc + ChildStatuses []string GroupCountSum int DeploymentID string Version uint64 From 07c0c9bbdbeb5a1c7eb2a2cf7ba8b72e6072ae78 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Jan 2024 11:59:13 -0600 Subject: [PATCH 17/98] probably unnecessary namespace optimization --- nomad/jobs_endpoint.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index c5fe5cfa3cf..0abf514866c 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -300,6 +300,18 @@ func (j *Jobs) Statuses3( defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) namespace := args.RequestNamespace() + // The ns from the UI by default is "*" which scans the whole "jobs" table. + // If specific jobs are requested, all with the same namespace, + // we may get some extra efficiency, especially for non-contiguous job IDs. + if len(args.Jobs) > 0 { + nses := set.New[string](0) + for _, j := range args.Jobs { + nses.Insert(j.Namespace) + } + if nses.Size() == 1 { + namespace = nses.Slice()[0] + } + } // Check for list-job permissions aclObj, err := j.srv.ResolveACL(args) From cbc9344bc0137f146ce66a0dad80ca91a574e5b9 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Jan 2024 15:28:10 -0600 Subject: [PATCH 18/98] server benchmarking testing.TB and UpsertAllocsRaw() --- nomad/rpc_test.go | 2 +- nomad/state/testing.go | 16 ++++++++++++++++ nomad/testing.go | 6 +++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go index 47c33d28883..37c05ce1dcc 100644 --- a/nomad/rpc_test.go +++ b/nomad/rpc_test.go @@ -41,7 +41,7 @@ import ( // rpcClient is a test helper method to return a ClientCodec to use to make rpc // calls to the passed server. -func rpcClient(t *testing.T, s *Server) rpc.ClientCodec { +func rpcClient(t testing.TB, s *Server) rpc.ClientCodec { t.Helper() addr := s.config.RPCAddr conn, err := net.DialTimeout("tcp", addr.String(), time.Second) diff --git a/nomad/state/testing.go b/nomad/state/testing.go index cb955ffa46c..72e4812312e 100644 --- a/nomad/state/testing.go +++ b/nomad/state/testing.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/shoenig/test/must" ) func TestStateStore(t testing.TB) *StateStore { @@ -318,3 +319,18 @@ func TestBadCSIState(t testing.TB, store *StateStore) error { return nil } + +func (s *StateStore) UpsertAllocsRaw(t testing.TB, idx uint64, allocs []*structs.Allocation) { + t.Helper() + txn := s.db.WriteTxn(idx) + defer txn.Abort() + var err error + for _, a := range allocs { + err = txn.Insert("allocs", a) + must.NoError(t, err, must.Sprint("error inserting alloc")) + } + err = txn.Insert("index", &IndexEntry{"allocs", idx}) + must.NoError(t, err, must.Sprint("error inserting index")) + err = txn.Commit() + must.NoError(t, err, must.Sprint("error committing transaction")) +} diff --git a/nomad/testing.go b/nomad/testing.go index 449d33fb819..e1c90f4384c 100644 --- a/nomad/testing.go +++ b/nomad/testing.go @@ -40,7 +40,7 @@ func TestACLServer(t testing.T, cb func(*Config)) (*Server, *structs.ACLToken, f return server, token, cleanup } -func TestServer(t testing.T, cb func(*Config)) (*Server, func()) { +func TestServer(t testing.TB, cb func(*Config)) (*Server, func()) { s, c, err := TestServerErr(t, cb) must.NoError(t, err, must.Sprint("failed to start test server")) return s, c @@ -48,7 +48,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) { // TestConfigForServer provides a fully functional Config to pass to NewServer() // It can be changed beforehand to induce different behavior such as specific errors. -func TestConfigForServer(t testing.T) *Config { +func TestConfigForServer(t testing.TB) *Config { t.Helper() // Setup the default settings @@ -121,7 +121,7 @@ func TestConfigForServer(t testing.T) *Config { return config } -func TestServerErr(t testing.T, cb func(*Config)) (*Server, func(), error) { +func TestServerErr(t testing.TB, cb func(*Config)) (*Server, func(), error) { config := TestConfigForServer(t) // Invoke the callback if any if cb != nil { From 17b7d3d73246923d2c98003e6c23b0a3efa973f7 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Jan 2024 16:37:06 -0600 Subject: [PATCH 19/98] smart alloc akin to jobsummary --- command/agent/job_endpoint.go | 5 +++++ nomad/jobs_endpoint.go | 22 +++++++++++++++++----- nomad/structs/job.go | 6 ++++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index beb0733f056..1fbfb25d05b 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2168,6 +2168,11 @@ func (s *HTTPServer) JobsStatuses3Request(resp http.ResponseWriter, req *http.Re if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil // seems whack } + if smartOnly, err := parseBool(req, "smart_only"); err != nil { + return nil, err + } else if smartOnly != nil { + args.SmartOnly = *smartOnly + } switch req.Method { case http.MethodGet: // GET requests will be treated as "get all jobs" but also with filtering and pagination and such diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 0abf514866c..9d4236107ea 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -51,7 +51,7 @@ func (j *Jobs) Statuses( if job == nil { continue } - uiJob, idx, err := UIJobFromJob(ws, state, job) + uiJob, idx, err := UIJobFromJob(ws, state, job, false) if err != nil { return err } @@ -153,11 +153,12 @@ func (j *Jobs) Statuses2( pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) + //summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) //if err != nil || summary == nil { // return fmt.Errorf("unable to look up summary for job: %v", job.ID) //} - uiJob, idx, err := UIJobFromJob(ws, state, job) + uiJob, idx, err := UIJobFromJob(ws, state, job, false) if err != nil { return err } @@ -200,7 +201,7 @@ func (j *Jobs) Statuses2( return j.srv.blockingRPC(&opts) } -func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) (structs.UIJob, uint64, error) { +func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { idx := job.ModifyIndex uiJob := structs.UIJob{ @@ -216,8 +217,9 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) Version: job.Version, // included here for completeness, populated below. Allocs: nil, - ChildStatuses: nil, + SmartAlloc: make(map[string]int), GroupCountSum: 0, + ChildStatuses: nil, DeploymentID: "", } for _, tg := range job.TaskGroups { @@ -249,7 +251,17 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job) if err != nil { return uiJob, idx, err } + for _, a := range allocs { + uiJob.SmartAlloc["total"]++ + uiJob.SmartAlloc[a.ClientStatus]++ + if a.DeploymentStatus.Canary { + uiJob.SmartAlloc["canary"]++ + } + if smartOnly { + continue + } + alloc := structs.JobStatusAlloc{ ID: a.ID, Group: a.TaskGroup, @@ -392,7 +404,7 @@ func (j *Jobs) Statuses3( return nil } - uiJob, idx, err := UIJobFromJob(ws, state, job) + uiJob, idx, err := UIJobFromJob(ws, state, job, args.SmartOnly) if err != nil { return err } diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 762312b5bb8..d4c28eca632 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -33,14 +33,16 @@ type UIJob struct { Datacenters []string Priority int Allocs []JobStatusAlloc - ChildStatuses []string + SmartAlloc map[string]int GroupCountSum int + ChildStatuses []string DeploymentID string Version uint64 } type JobsStatusesRequest struct { - Jobs []NamespacedID + Jobs []NamespacedID + SmartOnly bool QueryOptions } From 62eba5052c1dd5f19327893a19e44869b4ec72a1 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Jan 2024 17:32:37 -0600 Subject: [PATCH 20/98] enable reverse sort in statuses3 --- nomad/jobs_endpoint.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 9d4236107ea..4130fddc1ad 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -105,6 +105,7 @@ func (j *Jobs) Statuses2( // compare between state unblocks to see if the RPC should unblock (namely, if any jobs have gone away) prevJobs := set.New[structs.NamespacedID](0) + sort := state.SortDefault // Setup the blocking query opts := blockingOptions{ @@ -125,11 +126,11 @@ func (j *Jobs) Statuses2( return err } else { if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix) + iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace) + iter, err = state.JobsByNamespace(ws, namespace, sort) } else { - iter, err = state.Jobs(ws) + iter, err = state.Jobs(ws, sort) } if err != nil { return err @@ -201,7 +202,7 @@ func (j *Jobs) Statuses2( return j.srv.blockingRPC(&opts) } -func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { +func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { idx := job.ModifyIndex uiJob := structs.UIJob{ @@ -227,7 +228,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job, } if job.IsParameterized() || job.IsPeriodic() { - children, err := state.JobsByIDPrefix(ws, job.Namespace, job.ID) + children, err := store.JobsByIDPrefix(ws, job.Namespace, job.ID, state.SortDefault) if err != nil { return uiJob, idx, err } @@ -247,7 +248,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job, } } - allocs, err := state.AllocsByJob(ws, job.Namespace, job.ID, false) + allocs, err := store.AllocsByJob(ws, job.Namespace, job.ID, true) // TODO: anyCreateIndex? if err != nil { return uiJob, idx, err } @@ -281,7 +282,7 @@ func UIJobFromJob(ws memdb.WatchSet, state *state.StateStore, job *structs.Job, } } - deploys, err := state.DeploymentsByJobID(ws, job.Namespace, job.ID, false) + deploys, err := store.DeploymentsByJobID(ws, job.Namespace, job.ID, true) if err != nil { return uiJob, idx, err } @@ -339,6 +340,8 @@ func (j *Jobs) Statuses3( // i.e. if new job(s) shift the page, or when job(s) go away. prevJobs := set.New[structs.NamespacedID](0) + sort := state.QueryOptionSort(args.QueryOptions) + // Setup the blocking query opts := blockingOptions{ queryOpts: &args.QueryOptions, @@ -357,11 +360,11 @@ func (j *Jobs) Statuses3( return err } else { if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix) + iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace) + iter, err = state.JobsByNamespace(ws, namespace, sort) } else { - iter, err = state.Jobs(ws) + iter, err = state.Jobs(ws, sort) } if err != nil { return err From 6b226186d680a3034469259fe1afd276e168d089 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 24 Jan 2024 11:57:28 -0600 Subject: [PATCH 21/98] fix a panic from DeploymentStatus being nil --- nomad/jobs_endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 4130fddc1ad..ab93d1ea176 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -256,7 +256,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, for _, a := range allocs { uiJob.SmartAlloc["total"]++ uiJob.SmartAlloc[a.ClientStatus]++ - if a.DeploymentStatus.Canary { + if a.DeploymentStatus != nil && a.DeploymentStatus.Canary { uiJob.SmartAlloc["canary"]++ } if smartOnly { From 5fc64a0f492a6fd64001794868d5f1bfd3acb19c Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 24 Jan 2024 12:12:37 -0600 Subject: [PATCH 22/98] clean up; statuses3 -> statuses --- command/agent/http.go | 2 - command/agent/job_endpoint.go | 56 +---------- nomad/jobs_endpoint.go | 181 +--------------------------------- nomad/structs/job.go | 18 ---- 4 files changed, 3 insertions(+), 254 deletions(-) diff --git a/command/agent/http.go b/command/agent/http.go index 3cd47c321b0..679948b591e 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -383,8 +383,6 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest)) s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobsStatusesRequest)) - s.mux.HandleFunc("/v1/jobs/statuses2", s.wrap(s.JobsStatuses2Request)) - s.mux.HandleFunc("/v1/jobs/statuses3", s.wrap(s.JobsStatuses3Request)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 1fbfb25d05b..f4507c1fabc 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2110,60 +2110,6 @@ func validateEvalPriorityOpt(priority int) HTTPCodedError { } func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { - if req.Method != http.MethodPost { - return nil, CodedError(405, ErrInvalidMethod) - } - - args := structs.JobsStatusesRequest{} - if parseWait(resp, req, &args.QueryOptions) { - return nil, nil // seems whack. - } - - var in api.JobsStatusesRequest - if err := decodeBody(req, &in); err != nil { - return nil, err - } - - for _, j := range in.Jobs { - if j.Namespace == "" { - j.Namespace = "default" - } - args.Jobs = append(args.Jobs, structs.NamespacedID{ - ID: j.ID, - // note: can't just use QueryOptions.Namespace, because each job may have a different NS - Namespace: j.Namespace, - }) - } - - var out structs.JobsStatusesResponse - if err := s.agent.RPC("Jobs.Statuses", &args, &out); err != nil { - return nil, err - } - - setMeta(resp, &out.QueryMeta) - return out.Jobs, nil -} - -func (s *HTTPServer) JobsStatuses2Request(resp http.ResponseWriter, req *http.Request) (interface{}, error) { - if req.Method != http.MethodGet { - return nil, CodedError(405, ErrInvalidMethod) - } - - args := structs.JobsStatuses2Request{} - if s.parse(resp, req, &args.Region, &args.QueryOptions) { - return nil, nil // seems whack - } - - var out structs.JobsStatuses2Response - if err := s.agent.RPC("Jobs.Statuses2", &args, &out); err != nil { - return nil, err - } - - setMeta(resp, &out.QueryMeta) - return out.Jobs, nil -} - -func (s *HTTPServer) JobsStatuses3Request(resp http.ResponseWriter, req *http.Request) (interface{}, error) { args := structs.JobsStatusesRequest{} if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil // seems whack @@ -2200,7 +2146,7 @@ func (s *HTTPServer) JobsStatuses3Request(resp http.ResponseWriter, req *http.Re } var out structs.JobsStatusesResponse - if err := s.agent.RPC("Jobs.Statuses3", &args, &out); err != nil { + if err := s.agent.RPC("Jobs.Statuses", &args, &out); err != nil { return nil, err } diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index ab93d1ea176..582d32a947b 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -28,180 +28,6 @@ type Jobs struct { logger hclog.Logger } -func (j *Jobs) Statuses( - args *structs.JobsStatusesRequest, - reply *structs.JobsStatusesResponse) error { - // TODO: auth, rate limiting, etc... - - // compare between state unblocks to see if the RPC should unblock (namely, if any jobs have gone away) - prevJobs := set.New[structs.NamespacedID](0) - - opts := blockingOptions{ - queryOpts: &args.QueryOptions, - queryMeta: &reply.QueryMeta, - run: func(ws memdb.WatchSet, state *state.StateStore) error { - var err error - jobs := make([]structs.UIJob, 0) - newJobs := set.New[structs.NamespacedID](0) - for _, j := range args.Jobs { - job, err := state.JobByID(ws, j.Namespace, j.ID) - if err != nil { - return err - } - if job == nil { - continue - } - uiJob, idx, err := UIJobFromJob(ws, state, job, false) - if err != nil { - return err - } - jobs = append(jobs, uiJob) - newJobs.Insert(job.NamespacedID()) - if idx > reply.Index { - reply.Index = idx - } - } - // mainly for if a job goes away - if !newJobs.Equal(prevJobs) { - reply.Index, err = state.Index("jobs") - if err != nil { - return err - } - } - prevJobs = newJobs - reply.Jobs = jobs - j.srv.setQueryMeta(&reply.QueryMeta) - return nil - }} - return j.srv.blockingRPC(&opts) -} - -func (j *Jobs) Statuses2( - args *structs.JobsStatuses2Request, - reply *structs.JobsStatuses2Response) error { - - // totally lifted from Job.List - authErr := j.srv.Authenticate(j.ctx, args) - if done, err := j.srv.forward("Jobs.Statuses2", args, args, reply); done { - return err - } - j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) - if authErr != nil { - return structs.ErrPermissionDenied - } - defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) - - namespace := args.RequestNamespace() - - // Check for list-job permissions - aclObj, err := j.srv.ResolveACL(args) - if err != nil { - return err - } - if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { - return structs.ErrPermissionDenied - } - allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) - - // compare between state unblocks to see if the RPC should unblock (namely, if any jobs have gone away) - prevJobs := set.New[structs.NamespacedID](0) - sort := state.SortDefault - - // Setup the blocking query - opts := blockingOptions{ - queryOpts: &args.QueryOptions, - queryMeta: &reply.QueryMeta, - run: func(ws memdb.WatchSet, state *state.StateStore) error { - // Capture all the jobs - var err error - var iter memdb.ResultIterator - - // Get the namespaces the user is allowed to access. - allowableNamespaces, err := allowedNSes(aclObj, state, allow) - if err == structs.ErrPermissionDenied { - // return empty jobs if token isn't authorized for any - // namespace, matching other endpoints - reply.Jobs = make([]structs.UIJob, 0) - } else if err != nil { - return err - } else { - if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) - } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace, sort) - } else { - iter, err = state.Jobs(ws, sort) - } - if err != nil { - return err - } - - tokenizer := paginator.NewStructsTokenizer( - iter, - paginator.StructsTokenizerOptions{ - WithNamespace: true, - WithID: true, - }, - ) - filters := []paginator.Filter{ - paginator.NamespaceFilter{ - AllowableNamespaces: allowableNamespaces, - }, - } - - jobs := make([]structs.UIJob, 0) - newJobs := set.New[structs.NamespacedID](0) - pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, - func(raw interface{}) error { - job := raw.(*structs.Job) - - //summary, err := state.JobSummaryByID(ws, job.Namespace, job.ID) - //if err != nil || summary == nil { - // return fmt.Errorf("unable to look up summary for job: %v", job.ID) - //} - uiJob, idx, err := UIJobFromJob(ws, state, job, false) - if err != nil { - return err - } - - jobs = append(jobs, uiJob) - newJobs.Insert(job.NamespacedID()) - - if idx > reply.Index { - reply.Index = idx - } - return nil - }) - if err != nil { - return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to create result paginator: %v", err) - } - - nextToken, err := pager.Page() - if err != nil { - return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to read result page: %v", err) - } - - if !newJobs.Equal(prevJobs) { - reply.Index, err = state.Index("jobs") - if err != nil { - return err - } - } - prevJobs = newJobs - - reply.QueryMeta.NextToken = nextToken - reply.Jobs = jobs - } - - // Set the query response - j.srv.setQueryMeta(&reply.QueryMeta) - return nil - }} - return j.srv.blockingRPC(&opts) -} - func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { idx := job.ModifyIndex @@ -297,13 +123,13 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, return uiJob, idx, nil } -func (j *Jobs) Statuses3( +func (j *Jobs) Statuses( args *structs.JobsStatusesRequest, reply *structs.JobsStatusesResponse) error { // totally lifted from Job.List authErr := j.srv.Authenticate(j.ctx, args) - if done, err := j.srv.forward("Jobs.Statuses3", args, args, reply); done { + if done, err := j.srv.forward("Jobs.Statuses", args, args, reply); done { return err } j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) @@ -443,9 +269,6 @@ func (j *Jobs) Statuses3( reply.QueryMeta.NextToken = nextToken reply.Jobs = jobs } - - // Set the query response - j.srv.setQueryMeta(&reply.QueryMeta) return nil }} return j.srv.blockingRPC(&opts) diff --git a/nomad/structs/job.go b/nomad/structs/job.go index d4c28eca632..7e82f08f31d 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -16,15 +16,6 @@ const ( JobServiceRegistrationsRPCMethod = "Job.GetServiceRegistrations" ) -type JobsStatuses2Request struct { - QueryOptions -} - -type JobsStatuses2Response struct { - Jobs []UIJob - QueryMeta -} - type UIJob struct { NamespacedID Name string @@ -51,15 +42,6 @@ type JobsStatusesResponse struct { QueryMeta } -//type JobStatus struct { -// ID string -// Namespace string -// Type string -// Allocs []JobStatusAlloc -// GroupCountSum int -// DeploymentID string -//} - type JobStatusAlloc struct { ID string Group string From 9fa3de841b20dc9f290adecbb8a0e4dd50f77c7e Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 24 Jan 2024 12:25:14 -0600 Subject: [PATCH 23/98] fine, ok, copywrite header --- nomad/jobs_endpoint.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 582d32a947b..8ec45a1df05 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package nomad import ( From 892318a169a4cfea9420b45707127033327be2a3 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 1 Feb 2024 14:54:41 -0600 Subject: [PATCH 24/98] add NodeID to allocs mainly to for system job accounting --- nomad/jobs_endpoint.go | 1 + nomad/structs/job.go | 1 + 2 files changed, 2 insertions(+) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 8ec45a1df05..7f53a9d7260 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -96,6 +96,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, ID: a.ID, Group: a.TaskGroup, ClientStatus: a.ClientStatus, + NodeID: a.NodeID, JobVersion: a.Job.Version, } // TODO: use methods instead of fields directly? diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 7e82f08f31d..88335c770af 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -46,6 +46,7 @@ type JobStatusAlloc struct { ID string Group string ClientStatus string + NodeID string DeploymentStatus JobStatusDeployment JobVersion uint64 } From 64cf698cca1617f665a889b3a004d3085eac70c2 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 13 Feb 2024 12:00:50 -0500 Subject: [PATCH 25/98] misc tidying and rearranging --- nomad/jobs_endpoint.go | 311 +++++++++++++++++++++-------------------- 1 file changed, 159 insertions(+), 152 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 7f53a9d7260..ebf69a24c88 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -4,6 +4,7 @@ package nomad import ( + "errors" "net/http" "time" @@ -31,6 +32,163 @@ type Jobs struct { logger hclog.Logger } +func (j *Jobs) Statuses( + args *structs.JobsStatusesRequest, + reply *structs.JobsStatusesResponse) error { + + authErr := j.srv.Authenticate(j.ctx, args) + if done, err := j.srv.forward("Jobs.Statuses", args, args, reply); done { + return err + } + j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) + if authErr != nil { + return structs.ErrPermissionDenied + } + defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) + + namespace := args.RequestNamespace() + // The ns from the UI by default is "*" which scans the whole "jobs" table. + // If specific jobs are requested, all with the same namespace, + // we may get some extra efficiency, especially for non-contiguous job IDs. + if len(args.Jobs) > 0 { + nses := set.New[string](0) + for _, j := range args.Jobs { + nses.Insert(j.Namespace) + } + if nses.Size() == 1 { + namespace = nses.Slice()[0] + } + } + + // Check for list-job permissions + aclObj, err := j.srv.ResolveACL(args) + if err != nil { + return err + } + if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { + return structs.ErrPermissionDenied + } + allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) + + // compare between state run() unblocks to see if the RPC should unblock. + // i.e. if new job(s) shift the page, or when job(s) go away. + prevJobs := set.New[structs.NamespacedID](0) + + sort := state.QueryOptionSort(args.QueryOptions) + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + var err error + var iter memdb.ResultIterator + + // Get the namespaces the user is allowed to access. + allowableNamespaces, err := allowedNSes(aclObj, state, allow) + if errors.Is(err, structs.ErrPermissionDenied) { + // return empty jobs if token isn't authorized for any + // namespace, matching other endpoints + reply.Jobs = make([]structs.UIJob, 0) + return nil + } else if err != nil { + return err + } + + if prefix := args.QueryOptions.Prefix; prefix != "" { + iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) + } else if namespace != structs.AllNamespacesSentinel { + iter, err = state.JobsByNamespace(ws, namespace, sort) + } else { + iter, err = state.Jobs(ws, sort) + } + if err != nil { + return err + } + + tokenizer := paginator.NewStructsTokenizer( + iter, + paginator.StructsTokenizerOptions{ + WithNamespace: true, + WithID: true, + }, + ) + filters := []paginator.Filter{ + paginator.NamespaceFilter{ + AllowableNamespaces: allowableNamespaces, + }, + // don't include child jobs; we'll look them up later, per parent. + paginator.GenericFilter{Allow: func(i interface{}) (bool, error) { + job := i.(*structs.Job) + return job.ParentID == "", nil + }}, + } + // only provide specific jobs if requested. + if len(args.Jobs) > 0 { + // set per-page to avoid iterating the whole table + args.QueryOptions.PerPage = int32(len(args.Jobs)) + // filter in the requested jobs + jobSet := set.From[structs.NamespacedID](args.Jobs) + filters = append(filters, paginator.GenericFilter{ + Allow: func(i interface{}) (bool, error) { + job := i.(*structs.Job) + return jobSet.Contains(job.NamespacedID()), nil + }, + }) + } + + jobs := make([]structs.UIJob, 0) + newJobs := set.New[structs.NamespacedID](0) + pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, + func(raw interface{}) error { + job := raw.(*structs.Job) + if job == nil { + return nil + } + + // this is where the sausage is made + uiJob, idx, err := UIJobFromJob(ws, state, job, args.SmartOnly) + if err != nil { + return err + } + + jobs = append(jobs, uiJob) + newJobs.Insert(job.NamespacedID()) + + if idx > reply.Index { + reply.Index = idx + } + return nil + }) + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to create result paginator: %v", err) + } + + nextToken, err := pager.Page() + if err != nil { + return structs.NewErrRPCCodedf( + http.StatusBadRequest, "failed to read result page: %v", err) + } + + // if the page has updated, or a job has gone away, + // bump the index to latest jobs entry. + if !newJobs.Equal(prevJobs) { + reply.Index, err = state.Index("jobs") + if err != nil { + return err + } + } + prevJobs = newJobs + + reply.QueryMeta.NextToken = nextToken + reply.Jobs = jobs + + return nil + }} + return j.srv.blockingRPC(&opts) +} + func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { idx := job.ModifyIndex @@ -107,7 +265,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, } } uiJob.Allocs = append(uiJob.Allocs, alloc) - if a.ModifyIndex > idx { // TODO: unblock if an alloc goes away (like GC) + if a.ModifyIndex > idx { idx = a.ModifyIndex } } @@ -126,154 +284,3 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, } return uiJob, idx, nil } - -func (j *Jobs) Statuses( - args *structs.JobsStatusesRequest, - reply *structs.JobsStatusesResponse) error { - - // totally lifted from Job.List - authErr := j.srv.Authenticate(j.ctx, args) - if done, err := j.srv.forward("Jobs.Statuses", args, args, reply); done { - return err - } - j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) - if authErr != nil { - return structs.ErrPermissionDenied - } - defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) - - namespace := args.RequestNamespace() - // The ns from the UI by default is "*" which scans the whole "jobs" table. - // If specific jobs are requested, all with the same namespace, - // we may get some extra efficiency, especially for non-contiguous job IDs. - if len(args.Jobs) > 0 { - nses := set.New[string](0) - for _, j := range args.Jobs { - nses.Insert(j.Namespace) - } - if nses.Size() == 1 { - namespace = nses.Slice()[0] - } - } - - // Check for list-job permissions - aclObj, err := j.srv.ResolveACL(args) - if err != nil { - return err - } - if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { - return structs.ErrPermissionDenied - } - allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) - - // compare between state run() unblocks to see if the RPC should unblock. - // i.e. if new job(s) shift the page, or when job(s) go away. - prevJobs := set.New[structs.NamespacedID](0) - - sort := state.QueryOptionSort(args.QueryOptions) - - // Setup the blocking query - opts := blockingOptions{ - queryOpts: &args.QueryOptions, - queryMeta: &reply.QueryMeta, - run: func(ws memdb.WatchSet, state *state.StateStore) error { - var err error - var iter memdb.ResultIterator - - // Get the namespaces the user is allowed to access. - allowableNamespaces, err := allowedNSes(aclObj, state, allow) - if err == structs.ErrPermissionDenied { - // return empty jobs if token isn't authorized for any - // namespace, matching other endpoints - reply.Jobs = make([]structs.UIJob, 0) - } else if err != nil { - return err - } else { - if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) - } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace, sort) - } else { - iter, err = state.Jobs(ws, sort) - } - if err != nil { - return err - } - - tokenizer := paginator.NewStructsTokenizer( - iter, - paginator.StructsTokenizerOptions{ - WithNamespace: true, - WithID: true, - }, - ) - filters := []paginator.Filter{ - paginator.NamespaceFilter{ - AllowableNamespaces: allowableNamespaces, - }, - // don't include child jobs; we'll look them up later, per parent. - paginator.GenericFilter{Allow: func(i interface{}) (bool, error) { - job := i.(*structs.Job) - return job.ParentID == "", nil - }}, - } - // only provide specific jobs if requested. - if len(args.Jobs) > 0 { - jobSet := set.From[structs.NamespacedID](args.Jobs) - filters = append(filters, paginator.GenericFilter{ - Allow: func(i interface{}) (bool, error) { - job := i.(*structs.Job) - return jobSet.Contains(job.NamespacedID()), nil - }, - }) - } - - jobs := make([]structs.UIJob, 0) - newJobs := set.New[structs.NamespacedID](0) - pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, - func(raw interface{}) error { - job := raw.(*structs.Job) - if job == nil { - return nil - } - - uiJob, idx, err := UIJobFromJob(ws, state, job, args.SmartOnly) - if err != nil { - return err - } - - jobs = append(jobs, uiJob) - newJobs.Insert(job.NamespacedID()) - - if idx > reply.Index { - reply.Index = idx - } - return nil - }) - if err != nil { - return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to create result paginator: %v", err) - } - - nextToken, err := pager.Page() - if err != nil { - return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to read result page: %v", err) - } - - // if the page has updated, or a job has gone away, bump the index to latest jobs entry. - if !newJobs.Equal(prevJobs) { - reply.Index, err = state.Index("jobs") - if err != nil { - return err - } - } - prevJobs = newJobs - - reply.QueryMeta.NextToken = nextToken - reply.Jobs = jobs - } - return nil - }} - return j.srv.blockingRPC(&opts) -} From 544b57f15a45877e88a4e67b0b010a872929eb94 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 14 Feb 2024 13:44:02 -0500 Subject: [PATCH 26/98] require read-job acl instead of list --- nomad/jobs_endpoint.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index ebf69a24c88..5ef9e3d90b2 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -60,15 +60,16 @@ func (j *Jobs) Statuses( } } - // Check for list-job permissions + // Check for read-job permissions, since this endpoint includes alloc info + // and possibly a deployment ID, and those APIs require read-job. aclObj, err := j.srv.ResolveACL(args) if err != nil { return err } - if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityListJobs) { + if !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityReadJob) { return structs.ErrPermissionDenied } - allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityListJobs) + allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityReadJob) // compare between state run() unblocks to see if the RPC should unblock. // i.e. if new job(s) shift the page, or when job(s) go away. From ea568745b1ca78957589e2de27d84c456a74992e Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 13 Mar 2024 14:57:13 -0500 Subject: [PATCH 27/98] rpc tests --- nomad/jobs_endpoint_test.go | 347 ++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 nomad/jobs_endpoint_test.go diff --git a/nomad/jobs_endpoint_test.go b/nomad/jobs_endpoint_test.go new file mode 100644 index 00000000000..46327324f94 --- /dev/null +++ b/nomad/jobs_endpoint_test.go @@ -0,0 +1,347 @@ +package nomad + +import ( + "context" + "testing" + "time" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" +) + +func TestJobs_Statuses_ACL(t *testing.T) { + s, _, cleanup := TestACLServer(t, nil) + t.Cleanup(cleanup) + testutil.WaitForLeader(t, s.RPC) + + insufficientToken := mock.CreatePolicyAndToken(t, s.State(), 1, "job-lister", + mock.NamespacePolicy("default", "", []string{"list-jobs"})) + happyToken := mock.CreatePolicyAndToken(t, s.State(), 2, "job-reader", + mock.NamespacePolicy("default", "", []string{"read-job"})) + + for _, tc := range []struct { + name, token, err string + }{ + {"no token", "", "Permission denied"}, + {"insufficient perms", insufficientToken.SecretID, "Permission denied"}, + {"happy token", happyToken.SecretID, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + req := &structs.JobsStatusesRequest{} + req.QueryOptions.Region = "global" + req.QueryOptions.AuthToken = tc.token + + var resp structs.JobsStatusesResponse + err := s.RPC("Jobs.Statuses", &req, &resp) + + if tc.err != "" { + must.ErrorContains(t, err, tc.err) + } else { + must.NoError(t, err) + } + }) + } +} + +func TestJobs_Statuses(t *testing.T) { + s, cleanup := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + t.Cleanup(cleanup) + testutil.WaitForLeader(t, s.RPC) + + // method under test + doRequest := func(t *testing.T, req *structs.JobsStatusesRequest) (resp structs.JobsStatusesResponse) { + t.Helper() + must.NotNil(t, req, must.Sprint("request must not be nil")) + req.QueryOptions.Region = "global" + must.NoError(t, s.RPC("Jobs.Statuses", req, &resp)) + return resp + } + + // job helpers + deleteJob := func(t *testing.T, job *structs.Job) { + t.Helper() + idx, err := s.State().LatestIndex() + must.NoError(t, err) + err = s.State().DeleteJob(idx+1, job.Namespace, job.ID) + if err != nil && err.Error() == "job not found" { + return + } + must.NoError(t, err) + } + upsertJob := func(t *testing.T, job *structs.Job) { + idx, err := s.State().LatestIndex() + must.NoError(t, err) + err = s.State().UpsertJob(structs.MsgTypeTestSetup, idx+1, nil, job) + must.NoError(t, err) + } + createJob := func(t *testing.T, id string) (job *structs.Job, cleanup func()) { + t.Helper() + job = mock.MinJob() + if id != "" { + job.ID = id + } + upsertJob(t, job) + cleanup = func() { + deleteJob(t, job) + } + t.Cleanup(cleanup) + return job, cleanup + } + + // set up 5 jobs + // they should be in order in state, due to lexicographical indexing + jobs := make([]*structs.Job, 5) + var deleteJob0, deleteJob1, deleteJob2 func() + jobs[0], deleteJob0 = createJob(t, "job0") + jobs[1], deleteJob1 = createJob(t, "job1") + jobs[2], deleteJob2 = createJob(t, "job2") + jobs[3], _ = createJob(t, "job3") + jobs[4], _ = createJob(t, "job4") + + // request all jobs + resp := doRequest(t, &structs.JobsStatusesRequest{}) + must.Len(t, 5, resp.Jobs) + + // make sure our state order assumption is correct + for i, j := range resp.Jobs { + must.Eq(t, jobs[i].ID, j.ID, must.Sprintf("jobs not in order; idx=%d", i)) + } + + // test various single-job requests + + for _, tc := range []struct { + name string + qo structs.QueryOptions + jobs []structs.NamespacedID + expect *structs.Job + }{ + { + name: "page 1", + qo: structs.QueryOptions{ + PerPage: 1, + }, + expect: jobs[0], + }, + { + name: "page 2", + qo: structs.QueryOptions{ + PerPage: 1, + NextToken: "default." + jobs[1].ID, + }, + expect: jobs[1], + }, + { + name: "reverse", + qo: structs.QueryOptions{ + PerPage: 1, + Reverse: true, + }, + expect: jobs[len(jobs)-1], + }, + { + name: "filter", + qo: structs.QueryOptions{ + Filter: "ID == " + jobs[0].ID, + }, + expect: jobs[0], + }, + { + name: "specific", + jobs: []structs.NamespacedID{ + jobs[0].NamespacedID(), + }, + expect: jobs[0], + }, + } { + t.Run(tc.name, func(t *testing.T) { + resp = doRequest(t, &structs.JobsStatusesRequest{ + QueryOptions: tc.qo, + Jobs: tc.jobs, + }) + must.Len(t, 1, resp.Jobs, must.Sprint("expect only one job")) + must.Eq(t, tc.expect.ID, resp.Jobs[0].ID) + }) + } + + // test blocking queries + + // this endpoint should only unblock if something relevant changes. + // "something relevant" is why this seemingly redundant blocking-query + // testing is done here, as the logic to determine what is "relevant" is + // specific to this endpoint, meaning the latest ModifyIndex on each + // job/alloc/deployment seen while iterating, i.e. those "on-page". + + // blocking query helpers + startQuery := func(t *testing.T, req *structs.JobsStatusesRequest) context.Context { + t.Helper() + if req == nil { + req = &structs.JobsStatusesRequest{} + } + // context to signal when the query unblocks + // mustBlock and mustUnblock below work by checking ctx.Done() + ctx, cancel := context.WithCancel(context.Background()) + // default latest index + if req.QueryOptions.MinQueryIndex == 0 { + idx, err := s.State().LatestIndex() + must.NoError(t, err) + req.QueryOptions.MinQueryIndex = idx + } + // start the query + // note: queries that are expected to remain blocked leak this goroutine + // unless some other test coincidentally frees it up + go func() { + resp = doRequest(t, req) + cancel() + }() + // give it a moment for the rpc to actually start up + // FLAKE ALERT: if this job is flaky, this might be why. + time.Sleep(time.Millisecond * 10) + return ctx + } + mustBlock := func(t *testing.T, ctx context.Context) { + t.Helper() + timer := time.NewTimer(time.Millisecond * 200) + defer timer.Stop() + select { + case <-ctx.Done(): + t.Fatal("query should be blocked") + case <-timer.C: + } + } + mustUnblock := func(t *testing.T, ctx context.Context) { + t.Helper() + timer := time.NewTimer(time.Millisecond * 200) // may take a sec for a job to get deleted + //timer := time.NewTimer(time.Second * 2) // may take a sec for a job to get deleted + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + t.Fatal("query should have unblocked") + } + } + + // alloc and deployment helpers + createAlloc := func(t *testing.T, job *structs.Job) { + idx, err := s.State().LatestIndex() + must.NoError(t, err) + a := mock.MinAllocForJob(job) + must.NoError(t, + s.State().UpsertAllocs(structs.AllocUpdateRequestType, idx+1, []*structs.Allocation{a})) + //t.Cleanup(func() { + // t.Helper() + // idx, err = s.State().Index("allocs") + // test.NoError(t, err) + // test.NoError(t, s.State().DeleteEval(idx, []string{}, []string{a.ID}, false)) + //}) + } + createDeployment := func(t *testing.T, job *structs.Job) { + idx, err := s.State().LatestIndex() + must.NoError(t, err) + deploy := mock.Deployment() + deploy.JobID = job.ID + must.NoError(t, s.State().UpsertDeployment(idx+1, deploy)) + //t.Cleanup(func(){ + // s.State().DeleteDeployment(idx+) + //}) + } + + // these must be run in order, as they affect outer-scope state. + + for _, tc := range []struct { + name string + watch *structs.Job // optional specific job to query + run func(*testing.T) // run after starting the blocking query + check func(*testing.T, context.Context) // mustBlock or mustUnblock + }{ + { + name: "get all jobs", + check: mustBlock, // TODO: these leak goroutines... until the server goes away, breaking the RPC? + }, + { + name: "delete job", + run: func(_ *testing.T) { + deleteJob0() + }, + check: mustUnblock, + }, + { + name: "change job", + run: func(t *testing.T) { + jobs[1].Name = "job1-new-name" + upsertJob(t, jobs[1]) + }, + check: mustUnblock, + }, + { + name: "new job", + run: func(t *testing.T) { + createJob(t, "new1") + }, + check: mustUnblock, + }, + { + name: "delete job off page", + watch: jobs[2], + run: func(_ *testing.T) { + deleteJob1() + }, + check: mustBlock, + }, + { + name: "delete job on page", + watch: jobs[2], + run: func(_ *testing.T) { + deleteJob2() + }, + check: mustUnblock, + }, + { + name: "new alloc on page", + watch: jobs[3], + run: func(t *testing.T) { + createAlloc(t, jobs[3]) + }, + check: mustUnblock, + }, + { + name: "new alloc off page", + watch: jobs[3], + run: func(t *testing.T) { + createAlloc(t, jobs[4]) + }, + check: mustBlock, + }, + { + name: "new deployment on page", + watch: jobs[3], + run: func(t *testing.T) { + createDeployment(t, jobs[3]) + }, + check: mustUnblock, + }, + { + name: "new deployment off page", + watch: jobs[3], + run: func(t *testing.T) { + createDeployment(t, jobs[4]) + }, + check: mustBlock, + }, + } { + t.Run(tc.name, func(t *testing.T) { + req := &structs.JobsStatusesRequest{} + if tc.watch != nil { + req.Jobs = []structs.NamespacedID{tc.watch.NamespacedID()} + } + ctx := startQuery(t, req) + if tc.run != nil { + tc.run(t) + } + tc.check(t, ctx) + }) + } +} From a527a53b1e8ea69725caff9e2c3661fe0a6bc121 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 13 Mar 2024 15:16:21 -0500 Subject: [PATCH 28/98] misc... --- nomad/jobs_endpoint.go | 62 +++++++++++++++++++++---------------- nomad/jobs_endpoint_test.go | 30 +++++++++--------- nomad/mock/alloc.go | 7 ++++- 3 files changed, 58 insertions(+), 41 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 5ef9e3d90b2..e912deddd02 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -71,8 +71,8 @@ func (j *Jobs) Statuses( } allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityReadJob) - // compare between state run() unblocks to see if the RPC should unblock. - // i.e. if new job(s) shift the page, or when job(s) go away. + // Compare between state run() unblocks to see if the RPC, as a whole, + // should unblock. i.e. if new jobs shift the page, or when jobs go away. prevJobs := set.New[structs.NamespacedID](0) sort := state.QueryOptionSort(args.QueryOptions) @@ -118,7 +118,7 @@ func (j *Jobs) Statuses( paginator.NamespaceFilter{ AllowableNamespaces: allowableNamespaces, }, - // don't include child jobs; we'll look them up later, per parent. + // skip child jobs; we'll look them up later, per parent. paginator.GenericFilter{Allow: func(i interface{}) (bool, error) { job := i.(*structs.Job) return job.ParentID == "", nil @@ -143,12 +143,9 @@ func (j *Jobs) Statuses( pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) - if job == nil { - return nil - } // this is where the sausage is made - uiJob, idx, err := UIJobFromJob(ws, state, job, args.SmartOnly) + uiJob, highestIndexOnPage, err := UIJobFromJob(ws, state, job, args.SmartOnly) if err != nil { return err } @@ -156,8 +153,12 @@ func (j *Jobs) Statuses( jobs = append(jobs, uiJob) newJobs.Insert(job.NamespacedID()) - if idx > reply.Index { - reply.Index = idx + // by using the highest index we find on any job/alloc/ + // deployment among the jobs on the page, instead of the + // latest index for any particular state table, we can + // avoid unblocking the RPC if something changes "off page" + if highestIndexOnPage > reply.Index { + reply.Index = highestIndexOnPage } return nil }) @@ -174,7 +175,7 @@ func (j *Jobs) Statuses( // if the page has updated, or a job has gone away, // bump the index to latest jobs entry. - if !newJobs.Equal(prevJobs) { + if !prevJobs.Empty() && !newJobs.Equal(prevJobs) { reply.Index, err = state.Index("jobs") if err != nil { return err @@ -211,10 +212,14 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, ChildStatuses: nil, DeploymentID: "", } + + // the GroupCountSum will map to how many allocations we expect to run + // (for service jobs) for _, tg := range job.TaskGroups { uiJob.GroupCountSum += tg.Count } + // collect the statuses of child jobs if job.IsParameterized() || job.IsPeriodic() { children, err := store.JobsByIDPrefix(ws, job.Namespace, job.ID, state.SortDefault) if err != nil { @@ -226,6 +231,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, break } j := child.(*structs.Job) + // note: this filters out grandchildren jobs (children of children) if j.ParentID != job.ID { continue } @@ -234,19 +240,28 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, } uiJob.ChildStatuses = append(uiJob.ChildStatuses, j.Status) } + // no allocs or deployments for parameterized/period jobs, + // so we're done here. + return uiJob, idx, err } - allocs, err := store.AllocsByJob(ws, job.Namespace, job.ID, true) // TODO: anyCreateIndex? + // collect info about allocations + allocs, err := store.AllocsByJob(ws, job.Namespace, job.ID, true) if err != nil { return uiJob, idx, err } - for _, a := range allocs { + if a.ModifyIndex > idx { + idx = a.ModifyIndex + } + uiJob.SmartAlloc["total"]++ uiJob.SmartAlloc[a.ClientStatus]++ if a.DeploymentStatus != nil && a.DeploymentStatus.Canary { uiJob.SmartAlloc["canary"]++ } + // callers may wish to keep response body size smaller by excluding + // details about allocations. if smartOnly { continue } @@ -258,29 +273,24 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, NodeID: a.NodeID, JobVersion: a.Job.Version, } - // TODO: use methods instead of fields directly? if a.DeploymentStatus != nil { - alloc.DeploymentStatus.Canary = a.DeploymentStatus.Canary - if a.DeploymentStatus.Healthy != nil { - alloc.DeploymentStatus.Healthy = *a.DeploymentStatus.Healthy - } + alloc.DeploymentStatus.Canary = a.DeploymentStatus.IsCanary() + alloc.DeploymentStatus.Healthy = a.DeploymentStatus.IsHealthy() } uiJob.Allocs = append(uiJob.Allocs, alloc) - if a.ModifyIndex > idx { - idx = a.ModifyIndex - } } - deploys, err := store.DeploymentsByJobID(ws, job.Namespace, job.ID, true) + // look for active deployment + deploy, err := store.LatestDeploymentByJobID(ws, job.Namespace, job.ID) if err != nil { return uiJob, idx, err } - for _, d := range deploys { - if d.Active() { - uiJob.DeploymentID = d.ID + if deploy != nil { + if deploy.Active() { + uiJob.DeploymentID = deploy.ID } - if d.ModifyIndex > idx { - idx = d.ModifyIndex + if deploy.ModifyIndex > idx { + idx = deploy.ModifyIndex } } return uiJob, idx, nil diff --git a/nomad/jobs_endpoint_test.go b/nomad/jobs_endpoint_test.go index 46327324f94..a13ea4f9244 100644 --- a/nomad/jobs_endpoint_test.go +++ b/nomad/jobs_endpoint_test.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test" "github.com/shoenig/test/must" ) @@ -184,7 +185,7 @@ func TestJobs_Statuses(t *testing.T) { // context to signal when the query unblocks // mustBlock and mustUnblock below work by checking ctx.Done() ctx, cancel := context.WithCancel(context.Background()) - // default latest index + // default latest index to induce blocking if req.QueryOptions.MinQueryIndex == 0 { idx, err := s.State().LatestIndex() must.NoError(t, err) @@ -214,8 +215,7 @@ func TestJobs_Statuses(t *testing.T) { } mustUnblock := func(t *testing.T, ctx context.Context) { t.Helper() - timer := time.NewTimer(time.Millisecond * 200) // may take a sec for a job to get deleted - //timer := time.NewTimer(time.Second * 2) // may take a sec for a job to get deleted + timer := time.NewTimer(time.Millisecond * 200) defer timer.Stop() select { case <-ctx.Done(): @@ -226,27 +226,29 @@ func TestJobs_Statuses(t *testing.T) { // alloc and deployment helpers createAlloc := func(t *testing.T, job *structs.Job) { + t.Helper() idx, err := s.State().LatestIndex() must.NoError(t, err) a := mock.MinAllocForJob(job) must.NoError(t, - s.State().UpsertAllocs(structs.AllocUpdateRequestType, idx+1, []*structs.Allocation{a})) - //t.Cleanup(func() { - // t.Helper() - // idx, err = s.State().Index("allocs") - // test.NoError(t, err) - // test.NoError(t, s.State().DeleteEval(idx, []string{}, []string{a.ID}, false)) - //}) + s.State().UpsertAllocs(structs.AllocUpdateRequestType, idx+1, []*structs.Allocation{a}), + must.Sprintf("error creating alloc for job %s", job.ID)) + t.Cleanup(func() { + idx, err = s.State().Index("allocs") + test.NoError(t, err) + test.NoError(t, s.State().DeleteEval(idx, []string{}, []string{a.ID}, false)) + }) } createDeployment := func(t *testing.T, job *structs.Job) { + t.Helper() idx, err := s.State().LatestIndex() must.NoError(t, err) deploy := mock.Deployment() deploy.JobID = job.ID must.NoError(t, s.State().UpsertDeployment(idx+1, deploy)) - //t.Cleanup(func(){ - // s.State().DeleteDeployment(idx+) - //}) + t.Cleanup(func() { + test.NoError(t, s.State().DeleteDeployment(idx+1, []string{deploy.ID})) + }) } // these must be run in order, as they affect outer-scope state. @@ -259,7 +261,7 @@ func TestJobs_Statuses(t *testing.T) { }{ { name: "get all jobs", - check: mustBlock, // TODO: these leak goroutines... until the server goes away, breaking the RPC? + check: mustBlock, }, { name: "delete job", diff --git a/nomad/mock/alloc.go b/nomad/mock/alloc.go index bfdd6ba3514..170c3b54b27 100644 --- a/nomad/mock/alloc.go +++ b/nomad/mock/alloc.go @@ -87,7 +87,10 @@ func Alloc() *structs.Allocation { } func MinAlloc() *structs.Allocation { - job := MinJob() + return MinAllocForJob(MinJob()) +} + +func MinAllocForJob(job *structs.Job) *structs.Allocation { group := job.TaskGroups[0] task := group.Tasks[0] return &structs.Allocation{ @@ -95,6 +98,8 @@ func MinAlloc() *structs.Allocation { EvalID: uuid.Generate(), NodeID: uuid.Generate(), Job: job, + JobID: job.ID, + Namespace: job.Namespace, TaskGroup: group.Name, ClientStatus: structs.AllocClientStatusPending, DesiredStatus: structs.AllocDesiredStatusRun, From 817e354c39738bd2e9007098f02cdc42de80b437 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 13 Mar 2024 15:30:32 -0500 Subject: [PATCH 29/98] copyright test file --- nomad/jobs_endpoint_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nomad/jobs_endpoint_test.go b/nomad/jobs_endpoint_test.go index a13ea4f9244..d7edfffb995 100644 --- a/nomad/jobs_endpoint_test.go +++ b/nomad/jobs_endpoint_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package nomad import ( From 312c6d0ddcf36fabcc138a9fee0ea74209cd6e16 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 13 Mar 2024 16:01:39 -0500 Subject: [PATCH 30/98] revert "add JobSummary.Children.Desired" This reverts commit 6a8b132408a19e2f46eb2757107e5c7471df93db. --- nomad/state/state_store.go | 9 --------- nomad/structs/structs.go | 1 - 2 files changed, 10 deletions(-) diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 1ba97a143b6..af538fa8888 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -5517,9 +5517,7 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, hasSummaryChanged = true } - var totalCount int64 for _, tg := range job.TaskGroups { - totalCount += int64(tg.Count) if _, ok := summary.Summary[tg.Name]; !ok { newSummary := structs.TaskGroupSummary{ Complete: 0, @@ -5531,7 +5529,6 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, hasSummaryChanged = true } } - summary.Children.Desired = totalCount // The job summary has changed, so update the modify index. if hasSummaryChanged { @@ -5890,12 +5887,6 @@ func (s *StateStore) updateSummaryWithAlloc(index uint64, alloc *structs.Allocat return nil } - var totalCount int64 - for _, tg := range alloc.Job.TaskGroups { - totalCount += int64(tg.Count) - } - jobSummary.Children.Desired = totalCount - tgSummary, ok := jobSummary.Summary[alloc.TaskGroup] if !ok { return fmt.Errorf("unable to find task group in the job summary: %v", alloc.TaskGroup) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 8c1431d321e..5611647090f 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5112,7 +5112,6 @@ type JobChildrenSummary struct { Pending int64 Running int64 Dead int64 - Desired int64 } // Copy returns a new copy of a JobChildrenSummary From 5a4665b5148e311f9deadfca3e358fab062cfb03 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 25 Mar 2024 16:32:51 -0400 Subject: [PATCH 31/98] sort jobs by ModifyIndex --- nomad/jobs_endpoint.go | 16 +++++++++++----- nomad/state/schema.go | 9 +++++++++ nomad/state/state_store.go | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index e912deddd02..5f8c4bdcf47 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -96,12 +96,18 @@ func (j *Jobs) Statuses( return err } - if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) - } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace, sort) + // the UI jobs index page shows most-recently changed at the top, + // and with pagination, we need to sort here on the backend. + if true { // TODO: parameterize...? + iter, err = state.JobsByModifyIndex(ws, sort) } else { - iter, err = state.Jobs(ws, sort) + if prefix := args.QueryOptions.Prefix; prefix != "" { + iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) + } else if namespace != structs.AllNamespacesSentinel { + iter, err = state.JobsByNamespace(ws, namespace, sort) + } else { + iter, err = state.Jobs(ws, sort) + } } if err != nil { return err diff --git a/nomad/state/schema.go b/nomad/state/schema.go index 93c77d0e4e7..18e485e6dc5 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -254,6 +254,15 @@ func jobTableSchema() *memdb.TableSchema { Field: "NodePool", }, }, + // ModifyIndex allows sorting by last-changed + "modify_index": { + Name: "modify_index", + AllowMissing: false, + Unique: false, + Indexer: &memdb.UintFieldIndex{ + Field: "ModifyIndex", + }, + }, }, } } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index af538fa8888..c6102950a18 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -2475,6 +2475,20 @@ func (s *StateStore) JobsByPool(ws memdb.WatchSet, pool string) (memdb.ResultIte return iter, nil } +// JobsByModifyIndex returns an iterator over all jobs, sorted by ModifyIndex. +func (s *StateStore) JobsByModifyIndex(ws memdb.WatchSet, sort SortOption) (memdb.ResultIterator, error) { + txn := s.db.ReadTxn() + + iter, err := getSorted(txn, sort, "jobs", "modify_index") + if err != nil { + return nil, err + } + + ws.Add(iter.WatchCh()) + + return iter, nil +} + // JobSummaryByID returns a job summary object which matches a specific id. func (s *StateStore) JobSummaryByID(ws memdb.WatchSet, namespace, jobID string) (*structs.JobSummary, error) { txn := s.db.ReadTxn() From 485f45b02fb3f65b923b8499d625652761b9d4e3 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 25 Mar 2024 16:34:09 -0400 Subject: [PATCH 32/98] add submit/modify time and rename ActiveDeploymentID --- nomad/jobs_endpoint.go | 14 ++++++++------ nomad/structs/job.go | 24 +++++++++++++----------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 5f8c4bdcf47..8816e8fe624 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -212,11 +212,13 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, Priority: job.Priority, Version: job.Version, // included here for completeness, populated below. - Allocs: nil, - SmartAlloc: make(map[string]int), - GroupCountSum: 0, - ChildStatuses: nil, - DeploymentID: "", + Allocs: nil, + SmartAlloc: make(map[string]int), + GroupCountSum: 0, + ChildStatuses: nil, + ActiveDeploymentID: "", + SubmitTime: job.SubmitTime, + ModifyIndex: job.ModifyIndex, } // the GroupCountSum will map to how many allocations we expect to run @@ -293,7 +295,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, } if deploy != nil { if deploy.Active() { - uiJob.DeploymentID = deploy.ID + uiJob.ActiveDeploymentID = deploy.ID } if deploy.ModifyIndex > idx { idx = deploy.ModifyIndex diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 88335c770af..e288b62ed3e 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -18,17 +18,19 @@ const ( type UIJob struct { NamespacedID - Name string - Type string - NodePool string - Datacenters []string - Priority int - Allocs []JobStatusAlloc - SmartAlloc map[string]int - GroupCountSum int - ChildStatuses []string - DeploymentID string - Version uint64 + Name string + Type string + NodePool string + Datacenters []string + Priority int + Allocs []JobStatusAlloc + SmartAlloc map[string]int + GroupCountSum int + ChildStatuses []string + ActiveDeploymentID string + Version uint64 + SubmitTime int64 + ModifyIndex uint64 } type JobsStatusesRequest struct { From 4447725dcaee992fb8660229d0c79c20079f2ea3 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 25 Mar 2024 16:34:36 -0400 Subject: [PATCH 33/98] internal errors are not bad requests --- nomad/jobs_endpoint.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 8816e8fe624..746099a6931 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -170,13 +170,13 @@ func (j *Jobs) Statuses( }) if err != nil { return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to create result paginator: %v", err) + http.StatusInternalServerError, "failed to create result paginator: %v", err) } nextToken, err := pager.Page() if err != nil { return structs.NewErrRPCCodedf( - http.StatusBadRequest, "failed to read result page: %v", err) + http.StatusInternalServerError, "failed to read result page: %v", err) } // if the page has updated, or a job has gone away, From 1b1944b2ed11dfe1d6cc8904e5ca348f458a1412 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 5 Apr 2024 13:17:20 -0400 Subject: [PATCH 34/98] sort by ModifyIndex *only*, and default reverse and move a namespace check and tweak some comments --- nomad/jobs_endpoint.go | 70 +++++++++++++++++++++++------------------- nomad/state/schema.go | 2 +- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 746099a6931..cf426896b83 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -47,9 +47,10 @@ func (j *Jobs) Statuses( defer metrics.MeasureSince([]string{"nomad", "jobs", "statuses"}, time.Now()) namespace := args.RequestNamespace() - // The ns from the UI by default is "*" which scans the whole "jobs" table. - // If specific jobs are requested, all with the same namespace, - // we may get some extra efficiency, especially for non-contiguous job IDs. + // the namespace from the UI by default is "*", but if specific jobs are + // requested, all with the same namespace, AllowNsOp() below may be able + // to quickly deny the request if the token lacks permissions for that ns, + // rather than iterating the whole jobs table and filtering out every job. if len(args.Jobs) > 0 { nses := set.New[string](0) for _, j := range args.Jobs { @@ -60,7 +61,7 @@ func (j *Jobs) Statuses( } } - // Check for read-job permissions, since this endpoint includes alloc info + // check for read-job permissions, since this endpoint includes alloc info // and possibly a deployment ID, and those APIs require read-job. aclObj, err := j.srv.ResolveACL(args) if err != nil { @@ -71,13 +72,40 @@ func (j *Jobs) Statuses( } allow := aclObj.AllowNsOpFunc(acl.NamespaceCapabilityReadJob) - // Compare between state run() unblocks to see if the RPC, as a whole, + store := j.srv.State() + + // get the namespaces the user is allowed to access. + allowableNamespaces, err := allowedNSes(aclObj, store, allow) + if errors.Is(err, structs.ErrPermissionDenied) { + // return empty jobs if token isn't authorized for any + // namespace, matching other endpoints + reply.Jobs = make([]structs.UIJob, 0) + return nil + } else if err != nil { + return err + } + // since the state index we're using doesn't include namespace, + // explicitly add the user-provided ns to our filter if needed. + // (allowableNamespaces will be nil if the caller sent a mgmt token) + if allowableNamespaces == nil && + namespace != "" && + namespace != structs.AllNamespacesSentinel { + allowableNamespaces = map[string]bool{ + namespace: true, + } + } + + // compare between state run() unblocks to see if the RPC, as a whole, // should unblock. i.e. if new jobs shift the page, or when jobs go away. prevJobs := set.New[structs.NamespacedID](0) + // because the state index is in order of ModifyIndex, lowest to highest, + // SortDefault would show oldest jobs first, so instead invert the default + // to show most recent job changes first. + args.QueryOptions.Reverse = !args.QueryOptions.Reverse sort := state.QueryOptionSort(args.QueryOptions) - // Setup the blocking query + // setup the blocking query opts := blockingOptions{ queryOpts: &args.QueryOptions, queryMeta: &reply.QueryMeta, @@ -85,39 +113,17 @@ func (j *Jobs) Statuses( var err error var iter memdb.ResultIterator - // Get the namespaces the user is allowed to access. - allowableNamespaces, err := allowedNSes(aclObj, state, allow) - if errors.Is(err, structs.ErrPermissionDenied) { - // return empty jobs if token isn't authorized for any - // namespace, matching other endpoints - reply.Jobs = make([]structs.UIJob, 0) - return nil - } else if err != nil { - return err - } - - // the UI jobs index page shows most-recently changed at the top, - // and with pagination, we need to sort here on the backend. - if true { // TODO: parameterize...? - iter, err = state.JobsByModifyIndex(ws, sort) - } else { - if prefix := args.QueryOptions.Prefix; prefix != "" { - iter, err = state.JobsByIDPrefix(ws, namespace, prefix, sort) - } else if namespace != structs.AllNamespacesSentinel { - iter, err = state.JobsByNamespace(ws, namespace, sort) - } else { - iter, err = state.Jobs(ws, sort) - } - } + // the UI jobs index page shows most-recently changed first. + iter, err = state.JobsByModifyIndex(ws, sort) if err != nil { return err } + // set up tokenizer and filters tokenizer := paginator.NewStructsTokenizer( iter, paginator.StructsTokenizerOptions{ - WithNamespace: true, - WithID: true, + OnlyModifyIndex: true, }, ) filters := []paginator.Filter{ diff --git a/nomad/state/schema.go b/nomad/state/schema.go index 18e485e6dc5..e27261bf111 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -258,7 +258,7 @@ func jobTableSchema() *memdb.TableSchema { "modify_index": { Name: "modify_index", AllowMissing: false, - Unique: false, + Unique: true, Indexer: &memdb.UintFieldIndex{ Field: "ModifyIndex", }, From e125aeb529378de1f9314e9dffe0c4c1928229be Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 9 Apr 2024 10:51:01 -0400 Subject: [PATCH 35/98] minimally fix tests (uncomplicated) --- nomad/jobs_endpoint_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/nomad/jobs_endpoint_test.go b/nomad/jobs_endpoint_test.go index d7edfffb995..d204f0f92c2 100644 --- a/nomad/jobs_endpoint_test.go +++ b/nomad/jobs_endpoint_test.go @@ -5,6 +5,7 @@ package nomad import ( "context" + "strconv" "testing" "time" @@ -112,7 +113,8 @@ func TestJobs_Statuses(t *testing.T) { // make sure our state order assumption is correct for i, j := range resp.Jobs { - must.Eq(t, jobs[i].ID, j.ID, must.Sprintf("jobs not in order; idx=%d", i)) + reverse := len(jobs) - i - 1 + must.Eq(t, jobs[reverse].ID, j.ID, must.Sprintf("jobs not in order; idx=%d", i)) } // test various single-job requests @@ -128,15 +130,17 @@ func TestJobs_Statuses(t *testing.T) { qo: structs.QueryOptions{ PerPage: 1, }, - expect: jobs[0], + expect: jobs[4], }, { name: "page 2", qo: structs.QueryOptions{ - PerPage: 1, - NextToken: "default." + jobs[1].ID, + PerPage: 1, + // paginator/tokenizer will produce a uint64 of the next job's + // ModifyIndex, so here we just fake that. + NextToken: strconv.FormatUint(jobs[3].ModifyIndex, 10), }, - expect: jobs[1], + expect: jobs[3], }, { name: "reverse", @@ -144,7 +148,7 @@ func TestJobs_Statuses(t *testing.T) { PerPage: 1, Reverse: true, }, - expect: jobs[len(jobs)-1], + expect: jobs[0], }, { name: "filter", From 404c5c5d99db010a8a4027e4b5620e6a46557dd5 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 9 Apr 2024 13:31:38 -0400 Subject: [PATCH 36/98] test: extra uint64 pagination assurance --- nomad/jobs_endpoint_test.go | 68 ++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/nomad/jobs_endpoint_test.go b/nomad/jobs_endpoint_test.go index d204f0f92c2..d3faa1252ad 100644 --- a/nomad/jobs_endpoint_test.go +++ b/nomad/jobs_endpoint_test.go @@ -66,21 +66,26 @@ func TestJobs_Statuses(t *testing.T) { return resp } - // job helpers - deleteJob := func(t *testing.T, job *structs.Job) { + // increment state index helper + incIdx := func(t *testing.T) uint64 { t.Helper() idx, err := s.State().LatestIndex() must.NoError(t, err) - err = s.State().DeleteJob(idx+1, job.Namespace, job.ID) + return idx + 1 + } + + // job helpers + deleteJob := func(t *testing.T, job *structs.Job) { + t.Helper() + err := s.State().DeleteJob(incIdx(t), job.Namespace, job.ID) if err != nil && err.Error() == "job not found" { return } must.NoError(t, err) } upsertJob := func(t *testing.T, job *structs.Job) { - idx, err := s.State().LatestIndex() - must.NoError(t, err) - err = s.State().UpsertJob(structs.MsgTypeTestSetup, idx+1, nil, job) + t.Helper() + err := s.State().UpsertJob(structs.MsgTypeTestSetup, incIdx(t), nil, job) must.NoError(t, err) } createJob := func(t *testing.T, id string) (job *structs.Job, cleanup func()) { @@ -97,8 +102,14 @@ func TestJobs_Statuses(t *testing.T) { return job, cleanup } - // set up 5 jobs - // they should be in order in state, due to lexicographical indexing + // this little cutie sets the latest state index to a predictable value, + // to ensure the below jobs span the boundary from 999->1000 which would + // break pagination without proper uint64 NextToken (ModifyIndex) comparison + must.NoError(t, s.State().UpsertNamespaces(996, nil)) + + // set up some jobs + // they should be in this order in state using the "modify_index" index, + // but the RPC will return them in reverse order by default. jobs := make([]*structs.Job, 5) var deleteJob0, deleteJob1, deleteJob2 func() jobs[0], deleteJob0 = createJob(t, "job0") @@ -120,27 +131,28 @@ func TestJobs_Statuses(t *testing.T) { // test various single-job requests for _, tc := range []struct { - name string - qo structs.QueryOptions - jobs []structs.NamespacedID - expect *structs.Job + name string + qo structs.QueryOptions + jobs []structs.NamespacedID + expect *structs.Job + expectNext uint64 // NextToken (ModifyIndex) }{ { name: "page 1", qo: structs.QueryOptions{ PerPage: 1, }, - expect: jobs[4], + expect: jobs[4], + expectNext: jobs[3].ModifyIndex, }, { name: "page 2", qo: structs.QueryOptions{ - PerPage: 1, - // paginator/tokenizer will produce a uint64 of the next job's - // ModifyIndex, so here we just fake that. + PerPage: 1, NextToken: strconv.FormatUint(jobs[3].ModifyIndex, 10), }, - expect: jobs[3], + expect: jobs[3], + expectNext: jobs[2].ModifyIndex, }, { name: "reverse", @@ -148,7 +160,8 @@ func TestJobs_Statuses(t *testing.T) { PerPage: 1, Reverse: true, }, - expect: jobs[0], + expect: jobs[0], + expectNext: jobs[1].ModifyIndex, }, { name: "filter", @@ -172,6 +185,11 @@ func TestJobs_Statuses(t *testing.T) { }) must.Len(t, 1, resp.Jobs, must.Sprint("expect only one job")) must.Eq(t, tc.expect.ID, resp.Jobs[0].ID) + expectToken := "" + if tc.expectNext > 0 { + expectToken = strconv.FormatUint(tc.expectNext, 10) + } + must.Eq(t, expectToken, resp.NextToken) }) } @@ -234,27 +252,21 @@ func TestJobs_Statuses(t *testing.T) { // alloc and deployment helpers createAlloc := func(t *testing.T, job *structs.Job) { t.Helper() - idx, err := s.State().LatestIndex() - must.NoError(t, err) a := mock.MinAllocForJob(job) must.NoError(t, - s.State().UpsertAllocs(structs.AllocUpdateRequestType, idx+1, []*structs.Allocation{a}), + s.State().UpsertAllocs(structs.AllocUpdateRequestType, incIdx(t), []*structs.Allocation{a}), must.Sprintf("error creating alloc for job %s", job.ID)) t.Cleanup(func() { - idx, err = s.State().Index("allocs") - test.NoError(t, err) - test.NoError(t, s.State().DeleteEval(idx, []string{}, []string{a.ID}, false)) + test.NoError(t, s.State().DeleteEval(incIdx(t), []string{}, []string{a.ID}, false)) }) } createDeployment := func(t *testing.T, job *structs.Job) { t.Helper() - idx, err := s.State().LatestIndex() - must.NoError(t, err) deploy := mock.Deployment() deploy.JobID = job.ID - must.NoError(t, s.State().UpsertDeployment(idx+1, deploy)) + must.NoError(t, s.State().UpsertDeployment(incIdx(t), deploy)) t.Cleanup(func() { - test.NoError(t, s.State().DeleteDeployment(idx+1, []string{deploy.ID})) + test.NoError(t, s.State().DeleteDeployment(incIdx(t), []string{deploy.ID})) }) } From b681e8b8ac207d1be8cb9bc4cad1086b11475bc1 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 9 Apr 2024 16:11:25 -0400 Subject: [PATCH 37/98] add LatestDeployment probably will remove ActiveDeploymentID later --- nomad/jobs_endpoint.go | 13 ++++++++++++- nomad/structs/job.go | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index cf426896b83..2943bfea766 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -294,7 +294,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, uiJob.Allocs = append(uiJob.Allocs, alloc) } - // look for active deployment + // look for latest deployment deploy, err := store.LatestDeploymentByJobID(ws, job.Namespace, job.ID) if err != nil { return uiJob, idx, err @@ -303,6 +303,17 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, if deploy.Active() { uiJob.ActiveDeploymentID = deploy.ID } + + uiJob.LatestDeployment = &structs.JobStatusLatestDeployment{ + ID: deploy.ID, + IsActive: deploy.Active(), + JobVersion: deploy.JobVersion, + Status: deploy.Status, + StatusDescription: deploy.StatusDescription, + AllAutoPromote: deploy.HasAutoPromote(), + RequiresPromotion: deploy.RequiresPromotion(), + } + if deploy.ModifyIndex > idx { idx = deploy.ModifyIndex } diff --git a/nomad/structs/job.go b/nomad/structs/job.go index e288b62ed3e..a8ce693b271 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -28,6 +28,7 @@ type UIJob struct { GroupCountSum int ChildStatuses []string ActiveDeploymentID string + LatestDeployment *JobStatusLatestDeployment Version uint64 SubmitTime int64 ModifyIndex uint64 @@ -58,6 +59,16 @@ type JobStatusDeployment struct { Healthy bool } +type JobStatusLatestDeployment struct { + ID string + IsActive bool + JobVersion uint64 + Status string + StatusDescription string + AllAutoPromote bool + RequiresPromotion bool +} + // JobServiceRegistrationsRequest is the request object used to list all // service registrations belonging to the specified Job.ID. type JobServiceRegistrationsRequest struct { From f8ccbd02da0141a2c3f3d18ae6b0635bd730178d Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 22 Apr 2024 14:53:14 -0400 Subject: [PATCH 38/98] remove SmartAlloc --- command/agent/job_endpoint.go | 5 ----- nomad/jobs_endpoint.go | 24 ++++++------------------ nomad/structs/job.go | 4 +--- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index f4507c1fabc..577e9ef9040 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2114,11 +2114,6 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil // seems whack } - if smartOnly, err := parseBool(req, "smart_only"); err != nil { - return nil, err - } else if smartOnly != nil { - args.SmartOnly = *smartOnly - } switch req.Method { case http.MethodGet: // GET requests will be treated as "get all jobs" but also with filtering and pagination and such diff --git a/nomad/jobs_endpoint.go b/nomad/jobs_endpoint.go index 2943bfea766..6f9ca62d2e9 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/jobs_endpoint.go @@ -157,7 +157,7 @@ func (j *Jobs) Statuses( job := raw.(*structs.Job) // this is where the sausage is made - uiJob, highestIndexOnPage, err := UIJobFromJob(ws, state, job, args.SmartOnly) + uiJob, highestIndexOnPage, err := UIJobFromJob(ws, state, job) if err != nil { return err } @@ -203,7 +203,7 @@ func (j *Jobs) Statuses( return j.srv.blockingRPC(&opts) } -func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, smartOnly bool) (structs.UIJob, uint64, error) { +func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) (structs.UIJob, uint64, error) { idx := job.ModifyIndex uiJob := structs.UIJob{ @@ -219,7 +219,6 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, Version: job.Version, // included here for completeness, populated below. Allocs: nil, - SmartAlloc: make(map[string]int), GroupCountSum: 0, ChildStatuses: nil, ActiveDeploymentID: "", @@ -265,21 +264,6 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, return uiJob, idx, err } for _, a := range allocs { - if a.ModifyIndex > idx { - idx = a.ModifyIndex - } - - uiJob.SmartAlloc["total"]++ - uiJob.SmartAlloc[a.ClientStatus]++ - if a.DeploymentStatus != nil && a.DeploymentStatus.Canary { - uiJob.SmartAlloc["canary"]++ - } - // callers may wish to keep response body size smaller by excluding - // details about allocations. - if smartOnly { - continue - } - alloc := structs.JobStatusAlloc{ ID: a.ID, Group: a.TaskGroup, @@ -292,6 +276,10 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job, alloc.DeploymentStatus.Healthy = a.DeploymentStatus.IsHealthy() } uiJob.Allocs = append(uiJob.Allocs, alloc) + + if a.ModifyIndex > idx { + idx = a.ModifyIndex + } } // look for latest deployment diff --git a/nomad/structs/job.go b/nomad/structs/job.go index a8ce693b271..0353109a3a5 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -24,7 +24,6 @@ type UIJob struct { Datacenters []string Priority int Allocs []JobStatusAlloc - SmartAlloc map[string]int GroupCountSum int ChildStatuses []string ActiveDeploymentID string @@ -35,8 +34,7 @@ type UIJob struct { } type JobsStatusesRequest struct { - Jobs []NamespacedID - SmartOnly bool + Jobs []NamespacedID QueryOptions } From 21baaaee8cc0b95610f405cebccf3dab838ecb0f Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Apr 2024 16:06:49 -0400 Subject: [PATCH 39/98] remove vestigial test helper --- nomad/state/testing.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/nomad/state/testing.go b/nomad/state/testing.go index 72e4812312e..cb955ffa46c 100644 --- a/nomad/state/testing.go +++ b/nomad/state/testing.go @@ -12,7 +12,6 @@ import ( "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" - "github.com/shoenig/test/must" ) func TestStateStore(t testing.TB) *StateStore { @@ -319,18 +318,3 @@ func TestBadCSIState(t testing.TB, store *StateStore) error { return nil } - -func (s *StateStore) UpsertAllocsRaw(t testing.TB, idx uint64, allocs []*structs.Allocation) { - t.Helper() - txn := s.db.WriteTxn(idx) - defer txn.Abort() - var err error - for _, a := range allocs { - err = txn.Insert("allocs", a) - must.NoError(t, err, must.Sprint("error inserting alloc")) - } - err = txn.Insert("index", &IndexEntry{"allocs", idx}) - must.NoError(t, err, must.Sprint("error inserting index")) - err = txn.Commit() - must.NoError(t, err, must.Sprint("error committing transaction")) -} From 1f593d937dc49e887ab73311af9c31084c6a3823 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 23 Apr 2024 16:45:00 -0400 Subject: [PATCH 40/98] remove separate Jobs RPC type put Statuses on Job instead and rename JobsStatuses* to JobStatuses* --- api/jobs.go | 2 +- command/agent/http.go | 2 +- command/agent/job_endpoint.go | 10 ++++---- ...s_endpoint.go => job_endpoint_statuses.go} | 23 ++++-------------- ..._test.go => job_endpoint_statuses_test.go} | 24 +++++++++---------- nomad/server.go | 1 - nomad/structs/job.go | 4 ++-- 7 files changed, 25 insertions(+), 41 deletions(-) rename nomad/{jobs_endpoint.go => job_endpoint_statuses.go} (95%) rename nomad/{jobs_endpoint_test.go => job_endpoint_statuses_test.go} (93%) diff --git a/api/jobs.go b/api/jobs.go index 6aa2ad9f399..239e8d94581 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -1545,7 +1545,7 @@ func (j *Jobs) ActionExec(ctx context.Context, return s.run(ctx) } -type JobsStatusesRequest struct { +type JobStatusesRequest struct { Jobs []struct { // TODO: proper type ID string `json:"id"` Namespace string `json:"namespace"` diff --git a/command/agent/http.go b/command/agent/http.go index 679948b591e..77394e8d805 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -382,7 +382,7 @@ func (s *HTTPServer) ResolveToken(req *http.Request) (*acl.ACL, error) { func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest)) s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) - s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobsStatusesRequest)) + s.mux.HandleFunc("/v1/jobs/statuses", s.wrap(s.JobStatusesRequest)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 577e9ef9040..559cbb0f90d 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2109,8 +2109,8 @@ func validateEvalPriorityOpt(priority int) HTTPCodedError { return nil } -func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { - args := structs.JobsStatusesRequest{} +func (s *HTTPServer) JobStatusesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.JobStatusesRequest{} if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil // seems whack } @@ -2119,7 +2119,7 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req // GET requests will be treated as "get all jobs" but also with filtering and pagination and such case http.MethodPost: // POST requests expect a list of Jobs in the request body, which will then be filtered/paginated, etc. - var in api.JobsStatusesRequest + var in api.JobStatusesRequest if err := decodeBody(req, &in); err != nil { return nil, err } @@ -2140,8 +2140,8 @@ func (s *HTTPServer) JobsStatusesRequest(resp http.ResponseWriter, req *http.Req return nil, CodedError(405, ErrInvalidMethod) } - var out structs.JobsStatusesResponse - if err := s.agent.RPC("Jobs.Statuses", &args, &out); err != nil { + var out structs.JobStatusesResponse + if err := s.agent.RPC("Job.Statuses", &args, &out); err != nil { return nil, err } diff --git a/nomad/jobs_endpoint.go b/nomad/job_endpoint_statuses.go similarity index 95% rename from nomad/jobs_endpoint.go rename to nomad/job_endpoint_statuses.go index 6f9ca62d2e9..8af55a51c8b 100644 --- a/nomad/jobs_endpoint.go +++ b/nomad/job_endpoint_statuses.go @@ -9,7 +9,6 @@ import ( "time" "github.com/armon/go-metrics" - "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-set/v2" "github.com/hashicorp/nomad/acl" @@ -18,26 +17,12 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) -func NewJobsEndpoint(s *Server, ctx *RPCContext) *Jobs { - return &Jobs{ - srv: s, - ctx: ctx, - logger: s.logger.Named("jobs"), - } -} - -type Jobs struct { - srv *Server - ctx *RPCContext - logger hclog.Logger -} - -func (j *Jobs) Statuses( - args *structs.JobsStatusesRequest, - reply *structs.JobsStatusesResponse) error { +func (j *Job) Statuses( + args *structs.JobStatusesRequest, + reply *structs.JobStatusesResponse) error { authErr := j.srv.Authenticate(j.ctx, args) - if done, err := j.srv.forward("Jobs.Statuses", args, args, reply); done { + if done, err := j.srv.forward("Job.Statuses", args, args, reply); done { return err } j.srv.MeasureRPCRate("jobs", structs.RateMetricList, args) diff --git a/nomad/jobs_endpoint_test.go b/nomad/job_endpoint_statuses_test.go similarity index 93% rename from nomad/jobs_endpoint_test.go rename to nomad/job_endpoint_statuses_test.go index d3faa1252ad..3f477534553 100644 --- a/nomad/jobs_endpoint_test.go +++ b/nomad/job_endpoint_statuses_test.go @@ -16,7 +16,7 @@ import ( "github.com/shoenig/test/must" ) -func TestJobs_Statuses_ACL(t *testing.T) { +func TestJob_Statuses_ACL(t *testing.T) { s, _, cleanup := TestACLServer(t, nil) t.Cleanup(cleanup) testutil.WaitForLeader(t, s.RPC) @@ -34,12 +34,12 @@ func TestJobs_Statuses_ACL(t *testing.T) { {"happy token", happyToken.SecretID, ""}, } { t.Run(tc.name, func(t *testing.T) { - req := &structs.JobsStatusesRequest{} + req := &structs.JobStatusesRequest{} req.QueryOptions.Region = "global" req.QueryOptions.AuthToken = tc.token - var resp structs.JobsStatusesResponse - err := s.RPC("Jobs.Statuses", &req, &resp) + var resp structs.JobStatusesResponse + err := s.RPC("Job.Statuses", &req, &resp) if tc.err != "" { must.ErrorContains(t, err, tc.err) @@ -50,7 +50,7 @@ func TestJobs_Statuses_ACL(t *testing.T) { } } -func TestJobs_Statuses(t *testing.T) { +func TestJob_Statuses(t *testing.T) { s, cleanup := TestServer(t, func(c *Config) { c.NumSchedulers = 0 // Prevent automatic dequeue }) @@ -58,11 +58,11 @@ func TestJobs_Statuses(t *testing.T) { testutil.WaitForLeader(t, s.RPC) // method under test - doRequest := func(t *testing.T, req *structs.JobsStatusesRequest) (resp structs.JobsStatusesResponse) { + doRequest := func(t *testing.T, req *structs.JobStatusesRequest) (resp structs.JobStatusesResponse) { t.Helper() must.NotNil(t, req, must.Sprint("request must not be nil")) req.QueryOptions.Region = "global" - must.NoError(t, s.RPC("Jobs.Statuses", req, &resp)) + must.NoError(t, s.RPC("Job.Statuses", req, &resp)) return resp } @@ -119,7 +119,7 @@ func TestJobs_Statuses(t *testing.T) { jobs[4], _ = createJob(t, "job4") // request all jobs - resp := doRequest(t, &structs.JobsStatusesRequest{}) + resp := doRequest(t, &structs.JobStatusesRequest{}) must.Len(t, 5, resp.Jobs) // make sure our state order assumption is correct @@ -179,7 +179,7 @@ func TestJobs_Statuses(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - resp = doRequest(t, &structs.JobsStatusesRequest{ + resp = doRequest(t, &structs.JobStatusesRequest{ QueryOptions: tc.qo, Jobs: tc.jobs, }) @@ -202,10 +202,10 @@ func TestJobs_Statuses(t *testing.T) { // job/alloc/deployment seen while iterating, i.e. those "on-page". // blocking query helpers - startQuery := func(t *testing.T, req *structs.JobsStatusesRequest) context.Context { + startQuery := func(t *testing.T, req *structs.JobStatusesRequest) context.Context { t.Helper() if req == nil { - req = &structs.JobsStatusesRequest{} + req = &structs.JobStatusesRequest{} } // context to signal when the query unblocks // mustBlock and mustUnblock below work by checking ctx.Done() @@ -354,7 +354,7 @@ func TestJobs_Statuses(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - req := &structs.JobsStatusesRequest{} + req := &structs.JobStatusesRequest{} if tc.watch != nil { req.Jobs = []structs.NamespacedID{tc.watch.NamespacedID()} } diff --git a/nomad/server.go b/nomad/server.go index 08f5d775848..a488481d997 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1341,7 +1341,6 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { _ = server.Register(NewDeploymentEndpoint(s, ctx)) _ = server.Register(NewEvalEndpoint(s, ctx)) _ = server.Register(NewJobEndpoints(s, ctx)) - _ = server.Register(NewJobsEndpoint(s, ctx)) _ = server.Register(NewKeyringEndpoint(s, ctx, s.encrypter)) _ = server.Register(NewNamespaceEndpoint(s, ctx)) _ = server.Register(NewNodeEndpoint(s, ctx)) diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 0353109a3a5..ba60af6725d 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -33,12 +33,12 @@ type UIJob struct { ModifyIndex uint64 } -type JobsStatusesRequest struct { +type JobStatusesRequest struct { Jobs []NamespacedID QueryOptions } -type JobsStatusesResponse struct { +type JobStatusesResponse struct { Jobs []UIJob QueryMeta } From 924935a54f0882fd1fe92dddf5481d2bac5943a6 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 24 Apr 2024 11:19:44 -0400 Subject: [PATCH 41/98] treat GET/POST the same --- command/agent/job_endpoint.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 559cbb0f90d..b7e4f342ba6 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2114,11 +2114,17 @@ func (s *HTTPServer) JobStatusesRequest(resp http.ResponseWriter, req *http.Requ if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil // seems whack } + switch req.Method { - case http.MethodGet: - // GET requests will be treated as "get all jobs" but also with filtering and pagination and such - case http.MethodPost: - // POST requests expect a list of Jobs in the request body, which will then be filtered/paginated, etc. + case http.MethodGet, http.MethodPost: + break + default: + return nil, CodedError(405, ErrInvalidMethod) + } + + // ostensibly GETs should not accept structured body, but the HTTP spec + // on this is more what you'd call "guidelines" than actual rules. + if req.Body != http.NoBody { var in api.JobStatusesRequest if err := decodeBody(req, &in); err != nil { return nil, err @@ -2126,18 +2132,22 @@ func (s *HTTPServer) JobStatusesRequest(resp http.ResponseWriter, req *http.Requ if len(in.Jobs) == 0 { return nil, CodedError(http.StatusBadRequest, "no jobs in request") } + + // each job has a separate namespace, so default to wildcard + // in case the NSes are mixed. + if args.QueryOptions.Namespace == "" { + args.QueryOptions.Namespace = "*" + } + for _, j := range in.Jobs { if j.Namespace == "" { j.Namespace = "default" } args.Jobs = append(args.Jobs, structs.NamespacedID{ - ID: j.ID, - // note: can't just use QueryOptions.Namespace, because each job may have a different NS + ID: j.ID, Namespace: j.Namespace, }) } - default: - return nil, CodedError(405, ErrInvalidMethod) } var out structs.JobStatusesResponse From 3e0ab6718e5eda8275bc399f53c2a423393cc221 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Mon, 29 Apr 2024 17:58:06 -0400 Subject: [PATCH 42/98] set/slice init size optimization --- nomad/job_endpoint_statuses.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/nomad/job_endpoint_statuses.go b/nomad/job_endpoint_statuses.go index 8af55a51c8b..633fce0041c 100644 --- a/nomad/job_endpoint_statuses.go +++ b/nomad/job_endpoint_statuses.go @@ -135,8 +135,11 @@ func (j *Job) Statuses( }) } - jobs := make([]structs.UIJob, 0) - newJobs := set.New[structs.NamespacedID](0) + // little dance to avoid extra memory allocations + i := 0 + jobs := make([]structs.UIJob, len(args.Jobs)) + newJobs := set.New[structs.NamespacedID](len(args.Jobs)) + pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) @@ -147,7 +150,12 @@ func (j *Job) Statuses( return err } - jobs = append(jobs, uiJob) + if len(args.Jobs) > 0 { + jobs[i] = uiJob + i++ + } else { + jobs = append(jobs, uiJob) + } newJobs.Insert(job.NamespacedID()) // by using the highest index we find on any job/alloc/ @@ -170,6 +178,15 @@ func (j *Job) Statuses( http.StatusInternalServerError, "failed to read result page: %v", err) } + // remove empty jobs (caller requested jobs that don't exist) + prevJobs.Remove(structs.NamespacedID{}) + jobsClean := make([]structs.UIJob, 0) + for _, j := range jobs { + if j.ID != "" && j.Namespace != "" { + jobsClean = append(jobsClean, j) + } + } + // if the page has updated, or a job has gone away, // bump the index to latest jobs entry. if !prevJobs.Empty() && !newJobs.Equal(prevJobs) { @@ -181,7 +198,7 @@ func (j *Job) Statuses( prevJobs = newJobs reply.QueryMeta.NextToken = nextToken - reply.Jobs = jobs + reply.Jobs = jobsClean return nil }} From 869b4fcb15cfea27bf41bce8ec0a137bf76926bc Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 30 Apr 2024 11:40:11 -0400 Subject: [PATCH 43/98] test: caller requests nonexistent job --- nomad/job_endpoint_statuses_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nomad/job_endpoint_statuses_test.go b/nomad/job_endpoint_statuses_test.go index 3f477534553..2cf9f48f4de 100644 --- a/nomad/job_endpoint_statuses_test.go +++ b/nomad/job_endpoint_statuses_test.go @@ -177,14 +177,28 @@ func TestJob_Statuses(t *testing.T) { }, expect: jobs[0], }, + { + name: "missing", + jobs: []structs.NamespacedID{ + { + ID: "do-not-exist", + Namespace: "anywhere", + }, + }, + expect: nil, + }, } { t.Run(tc.name, func(t *testing.T) { resp = doRequest(t, &structs.JobStatusesRequest{ QueryOptions: tc.qo, Jobs: tc.jobs, }) - must.Len(t, 1, resp.Jobs, must.Sprint("expect only one job")) - must.Eq(t, tc.expect.ID, resp.Jobs[0].ID) + if tc.expect == nil { + must.Len(t, 0, resp.Jobs, must.Sprint("expect no jobs")) + } else { + must.Len(t, 1, resp.Jobs, must.Sprint("expect only one job")) + must.Eq(t, tc.expect.ID, resp.Jobs[0].ID) + } expectToken := "" if tc.expectNext > 0 { expectToken = strconv.FormatUint(tc.expectNext, 10) From 2b726dc8fc44dfe3c2b9623be1fc5cc934139a64 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 30 Apr 2024 11:42:33 -0400 Subject: [PATCH 44/98] Revert "set/slice init size optimization" This reverts commit 3e0ab6718e5eda8275bc399f53c2a423393cc221. --- nomad/job_endpoint_statuses.go | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/nomad/job_endpoint_statuses.go b/nomad/job_endpoint_statuses.go index 633fce0041c..8af55a51c8b 100644 --- a/nomad/job_endpoint_statuses.go +++ b/nomad/job_endpoint_statuses.go @@ -135,11 +135,8 @@ func (j *Job) Statuses( }) } - // little dance to avoid extra memory allocations - i := 0 - jobs := make([]structs.UIJob, len(args.Jobs)) - newJobs := set.New[structs.NamespacedID](len(args.Jobs)) - + jobs := make([]structs.UIJob, 0) + newJobs := set.New[structs.NamespacedID](0) pager, err := paginator.NewPaginator(iter, tokenizer, filters, args.QueryOptions, func(raw interface{}) error { job := raw.(*structs.Job) @@ -150,12 +147,7 @@ func (j *Job) Statuses( return err } - if len(args.Jobs) > 0 { - jobs[i] = uiJob - i++ - } else { - jobs = append(jobs, uiJob) - } + jobs = append(jobs, uiJob) newJobs.Insert(job.NamespacedID()) // by using the highest index we find on any job/alloc/ @@ -178,15 +170,6 @@ func (j *Job) Statuses( http.StatusInternalServerError, "failed to read result page: %v", err) } - // remove empty jobs (caller requested jobs that don't exist) - prevJobs.Remove(structs.NamespacedID{}) - jobsClean := make([]structs.UIJob, 0) - for _, j := range jobs { - if j.ID != "" && j.Namespace != "" { - jobsClean = append(jobsClean, j) - } - } - // if the page has updated, or a job has gone away, // bump the index to latest jobs entry. if !prevJobs.Empty() && !newJobs.Equal(prevJobs) { @@ -198,7 +181,7 @@ func (j *Job) Statuses( prevJobs = newJobs reply.QueryMeta.NextToken = nextToken - reply.Jobs = jobsClean + reply.Jobs = jobs return nil }} From 73440c2efb5214d687f2fa89a09b1eced1582368 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 30 Apr 2024 11:43:30 -0400 Subject: [PATCH 45/98] drop ActiveDeploymentID in favor of LatestDeployment which has more info --- nomad/job_endpoint_statuses.go | 16 ++++++---------- nomad/structs/job.go | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/nomad/job_endpoint_statuses.go b/nomad/job_endpoint_statuses.go index 8af55a51c8b..faab86b20bd 100644 --- a/nomad/job_endpoint_statuses.go +++ b/nomad/job_endpoint_statuses.go @@ -203,12 +203,12 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) Priority: job.Priority, Version: job.Version, // included here for completeness, populated below. - Allocs: nil, - GroupCountSum: 0, - ChildStatuses: nil, - ActiveDeploymentID: "", - SubmitTime: job.SubmitTime, - ModifyIndex: job.ModifyIndex, + Allocs: nil, + GroupCountSum: 0, + ChildStatuses: nil, + LatestDeployment: nil, + SubmitTime: job.SubmitTime, + ModifyIndex: job.ModifyIndex, } // the GroupCountSum will map to how many allocations we expect to run @@ -273,10 +273,6 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) return uiJob, idx, err } if deploy != nil { - if deploy.Active() { - uiJob.ActiveDeploymentID = deploy.ID - } - uiJob.LatestDeployment = &structs.JobStatusLatestDeployment{ ID: deploy.ID, IsActive: deploy.Active(), diff --git a/nomad/structs/job.go b/nomad/structs/job.go index ba60af6725d..6987c716bc4 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -18,19 +18,18 @@ const ( type UIJob struct { NamespacedID - Name string - Type string - NodePool string - Datacenters []string - Priority int - Allocs []JobStatusAlloc - GroupCountSum int - ChildStatuses []string - ActiveDeploymentID string - LatestDeployment *JobStatusLatestDeployment - Version uint64 - SubmitTime int64 - ModifyIndex uint64 + Name string + Type string + NodePool string + Datacenters []string + Priority int + Allocs []JobStatusAlloc + GroupCountSum int + ChildStatuses []string + LatestDeployment *JobStatusLatestDeployment + Version uint64 + SubmitTime int64 + ModifyIndex uint64 } type JobStatusesRequest struct { From 13b1d771fdb8271351ea8d6360e588b12d51b77c Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Tue, 30 Apr 2024 15:46:04 -0400 Subject: [PATCH 46/98] child job changes * add ability to include them at all: ?include_children=true * add ParentID to UIJob response, so they can be associated with the parent job * for parent jobs, ChildStatuses should always be an array, rather than nil (null) --- command/agent/job_endpoint.go | 6 ++++++ nomad/job_endpoint_statuses.go | 7 ++++++- nomad/structs/job.go | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b7e4f342ba6..49419d5cc13 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2122,6 +2122,12 @@ func (s *HTTPServer) JobStatusesRequest(resp http.ResponseWriter, req *http.Requ return nil, CodedError(405, ErrInvalidMethod) } + if includeChildren, err := parseBool(req, "include_children"); err != nil { + return nil, err + } else if includeChildren != nil { + args.IncludeChildren = *includeChildren + } + // ostensibly GETs should not accept structured body, but the HTTP spec // on this is more what you'd call "guidelines" than actual rules. if req.Body != http.NoBody { diff --git a/nomad/job_endpoint_statuses.go b/nomad/job_endpoint_statuses.go index faab86b20bd..d580b2223af 100644 --- a/nomad/job_endpoint_statuses.go +++ b/nomad/job_endpoint_statuses.go @@ -115,8 +115,11 @@ func (j *Job) Statuses( paginator.NamespaceFilter{ AllowableNamespaces: allowableNamespaces, }, - // skip child jobs; we'll look them up later, per parent. + // skip child jobs unless requested to include them paginator.GenericFilter{Allow: func(i interface{}) (bool, error) { + if args.IncludeChildren { + return true, nil + } job := i.(*structs.Job) return job.ParentID == "", nil }}, @@ -202,6 +205,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) Datacenters: job.Datacenters, Priority: job.Priority, Version: job.Version, + ParentID: job.ParentID, // included here for completeness, populated below. Allocs: nil, GroupCountSum: 0, @@ -219,6 +223,7 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) // collect the statuses of child jobs if job.IsParameterized() || job.IsPeriodic() { + uiJob.ChildStatuses = make([]string, 0) // set to not-nil children, err := store.JobsByIDPrefix(ws, job.Namespace, job.ID, state.SortDefault) if err != nil { return uiJob, idx, err diff --git a/nomad/structs/job.go b/nomad/structs/job.go index 6987c716bc4..f34a3613c85 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -26,6 +26,7 @@ type UIJob struct { Allocs []JobStatusAlloc GroupCountSum int ChildStatuses []string + ParentID string LatestDeployment *JobStatusLatestDeployment Version uint64 SubmitTime int64 @@ -33,7 +34,8 @@ type UIJob struct { } type JobStatusesRequest struct { - Jobs []NamespacedID + Jobs []NamespacedID + IncludeChildren bool QueryOptions } From f999fbf02ee606221ca20496e1ff096150a89fa4 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Thu, 2 May 2024 08:48:08 -0400 Subject: [PATCH 47/98] Healthy *bool and add FollowupEvalID --- nomad/job_endpoint_statuses.go | 13 +++++++------ nomad/structs/job.go | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/nomad/job_endpoint_statuses.go b/nomad/job_endpoint_statuses.go index d580b2223af..3976a6ea44d 100644 --- a/nomad/job_endpoint_statuses.go +++ b/nomad/job_endpoint_statuses.go @@ -255,15 +255,16 @@ func UIJobFromJob(ws memdb.WatchSet, store *state.StateStore, job *structs.Job) } for _, a := range allocs { alloc := structs.JobStatusAlloc{ - ID: a.ID, - Group: a.TaskGroup, - ClientStatus: a.ClientStatus, - NodeID: a.NodeID, - JobVersion: a.Job.Version, + ID: a.ID, + Group: a.TaskGroup, + ClientStatus: a.ClientStatus, + NodeID: a.NodeID, + JobVersion: a.Job.Version, + FollowupEvalID: a.FollowupEvalID, } if a.DeploymentStatus != nil { alloc.DeploymentStatus.Canary = a.DeploymentStatus.IsCanary() - alloc.DeploymentStatus.Healthy = a.DeploymentStatus.IsHealthy() + alloc.DeploymentStatus.Healthy = a.DeploymentStatus.Healthy } uiJob.Allocs = append(uiJob.Allocs, alloc) diff --git a/nomad/structs/job.go b/nomad/structs/job.go index f34a3613c85..2aa653d1520 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -51,11 +51,12 @@ type JobStatusAlloc struct { NodeID string DeploymentStatus JobStatusDeployment JobVersion uint64 + FollowupEvalID string } type JobStatusDeployment struct { Canary bool - Healthy bool + Healthy *bool } type JobStatusLatestDeployment struct { From 1e675f8db919504267a5c97129d2056a163a395c Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 18 Jan 2024 13:35:54 -0500 Subject: [PATCH 48/98] Hook and latch on the initial index --- ui/app/adapters/job.js | 26 +++++++++ ui/app/adapters/watchable.js | 99 +++++++++++++++++++++++++++----- ui/app/routes/jobs/index.js | 36 +++++++++--- ui/app/services/watch-list.js | 1 + ui/app/utils/properties/watch.js | 2 +- 5 files changed, 142 insertions(+), 22 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 6441d57391c..e3899ef939c 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -202,4 +202,30 @@ export default class JobAdapter extends WatchableNamespaceIDs { return wsUrl; } + + query(store, type, query, snapshotRecordArray, options) { + console.log('querying', query); + let { queryType } = query; + options = options || {}; + options.adapterOptions = options.adapterOptions || {}; + if (queryType === 'initialize') { + // options.url = this.urlForQuery(query, type.modelName); + options.adapterOptions.method = 'GET'; + } else if (queryType === 'update') { + // options.url = this.urlForUpdateQuery(query, type.modelName); + options.adapterOptions.method = 'POST'; + options.adapterOptions.watch = true; + // TODO: probably use watchList to get the index of "/v1/jobs/statuses3?meta=true&queryType=initialize" presuming it's already been set there. + // TODO: a direct lookup like this is the wrong way to do it. Gotta getIndexFor os something. + // options.adapterOptions.knownIndex = + // this.watchList.list[ + // '/v1/jobs/statuses3?meta=true&queryType=initialize' + // ]; + } + return super.query(store, type, query, snapshotRecordArray, options); + } + + urlForQuery(query, modelName) { + return `/${this.namespace}/jobs/statuses3`; + } } diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 792e9f4bc0c..62d2eb79044 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -25,18 +25,46 @@ export default class Watchable extends ApplicationAdapter { // It's either this weird side-effecting thing that also requires a change // to ajaxOptions or overriding ajax completely. ajax(url, type, options) { + console.log('ajaxing', url, type, options); const hasParams = hasNonBlockingQueryParams(options); - if (!hasParams || type !== 'GET') return super.ajax(url, type, options); - - const params = { ...options.data }; - delete params.index; + console.log('hasParams', hasParams); + // if (!hasParams || type !== 'GET') return super.ajax(url, type, options); + console.log('LATCHING ON', url, options.data.index); + if (!hasParams) return super.ajax(url, type, options); + let params = { ...options.data }; + // delete params.queryType; + // TODO: TEMP; + if (type === 'POST') { + console.log( + 'ummm, maybe affect url here?', + url, + params, + queryString.stringify(params), + queryString + ); + let index = params.index; + delete params.index; + // Delete everything but index from params + // params = { index: params.index }; + // delete params.jobs; + // delete params.index; + // params = {}; + // url = `${url}?${queryString.stringify(params)}`; + url = `${url}?hash=${btoa(JSON.stringify(params))}&index=${index}`; + console.log('xxx url on the way out is', url); + console.log('xxx atob, ', JSON.stringify(params)); + } else { + // Options data gets appended as query params as part of ajaxOptions. + // In order to prevent doubling params, data should only include index + // at this point since everything else is added to the URL in advance. + options.data = options.data.index ? { index: options.data.index } : {}; - // Options data gets appended as query params as part of ajaxOptions. - // In order to prevent doubling params, data should only include index - // at this point since everything else is added to the URL in advance. - options.data = options.data.index ? { index: options.data.index } : {}; + delete params.index; + url = `${url}?${queryString.stringify(params)}`; + } + // debugger; - return super.ajax(`${url}?${queryString.stringify(params)}`, type, options); + return super.ajax(url, type, options); } findAll(store, type, sinceToken, snapshotRecordArray, additionalParams = {}) { @@ -96,6 +124,7 @@ export default class Watchable extends ApplicationAdapter { additionalParams = {} ) { const url = this.buildURL(type.modelName, null, null, 'query', query); + const method = get(options, 'adapterOptions.method') || 'GET'; let [urlPath, params] = url.split('?'); params = assign( queryString.parse(params) || {}, @@ -107,13 +136,43 @@ export default class Watchable extends ApplicationAdapter { if (get(options, 'adapterOptions.watch')) { // The intended query without additional blocking query params is used // to track the appropriate query index. - params.index = this.watchList.getIndexFor( - `${urlPath}?${queryString.stringify(query)}` - ); + + // if POST, dont get whole queryString, just the index + if (method === 'POST') { + // TODO: THURSDAY MORNING: THE CLUE IS ABOUT HERE. If I hardcode the index with meta value, it works. + // What I think I probably ought to be doing is, for posts, setIndexFor should take a signature of the body, rather than just the url. + // Even after I do that, though, I'm worried about the index "sticking" right. + + // params.index = this.watchList.getIndexFor(urlPath); + // TODO: TEMP HARDCODE + // If the hashed version already exists, use it: + let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; + console.log( + 'xxx urlPath', + hashifiedURL, + this.watchList.getIndexFor(hashifiedURL), + { params } + ); + // debugger; + if (this.watchList.getIndexFor(hashifiedURL) > 1) { + console.log('xxx HASHIFIED INDEX FOUND'); + params.index = this.watchList.getIndexFor(hashifiedURL); + } else { + console.log('xxx NO HASHIFIED INDEX FOUND. WHAT ABOUT STATUSES3?'); + params.index = this.watchList.getIndexFor( + '/v1/jobs/statuses3?meta=true&queryType=initialize' + ); + console.log('xxx params.index', params.index); + } + } else { + params.index = this.watchList.getIndexFor( + `${urlPath}?${queryString.stringify(query)}` + ); + } } const signal = get(options, 'adapterOptions.abortController.signal'); - return this.ajax(urlPath, 'GET', { + return this.ajax(urlPath, method, { signal, data: params, }).then((payload) => { @@ -206,11 +265,23 @@ export default class Watchable extends ApplicationAdapter { } handleResponse(status, headers, payload, requestData) { + console.log('handling response', requestData, payload); // Some browsers lowercase all headers. Others keep them // case sensitive. const newIndex = headers['x-nomad-index'] || headers['X-Nomad-Index']; if (newIndex) { - this.watchList.setIndexFor(requestData.url, newIndex); + if (requestData.method === 'POST') { + // without the last &index= bit + this.watchList.setIndexFor(requestData.url.split('&')[0], newIndex); + console.log( + 'watchlist updated for', + requestData.url.split('&')[0], + newIndex + ); + } else { + this.watchList.setIndexFor(requestData.url, newIndex); + console.log('watchlist updated for', requestData.url, newIndex); + } } return super.handleResponse(...arguments); diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index fd81cf89e78..437f1b85195 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -24,25 +24,47 @@ export default class IndexRoute extends Route.extend( }, }; - model(params) { + async model(params) { + const jobs = await this.store + .query('job', { + namespace: params.qpNamespace, + meta: true, + queryType: 'initialize', + }) + .catch(notifyForbidden(this)); + return RSVP.hash({ - jobs: this.store - .query('job', { namespace: params.qpNamespace, meta: true }) - .catch(notifyForbidden(this)), + jobs, namespaces: this.store.findAll('namespace'), nodePools: this.store.findAll('node-pool'), }); } - startWatchers(controller) { + startWatchers(controller, model) { controller.set('namespacesWatch', this.watchNamespaces.perform()); + // controller.set( + // 'modelWatch', + // this.watchJobs.perform({ namespace: controller.qpNamespace, meta: true }) + // ); controller.set( - 'modelWatch', - this.watchJobs.perform({ namespace: controller.qpNamespace, meta: true }) + 'jobsWatch', + this.watchJobs.perform({ + namespace: controller.qpNamespace, + meta: true, + queryType: 'update', + jobs: model.jobs.map((job) => { + // TODO: maybe this should be set on controller for user-controlled updates? + return { + id: job.plainId, + namespace: job.belongsTo('namespace').id(), + }; + }), + }) ); } @watchQuery('job') watchJobs; + // @watchQuery('job', { queryType: 'update' }) watchJobsUpdate; @watchAll('namespace') watchNamespaces; @collect('watchJobs', 'watchNamespaces') watchers; } diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 80a44d138c1..b52b6041e6e 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -29,5 +29,6 @@ export default class WatchListService extends Service { setIndexFor(url, value) { list[url] = +value; + console.log('xxx total list is now', list); } } diff --git a/ui/app/utils/properties/watch.js b/ui/app/utils/properties/watch.js index 6ff96e18d57..a63aa89318a 100644 --- a/ui/app/utils/properties/watch.js +++ b/ui/app/utils/properties/watch.js @@ -135,7 +135,7 @@ export function watchAll(modelName) { } export function watchQuery(modelName) { - return task(function* (params, throttle = 10000) { + return task(function* (params, throttle = 5000) { assert( 'To watch a query, the adapter for the type being queried MUST extend Watchable', this.store.adapterFor(modelName) instanceof Watchable From 4c415a3c30b14c411787fbe0cca4f3efac21c2e6 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 18 Jan 2024 21:24:47 -0500 Subject: [PATCH 49/98] Serialization and restart of controller and table --- ui/app/adapters/watchable.js | 4 +- ui/app/controllers/jobs/index.js | 361 ++------------------------- ui/app/controllers/jobs/index_old.js | 352 ++++++++++++++++++++++++++ ui/app/routes/jobs/index.js | 19 +- ui/app/serializers/job.js | 52 ++++ ui/app/services/watch-list.js | 2 +- ui/app/templates/jobs/index.hbs | 247 +++--------------- ui/app/templates/jobs/index_old.hbs | 222 ++++++++++++++++ ui/app/utils/properties/watch.js | 2 +- 9 files changed, 706 insertions(+), 555 deletions(-) create mode 100644 ui/app/controllers/jobs/index_old.js create mode 100644 ui/app/templates/jobs/index_old.hbs diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 62d2eb79044..3def92e5a09 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -29,9 +29,9 @@ export default class Watchable extends ApplicationAdapter { const hasParams = hasNonBlockingQueryParams(options); console.log('hasParams', hasParams); // if (!hasParams || type !== 'GET') return super.ajax(url, type, options); - console.log('LATCHING ON', url, options.data.index); + console.log('LATCHING ON', url, options?.data.index); if (!hasParams) return super.ajax(url, type, options); - let params = { ...options.data }; + let params = { ...options?.data }; // delete params.queryType; // TODO: TEMP; if (type === 'POST') { diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index a095e5154fc..6eb25b9d60b 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -3,355 +3,40 @@ * SPDX-License-Identifier: BUSL-1.1 */ -//@ts-check +// @ts-check -/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ +import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; -import { alias, readOnly } from '@ember/object/computed'; -import Controller from '@ember/controller'; -import { computed, action } from '@ember/object'; -import { scheduleOnce } from '@ember/runloop'; -import intersection from 'lodash.intersection'; -import Sortable from 'nomad-ui/mixins/sortable'; -import Searchable from 'nomad-ui/mixins/searchable'; -import { - serialize, - deserializedQueryParam as selection, -} from 'nomad-ui/utils/qp-serialize'; -import classic from 'ember-classic-decorator'; +import { action } from '@ember/object'; -const DEFAULT_SORT_PROPERTY = 'modifyIndex'; -const DEFAULT_SORT_DESCENDING = true; +const ALL_NAMESPACE_WILDCARD = '*'; -@classic -export default class IndexController extends Controller.extend( - Sortable, - Searchable -) { +export default class JobsIndexController extends Controller { + // @service router; + // @service store; @service system; - @service userSettings; - @service router; isForbidden = false; - queryParams = [ - { - currentPage: 'page', - }, - { - searchTerm: 'search', - }, - { - sortProperty: 'sort', - }, - { - sortDescending: 'desc', - }, - { - qpType: 'type', - }, - { - qpStatus: 'status', - }, - { - qpDatacenter: 'dc', - }, - { - qpPrefix: 'prefix', - }, - { - qpNamespace: 'namespace', - }, - { - qpNodePool: 'nodePool', - }, - ]; - - qpNamespace = '*'; - - currentPage = 1; - @readOnly('userSettings.pageSize') pageSize; - - sortProperty = DEFAULT_SORT_PROPERTY; - sortDescending = DEFAULT_SORT_DESCENDING; - - @computed - get searchProps() { - return ['id', 'name']; - } - - @computed - get fuzzySearchProps() { - return ['name']; - } - - fuzzySearchEnabled = true; - - qpType = ''; - qpStatus = ''; - qpDatacenter = ''; - qpPrefix = ''; - qpNodePool = ''; - - @selection('qpType') selectionType; - @selection('qpStatus') selectionStatus; - @selection('qpDatacenter') selectionDatacenter; - @selection('qpPrefix') selectionPrefix; - @selection('qpNodePool') selectionNodePool; - - @computed - get optionsType() { - return [ - { key: 'batch', label: 'Batch' }, - { key: 'pack', label: 'Pack' }, - { key: 'parameterized', label: 'Parameterized' }, - { key: 'periodic', label: 'Periodic' }, - { key: 'service', label: 'Service' }, - { key: 'system', label: 'System' }, - { key: 'sysbatch', label: 'System Batch' }, - ]; - } - - @computed - get optionsStatus() { + get tableColumns() { return [ - { key: 'pending', label: 'Pending' }, - { key: 'running', label: 'Running' }, - { key: 'dead', label: 'Dead' }, - ]; - } - - @computed('selectionDatacenter', 'visibleJobs.[]') - get optionsDatacenter() { - const flatten = (acc, val) => acc.concat(val); - const allDatacenters = new Set( - this.visibleJobs.mapBy('datacenters').reduce(flatten, []) - ); - - // Remove any invalid datacenters from the query param/selection - const availableDatacenters = Array.from(allDatacenters).compact(); - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpDatacenter', - serialize(intersection(availableDatacenters, this.selectionDatacenter)) - ); - }); - - return availableDatacenters.sort().map((dc) => ({ key: dc, label: dc })); - } - - @computed('selectionPrefix', 'visibleJobs.[]') - get optionsPrefix() { - // A prefix is defined as the start of a job name up to the first - or . - // ex: mktg-analytics -> mktg, ds.supermodel.classifier -> ds - const hasPrefix = /.[-._]/; - - // Collect and count all the prefixes - const allNames = this.visibleJobs.mapBy('name'); - const nameHistogram = allNames.reduce((hist, name) => { - if (hasPrefix.test(name)) { - const prefix = name.match(/(.+?)[-._]/)[1]; - hist[prefix] = hist[prefix] ? hist[prefix] + 1 : 1; - } - return hist; - }, {}); - - // Convert to an array - const nameTable = Object.keys(nameHistogram).map((key) => ({ - prefix: key, - count: nameHistogram[key], - })); - - // Only consider prefixes that match more than one name - const prefixes = nameTable.filter((name) => name.count > 1); - - // Remove any invalid prefixes from the query param/selection - const availablePrefixes = prefixes.mapBy('prefix'); - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpPrefix', - serialize(intersection(availablePrefixes, this.selectionPrefix)) - ); - }); - - // Sort, format, and include the count in the label - return prefixes.sortBy('prefix').map((name) => ({ - key: name.prefix, - label: `${name.prefix} (${name.count})`, - })); - } - - @computed('qpNamespace', 'model.namespaces.[]') - get optionsNamespaces() { - const availableNamespaces = this.model.namespaces.map((namespace) => ({ - key: namespace.name, - label: namespace.name, - })); - - availableNamespaces.unshift({ - key: '*', - label: 'All (*)', - }); - - // Unset the namespace selection if it was server-side deleted - if ( - this.qpNamespace && - !availableNamespaces.mapBy('key').includes(this.qpNamespace) - ) { - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set('qpNamespace', '*'); + 'name', + this.system.shouldShowNamespaces ? 'namespace' : null, + this.system.shouldShowNodepools ? 'node pools' : null, // TODO: implement on system service + 'status', + 'type', + 'priority', + 'summary', + ] + .filter((c) => !!c) + .map((c) => { + return { + label: c.charAt(0).toUpperCase() + c.slice(1), + }; }); - } - - return availableNamespaces; - } - - @computed('selectionNodePool', 'model.nodePools.[]') - get optionsNodePool() { - const availableNodePools = this.model.nodePools; - - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpNodePool', - serialize( - intersection( - availableNodePools.map(({ name }) => name), - this.selectionNodePool - ) - ) - ); - }); - - return availableNodePools.map((nodePool) => ({ - key: nodePool.name, - label: nodePool.name, - })); - } - - /** - Visible jobs are those that match the selected namespace and aren't children - of periodic or parameterized jobs. - */ - @computed('model.jobs.@each.parent') - get visibleJobs() { - if (!this.model || !this.model.jobs) return []; - return this.model.jobs - .compact() - .filter((job) => !job.isNew) - .filter((job) => !job.get('parent.content')); - } - - @computed( - 'visibleJobs.[]', - 'selectionType', - 'selectionStatus', - 'selectionDatacenter', - 'selectionNodePool', - 'selectionPrefix' - ) - get filteredJobs() { - const { - selectionType: types, - selectionStatus: statuses, - selectionDatacenter: datacenters, - selectionPrefix: prefixes, - selectionNodePool: nodePools, - } = this; - - // A job must match ALL filter facets, but it can match ANY selection within a facet - // Always return early to prevent unnecessary facet predicates. - return this.visibleJobs.filter((job) => { - const shouldShowPack = types.includes('pack') && job.displayType.isPack; - - if (types.length && shouldShowPack) { - return true; - } - - if (types.length && !types.includes(job.get('displayType.type'))) { - return false; - } - - if (statuses.length && !statuses.includes(job.get('status'))) { - return false; - } - - if ( - datacenters.length && - !job.get('datacenters').find((dc) => datacenters.includes(dc)) - ) { - return false; - } - - if (nodePools.length && !nodePools.includes(job.get('nodePool'))) { - return false; - } - - const name = job.get('name'); - if ( - prefixes.length && - !prefixes.find((prefix) => name.startsWith(prefix)) - ) { - return false; - } - - return true; - }); - } - - // eslint-disable-next-line ember/require-computed-property-dependencies - @computed('searchTerm') - get sortAtLastSearch() { - return { - sortProperty: this.sortProperty, - sortDescending: this.sortDescending, - searchTerm: this.searchTerm, - }; - } - - @computed( - 'searchTerm', - 'sortAtLastSearch.{sortDescending,sortProperty}', - 'sortDescending', - 'sortProperty' - ) - get prioritizeSearchOrder() { - let shouldPrioritizeSearchOrder = - !!this.searchTerm && - this.sortAtLastSearch.sortProperty === this.sortProperty && - this.sortAtLastSearch.sortDescending === this.sortDescending; - if (shouldPrioritizeSearchOrder) { - /* eslint-disable ember/no-side-effects */ - this.set('sortDescending', DEFAULT_SORT_DESCENDING); - this.set('sortProperty', DEFAULT_SORT_PROPERTY); - this.set('sortAtLastSearch.sortProperty', DEFAULT_SORT_PROPERTY); - this.set('sortAtLastSearch.sortDescending', DEFAULT_SORT_DESCENDING); - } - /* eslint-enable ember/no-side-effects */ - return shouldPrioritizeSearchOrder; - } - - @alias('filteredJobs') listToSearch; - @alias('listSearched') listToSort; - - // sortedJobs is what we use to populate the table; - // If the user has searched but not sorted, we return the (fuzzy) searched list verbatim - // If the user has sorted, we allow the fuzzy search to filter down the list, but return it in a sorted order. - get sortedJobs() { - return this.prioritizeSearchOrder ? this.listSearched : this.listSorted; - } - - isShowingDeploymentDetails = false; - - setFacetQueryParam(queryParam, selection) { - this.set(queryParam, serialize(selection)); } - @action - goToRun() { - this.router.transitionTo('jobs.run'); + get jobs() { + return this.model.jobs; } } diff --git a/ui/app/controllers/jobs/index_old.js b/ui/app/controllers/jobs/index_old.js new file mode 100644 index 00000000000..86ca64abed7 --- /dev/null +++ b/ui/app/controllers/jobs/index_old.js @@ -0,0 +1,352 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +//@ts-check + +/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ +import { inject as service } from '@ember/service'; +import { alias, readOnly } from '@ember/object/computed'; +import Controller from '@ember/controller'; +import { computed, action } from '@ember/object'; +import { scheduleOnce } from '@ember/runloop'; +import intersection from 'lodash.intersection'; +import Sortable from 'nomad-ui/mixins/sortable'; +import Searchable from 'nomad-ui/mixins/searchable'; +import { + serialize, + deserializedQueryParam as selection, +} from 'nomad-ui/utils/qp-serialize'; +import classic from 'ember-classic-decorator'; + +const DEFAULT_SORT_PROPERTY = 'modifyIndex'; +const DEFAULT_SORT_DESCENDING = true; + +@classic +export default class IndexController extends Controller.extend( + Sortable, + Searchable +) { + @service system; + @service userSettings; + @service router; + + isForbidden = false; + + queryParams = [ + { + currentPage: 'page', + }, + { + searchTerm: 'search', + }, + { + sortProperty: 'sort', + }, + { + sortDescending: 'desc', + }, + { + qpType: 'type', + }, + { + qpStatus: 'status', + }, + { + qpDatacenter: 'dc', + }, + { + qpPrefix: 'prefix', + }, + { + qpNamespace: 'namespace', + }, + { + qpNodePool: 'nodePool', + }, + ]; + + currentPage = 1; + @readOnly('userSettings.pageSize') pageSize; + + sortProperty = DEFAULT_SORT_PROPERTY; + sortDescending = DEFAULT_SORT_DESCENDING; + + @computed + get searchProps() { + return ['id', 'name']; + } + + @computed + get fuzzySearchProps() { + return ['name']; + } + + fuzzySearchEnabled = true; + + qpType = ''; + qpStatus = ''; + qpDatacenter = ''; + qpPrefix = ''; + qpNodePool = ''; + + @selection('qpType') selectionType; + @selection('qpStatus') selectionStatus; + @selection('qpDatacenter') selectionDatacenter; + @selection('qpPrefix') selectionPrefix; + @selection('qpNodePool') selectionNodePool; + + @computed + get optionsType() { + return [ + { key: 'batch', label: 'Batch' }, + { key: 'pack', label: 'Pack' }, + { key: 'parameterized', label: 'Parameterized' }, + { key: 'periodic', label: 'Periodic' }, + { key: 'service', label: 'Service' }, + { key: 'system', label: 'System' }, + { key: 'sysbatch', label: 'System Batch' }, + ]; + } + + @computed + get optionsStatus() { + return [ + { key: 'pending', label: 'Pending' }, + { key: 'running', label: 'Running' }, + { key: 'dead', label: 'Dead' }, + ]; + } + + @computed('selectionDatacenter', 'visibleJobs.[]') + get optionsDatacenter() { + const flatten = (acc, val) => acc.concat(val); + const allDatacenters = new Set( + this.visibleJobs.mapBy('datacenters').reduce(flatten, []) + ); + + // Remove any invalid datacenters from the query param/selection + const availableDatacenters = Array.from(allDatacenters).compact(); + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set( + 'qpDatacenter', + serialize(intersection(availableDatacenters, this.selectionDatacenter)) + ); + }); + + return availableDatacenters.sort().map((dc) => ({ key: dc, label: dc })); + } + + @computed('selectionPrefix', 'visibleJobs.[]') + get optionsPrefix() { + // A prefix is defined as the start of a job name up to the first - or . + // ex: mktg-analytics -> mktg, ds.supermodel.classifier -> ds + const hasPrefix = /.[-._]/; + + // Collect and count all the prefixes + const allNames = this.visibleJobs.mapBy('name'); + const nameHistogram = allNames.reduce((hist, name) => { + if (hasPrefix.test(name)) { + const prefix = name.match(/(.+?)[-._]/)[1]; + hist[prefix] = hist[prefix] ? hist[prefix] + 1 : 1; + } + return hist; + }, {}); + + // Convert to an array + const nameTable = Object.keys(nameHistogram).map((key) => ({ + prefix: key, + count: nameHistogram[key], + })); + + // Only consider prefixes that match more than one name + const prefixes = nameTable.filter((name) => name.count > 1); + + // Remove any invalid prefixes from the query param/selection + const availablePrefixes = prefixes.mapBy('prefix'); + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set( + 'qpPrefix', + serialize(intersection(availablePrefixes, this.selectionPrefix)) + ); + }); + + // Sort, format, and include the count in the label + return prefixes.sortBy('prefix').map((name) => ({ + key: name.prefix, + label: `${name.prefix} (${name.count})`, + })); + } + + @computed('qpNamespace', 'model.namespaces.[]') + get optionsNamespaces() { + const availableNamespaces = this.model.namespaces.map((namespace) => ({ + key: namespace.name, + label: namespace.name, + })); + + availableNamespaces.unshift({ + key: '*', + label: 'All (*)', + }); + + // Unset the namespace selection if it was server-side deleted + if (!availableNamespaces.mapBy('key').includes(this.qpNamespace)) { + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set('qpNamespace', '*'); + }); + } + + return availableNamespaces; + } + + @computed('selectionNodePool', 'model.nodePools.[]') + get optionsNodePool() { + const availableNodePools = this.model.nodePools; + + scheduleOnce('actions', () => { + // eslint-disable-next-line ember/no-side-effects + this.set( + 'qpNodePool', + serialize( + intersection( + availableNodePools.map(({ name }) => name), + this.selectionNodePool + ) + ) + ); + }); + + return availableNodePools.map((nodePool) => ({ + key: nodePool.name, + label: nodePool.name, + })); + } + + /** + Visible jobs are those that match the selected namespace and aren't children + of periodic or parameterized jobs. + */ + @computed('model.jobs.@each.parent') + get visibleJobs() { + if (!this.model || !this.model.jobs) return []; + return this.model.jobs + .compact() + .filter((job) => !job.isNew) + .filter((job) => !job.get('parent.content')); + } + + @computed( + 'visibleJobs.[]', + 'selectionType', + 'selectionStatus', + 'selectionDatacenter', + 'selectionNodePool', + 'selectionPrefix' + ) + get filteredJobs() { + const { + selectionType: types, + selectionStatus: statuses, + selectionDatacenter: datacenters, + selectionPrefix: prefixes, + selectionNodePool: nodePools, + } = this; + + // A job must match ALL filter facets, but it can match ANY selection within a facet + // Always return early to prevent unnecessary facet predicates. + return this.visibleJobs.filter((job) => { + const shouldShowPack = types.includes('pack') && job.displayType.isPack; + + if (types.length && shouldShowPack) { + return true; + } + + if (types.length && !types.includes(job.get('displayType.type'))) { + return false; + } + + if (statuses.length && !statuses.includes(job.get('status'))) { + return false; + } + + if ( + datacenters.length && + !job.get('datacenters').find((dc) => datacenters.includes(dc)) + ) { + return false; + } + + if (nodePools.length && !nodePools.includes(job.get('nodePool'))) { + return false; + } + + const name = job.get('name'); + if ( + prefixes.length && + !prefixes.find((prefix) => name.startsWith(prefix)) + ) { + return false; + } + + return true; + }); + } + + // eslint-disable-next-line ember/require-computed-property-dependencies + @computed('searchTerm') + get sortAtLastSearch() { + return { + sortProperty: this.sortProperty, + sortDescending: this.sortDescending, + searchTerm: this.searchTerm, + }; + } + + @computed( + 'searchTerm', + 'sortAtLastSearch.{sortDescending,sortProperty}', + 'sortDescending', + 'sortProperty' + ) + get prioritizeSearchOrder() { + let shouldPrioritizeSearchOrder = + !!this.searchTerm && + this.sortAtLastSearch.sortProperty === this.sortProperty && + this.sortAtLastSearch.sortDescending === this.sortDescending; + if (shouldPrioritizeSearchOrder) { + /* eslint-disable ember/no-side-effects */ + this.set('sortDescending', DEFAULT_SORT_DESCENDING); + this.set('sortProperty', DEFAULT_SORT_PROPERTY); + this.set('sortAtLastSearch.sortProperty', DEFAULT_SORT_PROPERTY); + this.set('sortAtLastSearch.sortDescending', DEFAULT_SORT_DESCENDING); + } + /* eslint-enable ember/no-side-effects */ + return shouldPrioritizeSearchOrder; + } + + @alias('filteredJobs') listToSearch; + @alias('listSearched') listToSort; + + // sortedJobs is what we use to populate the table; + // If the user has searched but not sorted, we return the (fuzzy) searched list verbatim + // If the user has sorted, we allow the fuzzy search to filter down the list, but return it in a sorted order. + get sortedJobs() { + return this.prioritizeSearchOrder ? this.listSearched : this.listSorted; + } + + isShowingDeploymentDetails = false; + + setFacetQueryParam(queryParam, selection) { + this.set(queryParam, serialize(selection)); + } + + @action + goToRun() { + this.router.transitionTo('jobs.run'); + } +} diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 437f1b85195..2d688f23311 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -29,6 +29,7 @@ export default class IndexRoute extends Route.extend( .query('job', { namespace: params.qpNamespace, meta: true, + per_page: 10, queryType: 'initialize', }) .catch(notifyForbidden(this)); @@ -42,13 +43,18 @@ export default class IndexRoute extends Route.extend( startWatchers(controller, model) { controller.set('namespacesWatch', this.watchNamespaces.perform()); - // controller.set( - // 'modelWatch', - // this.watchJobs.perform({ namespace: controller.qpNamespace, meta: true }) - // ); controller.set( - 'jobsWatch', + 'modelWatch', this.watchJobs.perform({ + namespace: controller.qpNamespace, + per_page: 10, + meta: true, + queryType: 'initialize', + }) + ); + controller.set( + 'jobsWatch', + this.watchJobsAllocs.perform({ namespace: controller.qpNamespace, meta: true, queryType: 'update', @@ -64,7 +70,8 @@ export default class IndexRoute extends Route.extend( } @watchQuery('job') watchJobs; + @watchQuery('job', { queryType: 'update' }) watchJobsAllocs; // @watchQuery('job', { queryType: 'update' }) watchJobsUpdate; @watchAll('namespace') watchNamespaces; - @collect('watchJobs', 'watchNamespaces') watchers; + @collect('watchJobs', 'watchJobsAllocs', 'watchNamespaces') watchers; } diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index b40e84ce9b9..bf728714a05 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -60,6 +60,33 @@ export default class JobSerializer extends ApplicationSerializer { return super.normalize(typeHash, hash); } + normalizeQueryResponse(store, primaryModelClass, payload, id, requestType) { + // const jobs = Object.values(payload.Jobs); + console.log('normalized', payload); + const jobs = payload; + // Signal that it's a query response at individual normalization level for allocation placement + jobs.forEach((job) => { + if (job.Allocs) { + job.relationships = { + allocations: { + data: job.Allocs.map((alloc) => ({ + id: alloc.id, + type: 'allocation', + })), + }, + }; + } + job._aggregate = true; + }); + return super.normalizeQueryResponse( + store, + primaryModelClass, + jobs, + id, + requestType + ); + } + extractRelationships(modelClass, hash) { const namespace = !hash.NamespaceID || hash.NamespaceID === 'default' @@ -80,8 +107,33 @@ export default class JobSerializer extends ApplicationSerializer { ? JSON.parse(hash.ParentID)[0] : hash.PlainId; + if (hash._aggregate && hash.Allocs) { + // Manually push allocations to store + hash.Allocs.forEach((alloc) => { + this.store.push({ + data: { + id: alloc.ID, + type: 'allocation', + attributes: { + clientStatus: alloc.ClientStatus, + deploymentStatus: { + Healthy: alloc.DeploymentStatus.Healthy, + Canary: alloc.DeploymentStatus.Canary, + }, + }, + }, + }); + }); + + delete hash._aggregate; + } + return assign(super.extractRelationships(...arguments), { allocations: { + data: hash.Allocs?.map((alloc) => ({ + id: alloc.ID, + type: 'allocation', + })), links: { related: buildURL(`${jobURL}/allocations`, { namespace }), }, diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index b52b6041e6e..197bdc6124f 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -29,6 +29,6 @@ export default class WatchListService extends Service { setIndexFor(url, value) { list[url] = +value; - console.log('xxx total list is now', list); + console.log('total list is now', list); } } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 0c577a7e7a7..97f96df340c 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -5,218 +5,51 @@ {{page-title "Jobs"}}
-
-
- {{#if this.visibleJobs.length}} - - {{/if}} -
- {{#if (media "isMobile")}} -
- {{#if (can "run job" namespace=this.qpNamespace)}} - - Run Job - - {{else}} - - {{/if}} -
- {{/if}} -
-
- {{#if this.system.shouldShowNamespaces}} - - {{/if}} - - - - - -
-
- {{#if (not (media "isMobile"))}} -
- {{#if (can "run job" namespace=this.qpNamespace)}} - - Run Job - - {{else}} - - {{/if}} -
- {{/if}} -
{{#if this.isForbidden}} - {{else if this.sortedJobs}} - - - - - Name - + <:body as |B|> + + {{!-- {{#each this.tableColumns as |column|}} + {{get B.data (lowercase column.label)}} + {{/each}} --}} + {{B.data.name}} {{#if this.system.shouldShowNamespaces}} - - Namespace - + {{B.data.namespace}} + {{/if}} + {{#if this.system.shouldShowNodepools}} + {{B.data.nodepool}} {{/if}} - - Status - - - Type - - - Node Pool - - - Priority - - - Summary - - - - - - -
- - -
- + + STATUS PLACEHOLDER + + + {{B.data.type}} + + + {{B.data.priority}} + + + {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}}running
+ {{B.data.allocations.length}} total + + + + + + + {{else}} -
- {{#if (eq this.visibleJobs.length 0)}} -

- No Jobs -

-

- The cluster is currently empty. -

- {{else if (eq this.filteredJobs.length 0)}} -

- No Matches -

-

- No jobs match your current filter selection. -

- {{else if this.searchTerm}} -

- No Matches -

-

- No jobs match the term - - {{this.searchTerm}} - -

- {{/if}} -
+ No jobs : {{/if}}
\ No newline at end of file diff --git a/ui/app/templates/jobs/index_old.hbs b/ui/app/templates/jobs/index_old.hbs new file mode 100644 index 00000000000..4ed62d2b1b2 --- /dev/null +++ b/ui/app/templates/jobs/index_old.hbs @@ -0,0 +1,222 @@ +{{! + Copyright (c) HashiCorp, Inc. + SPDX-License-Identifier: BUSL-1.1 +~}} + +{{page-title "Jobs"}} +
+
+
+ {{#if this.visibleJobs.length}} + + {{/if}} +
+ {{#if (media "isMobile")}} +
+ {{#if (can "run job" namespace=this.qpNamespace)}} + + Run Job + + {{else}} + + {{/if}} +
+ {{/if}} +
+
+ {{#if this.system.shouldShowNamespaces}} + + {{/if}} + + + + + +
+
+ {{#if (not (media "isMobile"))}} +
+ {{#if (can "run job" namespace=this.qpNamespace)}} + + Run Job + + {{else}} + + {{/if}} +
+ {{/if}} +
+ {{#if this.isForbidden}} + + {{else if this.sortedJobs}} + + + + + Name + + {{#if this.system.shouldShowNamespaces}} + + Namespace + + {{/if}} + + Status + + + Type + + + Node Pool + + + Priority + + + Summary + + + + + + +
+ + +
+ + {{else}} +
+ {{#if (eq this.visibleJobs.length 0)}} +

+ No Jobs +

+

+ The cluster is currently empty. +

+ {{else if (eq this.filteredJobs.length 0)}} +

+ No Matches +

+

+ No jobs match your current filter selection. +

+ {{else if this.searchTerm}} +

+ No Matches +

+

+ No jobs match the term + + {{this.searchTerm}} + +

+ {{/if}} +
+ {{/if}} +
diff --git a/ui/app/utils/properties/watch.js b/ui/app/utils/properties/watch.js index a63aa89318a..99f8aa888eb 100644 --- a/ui/app/utils/properties/watch.js +++ b/ui/app/utils/properties/watch.js @@ -135,7 +135,7 @@ export function watchAll(modelName) { } export function watchQuery(modelName) { - return task(function* (params, throttle = 5000) { + return task(function* (params, throttle = 2000) { assert( 'To watch a query, the adapter for the type being queried MUST extend Watchable', this.store.adapterFor(modelName) instanceof Watchable From 3355da2da24b36ee2208bcccc7774d232d6bc622 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 19 Jan 2024 10:06:01 -0500 Subject: [PATCH 50/98] de-log --- ui/app/adapters/job.js | 7 ------ ui/app/adapters/watchable.js | 46 +++-------------------------------- ui/app/serializers/job.js | 1 - ui/app/services/watch-list.js | 1 - 4 files changed, 3 insertions(+), 52 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index e3899ef939c..140d2921246 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -204,7 +204,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { } query(store, type, query, snapshotRecordArray, options) { - console.log('querying', query); let { queryType } = query; options = options || {}; options.adapterOptions = options.adapterOptions || {}; @@ -215,12 +214,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { // options.url = this.urlForUpdateQuery(query, type.modelName); options.adapterOptions.method = 'POST'; options.adapterOptions.watch = true; - // TODO: probably use watchList to get the index of "/v1/jobs/statuses3?meta=true&queryType=initialize" presuming it's already been set there. - // TODO: a direct lookup like this is the wrong way to do it. Gotta getIndexFor os something. - // options.adapterOptions.knownIndex = - // this.watchList.list[ - // '/v1/jobs/statuses3?meta=true&queryType=initialize' - // ]; } return super.query(store, type, query, snapshotRecordArray, options); } diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 3def92e5a09..588f6d3b2fc 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -25,34 +25,15 @@ export default class Watchable extends ApplicationAdapter { // It's either this weird side-effecting thing that also requires a change // to ajaxOptions or overriding ajax completely. ajax(url, type, options) { - console.log('ajaxing', url, type, options); const hasParams = hasNonBlockingQueryParams(options); - console.log('hasParams', hasParams); // if (!hasParams || type !== 'GET') return super.ajax(url, type, options); - console.log('LATCHING ON', url, options?.data.index); if (!hasParams) return super.ajax(url, type, options); let params = { ...options?.data }; - // delete params.queryType; - // TODO: TEMP; + // TODO: Creating a hash of the params as watchList key feels a little hacky if (type === 'POST') { - console.log( - 'ummm, maybe affect url here?', - url, - params, - queryString.stringify(params), - queryString - ); let index = params.index; delete params.index; - // Delete everything but index from params - // params = { index: params.index }; - // delete params.jobs; - // delete params.index; - // params = {}; - // url = `${url}?${queryString.stringify(params)}`; url = `${url}?hash=${btoa(JSON.stringify(params))}&index=${index}`; - console.log('xxx url on the way out is', url); - console.log('xxx atob, ', JSON.stringify(params)); } else { // Options data gets appended as query params as part of ajaxOptions. // In order to prevent doubling params, data should only include index @@ -139,30 +120,15 @@ export default class Watchable extends ApplicationAdapter { // if POST, dont get whole queryString, just the index if (method === 'POST') { - // TODO: THURSDAY MORNING: THE CLUE IS ABOUT HERE. If I hardcode the index with meta value, it works. - // What I think I probably ought to be doing is, for posts, setIndexFor should take a signature of the body, rather than just the url. - // Even after I do that, though, I'm worried about the index "sticking" right. - - // params.index = this.watchList.getIndexFor(urlPath); - // TODO: TEMP HARDCODE // If the hashed version already exists, use it: let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; - console.log( - 'xxx urlPath', - hashifiedURL, - this.watchList.getIndexFor(hashifiedURL), - { params } - ); - // debugger; if (this.watchList.getIndexFor(hashifiedURL) > 1) { - console.log('xxx HASHIFIED INDEX FOUND'); params.index = this.watchList.getIndexFor(hashifiedURL); } else { - console.log('xxx NO HASHIFIED INDEX FOUND. WHAT ABOUT STATUSES3?'); + // TODO: De-hardcode the initialize query, identify it in watchlist somehow? params.index = this.watchList.getIndexFor( '/v1/jobs/statuses3?meta=true&queryType=initialize' ); - console.log('xxx params.index', params.index); } } else { params.index = this.watchList.getIndexFor( @@ -265,22 +231,16 @@ export default class Watchable extends ApplicationAdapter { } handleResponse(status, headers, payload, requestData) { - console.log('handling response', requestData, payload); // Some browsers lowercase all headers. Others keep them // case sensitive. const newIndex = headers['x-nomad-index'] || headers['X-Nomad-Index']; if (newIndex) { if (requestData.method === 'POST') { // without the last &index= bit + // TODO: this is a weird way to save key. this.watchList.setIndexFor(requestData.url.split('&')[0], newIndex); - console.log( - 'watchlist updated for', - requestData.url.split('&')[0], - newIndex - ); } else { this.watchList.setIndexFor(requestData.url, newIndex); - console.log('watchlist updated for', requestData.url, newIndex); } } diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index bf728714a05..6cbe1c2cf12 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -62,7 +62,6 @@ export default class JobSerializer extends ApplicationSerializer { normalizeQueryResponse(store, primaryModelClass, payload, id, requestType) { // const jobs = Object.values(payload.Jobs); - console.log('normalized', payload); const jobs = payload; // Signal that it's a query response at individual normalization level for allocation placement jobs.forEach((job) => { diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 197bdc6124f..80a44d138c1 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -29,6 +29,5 @@ export default class WatchListService extends Service { setIndexFor(url, value) { list[url] = +value; - console.log('total list is now', list); } } From a137c3d99811b792c1e877a4e2e78a22790465ff Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Sun, 21 Jan 2024 22:13:54 -0500 Subject: [PATCH 51/98] allocBlocks reimplemented at job model level --- ui/app/components/job-status/panel/steady.js | 1 + ui/app/controllers/jobs/index.js | 7 +- ui/app/models/job.js | 205 +++++++++++++++++++ ui/app/templates/jobs/index.hbs | 31 ++- 4 files changed, 238 insertions(+), 6 deletions(-) diff --git a/ui/app/components/job-status/panel/steady.js b/ui/app/components/job-status/panel/steady.js index 9e6f9c40d51..048db7ac92f 100644 --- a/ui/app/components/job-status/panel/steady.js +++ b/ui/app/components/job-status/panel/steady.js @@ -11,6 +11,7 @@ import { jobAllocStatuses } from '../../../utils/allocation-client-statuses'; export default class JobStatusPanelSteadyComponent extends Component { @alias('args.job') job; + // TODO: use the job model for this get allocTypes() { return jobAllocStatuses[this.args.job.type].map((type) => { return { diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 6eb25b9d60b..e26deba9b8b 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -12,7 +12,7 @@ import { action } from '@ember/object'; const ALL_NAMESPACE_WILDCARD = '*'; export default class JobsIndexController extends Controller { - // @service router; + @service router; // @service store; @service system; @@ -39,4 +39,9 @@ export default class JobsIndexController extends Controller { get jobs() { return this.model.jobs; } + + @action + gotoJob(job) { + this.router.transitionTo('jobs.job.index', job.idWithNamespace); + } } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index b873c0a5199..798042417dd 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -3,6 +3,8 @@ * SPDX-License-Identifier: BUSL-1.1 */ +// @ts-check + import { alias, equal, or, and, mapBy } from '@ember/object/computed'; import { computed } from '@ember/object'; import Model from '@ember-data/model'; @@ -11,6 +13,7 @@ import { fragment, fragmentArray } from 'ember-data-model-fragments/attributes'; import RSVP from 'rsvp'; import { assert } from '@ember/debug'; import classic from 'ember-classic-decorator'; +import { jobAllocStatuses } from '../utils/allocation-client-statuses'; const JOB_TYPES = ['service', 'batch', 'system', 'sysbatch']; @@ -30,6 +33,208 @@ export default class Job extends Model { @attr('date') submitTime; @attr('string') nodePool; // Jobs are related to Node Pools either directly or via its Namespace, but no relationship. + @attr('number') groupCountSum; + @attr('string') deploymentID; + + get allocTypes() { + return jobAllocStatuses[this.type].map((type) => { + return { + label: type, + }; + }); + } + + /** + * @typedef {Object} CurrentStatus + * @property {"Healthy"|"Failed"|"Degraded"|"Recovering"|"Complete"|"Running"} label - The current status of the job + * @property {"highlight"|"success"|"warning"|"critical"} state - + */ + + /** + * @typedef {Object} HealthStatus + * @property {Array} nonCanary + * @property {Array} canary + */ + + /** + * @typedef {Object} AllocationStatus + * @property {HealthStatus} healthy + * @property {HealthStatus} unhealthy + * @property {HealthStatus} health unknown + */ + + /** + * @typedef {Object} AllocationBlock + * @property {AllocationStatus} [running] + * @property {AllocationStatus} [pending] + * @property {AllocationStatus} [failed] + * @property {AllocationStatus} [lost] + * @property {AllocationStatus} [unplaced] + * @property {AllocationStatus} [complete] + */ + + /** + * Looks through running/pending allocations with the aim of filling up your desired number of allocations. + * If any desired remain, it will walk backwards through job versions and other allocation types to build + * a picture of the job's overall status. + * + * @returns {AllocationBlock} An object containing healthy non-canary allocations + * for each clientStatus. + */ + get allocBlocks() { + let availableSlotsToFill = this.totalAllocs; + + // Initialize allocationsOfShowableType with empty arrays for each clientStatus + /** + * @type {AllocationBlock} + */ + let allocationsOfShowableType = this.allocTypes.reduce( + (accumulator, type) => { + accumulator[type.label] = { healthy: { nonCanary: [] } }; + return accumulator; + }, + {} + ); + + // First accumulate the Running/Pending allocations + for (const alloc of this.allocations.filter( + (a) => a.clientStatus === 'running' || a.clientStatus === 'pending' + )) { + if (availableSlotsToFill === 0) { + break; + } + + const status = alloc.clientStatus; + allocationsOfShowableType[status].healthy.nonCanary.push(alloc); + availableSlotsToFill--; + } + + // Sort all allocs by jobVersion in descending order + const sortedAllocs = this.allocations + .filter( + (a) => a.clientStatus !== 'running' && a.clientStatus !== 'pending' + ) + .sort((a, b) => { + // First sort by jobVersion + if (a.jobVersion > b.jobVersion) return 1; + if (a.jobVersion < b.jobVersion) return -1; + + // If jobVersion is the same, sort by status order + if (a.jobVersion === b.jobVersion) { + return ( + jobAllocStatuses[this.type].indexOf(b.clientStatus) - + jobAllocStatuses[this.type].indexOf(a.clientStatus) + ); + } else { + return 0; + } + }) + .reverse(); + + // Iterate over the sorted allocs + for (const alloc of sortedAllocs) { + if (availableSlotsToFill === 0) { + break; + } + + const status = alloc.clientStatus; + // If the alloc has another clientStatus, add it to the corresponding list + // as long as we haven't reached the totalAllocs limit for that clientStatus + if ( + this.allocTypes.map(({ label }) => label).includes(status) && + allocationsOfShowableType[status].healthy.nonCanary.length < + this.totalAllocs + ) { + allocationsOfShowableType[status].healthy.nonCanary.push(alloc); + availableSlotsToFill--; + } + } + + // Handle unplaced allocs + if (availableSlotsToFill > 0) { + allocationsOfShowableType['unplaced'] = { + healthy: { + nonCanary: Array(availableSlotsToFill) + .fill() + .map(() => { + return { clientStatus: 'unplaced' }; + }), + }, + }; + } + + console.log('allocBlocks for', this.name, 'is', allocationsOfShowableType); + + return allocationsOfShowableType; + } + + /** + * A single status to indicate how a job is doing, based on running/healthy allocations vs desired. + * Possible statuses are: + * - Deploying: A deployment is actively taking place + * - Complete: (Batch/Sysbatch only) All expected allocations are complete + * - Running: (Batch/Sysbatch only) All expected allocations are running + * - Healthy: All expected allocations are running and healthy + * - Recovering: Some allocations are pending + * - Degraded: A deployment is not taking place, and some allocations are failed, lost, or unplaced + * - Failed: All allocations are failed, lost, or unplaced + * @returns {CurrentStatus} + */ + /** + * A general assessment for how a job is going, in a non-deployment state + * @returns {CurrentStatus} + */ + get aggregateAllocStatus() { + // If all allocs are running, the job is Healthy + const totalAllocs = this.totalAllocs; + console.log('groupCountSum is', totalAllocs); + console.log('ablocks are', this.allocBlocks); + + // If deploying: + if (this.deploymentID) { + return { label: 'Deploying', state: 'highlight' }; + } + + if (this.type === 'batch' || this.type === 'sysbatch') { + // If all the allocs are complete, the job is Complete + const completeAllocs = this.allocBlocks.complete?.healthy?.nonCanary; + if (completeAllocs?.length === totalAllocs) { + return { label: 'Complete', state: 'success' }; + } + + // If any allocations are running the job is "Running" + const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; + if (healthyAllocs?.length + completeAllocs?.length === totalAllocs) { + return { label: 'Running', state: 'success' }; + } + } + + const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; + if (healthyAllocs?.length === totalAllocs) { + return { label: 'Healthy', state: 'success' }; + } + + // If any allocations are pending the job is "Recovering" + const pendingAllocs = this.allocBlocks.pending?.healthy?.nonCanary; + if (pendingAllocs?.length > 0) { + return { label: 'Recovering', state: 'highlight' }; + } + + // If any allocations are failed, lost, or unplaced in a steady state, the job is "Degraded" + const failedOrLostAllocs = [ + ...this.allocBlocks.failed?.healthy?.nonCanary, + ...this.allocBlocks.lost?.healthy?.nonCanary, + ...this.allocBlocks.unplaced?.healthy?.nonCanary, + ]; + console.log('numFailedAllocs', failedOrLostAllocs.length); + // if (failedOrLostAllocs.length === totalAllocs) { + if (failedOrLostAllocs.length >= totalAllocs) { + // TODO: when totalAllocs only cares about latest version, change back to === + return { label: 'Failed', state: 'critical' }; + } else { + return { label: 'Degraded', state: 'warning' }; + } + } @fragment('structured-attributes') meta; get isPack() { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 97f96df340c..7a2e5e5d424 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -14,11 +14,31 @@ @columns={{this.tableColumns}} > <:body as |B|> - + {{!-- TODO: use --}} + {{!-- {{#each this.tableColumns as |column|}} {{get B.data (lowercase column.label)}} {{/each}} --}} - {{B.data.name}} + + + {{B.data.name}} + {{#if B.data.meta.structured.pack}} + + {{x-icon "box" class= "test"}} + Pack + + {{/if}} + + {{#if this.system.shouldShowNamespaces}} {{B.data.namespace}} {{/if}} @@ -26,7 +46,7 @@ {{B.data.nodepool}} {{/if}} - STATUS PLACEHOLDER + {{B.data.type}} @@ -35,8 +55,9 @@ {{B.data.priority}} - {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}}running
- {{B.data.allocations.length}} total + {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
+ {{B.data.allocations.length}} total
+ {{B.data.groupCountSum}} desired From 207ea9c9dc4f7d0fefd2df5bd665c7d1f0a5d6e3 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Sun, 21 Jan 2024 22:41:13 -0500 Subject: [PATCH 52/98] totalAllocs doesnt mean on jobmodel what it did in steady.js --- ui/app/models/job.js | 14 ++++++++++---- ui/app/templates/jobs/index.hbs | 6 +++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 798042417dd..8d87740eb3a 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -82,7 +82,7 @@ export default class Job extends Model { * for each clientStatus. */ get allocBlocks() { - let availableSlotsToFill = this.totalAllocs; + let availableSlotsToFill = this.groupCountSum; // Initialize allocationsOfShowableType with empty arrays for each clientStatus /** @@ -139,11 +139,11 @@ export default class Job extends Model { const status = alloc.clientStatus; // If the alloc has another clientStatus, add it to the corresponding list - // as long as we haven't reached the totalAllocs limit for that clientStatus + // as long as we haven't reached the groupCountSum limit for that clientStatus if ( this.allocTypes.map(({ label }) => label).includes(status) && allocationsOfShowableType[status].healthy.nonCanary.length < - this.totalAllocs + this.groupCountSum ) { allocationsOfShowableType[status].healthy.nonCanary.push(alloc); availableSlotsToFill--; @@ -186,7 +186,7 @@ export default class Job extends Model { */ get aggregateAllocStatus() { // If all allocs are running, the job is Healthy - const totalAllocs = this.totalAllocs; + const totalAllocs = this.groupCountSum; console.log('groupCountSum is', totalAllocs); console.log('ablocks are', this.allocBlocks); @@ -210,11 +210,14 @@ export default class Job extends Model { } const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; + console.log('healthyAllocs', this.name, healthyAllocs, totalAllocs); if (healthyAllocs?.length === totalAllocs) { return { label: 'Healthy', state: 'success' }; } // If any allocations are pending the job is "Recovering" + // TODO: weird, but batch jobs (which do not have deployments!) go into "recovering" right away, since some of their statuses are "pending" as they come online. + // This feels a little wrong. const pendingAllocs = this.allocBlocks.pending?.healthy?.nonCanary; if (pendingAllocs?.length > 0) { return { label: 'Recovering', state: 'highlight' }; @@ -226,6 +229,9 @@ export default class Job extends Model { ...this.allocBlocks.lost?.healthy?.nonCanary, ...this.allocBlocks.unplaced?.healthy?.nonCanary, ]; + // TODO: GroupCountSum for a parameterized parent job is the count present at group level, but that's not quite true, as the parent job isn't expecting any allocs, its children are. Chat with BFF about this. + // TODO: handle garbage collected cases not showing "failed" for batch jobs here maybe? + console.log('numFailedAllocs', failedOrLostAllocs.length); // if (failedOrLostAllocs.length === totalAllocs) { if (failedOrLostAllocs.length >= totalAllocs) { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7a2e5e5d424..5958233d245 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -46,7 +46,7 @@ {{B.data.nodepool}} {{/if}} - + {{B.data.type}} @@ -58,6 +58,10 @@ {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
{{B.data.allocations.length}} total
{{B.data.groupCountSum}} desired +
+
+ +
From af1d3a7109e0664de56be9a209740a618d82689f Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 23 Jan 2024 20:16:46 -0500 Subject: [PATCH 53/98] Hamburgers to sausages --- ui/app/adapters/job.js | 14 ++++ .../job-status/allocation-status-block.js | 3 + .../job-status/allocation-status-row.hbs | 14 +++- ui/app/components/job-status/panel/steady.js | 10 +++ ui/app/controllers/jobs/index.js | 78 +++++++++++++++++++ ui/app/models/job.js | 13 +++- ui/app/routes/jobs/index.js | 17 +++- .../styles/components/job-status-panel.scss | 18 +++++ ui/app/templates/jobs/index.hbs | 30 +++---- 9 files changed, 176 insertions(+), 21 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 140d2921246..b6b7938e7da 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -218,6 +218,20 @@ export default class JobAdapter extends WatchableNamespaceIDs { return super.query(store, type, query, snapshotRecordArray, options); } + handleResponse(status, headers) { + /** + * @type {Object} + */ + const result = super.handleResponse(...arguments); + if (result) { + result.meta = result.meta || {}; + if (headers['x-nomad-nexttoken']) { + result.meta.nextToken = headers['x-nomad-nexttoken']; + } + } + return result; + } + urlForQuery(query, modelName) { return `/${this.namespace}/jobs/statuses3`; } diff --git a/ui/app/components/job-status/allocation-status-block.js b/ui/app/components/job-status/allocation-status-block.js index 189014a9aaa..f8866bad5f6 100644 --- a/ui/app/components/job-status/allocation-status-block.js +++ b/ui/app/components/job-status/allocation-status-block.js @@ -7,6 +7,9 @@ import Component from '@glimmer/component'; export default class JobStatusAllocationStatusBlockComponent extends Component { get countToShow() { + if (this.args.compact) { + return 0; + } const restWidth = 50; const restGap = 10; let cts = Math.floor((this.args.width - (restWidth + restGap)) / 42); diff --git a/ui/app/components/job-status/allocation-status-row.hbs b/ui/app/components/job-status/allocation-status-row.hbs index d0c32344f16..45dd528967f 100644 --- a/ui/app/components/job-status/allocation-status-row.hbs +++ b/ui/app/components/job-status/allocation-status-row.hbs @@ -3,8 +3,8 @@ SPDX-License-Identifier: BUSL-1.1 ~}} -
- {{#if this.showSummaries}} +
+ {{#if (or this.showSummaries @compact)}}
{{/if}} {{/each-in}} @@ -50,5 +54,9 @@ {{/each-in}}
{{/if}} + {{#if @compact}} + {{!-- TODO: @runningAllocs using the wrong thing --}} + {{@runningAllocs}}/{{@groupCountSum}} + {{/if}}
diff --git a/ui/app/components/job-status/panel/steady.js b/ui/app/components/job-status/panel/steady.js index 048db7ac92f..43ba9785aa4 100644 --- a/ui/app/components/job-status/panel/steady.js +++ b/ui/app/components/job-status/panel/steady.js @@ -141,7 +141,12 @@ export default class JobStatusPanelSteadyComponent extends Component { } get totalAllocs() { + console.log('ahum what is type here', this.args.job.type); if (this.args.job.type === 'service' || this.args.job.type === 'batch') { + console.log( + 'totalAllocs tally', + this.args.job.taskGroups.map((tg) => tg.count) + ); return this.args.job.taskGroups.reduce((sum, tg) => sum + tg.count, 0); } else if (this.atMostOneAllocPerNode) { return this.args.job.allocations.uniqBy('nodeID').length; @@ -214,6 +219,11 @@ export default class JobStatusPanelSteadyComponent extends Component { * @returns {CurrentStatus} */ get currentStatus() { + console.log('determine status for', this.job.name); + console.log( + this.totalAllocs, + this.allocBlocks.complete?.healthy?.nonCanary + ); // If all allocs are running, the job is Healthy const totalAllocs = this.totalAllocs; diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index e26deba9b8b..b89b640bb2c 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -8,6 +8,7 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action } from '@ember/object'; +import { tracked } from '@glimmer/tracking'; const ALL_NAMESPACE_WILDCARD = '*'; @@ -16,6 +17,16 @@ export default class JobsIndexController extends Controller { // @service store; @service system; + queryParams = [ + 'nextToken', + 'pageSize', + 'foo', + // 'status', + { qpNamespace: 'namespace' }, + // 'type', + // 'searchTerm', + ]; + isForbidden = false; get tableColumns() { @@ -32,6 +43,7 @@ export default class JobsIndexController extends Controller { .map((c) => { return { label: c.charAt(0).toUpperCase() + c.slice(1), + width: c === 'summary' ? '200px' : undefined, }; }); } @@ -44,4 +56,70 @@ export default class JobsIndexController extends Controller { gotoJob(job) { this.router.transitionTo('jobs.job.index', job.idWithNamespace); } + + // #region pagination + // @action + // onNext(nextToken) { + // this.previousTokens = [...this.previousTokens, this.nextToken]; + // this.nextToken = nextToken; + // } + + // get getPrevToken() { + // return "beep"; + // } + // get getNextToken() { + // return "boop"; + // } + + @tracked initialNextToken; + @tracked nextToken; + @tracked previousTokens = [null]; + + @action someFunc(a, b, c) { + console.log('someFunc called', a, b, c); + } + + /** + * + * @param {"prev"|"next"} page + */ + @action handlePageChange(page, event, c) { + console.log('hPC', page, event, c); + // event.preventDefault(); + if (page === 'prev') { + console.log('prev page'); + this.nextToken = this.previousTokens.pop(); + this.previousTokens = [...this.previousTokens]; + } else if (page === 'next') { + console.log('next page', this.model.jobs.meta); + this.previousTokens = [...this.previousTokens, this.nextToken]; + // this.nextToken = "boop"; + // random + // this.nextToken = Math.random().toString(36).substring(7); + this.nextToken = this.model.jobs.meta.nextToken; + // this.foo = "bar"; + } + } + + // get paginate() { + // console.log('paginating'); + // return (page,b,c) => { + // return { + // // nextToken: this.nextToken, + // nextToken: "boop", + // } + // } + // } + + // get demoQueryFunctionCompact() { + // return (page,b,c) => { + // console.log('demoQueryFunctionCompact', page, b,c); + // return { + // // demoCurrentToken: page === 'prev' ? this.getPrevToken : this.getNextToken, + // // demoExtraParam: 'hello', + // nextToken: page === 'prev' ? this.getPrevToken : this.getNextToken, + // }; + // }; + // } + // #endregion pagination } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 8d87740eb3a..a7a7731acf0 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -108,7 +108,7 @@ export default class Job extends Model { allocationsOfShowableType[status].healthy.nonCanary.push(alloc); availableSlotsToFill--; } - + // TODO: return early here if !availableSlotsToFill // Sort all allocs by jobVersion in descending order const sortedAllocs = this.allocations .filter( @@ -120,6 +120,9 @@ export default class Job extends Model { if (a.jobVersion < b.jobVersion) return -1; // If jobVersion is the same, sort by status order + // For example, we may have some allocBlock slots to fill, and need to determine + // if the user expects to see, from non-running/non-pending allocs, some old "failed" ones + // or "lost" or "complete" ones, etc. jobAllocStatuses give us this order. if (a.jobVersion === b.jobVersion) { return ( jobAllocStatuses[this.type].indexOf(b.clientStatus) - @@ -196,6 +199,7 @@ export default class Job extends Model { } if (this.type === 'batch' || this.type === 'sysbatch') { + // TODO: showing as failed when long-complete // If all the allocs are complete, the job is Complete const completeAllocs = this.allocBlocks.complete?.healthy?.nonCanary; if (completeAllocs?.length === totalAllocs) { @@ -232,7 +236,12 @@ export default class Job extends Model { // TODO: GroupCountSum for a parameterized parent job is the count present at group level, but that's not quite true, as the parent job isn't expecting any allocs, its children are. Chat with BFF about this. // TODO: handle garbage collected cases not showing "failed" for batch jobs here maybe? - console.log('numFailedAllocs', failedOrLostAllocs.length); + console.log( + 'numFailedAllocs', + failedOrLostAllocs.length, + failedOrLostAllocs, + totalAllocs + ); // if (failedOrLostAllocs.length === totalAllocs) { if (failedOrLostAllocs.length >= totalAllocs) { // TODO: when totalAllocs only cares about latest version, change back to === diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 2d688f23311..8b53c015c76 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -18,10 +18,15 @@ export default class IndexRoute extends Route.extend( ) { @service store; + perPage = 10; + queryParams = { qpNamespace: { refreshModel: true, }, + nextToken: { + refreshModel: true, + }, }; async model(params) { @@ -29,11 +34,19 @@ export default class IndexRoute extends Route.extend( .query('job', { namespace: params.qpNamespace, meta: true, - per_page: 10, + per_page: this.perPage, + next_token: params.nextToken || '000', + queryType: 'initialize', }) .catch(notifyForbidden(this)); + console.log('did my header return a nextToken header?'); + console.log(this.store.adapterFor('job').headers); + console.log('what about meta', jobs.meta); + console.log('what about', params.nextToken); + // debugger; + return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -47,7 +60,7 @@ export default class IndexRoute extends Route.extend( 'modelWatch', this.watchJobs.perform({ namespace: controller.qpNamespace, - per_page: 10, + per_page: this.perPage, meta: true, queryType: 'initialize', }) diff --git a/ui/app/styles/components/job-status-panel.scss b/ui/app/styles/components/job-status-panel.scss index d488abf6595..917277ad595 100644 --- a/ui/app/styles/components/job-status-panel.scss +++ b/ui/app/styles/components/job-status-panel.scss @@ -405,6 +405,24 @@ } } + .allocation-status-row.compact { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + gap: 1rem; + max-width: 400px; + .alloc-status-summaries { + height: 6px; + gap: 6px; + .represented-allocation { + height: 6px; + .rest-count { + display: none; + } + } + } + } + .legend-item .represented-allocation .flight-icon { animation: none; } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 5958233d245..9d0fbe33002 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -5,13 +5,12 @@ {{page-title "Jobs"}}
- {{#if this.isForbidden}} - - {{else if this.jobs.length}} - + Next token is {{this.model.jobs.meta.nextToken}}
+ Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
<:body as |B|> {{!-- TODO: use --}} @@ -55,12 +54,19 @@ {{B.data.priority}} - {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
+ {{!-- {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
{{B.data.allocations.length}} total
{{B.data.groupCountSum}} desired -
+
--}}
- +
@@ -68,13 +74,9 @@ - {{else}} - No jobs : - {{/if}}
\ No newline at end of file From 0294e1656a78aac85c6f4a07cae363aa592bc8b2 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 24 Jan 2024 12:28:56 -0500 Subject: [PATCH 54/98] Hacky way to bring new jobs back around and parent job handling in list view --- ui/app/controllers/jobs/index.js | 13 ++++------- ui/app/models/job.js | 37 +++++++++++++++++++++++--------- ui/app/routes/jobs/index.js | 22 +++++++++---------- ui/app/templates/jobs/index.hbs | 29 +++++++++++++++++-------- ui/app/utils/properties/watch.js | 25 +++++++++++++++------ 5 files changed, 79 insertions(+), 47 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index b89b640bb2c..e2ee7b858d7 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -9,6 +9,7 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action } from '@ember/object'; import { tracked } from '@glimmer/tracking'; +import { alias } from '@ember/object/computed'; const ALL_NAMESPACE_WILDCARD = '*'; @@ -37,20 +38,18 @@ export default class JobsIndexController extends Controller { 'status', 'type', 'priority', - 'summary', + 'running allocations', ] .filter((c) => !!c) .map((c) => { return { label: c.charAt(0).toUpperCase() + c.slice(1), - width: c === 'summary' ? '200px' : undefined, + width: c === 'running allocations' ? '200px' : undefined, }; }); } - get jobs() { - return this.model.jobs; - } + @alias('model.jobs') jobs; @action gotoJob(job) { @@ -75,10 +74,6 @@ export default class JobsIndexController extends Controller { @tracked nextToken; @tracked previousTokens = [null]; - @action someFunc(a, b, c) { - console.log('someFunc called', a, b, c); - } - /** * * @param {"prev"|"next"} page diff --git a/ui/app/models/job.js b/ui/app/models/job.js index a7a7731acf0..1696237829e 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -36,6 +36,23 @@ export default class Job extends Model { @attr('number') groupCountSum; @attr('string') deploymentID; + @attr() childStatuses; + + get childStatusBreakdown() { + // child statuses is something like ['dead', 'dead', 'complete', 'running', 'running', 'dead']. + // Return an object counting by status, like {dead: 3, complete: 1, running: 2} + const breakdown = {}; + this.childStatuses.forEach((status) => { + if (breakdown[status]) { + breakdown[status]++; + } else { + breakdown[status] = 1; + } + }); + console.log('breakdown', breakdown); + return breakdown; + } + get allocTypes() { return jobAllocStatuses[this.type].map((type) => { return { @@ -166,7 +183,7 @@ export default class Job extends Model { }; } - console.log('allocBlocks for', this.name, 'is', allocationsOfShowableType); + // console.log('allocBlocks for', this.name, 'is', allocationsOfShowableType); return allocationsOfShowableType; } @@ -190,8 +207,8 @@ export default class Job extends Model { get aggregateAllocStatus() { // If all allocs are running, the job is Healthy const totalAllocs = this.groupCountSum; - console.log('groupCountSum is', totalAllocs); - console.log('ablocks are', this.allocBlocks); + // console.log('groupCountSum is', totalAllocs); + // console.log('ablocks are', this.allocBlocks); // If deploying: if (this.deploymentID) { @@ -214,7 +231,7 @@ export default class Job extends Model { } const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; - console.log('healthyAllocs', this.name, healthyAllocs, totalAllocs); + // console.log('healthyAllocs', this.name, healthyAllocs, totalAllocs); if (healthyAllocs?.length === totalAllocs) { return { label: 'Healthy', state: 'success' }; } @@ -236,12 +253,12 @@ export default class Job extends Model { // TODO: GroupCountSum for a parameterized parent job is the count present at group level, but that's not quite true, as the parent job isn't expecting any allocs, its children are. Chat with BFF about this. // TODO: handle garbage collected cases not showing "failed" for batch jobs here maybe? - console.log( - 'numFailedAllocs', - failedOrLostAllocs.length, - failedOrLostAllocs, - totalAllocs - ); + // console.log( + // 'numFailedAllocs', + // failedOrLostAllocs.length, + // failedOrLostAllocs, + // totalAllocs + // ); // if (failedOrLostAllocs.length === totalAllocs) { if (failedOrLostAllocs.length >= totalAllocs) { // TODO: when totalAllocs only cares about latest version, change back to === diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 8b53c015c76..d683b2d58c1 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -41,12 +41,6 @@ export default class IndexRoute extends Route.extend( }) .catch(notifyForbidden(this)); - console.log('did my header return a nextToken header?'); - console.log(this.store.adapterFor('job').headers); - console.log('what about meta', jobs.meta); - console.log('what about', params.nextToken); - // debugger; - return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -58,12 +52,16 @@ export default class IndexRoute extends Route.extend( controller.set('namespacesWatch', this.watchNamespaces.perform()); controller.set( 'modelWatch', - this.watchJobs.perform({ - namespace: controller.qpNamespace, - per_page: this.perPage, - meta: true, - queryType: 'initialize', - }) + this.watchJobs.perform( + { + namespace: controller.qpNamespace, + per_page: this.perPage, + meta: true, + queryType: 'initialize', + }, + 1000, + { model } + ) // TODO: VERY HACKY WAY TO PASS MODEL ); controller.set( 'jobsWatch', diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 9d0fbe33002..2c53f0e86cd 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -7,6 +7,8 @@
Next token is {{this.model.jobs.meta.nextToken}}
Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
+ Model.jobs length: {{this.model.jobs.length}}
+ Controller.jobs length: {{this.jobs.length}}
{{B.data.nodepool}} {{/if}} - + {{#unless B.data.childStatuses}} + + {{/unless}} {{B.data.type}} @@ -59,14 +63,21 @@ {{B.data.groupCountSum}} desired
--}}
- + {{#if B.data.childStatuses}} + {{B.data.childStatuses.length}} child jobs;
+ {{#each-in B.data.childStatusBreakdown as |status count|}} + {{count}} {{status}}
+ {{/each-in}} + {{else}} + + {{/if}}
diff --git a/ui/app/utils/properties/watch.js b/ui/app/utils/properties/watch.js index 99f8aa888eb..491c750d092 100644 --- a/ui/app/utils/properties/watch.js +++ b/ui/app/utils/properties/watch.js @@ -6,7 +6,7 @@ // @ts-check import Ember from 'ember'; -import { get } from '@ember/object'; +import { get, set } from '@ember/object'; import { assert } from '@ember/debug'; import RSVP from 'rsvp'; import { task } from 'ember-concurrency'; @@ -134,8 +134,11 @@ export function watchAll(modelName) { }).drop(); } -export function watchQuery(modelName) { - return task(function* (params, throttle = 2000) { +export function watchQuery(modelName, b, c) { + console.log('watchQuery', b, c); + return task(function* (params, throttle = 2000, options = {}) { + console.log('watchQuery of queryType', params.queryType); + let { model } = options; assert( 'To watch a query, the adapter for the type being queried MUST extend Watchable', this.store.adapterFor(modelName) instanceof Watchable @@ -144,10 +147,18 @@ export function watchQuery(modelName) { const controller = new AbortController(); try { yield RSVP.all([ - this.store.query(modelName, params, { - reload: true, - adapterOptions: { watch: true, abortController: controller }, - }), + this.store + .query(modelName, params, { + reload: true, + adapterOptions: { watch: true, abortController: controller }, + }) + .then((r) => { + console.log('do i have a model to attach this to?', model, r); + if (model) { + set(model, 'jobs', r); // woof, TODO: dont do this. It works but is hardcoded and hacky. Talk to Michael. + } + return r; + }), wait(throttle), ]); } catch (e) { From 0755feca8384379787fd640bfdc4d2124c992c66 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 26 Jan 2024 14:43:43 -0500 Subject: [PATCH 55/98] Getting closer to hook/latch --- ui/app/adapters/job.js | 110 ++++++++++++++-- ui/app/adapters/watchable.js | 106 +++++++++------ ui/app/controllers/jobs/index.js | 10 +- ui/app/models/job.js | 1 - ui/app/routes/jobs/index.js | 215 ++++++++++++++++++++++++------- ui/app/services/watch-list.js | 2 + ui/app/templates/jobs/index.hbs | 1 + ui/app/utils/properties/watch.js | 27 ++-- 8 files changed, 360 insertions(+), 112 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index b6b7938e7da..f2b6fd96d0c 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -10,10 +10,13 @@ import { base64EncodeString } from 'nomad-ui/utils/encode'; import classic from 'ember-classic-decorator'; import { inject as service } from '@ember/service'; import { getOwner } from '@ember/application'; +import { get } from '@ember/object'; +import queryString from 'query-string'; @classic export default class JobAdapter extends WatchableNamespaceIDs { @service system; + @service watchList; relationshipFallbackLinks = { summary: '/summary', @@ -204,35 +207,118 @@ export default class JobAdapter extends WatchableNamespaceIDs { } query(store, type, query, snapshotRecordArray, options) { - let { queryType } = query; + // let { queryType } = query; options = options || {}; options.adapterOptions = options.adapterOptions || {}; - if (queryType === 'initialize') { - // options.url = this.urlForQuery(query, type.modelName); - options.adapterOptions.method = 'GET'; - } else if (queryType === 'update') { - // options.url = this.urlForUpdateQuery(query, type.modelName); - options.adapterOptions.method = 'POST'; - options.adapterOptions.watch = true; + + // const url = this.buildURL(type.modelName, null, null, 'query', query); + const method = get(options, 'adapterOptions.method') || 'GET'; + const url = this.urlForQuery(query, type.modelName, method); + console.log('url, method', url, method, options); + + // if (queryType === 'initialize') { + // // // options.url = this.urlForQuery(query, type.modelName); + // options.adapterOptions.method = 'GET'; + // } else { + // options.adapterOptions.watch = true; + // } + // if (queryType === 'update') { + // options.adapterOptions.method = 'POST'; + // options.adapterOptions.watch = true; // TODO: probably? + // delete query.queryType; + // } + + // Let's establish the index, via watchList.getIndexFor. + + // url needs to have stringified params on it + let index = this.watchList.getIndexFor(url); + console.log('index for', url, 'is', index); + if (this.watchList.getIndexFor(url)) { + query.index = index; } - return super.query(store, type, query, snapshotRecordArray, options); + + // console.log('so then uh', query); + // } else if (queryType === 'update') { + // // options.url = this.urlForUpdateQuery(query, type.modelName); + // options.adapterOptions.method = 'POST'; + // options.adapterOptions.watch = true; + // } + // return super.query(store, type, query, snapshotRecordArray, options); + // let superQuery = super.query(store, type, query, snapshotRecordArray, options); + // console.log('superquery', superQuery); + // return superQuery; + + const signal = get(options, 'adapterOptions.abortController.signal'); + return this.ajax(url, method, { + signal, + data: query, + skipURLModification: true, + }); } - handleResponse(status, headers) { + handleResponse(status, headers, payload, requestData) { + // console.log('jobadapter handleResponse', status, headers, payload, requestData); /** * @type {Object} */ const result = super.handleResponse(...arguments); + // console.log('response', result, headers); if (result) { result.meta = result.meta || {}; if (headers['x-nomad-nexttoken']) { result.meta.nextToken = headers['x-nomad-nexttoken']; } } + + // If the url contains the urlForQuery, we should fire a new method that handles index tracking + if (requestData.url.includes(this.urlForQuery())) { + this.updateQueryIndex(headers['x-nomad-index']); + } + return result; } - urlForQuery(query, modelName) { - return `/${this.namespace}/jobs/statuses3`; + // urlForQuery(query, modelName) { + // return `/${this.namespace}/jobs/statuses3`; + // } + + urlForQuery(query, modelName, method) { + let baseUrl = `/${this.namespace}/jobs/statuses3`; + // let queryString = Object.keys(query).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); + if (method === 'POST') { + return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; + } else { + return `${baseUrl}?${queryString.stringify(query)}`; + } + } + + // urlForQuery(query, modelName) { + // let baseUrl = `/${this.namespace}/jobs/statuses3`; + // // let queryString = Object.keys(query).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); + // // Only include non-empty query params + // let queryString = Object.keys(query).filter(key => !!query[key]).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); + // console.log('+++ querystring', queryString) + // return `${baseUrl}?${queryString}`; + // } + + ajaxOptions(url, type, options) { + let hash = super.ajaxOptions(url, type, options); + console.log('+++ ajaxOptions', url, type, options, hash); + // debugger; + // console.log('options', options, hash); + + // Custom handling for POST requests to append 'index' as a query parameter + if (type === 'POST' && options.data && options.data.index) { + let index = encodeURIComponent(options.data.index); + hash.url = `${hash.url}&index=${index}`; + } + + return hash; + } + + updateQueryIndex(index) { + console.log('setQueryIndex', index); + // Is there an established index for jobs + // this.watchList.setIndexFor(url, index); } } diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 588f6d3b2fc..75fc5180213 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -25,27 +25,30 @@ export default class Watchable extends ApplicationAdapter { // It's either this weird side-effecting thing that also requires a change // to ajaxOptions or overriding ajax completely. ajax(url, type, options) { + console.log('+++ watchable ajax', url, type, options); const hasParams = hasNonBlockingQueryParams(options); + // console.log('hasParams', url, hasParams, options) // if (!hasParams || type !== 'GET') return super.ajax(url, type, options); - if (!hasParams) return super.ajax(url, type, options); + if (!hasParams || options.skipURLModification) + return super.ajax(url, type, options); let params = { ...options?.data }; - // TODO: Creating a hash of the params as watchList key feels a little hacky - if (type === 'POST') { - let index = params.index; - delete params.index; - url = `${url}?hash=${btoa(JSON.stringify(params))}&index=${index}`; - } else { - // Options data gets appended as query params as part of ajaxOptions. - // In order to prevent doubling params, data should only include index - // at this point since everything else is added to the URL in advance. - options.data = options.data.index ? { index: options.data.index } : {}; + delete params.index; + // // TODO: Creating a hash of the params as watchList key feels a little hacky + // if (type === 'POST') { + // // TODO: check that dispatch still works! + // // console.log('hashing params', params); + // url = `${url}?hash=${btoa(JSON.stringify(params))}`; + // } else { + // // Options data gets appended as query params as part of ajaxOptions. + // // In order to prevent doubling params, data should only include index + // // at this point since everything else is added to the URL in advance. + // options.data = options.data.index ? { index: options.data.index } : {}; - delete params.index; - url = `${url}?${queryString.stringify(params)}`; - } - // debugger; + // delete params.index; + // url = `${url}?${queryString.stringify(params)}`; + // } - return super.ajax(url, type, options); + return super.ajax(`${url}?${queryString.stringify(params)}`, type, options); } findAll(store, type, sinceToken, snapshotRecordArray, additionalParams = {}) { @@ -118,23 +121,45 @@ export default class Watchable extends ApplicationAdapter { // The intended query without additional blocking query params is used // to track the appropriate query index. - // if POST, dont get whole queryString, just the index - if (method === 'POST') { - // If the hashed version already exists, use it: - let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; - if (this.watchList.getIndexFor(hashifiedURL) > 1) { - params.index = this.watchList.getIndexFor(hashifiedURL); - } else { - // TODO: De-hardcode the initialize query, identify it in watchlist somehow? - params.index = this.watchList.getIndexFor( - '/v1/jobs/statuses3?meta=true&queryType=initialize' - ); - } - } else { - params.index = this.watchList.getIndexFor( - `${urlPath}?${queryString.stringify(query)}` - ); - } + // // if POST, dont get whole queryString, just the index + // if (method === 'POST') { + // // If the hashed version already exists, use it: + // let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; + // if (this.watchList.getIndexFor(hashifiedURL) > 1) { + // params.index = this.watchList.getIndexFor(hashifiedURL); + // } else { + // // TODO: De-hardcode the initialize query, identify it in watchlist somehow? + // params.index = this.watchList.getIndexFor( + // '/v1/jobs/statuses3?meta=true&queryType=initialize' + // ); + // } + // } else { + // params.index = this.watchList.getIndexFor( + // `${urlPath}?${queryString.stringify(query)}` + // ); + // } + + // if (method === 'POST') { + // // If the hashed version already exists, use it: + // let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; + // console.log('hashifiedURL', hashifiedURL) + // if (this.watchList.getIndexFor(hashifiedURL) > 1) { + // console.log('+++1 found index for hashifiedURL', this.watchList.getIndexFor(hashifiedURL)); + // params.index = this.watchList.getIndexFor(hashifiedURL); + // } else { + // console.log('+++2 didnt find index for hashifiedURL, checking initialize', this.watchList.getIndexFor(hashifiedURL)); + // // otherwise, try to get the index from the initialize query + // params.index = this.watchList.getIndexFor( + // '/v1/jobs/statuses3?meta=true&per_page=10&queryType=initialize' // TODO: fickle! + // ); + // } + // } else { + params.index = this.watchList.getIndexFor( + `${urlPath}?${queryString.stringify(query)}` + ); + // } + // console.log('paramsindex', params.index); + // // debugger; } const signal = get(options, 'adapterOptions.abortController.signal'); @@ -231,17 +256,18 @@ export default class Watchable extends ApplicationAdapter { } handleResponse(status, headers, payload, requestData) { + // console.log('handling response', requestData); // Some browsers lowercase all headers. Others keep them // case sensitive. const newIndex = headers['x-nomad-index'] || headers['X-Nomad-Index']; if (newIndex) { - if (requestData.method === 'POST') { - // without the last &index= bit - // TODO: this is a weird way to save key. - this.watchList.setIndexFor(requestData.url.split('&')[0], newIndex); - } else { - this.watchList.setIndexFor(requestData.url, newIndex); - } + // if (requestData.method === 'POST') { + // // without the last &index= bit + // // TODO: this is a weird way to save key. + // this.watchList.setIndexFor(requestData.url.split('&')[0], newIndex); + // } else { + this.watchList.setIndexFor(requestData.url, newIndex); + // } } return super.handleResponse(...arguments); diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index e2ee7b858d7..4a999cecd9e 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -9,7 +9,8 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action } from '@ember/object'; import { tracked } from '@glimmer/tracking'; -import { alias } from '@ember/object/computed'; +// import { alias } from '@ember/object/computed'; +import { computed } from '@ember/object'; const ALL_NAMESPACE_WILDCARD = '*'; @@ -49,7 +50,12 @@ export default class JobsIndexController extends Controller { }); } - @alias('model.jobs') jobs; + // @alias('model.jobs') jobs; + // @computed('model.jobs.[]') + // get jobs() { + // return this.model.jobs; + // } + jobs = []; @action gotoJob(job) { diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 1696237829e..eabe227074c 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -49,7 +49,6 @@ export default class Job extends Model { breakdown[status] = 1; } }); - console.log('breakdown', breakdown); return breakdown; } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index d683b2d58c1..9ff15904bb5 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -3,6 +3,8 @@ * SPDX-License-Identifier: BUSL-1.1 */ +// @ts-check + import { inject as service } from '@ember/service'; import Route from '@ember/routing/route'; import RSVP from 'rsvp'; @@ -11,12 +13,14 @@ import { watchAll, watchQuery } from 'nomad-ui/utils/properties/watch'; import WithWatchers from 'nomad-ui/mixins/with-watchers'; import notifyForbidden from 'nomad-ui/utils/notify-forbidden'; import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; +import { task, restartableTask, timeout } from 'ember-concurrency'; export default class IndexRoute extends Route.extend( - WithWatchers, + // WithWatchers, WithForbiddenState ) { @service store; + @service watchList; perPage = 10; @@ -29,60 +33,185 @@ export default class IndexRoute extends Route.extend( }, }; - async model(params) { - const jobs = await this.store - .query('job', { - namespace: params.qpNamespace, - meta: true, - per_page: this.perPage, - next_token: params.nextToken || '000', - - queryType: 'initialize', + defaultParams = { + meta: true, + per_page: this.perPage, + }; + + getCurrentParams() { + let queryParams = this.paramsFor(this.routeName); // Get current query params + return { ...this.defaultParams, ...queryParams }; + } + + jobQuery(params, options = {}) { + return this.store + .query('job', params, { + adapterOptions: { + method: 'GET', // TODO: default + queryType: options.queryType, + }, }) .catch(notifyForbidden(this)); + } + + jobAllocsQuery(jobIDs) { + return this.store + .query( + 'job', + { + jobs: jobIDs, + }, + { + adapterOptions: { + method: 'POST', + queryType: 'update', + }, + } + ) + .catch(notifyForbidden(this)); + } + + @restartableTask *watchJobIDs(params, throttle = 2000) { + while (true) { + let currentParams = this.getCurrentParams(); + const newJobs = yield this.jobQuery(currentParams, { + queryType: 'update_ids', + }); + + const jobIDs = newJobs.map((job) => ({ + id: job.plainId, + namespace: job.belongsTo('namespace').id(), + })); + this.controller.set('jobIDs', jobIDs); + // BIG TODO: MAKE ANY jobIDs UPDATES TRIGGER A NEW WATCHJOBS TASK + this.watchJobs.perform({}, 500); + + yield timeout(throttle); // Moved to the end of the loop + } + } + + // @restartableTask *watchJobIDs(params, throttle = 2000) { + // while (true) { + // // let currentParams = this.getCurrentParams(); + // // const jobs = yield this.jobQuery(currentParams); + // // yield timeout(throttle); + // let currentParams = this.getCurrentParams(); + // currentParams.queryType = 'update_ids'; + // const newJobs = yield this.jobQuery(currentParams); + + // const jobIDs = newJobs.map((job) => { + // return { + // id: job.plainId, + // namespace: job.belongsTo('namespace').id(), + // }; + // }); + // this.controller.set('jobIDs', jobIDs); + + // yield timeout(throttle); + + // // this.watchJobs.perform(params, 2000); // TODO mismatched throttle + // } + // } + + @restartableTask *watchJobs(params, throttle = 2000) { + // TODO: THURSDAY MORNING: + // Most of the ordering stuff feels disjointed! + // use the index from the watchList of the initial query here, too + + while (true) { + let jobIDs = this.controller.jobIDs; + console.log('watchJobs called', jobIDs); + // console.log('jobids in watchjobs', jobIDs); + // console.log('watchList list', this.watchList.list); + + // Either get index from watchlist entry for this particular hash, or + // get index from watchlist entry for the initial query + + if (jobIDs && jobIDs.length > 0) { + let jobDetails = yield this.jobAllocsQuery(jobIDs); + this.controller.set('jobs', jobDetails); + } + // TODO: might need an else condition here for if there are no jobIDs, + // which would indicate no jobs, but the updater above might not fire. + yield timeout(throttle); + } + } + + async model(params) { + // console.log('model firing'); + // console.log('sending off params', params); + let currentParams = this.getCurrentParams(); + // currentParams.queryType = 'initialize'; return RSVP.hash({ - jobs, + jobs: await this.jobQuery(currentParams, { queryType: 'initialize' }), namespaces: this.store.findAll('namespace'), nodePools: this.store.findAll('node-pool'), }); } - startWatchers(controller, model) { - controller.set('namespacesWatch', this.watchNamespaces.perform()); - controller.set( - 'modelWatch', - this.watchJobs.perform( - { - namespace: controller.qpNamespace, - per_page: this.perPage, - meta: true, - queryType: 'initialize', - }, - 1000, - { model } - ) // TODO: VERY HACKY WAY TO PASS MODEL - ); + setupController(controller, model) { + super.setupController(controller, model); + controller.set('jobs', model.jobs); controller.set( - 'jobsWatch', - this.watchJobsAllocs.perform({ - namespace: controller.qpNamespace, - meta: true, - queryType: 'update', - jobs: model.jobs.map((job) => { - // TODO: maybe this should be set on controller for user-controlled updates? - return { - id: job.plainId, - namespace: job.belongsTo('namespace').id(), - }; - }), + 'jobIDs', + model.jobs.map((job) => { + return { + id: job.plainId, + namespace: job.belongsTo('namespace').id(), + }; }) ); + + this.watchJobIDs.perform({}, 2000); + this.watchJobs.perform({}, 500); // Start watchJobs independently with its own throttle } - @watchQuery('job') watchJobs; - @watchQuery('job', { queryType: 'update' }) watchJobsAllocs; - // @watchQuery('job', { queryType: 'update' }) watchJobsUpdate; - @watchAll('namespace') watchNamespaces; - @collect('watchJobs', 'watchJobsAllocs', 'watchNamespaces') watchers; + // afterModel(model, transition) { + // console.log('afterModel firing', model, transition); + // // let jobs = this.watchJobsTask.perform(params); + // let params = this.getCurrentParams(); + // let jobs = this.watchJobsTask.perform(params, 200); + + // console.log('jobs', jobs); + // // model.jobs = jobs; + // } + + // startWatchers(controller, model) { + // controller.set('namespacesWatch', this.watchNamespaces.perform()); + // controller.set( + // 'modelWatch', + // this.watchJobs.perform( + // { + // namespace: controller.qpNamespace, + // per_page: this.perPage, + // meta: true, + // queryType: 'initialize', + // }, + // 1000, + // { model } + // ) // TODO: VERY HACKY WAY TO PASS MODEL + // ); + // controller.set( + // 'jobsWatch', + // this.watchJobsAllocs.perform({ + // namespace: controller.qpNamespace, + // meta: true, + // queryType: 'update', + // jobs: model.jobs.map((job) => { + // // TODO: maybe this should be set on controller for user-controlled updates? + // return { + // id: job.plainId, + // namespace: job.belongsTo('namespace').id(), + // }; + // }), + // }) + // ); + // } + + // @watchQuery('job') watchJobs; + // @watchQuery('job', { queryType: 'update' }) watchJobsAllocs; + // // @watchQuery('job', { queryType: 'update' }) watchJobsUpdate; + // @watchAll('namespace') watchNamespaces; + // @collect('watchJobs', 'watchJobsAllocs', 'watchNamespaces') watchers; } diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 80a44d138c1..4a1874b0772 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -29,5 +29,7 @@ export default class WatchListService extends Service { setIndexFor(url, value) { list[url] = +value; + console.log('total list is now', list); + console.log('==================='); } } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 2c53f0e86cd..edc09300bd6 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -9,6 +9,7 @@ Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
Model.jobs length: {{this.model.jobs.length}}
Controller.jobs length: {{this.jobs.length}}
+ Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
{ - console.log('do i have a model to attach this to?', model, r); - if (model) { - set(model, 'jobs', r); // woof, TODO: dont do this. It works but is hardcoded and hacky. Talk to Michael. - } - return r; - }), + this.store.query(modelName, params, { + reload: true, + adapterOptions: { watch: true, abortController: controller }, + }), + // .then((r) => { + // // console.log('do i have a model to attach this to?', model, r); + // if (model) { + // set(model, 'jobs', r); // woof, TODO: dont do this. It works but is hardcoded and hacky. Talk to Michael. + // } + // return r; + // }), wait(throttle), ]); } catch (e) { From 24cd8547f75ad30d3ebdcd9d20a7e4cc8a8f8441 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 26 Jan 2024 22:27:25 -0500 Subject: [PATCH 56/98] Latch from update on hook from initialize, but fickle --- ui/app/adapters/job.js | 15 +++++++++++++++ ui/app/routes/jobs/index.js | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index f2b6fd96d0c..1d3aaca8873 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -231,7 +231,22 @@ export default class JobAdapter extends WatchableNamespaceIDs { // Let's establish the index, via watchList.getIndexFor. // url needs to have stringified params on it + let index = this.watchList.getIndexFor(url); + + // In the case that this is of queryType update, + // and its index is found to be 1, + // check for the initialize query's index and use that instead + if (options.adapterOptions.queryType === 'update' && index === 1) { + let initializeQueryIndex = this.watchList.getIndexFor( + '/v1/jobs/statuses3?meta=true&per_page=10' + ); // TODO: fickle! + if (initializeQueryIndex) { + console.log('initializeQUeryIndex', initializeQueryIndex); + index = initializeQueryIndex; + } + } + console.log('index for', url, 'is', index); if (this.watchList.getIndexFor(url)) { query.index = index; diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 9ff15904bb5..68c2f62a3ba 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -84,6 +84,8 @@ export default class IndexRoute extends Route.extend( })); this.controller.set('jobIDs', jobIDs); // BIG TODO: MAKE ANY jobIDs UPDATES TRIGGER A NEW WATCHJOBS TASK + // And also cancel the current watchJobs! It may be watching for a different list than the new jobIDs and wouldn't naturally unblock. + this.watchJobs.perform({}, 500); yield timeout(throttle); // Moved to the end of the loop @@ -129,6 +131,12 @@ export default class IndexRoute extends Route.extend( if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); + console.log( + 'jobDetails fetched, about to set on controller', + jobDetails, + this.controller + ); + // debugger; this.controller.set('jobs', jobDetails); } // TODO: might need an else condition here for if there are no jobIDs, From bcb2419753bba41fe954d982b1278b1433b84917 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Sun, 28 Jan 2024 22:13:34 -0500 Subject: [PATCH 57/98] Note on multiple-watch problem --- ui/app/adapters/job.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 1d3aaca8873..47fa8468d1c 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -247,6 +247,9 @@ export default class JobAdapter extends WatchableNamespaceIDs { } } + // TODO: adding a new job hash will not necessarily cancel the old one. + // You could be holding open a POST on jobs AB and ABC at the same time. + console.log('index for', url, 'is', index); if (this.watchList.getIndexFor(url)) { query.index = index; From fcdc4800a6febcec81f9ebca34d06aad8f81db3b Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 29 Jan 2024 09:39:55 -0500 Subject: [PATCH 58/98] Sensible monday morning comment removal --- ui/app/adapters/job.js | 68 ++------------ ui/app/adapters/watchable.js | 64 -------------- ui/app/components/job-status/panel/steady.js | 10 --- ui/app/controllers/jobs/index.js | 49 +---------- ui/app/routes/jobs/index.js | 93 ++------------------ ui/app/utils/properties/watch.js | 12 +-- 6 files changed, 17 insertions(+), 279 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 47fa8468d1c..f6e78d07b83 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -207,31 +207,13 @@ export default class JobAdapter extends WatchableNamespaceIDs { } query(store, type, query, snapshotRecordArray, options) { - // let { queryType } = query; options = options || {}; options.adapterOptions = options.adapterOptions || {}; - // const url = this.buildURL(type.modelName, null, null, 'query', query); const method = get(options, 'adapterOptions.method') || 'GET'; const url = this.urlForQuery(query, type.modelName, method); - console.log('url, method', url, method, options); - - // if (queryType === 'initialize') { - // // // options.url = this.urlForQuery(query, type.modelName); - // options.adapterOptions.method = 'GET'; - // } else { - // options.adapterOptions.watch = true; - // } - // if (queryType === 'update') { - // options.adapterOptions.method = 'POST'; - // options.adapterOptions.watch = true; // TODO: probably? - // delete query.queryType; - // } // Let's establish the index, via watchList.getIndexFor. - - // url needs to have stringified params on it - let index = this.watchList.getIndexFor(url); // In the case that this is of queryType update, @@ -242,7 +224,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { '/v1/jobs/statuses3?meta=true&per_page=10' ); // TODO: fickle! if (initializeQueryIndex) { - console.log('initializeQUeryIndex', initializeQueryIndex); index = initializeQueryIndex; } } @@ -251,22 +232,13 @@ export default class JobAdapter extends WatchableNamespaceIDs { // You could be holding open a POST on jobs AB and ABC at the same time. console.log('index for', url, 'is', index); - if (this.watchList.getIndexFor(url)) { + + if (index && index > 1) { query.index = index; } - // console.log('so then uh', query); - // } else if (queryType === 'update') { - // // options.url = this.urlForUpdateQuery(query, type.modelName); - // options.adapterOptions.method = 'POST'; - // options.adapterOptions.watch = true; - // } - // return super.query(store, type, query, snapshotRecordArray, options); - // let superQuery = super.query(store, type, query, snapshotRecordArray, options); - // console.log('superquery', superQuery); - // return superQuery; - const signal = get(options, 'adapterOptions.abortController.signal'); + // TODO: use this signal to abort the request in the case of multiple update requests return this.ajax(url, method, { signal, data: query, @@ -275,12 +247,12 @@ export default class JobAdapter extends WatchableNamespaceIDs { } handleResponse(status, headers, payload, requestData) { - // console.log('jobadapter handleResponse', status, headers, payload, requestData); + // watchList.setIndexFor() happens in the watchable adapter, super'd here + /** * @type {Object} */ const result = super.handleResponse(...arguments); - // console.log('response', result, headers); if (result) { result.meta = result.meta || {}; if (headers['x-nomad-nexttoken']) { @@ -288,42 +260,24 @@ export default class JobAdapter extends WatchableNamespaceIDs { } } - // If the url contains the urlForQuery, we should fire a new method that handles index tracking - if (requestData.url.includes(this.urlForQuery())) { - this.updateQueryIndex(headers['x-nomad-index']); - } - return result; } - // urlForQuery(query, modelName) { - // return `/${this.namespace}/jobs/statuses3`; - // } - urlForQuery(query, modelName, method) { let baseUrl = `/${this.namespace}/jobs/statuses3`; - // let queryString = Object.keys(query).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); if (method === 'POST') { + // Setting a base64 hash to represent the body of the POST request + // (which is otherwise not represented in the URL) + // because the watchList uses the URL as a key for index lookups. return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; } else { return `${baseUrl}?${queryString.stringify(query)}`; } } - // urlForQuery(query, modelName) { - // let baseUrl = `/${this.namespace}/jobs/statuses3`; - // // let queryString = Object.keys(query).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); - // // Only include non-empty query params - // let queryString = Object.keys(query).filter(key => !!query[key]).map(key => `${encodeURIComponent(key)}=${encodeURIComponent(query[key])}`).join('&'); - // console.log('+++ querystring', queryString) - // return `${baseUrl}?${queryString}`; - // } - ajaxOptions(url, type, options) { let hash = super.ajaxOptions(url, type, options); console.log('+++ ajaxOptions', url, type, options, hash); - // debugger; - // console.log('options', options, hash); // Custom handling for POST requests to append 'index' as a query parameter if (type === 'POST' && options.data && options.data.index) { @@ -333,10 +287,4 @@ export default class JobAdapter extends WatchableNamespaceIDs { return hash; } - - updateQueryIndex(index) { - console.log('setQueryIndex', index); - // Is there an established index for jobs - // this.watchList.setIndexFor(url, index); - } } diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 75fc5180213..468e5f73829 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -25,29 +25,11 @@ export default class Watchable extends ApplicationAdapter { // It's either this weird side-effecting thing that also requires a change // to ajaxOptions or overriding ajax completely. ajax(url, type, options) { - console.log('+++ watchable ajax', url, type, options); const hasParams = hasNonBlockingQueryParams(options); - // console.log('hasParams', url, hasParams, options) - // if (!hasParams || type !== 'GET') return super.ajax(url, type, options); if (!hasParams || options.skipURLModification) return super.ajax(url, type, options); let params = { ...options?.data }; delete params.index; - // // TODO: Creating a hash of the params as watchList key feels a little hacky - // if (type === 'POST') { - // // TODO: check that dispatch still works! - // // console.log('hashing params', params); - // url = `${url}?hash=${btoa(JSON.stringify(params))}`; - // } else { - // // Options data gets appended as query params as part of ajaxOptions. - // // In order to prevent doubling params, data should only include index - // // at this point since everything else is added to the URL in advance. - // options.data = options.data.index ? { index: options.data.index } : {}; - - // delete params.index; - // url = `${url}?${queryString.stringify(params)}`; - // } - return super.ajax(`${url}?${queryString.stringify(params)}`, type, options); } @@ -118,48 +100,9 @@ export default class Watchable extends ApplicationAdapter { ); if (get(options, 'adapterOptions.watch')) { - // The intended query without additional blocking query params is used - // to track the appropriate query index. - - // // if POST, dont get whole queryString, just the index - // if (method === 'POST') { - // // If the hashed version already exists, use it: - // let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; - // if (this.watchList.getIndexFor(hashifiedURL) > 1) { - // params.index = this.watchList.getIndexFor(hashifiedURL); - // } else { - // // TODO: De-hardcode the initialize query, identify it in watchlist somehow? - // params.index = this.watchList.getIndexFor( - // '/v1/jobs/statuses3?meta=true&queryType=initialize' - // ); - // } - // } else { - // params.index = this.watchList.getIndexFor( - // `${urlPath}?${queryString.stringify(query)}` - // ); - // } - - // if (method === 'POST') { - // // If the hashed version already exists, use it: - // let hashifiedURL = `${urlPath}?hash=${btoa(JSON.stringify(params))}`; - // console.log('hashifiedURL', hashifiedURL) - // if (this.watchList.getIndexFor(hashifiedURL) > 1) { - // console.log('+++1 found index for hashifiedURL', this.watchList.getIndexFor(hashifiedURL)); - // params.index = this.watchList.getIndexFor(hashifiedURL); - // } else { - // console.log('+++2 didnt find index for hashifiedURL, checking initialize', this.watchList.getIndexFor(hashifiedURL)); - // // otherwise, try to get the index from the initialize query - // params.index = this.watchList.getIndexFor( - // '/v1/jobs/statuses3?meta=true&per_page=10&queryType=initialize' // TODO: fickle! - // ); - // } - // } else { params.index = this.watchList.getIndexFor( `${urlPath}?${queryString.stringify(query)}` ); - // } - // console.log('paramsindex', params.index); - // // debugger; } const signal = get(options, 'adapterOptions.abortController.signal'); @@ -256,18 +199,11 @@ export default class Watchable extends ApplicationAdapter { } handleResponse(status, headers, payload, requestData) { - // console.log('handling response', requestData); // Some browsers lowercase all headers. Others keep them // case sensitive. const newIndex = headers['x-nomad-index'] || headers['X-Nomad-Index']; if (newIndex) { - // if (requestData.method === 'POST') { - // // without the last &index= bit - // // TODO: this is a weird way to save key. - // this.watchList.setIndexFor(requestData.url.split('&')[0], newIndex); - // } else { this.watchList.setIndexFor(requestData.url, newIndex); - // } } return super.handleResponse(...arguments); diff --git a/ui/app/components/job-status/panel/steady.js b/ui/app/components/job-status/panel/steady.js index 43ba9785aa4..048db7ac92f 100644 --- a/ui/app/components/job-status/panel/steady.js +++ b/ui/app/components/job-status/panel/steady.js @@ -141,12 +141,7 @@ export default class JobStatusPanelSteadyComponent extends Component { } get totalAllocs() { - console.log('ahum what is type here', this.args.job.type); if (this.args.job.type === 'service' || this.args.job.type === 'batch') { - console.log( - 'totalAllocs tally', - this.args.job.taskGroups.map((tg) => tg.count) - ); return this.args.job.taskGroups.reduce((sum, tg) => sum + tg.count, 0); } else if (this.atMostOneAllocPerNode) { return this.args.job.allocations.uniqBy('nodeID').length; @@ -219,11 +214,6 @@ export default class JobStatusPanelSteadyComponent extends Component { * @returns {CurrentStatus} */ get currentStatus() { - console.log('determine status for', this.job.name); - console.log( - this.totalAllocs, - this.allocBlocks.complete?.healthy?.nonCanary - ); // If all allocs are running, the job is Healthy const totalAllocs = this.totalAllocs; diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 4a999cecd9e..0e85febeefd 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -9,20 +9,16 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action } from '@ember/object'; import { tracked } from '@glimmer/tracking'; -// import { alias } from '@ember/object/computed'; -import { computed } from '@ember/object'; const ALL_NAMESPACE_WILDCARD = '*'; export default class JobsIndexController extends Controller { @service router; - // @service store; @service system; queryParams = [ 'nextToken', 'pageSize', - 'foo', // 'status', { qpNamespace: 'namespace' }, // 'type', @@ -50,12 +46,7 @@ export default class JobsIndexController extends Controller { }); } - // @alias('model.jobs') jobs; - // @computed('model.jobs.[]') - // get jobs() { - // return this.model.jobs; - // } - jobs = []; + @tracked jobs = []; @action gotoJob(job) { @@ -63,19 +54,6 @@ export default class JobsIndexController extends Controller { } // #region pagination - // @action - // onNext(nextToken) { - // this.previousTokens = [...this.previousTokens, this.nextToken]; - // this.nextToken = nextToken; - // } - - // get getPrevToken() { - // return "beep"; - // } - // get getNextToken() { - // return "boop"; - // } - @tracked initialNextToken; @tracked nextToken; @tracked previousTokens = [null]; @@ -94,33 +72,8 @@ export default class JobsIndexController extends Controller { } else if (page === 'next') { console.log('next page', this.model.jobs.meta); this.previousTokens = [...this.previousTokens, this.nextToken]; - // this.nextToken = "boop"; - // random - // this.nextToken = Math.random().toString(36).substring(7); this.nextToken = this.model.jobs.meta.nextToken; - // this.foo = "bar"; } } - - // get paginate() { - // console.log('paginating'); - // return (page,b,c) => { - // return { - // // nextToken: this.nextToken, - // nextToken: "boop", - // } - // } - // } - - // get demoQueryFunctionCompact() { - // return (page,b,c) => { - // console.log('demoQueryFunctionCompact', page, b,c); - // return { - // // demoCurrentToken: page === 'prev' ? this.getPrevToken : this.getNextToken, - // // demoExtraParam: 'hello', - // nextToken: page === 'prev' ? this.getPrevToken : this.getNextToken, - // }; - // }; - // } // #endregion pagination } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 68c2f62a3ba..a90bd0ba8fb 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -16,7 +16,7 @@ import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; import { task, restartableTask, timeout } from 'ember-concurrency'; export default class IndexRoute extends Route.extend( - // WithWatchers, + WithWatchers, WithForbiddenState ) { @service store; @@ -92,43 +92,9 @@ export default class IndexRoute extends Route.extend( } } - // @restartableTask *watchJobIDs(params, throttle = 2000) { - // while (true) { - // // let currentParams = this.getCurrentParams(); - // // const jobs = yield this.jobQuery(currentParams); - // // yield timeout(throttle); - // let currentParams = this.getCurrentParams(); - // currentParams.queryType = 'update_ids'; - // const newJobs = yield this.jobQuery(currentParams); - - // const jobIDs = newJobs.map((job) => { - // return { - // id: job.plainId, - // namespace: job.belongsTo('namespace').id(), - // }; - // }); - // this.controller.set('jobIDs', jobIDs); - - // yield timeout(throttle); - - // // this.watchJobs.perform(params, 2000); // TODO mismatched throttle - // } - // } - @restartableTask *watchJobs(params, throttle = 2000) { - // TODO: THURSDAY MORNING: - // Most of the ordering stuff feels disjointed! - // use the index from the watchList of the initial query here, too - while (true) { let jobIDs = this.controller.jobIDs; - console.log('watchJobs called', jobIDs); - // console.log('jobids in watchjobs', jobIDs); - // console.log('watchList list', this.watchList.list); - - // Either get index from watchlist entry for this particular hash, or - // get index from watchlist entry for the initial query - if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); console.log( @@ -136,7 +102,6 @@ export default class IndexRoute extends Route.extend( jobDetails, this.controller ); - // debugger; this.controller.set('jobs', jobDetails); } // TODO: might need an else condition here for if there are no jobIDs, @@ -146,10 +111,7 @@ export default class IndexRoute extends Route.extend( } async model(params) { - // console.log('model firing'); - // console.log('sending off params', params); let currentParams = this.getCurrentParams(); - // currentParams.queryType = 'initialize'; return RSVP.hash({ jobs: await this.jobQuery(currentParams, { queryType: 'initialize' }), @@ -175,51 +137,10 @@ export default class IndexRoute extends Route.extend( this.watchJobs.perform({}, 500); // Start watchJobs independently with its own throttle } - // afterModel(model, transition) { - // console.log('afterModel firing', model, transition); - // // let jobs = this.watchJobsTask.perform(params); - // let params = this.getCurrentParams(); - // let jobs = this.watchJobsTask.perform(params, 200); - - // console.log('jobs', jobs); - // // model.jobs = jobs; - // } - - // startWatchers(controller, model) { - // controller.set('namespacesWatch', this.watchNamespaces.perform()); - // controller.set( - // 'modelWatch', - // this.watchJobs.perform( - // { - // namespace: controller.qpNamespace, - // per_page: this.perPage, - // meta: true, - // queryType: 'initialize', - // }, - // 1000, - // { model } - // ) // TODO: VERY HACKY WAY TO PASS MODEL - // ); - // controller.set( - // 'jobsWatch', - // this.watchJobsAllocs.perform({ - // namespace: controller.qpNamespace, - // meta: true, - // queryType: 'update', - // jobs: model.jobs.map((job) => { - // // TODO: maybe this should be set on controller for user-controlled updates? - // return { - // id: job.plainId, - // namespace: job.belongsTo('namespace').id(), - // }; - // }), - // }) - // ); - // } - - // @watchQuery('job') watchJobs; - // @watchQuery('job', { queryType: 'update' }) watchJobsAllocs; - // // @watchQuery('job', { queryType: 'update' }) watchJobsUpdate; - // @watchAll('namespace') watchNamespaces; - // @collect('watchJobs', 'watchJobsAllocs', 'watchNamespaces') watchers; + startWatchers(controller, model) { + controller.set('namespacesWatch', this.watchNamespaces.perform()); + } + + @watchAll('namespace') watchNamespaces; + @collect('watchNamespaces') watchers; } diff --git a/ui/app/utils/properties/watch.js b/ui/app/utils/properties/watch.js index 6c5e3858114..13192beb676 100644 --- a/ui/app/utils/properties/watch.js +++ b/ui/app/utils/properties/watch.js @@ -6,7 +6,7 @@ // @ts-check import Ember from 'ember'; -import { get, set } from '@ember/object'; +import { get } from '@ember/object'; import { assert } from '@ember/debug'; import RSVP from 'rsvp'; import { task } from 'ember-concurrency'; @@ -135,10 +135,7 @@ export function watchAll(modelName) { } export function watchQuery(modelName, b, c) { - // console.log('watchQuery', b, c); return task(function* (params, throttle = 2000, options = {}) { - // console.log('watchQuery of queryType', params.queryType); - let { model } = options; assert( 'To watch a query, the adapter for the type being queried MUST extend Watchable', this.store.adapterFor(modelName) instanceof Watchable @@ -151,13 +148,6 @@ export function watchQuery(modelName, b, c) { reload: true, adapterOptions: { watch: true, abortController: controller }, }), - // .then((r) => { - // // console.log('do i have a model to attach this to?', model, r); - // if (model) { - // set(model, 'jobs', r); // woof, TODO: dont do this. It works but is hardcoded and hacky. Talk to Michael. - // } - // return r; - // }), wait(throttle), ]); } catch (e) { From d586c572ec9c17c45929ceaf0de9e1518a6517e9 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 29 Jan 2024 16:16:29 -0500 Subject: [PATCH 59/98] use of abortController to handle transition and reset events --- ui/app/adapters/job.js | 21 +++++++++++------- ui/app/controllers/jobs/index.js | 1 + ui/app/routes/jobs/index.js | 38 +++++++++++++++++++++++++++----- ui/app/services/watch-list.js | 32 ++++++++++++++++----------- ui/app/templates/jobs/index.hbs | 8 +++++++ ui/app/utils/properties/watch.js | 2 +- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index f6e78d07b83..aac654206df 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -219,13 +219,18 @@ export default class JobAdapter extends WatchableNamespaceIDs { // In the case that this is of queryType update, // and its index is found to be 1, // check for the initialize query's index and use that instead - if (options.adapterOptions.queryType === 'update' && index === 1) { - let initializeQueryIndex = this.watchList.getIndexFor( - '/v1/jobs/statuses3?meta=true&per_page=10' - ); // TODO: fickle! - if (initializeQueryIndex) { - index = initializeQueryIndex; - } + // if (options.adapterOptions.queryType === 'update' && index === 1) { + // let initializeQueryIndex = this.watchList.getIndexFor( + // '/v1/jobs/statuses3?meta=true&per_page=10' + // ); // TODO: fickle! + // if (initializeQueryIndex) { + // index = initializeQueryIndex; + // } + // } + + // Disregard the index if this is an initialize query + if (options.adapterOptions.queryType === 'initialize') { + index = null; } // TODO: adding a new job hash will not necessarily cancel the old one. @@ -238,7 +243,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { } const signal = get(options, 'adapterOptions.abortController.signal'); - // TODO: use this signal to abort the request in the case of multiple update requests + return this.ajax(url, method, { signal, data: query, diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 0e85febeefd..2c860419ab5 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -15,6 +15,7 @@ const ALL_NAMESPACE_WILDCARD = '*'; export default class JobsIndexController extends Controller { @service router; @service system; + @service watchList; // TODO: temp queryParams = [ 'nextToken', diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index a90bd0ba8fb..db257901563 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -9,11 +9,12 @@ import { inject as service } from '@ember/service'; import Route from '@ember/routing/route'; import RSVP from 'rsvp'; import { collect } from '@ember/object/computed'; -import { watchAll, watchQuery } from 'nomad-ui/utils/properties/watch'; +import { watchAll } from 'nomad-ui/utils/properties/watch'; import WithWatchers from 'nomad-ui/mixins/with-watchers'; import notifyForbidden from 'nomad-ui/utils/notify-forbidden'; import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; import { task, restartableTask, timeout } from 'ember-concurrency'; +import { action } from '@ember/object'; export default class IndexRoute extends Route.extend( WithWatchers, @@ -22,7 +23,7 @@ export default class IndexRoute extends Route.extend( @service store; @service watchList; - perPage = 10; + perPage = 1; queryParams = { qpNamespace: { @@ -44,17 +45,23 @@ export default class IndexRoute extends Route.extend( } jobQuery(params, options = {}) { + this.watchList.jobsIndexIDsController.abort(); + this.watchList.jobsIndexIDsController = new AbortController(); + return this.store .query('job', params, { adapterOptions: { method: 'GET', // TODO: default queryType: options.queryType, + abortController: this.watchList.jobsIndexIDsController, }, }) .catch(notifyForbidden(this)); } jobAllocsQuery(jobIDs) { + this.watchList.jobsIndexDetailsController.abort(); + this.watchList.jobsIndexDetailsController = new AbortController(); return this.store .query( 'job', @@ -65,6 +72,7 @@ export default class IndexRoute extends Route.extend( adapterOptions: { method: 'POST', queryType: 'update', + abortController: this.watchList.jobsIndexDetailsController, }, } ) @@ -82,19 +90,27 @@ export default class IndexRoute extends Route.extend( id: job.plainId, namespace: job.belongsTo('namespace').id(), })); + this.controller.set('jobIDs', jobIDs); // BIG TODO: MAKE ANY jobIDs UPDATES TRIGGER A NEW WATCHJOBS TASK // And also cancel the current watchJobs! It may be watching for a different list than the new jobIDs and wouldn't naturally unblock. - this.watchJobs.perform({}, 500); + // this.watchJobs.perform({}, 500); + this.watchList.jobsIndexDetailsController.abort(); + console.log( + 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', + jobIDs + ); + this.watchList.jobsIndexDetailsController = new AbortController(); + this.watchJobs.perform(jobIDs, 500); yield timeout(throttle); // Moved to the end of the loop } } - @restartableTask *watchJobs(params, throttle = 2000) { + @restartableTask *watchJobs(jobIDs, throttle = 2000) { while (true) { - let jobIDs = this.controller.jobIDs; + // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); console.log( @@ -133,14 +149,24 @@ export default class IndexRoute extends Route.extend( }) ); + // Now that we've set the jobIDs, immediately start watching them + this.watchJobs.perform(controller.jobIDs, 500); + + // And also watch for any changes to the jobIDs list this.watchJobIDs.perform({}, 2000); - this.watchJobs.perform({}, 500); // Start watchJobs independently with its own throttle } startWatchers(controller, model) { controller.set('namespacesWatch', this.watchNamespaces.perform()); } + @action + willTransition() { + console.log('willtra'); + this.watchList.jobsIndexDetailsController.abort(); + this.watchList.jobsIndexIDsController.abort(); + } + @watchAll('namespace') watchNamespaces; @collect('watchNamespaces') watchers; } diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 4a1874b0772..9b140a3012d 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -7,29 +7,35 @@ import { computed } from '@ember/object'; import { readOnly } from '@ember/object/computed'; import { copy } from 'ember-copy'; import Service from '@ember/service'; +import { tracked } from '@glimmer/tracking'; -let list = {}; +// let list = {}; export default class WatchListService extends Service { - @computed - get _list() { - return copy(list, true); - } + // @computed + // get _list() { + // return copy(list, true); + // } - @readOnly('_list') list; + jobsIndexIDsController = new AbortController(); + jobsIndexDetailsController = new AbortController(); - constructor() { - super(...arguments); - list = {}; - } + // @readOnly('_list') list; + @tracked list = {}; + + // constructor() { + // super(...arguments); + // list = {}; + // } getIndexFor(url) { - return list[url] || 1; + return this.list[url] || 1; } setIndexFor(url, value) { - list[url] = +value; - console.log('total list is now', list); + this.list[url] = +value; + this.list = { ...this.list }; + console.log('total list is now', this.list); console.log('==================='); } } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index edc09300bd6..faf97470461 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -10,6 +10,14 @@ Model.jobs length: {{this.model.jobs.length}}
Controller.jobs length: {{this.jobs.length}}
Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
+
+ Watchlist: +
    + {{#each-in this.watchList.list as |key value|}} +
  • {{key}}: {{value}}
    • + {{/each-in}} +
+
Date: Tue, 30 Jan 2024 09:54:31 -0500 Subject: [PATCH 60/98] Next token will now update when there's an on-page shift --- ui/app/adapters/job.js | 2 -- ui/app/controllers/jobs/index.js | 16 ++++++++-------- ui/app/routes/jobs/index.js | 25 ++++++++++++++----------- ui/app/templates/jobs/index.hbs | 2 +- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index aac654206df..23edc7300cd 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -282,8 +282,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { ajaxOptions(url, type, options) { let hash = super.ajaxOptions(url, type, options); - console.log('+++ ajaxOptions', url, type, options, hash); - // Custom handling for POST requests to append 'index' as a query parameter if (type === 'POST' && options.data && options.data.index) { let index = encodeURIComponent(options.data.index); diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 2c860419ab5..01992a2348b 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -18,7 +18,7 @@ export default class JobsIndexController extends Controller { @service watchList; // TODO: temp queryParams = [ - 'nextToken', + 'cursorAt', 'pageSize', // 'status', { qpNamespace: 'namespace' }, @@ -55,9 +55,9 @@ export default class JobsIndexController extends Controller { } // #region pagination - @tracked initialNextToken; - @tracked nextToken; - @tracked previousTokens = [null]; + @tracked cursorAt; + @tracked nextToken; // route sets this when new data is fetched + @tracked previousTokens = []; /** * @@ -68,12 +68,12 @@ export default class JobsIndexController extends Controller { // event.preventDefault(); if (page === 'prev') { console.log('prev page'); - this.nextToken = this.previousTokens.pop(); + this.cursorAt = this.previousTokens.pop(); this.previousTokens = [...this.previousTokens]; } else if (page === 'next') { - console.log('next page', this.model.jobs.meta); - this.previousTokens = [...this.previousTokens, this.nextToken]; - this.nextToken = this.model.jobs.meta.nextToken; + console.log('next page', this.nextToken); + this.previousTokens = [...this.previousTokens, this.cursorAt]; + this.cursorAt = this.nextToken; } } // #endregion pagination diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index db257901563..16de23226c5 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -23,13 +23,13 @@ export default class IndexRoute extends Route.extend( @service store; @service watchList; - perPage = 1; + perPage = 2; queryParams = { qpNamespace: { refreshModel: true, }, - nextToken: { + cursorAt: { refreshModel: true, }, }; @@ -41,6 +41,8 @@ export default class IndexRoute extends Route.extend( getCurrentParams() { let queryParams = this.paramsFor(this.routeName); // Get current query params + queryParams.next_token = queryParams.cursorAt; + delete queryParams.cursorAt; // TODO: hacky, should be done in the serializer/adapter? return { ...this.defaultParams, ...queryParams }; } @@ -85,6 +87,9 @@ export default class IndexRoute extends Route.extend( const newJobs = yield this.jobQuery(currentParams, { queryType: 'update_ids', }); + if (newJobs.meta.nextToken) { + this.controller.set('nextToken', newJobs.meta.nextToken); + } const jobIDs = newJobs.map((job) => ({ id: job.plainId, @@ -101,6 +106,7 @@ export default class IndexRoute extends Route.extend( 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', jobIDs ); + // ^--- TODO: bad assumption! this.watchList.jobsIndexDetailsController = new AbortController(); this.watchJobs.perform(jobIDs, 500); @@ -113,11 +119,6 @@ export default class IndexRoute extends Route.extend( // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); - console.log( - 'jobDetails fetched, about to set on controller', - jobDetails, - this.controller - ); this.controller.set('jobs', jobDetails); } // TODO: might need an else condition here for if there are no jobIDs, @@ -139,6 +140,7 @@ export default class IndexRoute extends Route.extend( setupController(controller, model) { super.setupController(controller, model); controller.set('jobs', model.jobs); + controller.set('nextToken', model.jobs.meta.nextToken); controller.set( 'jobIDs', model.jobs.map((job) => { @@ -161,10 +163,11 @@ export default class IndexRoute extends Route.extend( } @action - willTransition() { - console.log('willtra'); - this.watchList.jobsIndexDetailsController.abort(); - this.watchList.jobsIndexIDsController.abort(); + willTransition(transition) { + if (transition.intent.name.startsWith(this.routeName)) { + this.watchList.jobsIndexDetailsController.abort(); + this.watchList.jobsIndexIDsController.abort(); + } } @watchAll('namespace') watchNamespaces; diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index faf97470461..0ffbeac2b06 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -5,7 +5,7 @@ {{page-title "Jobs"}}
- Next token is {{this.model.jobs.meta.nextToken}}
+ Next token is {{this.nextToken}}
Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
Model.jobs length: {{this.model.jobs.length}}
Controller.jobs length: {{this.jobs.length}}
From 5354cd2ff02fec5c89e9a763a0bee2d57bb20b87 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 30 Jan 2024 13:14:03 -0500 Subject: [PATCH 61/98] Very rough anti-jostle technique --- ui/app/controllers/jobs/index.js | 29 ++++++++++++++++++++++++++++- ui/app/routes/jobs/index.js | 20 ++++++++------------ ui/app/templates/jobs/index.hbs | 7 +++++++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 01992a2348b..e08ee0ba89c 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -7,7 +7,7 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; -import { action } from '@ember/object'; +import { action, computed } from '@ember/object'; import { tracked } from '@glimmer/tracking'; const ALL_NAMESPACE_WILDCARD = '*'; @@ -76,5 +76,32 @@ export default class JobsIndexController extends Controller { this.cursorAt = this.nextToken; } } + + /** + * If job_ids are different from jobs, it means our GET summaries has returned + * some new jobs. Instead of jostling the list for the user, give them the option + * to refresh the list. + */ + @computed('jobs.[]', 'jobIDs.[]') + get jobListChangePending() { + const stringifiedJobsEntries = JSON.stringify(this.jobs.map((j) => j.id)); + const stringifiedJobIDsEntries = JSON.stringify( + this.jobIDs.map((j) => JSON.stringify(Object.values(j))) + ); + console.log( + 'checking jobs list pending', + this.jobs, + this.jobIDs, + stringifiedJobsEntries, + stringifiedJobIDsEntries + ); + return stringifiedJobsEntries !== stringifiedJobIDsEntries; + // return this.jobs.map((j) => j.id).join() !== this.jobIDs.join(); + // return true; + } + + @action updateJobList() { + console.log('updating jobs list'); + } // #endregion pagination } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 16de23226c5..ce8a3fa9306 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -97,18 +97,14 @@ export default class IndexRoute extends Route.extend( })); this.controller.set('jobIDs', jobIDs); - // BIG TODO: MAKE ANY jobIDs UPDATES TRIGGER A NEW WATCHJOBS TASK - // And also cancel the current watchJobs! It may be watching for a different list than the new jobIDs and wouldn't naturally unblock. - - // this.watchJobs.perform({}, 500); - this.watchList.jobsIndexDetailsController.abort(); - console.log( - 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', - jobIDs - ); - // ^--- TODO: bad assumption! - this.watchList.jobsIndexDetailsController = new AbortController(); - this.watchJobs.perform(jobIDs, 500); + // this.watchList.jobsIndexDetailsController.abort(); + // console.log( + // 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', + // jobIDs + // ); + // // ^--- TODO: bad assumption! + // this.watchList.jobsIndexDetailsController = new AbortController(); + // this.watchJobs.perform(jobIDs, 500); yield timeout(throttle); // Moved to the end of the loop } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 0ffbeac2b06..7b45585118e 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -10,6 +10,13 @@ Model.jobs length: {{this.model.jobs.length}}
Controller.jobs length: {{this.jobs.length}}
Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
+ {{#if this.jobListChangePending}} + + Changes pending + There are changes to yer list of jobs! Click the button below to update yer list. + + + {{/if}}
Watchlist:
    From 63847e916e06657fbb0120ecb006d05d07aa42d7 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 31 Jan 2024 14:19:22 -0500 Subject: [PATCH 62/98] Demoable, now to move things out of route and into controller --- ui/app/adapters/job.js | 7 ++ ui/app/controllers/jobs/index.js | 69 +++++++++----- ui/app/controllers/settings/tokens.js | 6 ++ ui/app/models/job.js | 9 ++ ui/app/routes/jobs/index.js | 47 +++++++++- ui/app/serializers/job.js | 47 +++++++++- ui/app/templates/jobs/index.hbs | 126 +++++++++++++++++++------- ui/app/templates/settings/tokens.hbs | 30 ++++++ 8 files changed, 285 insertions(+), 56 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 23edc7300cd..c35676ddbb2 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -248,6 +248,13 @@ export default class JobAdapter extends WatchableNamespaceIDs { signal, data: query, skipURLModification: true, + }).then((payload) => { + console.log('thenner', payload, query); + // If there was a request body, append it to my payload + if (query.jobs) { + payload._requestBody = query; + } + return payload; }); } diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index e08ee0ba89c..54eb6fa4966 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -9,6 +9,8 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action, computed } from '@ember/object'; import { tracked } from '@glimmer/tracking'; +import { restartableTask } from 'ember-concurrency'; +import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; const ALL_NAMESPACE_WILDCARD = '*'; @@ -77,31 +79,56 @@ export default class JobsIndexController extends Controller { } } - /** - * If job_ids are different from jobs, it means our GET summaries has returned - * some new jobs. Instead of jostling the list for the user, give them the option - * to refresh the list. - */ - @computed('jobs.[]', 'jobIDs.[]') - get jobListChangePending() { - const stringifiedJobsEntries = JSON.stringify(this.jobs.map((j) => j.id)); - const stringifiedJobIDsEntries = JSON.stringify( - this.jobIDs.map((j) => JSON.stringify(Object.values(j))) - ); - console.log( - 'checking jobs list pending', - this.jobs, - this.jobIDs, - stringifiedJobsEntries, - stringifiedJobIDsEntries + // /** + // * If job_ids are different from jobs, it means our GET summaries has returned + // * some new jobs. Instead of jostling the list for the user, give them the option + // * to refresh the list. + // */ + // @computed('jobs.[]', 'jobIDs.[]') + // get jobListChangePending() { + // const stringifiedJobsEntries = JSON.stringify(this.jobs.map((j) => j.id)); + // const stringifiedJobIDsEntries = JSON.stringify( + // this.jobIDs.map((j) => JSON.stringify(Object.values(j))) + // ); + // console.log( + // 'checking jobs list pending', + // this.jobs, + // this.jobIDs, + // stringifiedJobsEntries, + // stringifiedJobIDsEntries + // ); + // return stringifiedJobsEntries !== stringifiedJobIDsEntries; + // // return this.jobs.map((j) => j.id).join() !== this.jobIDs.join(); + // // return true; + // } + + @tracked pendingJobs = null; + @tracked pendingJobIDs = null; + + get pendingJobIDDiff() { + console.log('pending job IDs', this.pendingJobIDs, this.jobIDs); + return ( + this.pendingJobIDs && + JSON.stringify( + this.pendingJobIDs.map((j) => `${j.namespace}.${j.id}`) + ) !== JSON.stringify(this.jobIDs.map((j) => `${j.namespace}.${j.id}`)) ); - return stringifiedJobsEntries !== stringifiedJobIDsEntries; - // return this.jobs.map((j) => j.id).join() !== this.jobIDs.join(); - // return true; } - @action updateJobList() { + @restartableTask *updateJobList() { console.log('updating jobs list'); + this.jobs = this.pendingJobs; + this.pendingJobs = null; + this.jobIDs = this.pendingJobIDs; + this.pendingJobIDs = null; + // TODO: need to re-kick-off the watchJobs task with updated jobIDs } + + @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; + + // @action updateJobList() { + // console.log('updating jobs list'); + // this.jobs = this.pendingJobs; + // } // #endregion pagination } diff --git a/ui/app/controllers/settings/tokens.js b/ui/app/controllers/settings/tokens.js index 74e2728f3d2..5cbfc3d3741 100644 --- a/ui/app/controllers/settings/tokens.js +++ b/ui/app/controllers/settings/tokens.js @@ -12,6 +12,7 @@ import { action } from '@ember/object'; import classic from 'ember-classic-decorator'; import { tracked } from '@glimmer/tracking'; import Ember from 'ember'; +import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; /** * @type {RegExp} @@ -279,4 +280,9 @@ export default class Tokens extends Controller { get shouldShowPolicies() { return this.tokenRecord; } + + // #region settings + @localStorageProperty('nomadShouldWrapCode', false) wordWrap; + @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdateJobsIndex; + // #endregion settings } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index eabe227074c..4ff5d17e571 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -52,6 +52,11 @@ export default class Job extends Model { return breakdown; } + // When we detect the deletion/purge of a job from within that job page, we kick the user out to the jobs index. + // But what about when that purge is detected from the jobs index? + // We set this flag to true to let the user know that the job has been removed without simply nixing it from view. + @attr('boolean', { defaultValue: false }) assumeGC; + get allocTypes() { return jobAllocStatuses[this.type].map((type) => { return { @@ -214,6 +219,10 @@ export default class Job extends Model { return { label: 'Deploying', state: 'highlight' }; } + if (this.assumeGC) { + return { label: 'Removed', state: 'neutral' }; + } + if (this.type === 'batch' || this.type === 'sysbatch') { // TODO: showing as failed when long-complete // If all the allocs are complete, the job is Complete diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index ce8a3fa9306..eb499bfb361 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -15,6 +15,7 @@ import notifyForbidden from 'nomad-ui/utils/notify-forbidden'; import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; import { task, restartableTask, timeout } from 'ember-concurrency'; import { action } from '@ember/object'; +import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; export default class IndexRoute extends Route.extend( WithWatchers, @@ -23,7 +24,7 @@ export default class IndexRoute extends Route.extend( @service store; @service watchList; - perPage = 2; + perPage = 3; queryParams = { qpNamespace: { @@ -81,6 +82,8 @@ export default class IndexRoute extends Route.extend( .catch(notifyForbidden(this)); } + @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; + @restartableTask *watchJobIDs(params, throttle = 2000) { while (true) { let currentParams = this.getCurrentParams(); @@ -96,7 +99,34 @@ export default class IndexRoute extends Route.extend( namespace: job.belongsTo('namespace').id(), })); - this.controller.set('jobIDs', jobIDs); + const okayToJostle = this.controller.get('liveUpdatesEnabled'); + console.log('okay to jostle?', okayToJostle); + if (okayToJostle) { + // this.controller.set('jobs', newJobs); + this.controller.set('jobIDs', jobIDs); + this.watchList.jobsIndexDetailsController.abort(); + console.log( + 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', + jobIDs + ); + this.watchList.jobsIndexDetailsController = new AbortController(); + this.watchJobs.perform(jobIDs, 500); + } else { + this.controller.set('pendingJobIDs', jobIDs); + this.controller.set('pendingJobs', newJobs); + } + + // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); + // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); + // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); + // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { + // this.controller.set('jobListChangePending', true); + // this.controller.set('pendingJobs', jobDetails); + // } else { + // this.controller.set('jobListChangePending', false); + // this.controller.set('jobs', jobDetails); + // } + // this.watchList.jobsIndexDetailsController.abort(); // console.log( // 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', @@ -115,7 +145,20 @@ export default class IndexRoute extends Route.extend( // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); + + // // Just a sec: what if the user doesnt want their list jostled? + // console.log('xxx jobIds and jobDetails', jobIDs, jobDetails); + + // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); + // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); + // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); + // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { + // this.controller.set('jobListChangePending', true); + // this.controller.set('pendingJobs', jobDetails); + // } else { + // this.controller.set('jobListChangePending', false); this.controller.set('jobs', jobDetails); + // } } // TODO: might need an else condition here for if there are no jobIDs, // which would indicate no jobs, but the updater above might not fire. diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index 6cbe1c2cf12..a3a327d96fb 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -61,7 +61,52 @@ export default class JobSerializer extends ApplicationSerializer { } normalizeQueryResponse(store, primaryModelClass, payload, id, requestType) { - // const jobs = Object.values(payload.Jobs); + // What jobs did we ask for? + console.log('normalizeQueryResponse', payload, id, requestType); + if (payload._requestBody?.jobs) { + let requestedJobIDs = payload._requestBody.jobs; + // If they dont match the jobIDs we got back, we need to create an empty one + // for the ones we didnt get back. + payload.forEach((job) => { + job.AssumeGC = false; + }); + let missingJobIDs = requestedJobIDs.filter( + (j) => + !payload.find((p) => p.ID === j.id && p.Namespace === j.namespace) + ); + missingJobIDs.forEach((job) => { + payload.push({ + ID: job.id, + Namespace: job.namespace, + Allocs: [], + AssumeGC: true, + }); + + job.relationships = { + allocations: { + data: [], + }, + }; + }); + console.log('missingJobIDs', missingJobIDs); + + // If any were missing, sort them in the order they were requested + if (missingJobIDs.length > 0) { + payload.sort((a, b) => { + return ( + requestedJobIDs.findIndex( + (j) => j.id === a.ID && j.namespace === a.Namespace + ) - + requestedJobIDs.findIndex( + (j) => j.id === b.ID && j.namespace === b.Namespace + ) + ); + }); + } + + delete payload._requestBody; + } + const jobs = payload; // Signal that it's a query response at individual normalization level for allocation placement jobs.forEach((job) => { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7b45585118e..a500fd62ec9 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -5,26 +5,54 @@ {{page-title "Jobs"}}
    - Next token is {{this.nextToken}}
    - Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    - Model.jobs length: {{this.model.jobs.length}}
    - Controller.jobs length: {{this.jobs.length}}
    - Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
    - {{#if this.jobListChangePending}} - + + + Jobs (soft J, like "yobs") + + + + + + +{{!-- + Search Jobs: + + --}} + {{#if this.pendingJobIDDiff}} + + + + {{!-- Changes pending There are changes to yer list of jobs! Click the button below to update yer list. - - + + --}} {{/if}} -
    - Watchlist: -
      - {{#each-in this.watchList.list as |key value|}} -
    • {{key}}: {{value}}
      • - {{/each-in}} -
    -
    + + {{!-- {{#unless this.tokenRecord.isExpired}} + + {{/unless}} --}} + + + {{!-- {{#each this.tableColumns as |column|}} {{get B.data (lowercase column.label)}} {{/each}} --}} + {{#if B.data.assumeGC}} + {{B.data.name}} + {{else}} {{/if}} + {{/if}} {{#if this.system.shouldShowNamespaces}} {{B.data.namespace}} @@ -79,21 +112,23 @@ {{B.data.groupCountSum}} desired
    --}}
    - {{#if B.data.childStatuses}} - {{B.data.childStatuses.length}} child jobs;
    - {{#each-in B.data.childStatusBreakdown as |status count|}} - {{count}} {{status}}
    - {{/each-in}} - {{else}} - - {{/if}} + {{#unless B.data.assumeGC}} + {{#if B.data.childStatuses}} + {{B.data.childStatuses.length}} child jobs;
    + {{#each-in B.data.childStatusBreakdown as |status count|}} + {{count}} {{status}}
    + {{/each-in}} + {{else}} + + {{/if}} + {{/unless}}
    @@ -106,4 +141,31 @@ @isDisabledNext={{not this.model.jobs.meta.nextToken}} /> +
    + Next token is {{this.nextToken}}
    + Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    + Model.jobs length: {{this.model.jobs.length}}
    + Controller.jobs length: {{this.jobs.length}}
    + Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
    + Pending Job IDs to watch for ({{this.pendingJobIDs.length}}): {{#each this.pendingJobIDs as |id|}}{{id.id}} | {{/each}}
    + + Live update new/removed jobs? + + {{this.liveUpdatesEnabled}}
    + +
    + Watchlist: +
      + {{#each-in this.watchList.list as |key value|}} +
    • {{key}}: {{value}}
      • + {{/each-in}} +
    +
    + +
    \ No newline at end of file diff --git a/ui/app/templates/settings/tokens.hbs b/ui/app/templates/settings/tokens.hbs index 22473668b55..513e3e64a53 100644 --- a/ui/app/templates/settings/tokens.hbs +++ b/ui/app/templates/settings/tokens.hbs @@ -35,6 +35,36 @@
    + + User Settings + These settings will be saved to your browser settings via Local Storage. + + + + UI + + Word Wrap + Wrap lines of text in logs and exec terminals in the UI + + + Live Updates to Jobs Index + When enabled, new or removed jobs will pop into and out of view on your jobs page. When disabled, you will be notified that changes are pending. + + + + + + {{#if (eq this.signInStatus "failure")}} Date: Wed, 31 Jan 2024 15:59:46 -0500 Subject: [PATCH 63/98] Into the controller, generally --- ui/app/controllers/jobs/index.js | 162 ++++++++++++++++++++++++------- ui/app/routes/jobs/index.js | 137 +++----------------------- 2 files changed, 142 insertions(+), 157 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 54eb6fa4966..ee6b85c7d9f 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -9,14 +9,13 @@ import Controller, { inject as controller } from '@ember/controller'; import { inject as service } from '@ember/service'; import { action, computed } from '@ember/object'; import { tracked } from '@glimmer/tracking'; -import { restartableTask } from 'ember-concurrency'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; - -const ALL_NAMESPACE_WILDCARD = '*'; +import { restartableTask, timeout } from 'ember-concurrency'; export default class JobsIndexController extends Controller { @service router; @service system; + @service store; @service watchList; // TODO: temp queryParams = [ @@ -50,6 +49,9 @@ export default class JobsIndexController extends Controller { } @tracked jobs = []; + @tracked jobIDs = []; + @tracked pendingJobs = null; + @tracked pendingJobIDs = null; @action gotoJob(job) { @@ -79,32 +81,6 @@ export default class JobsIndexController extends Controller { } } - // /** - // * If job_ids are different from jobs, it means our GET summaries has returned - // * some new jobs. Instead of jostling the list for the user, give them the option - // * to refresh the list. - // */ - // @computed('jobs.[]', 'jobIDs.[]') - // get jobListChangePending() { - // const stringifiedJobsEntries = JSON.stringify(this.jobs.map((j) => j.id)); - // const stringifiedJobIDsEntries = JSON.stringify( - // this.jobIDs.map((j) => JSON.stringify(Object.values(j))) - // ); - // console.log( - // 'checking jobs list pending', - // this.jobs, - // this.jobIDs, - // stringifiedJobsEntries, - // stringifiedJobIDsEntries - // ); - // return stringifiedJobsEntries !== stringifiedJobIDsEntries; - // // return this.jobs.map((j) => j.id).join() !== this.jobIDs.join(); - // // return true; - // } - - @tracked pendingJobs = null; - @tracked pendingJobIDs = null; - get pendingJobIDDiff() { console.log('pending job IDs', this.pendingJobIDs, this.jobIDs); return ( @@ -121,14 +97,132 @@ export default class JobsIndexController extends Controller { this.pendingJobs = null; this.jobIDs = this.pendingJobIDs; this.pendingJobIDs = null; - // TODO: need to re-kick-off the watchJobs task with updated jobIDs + this.watchJobs.perform(this.jobIDs, 500); } @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; - // @action updateJobList() { - // console.log('updating jobs list'); - // this.jobs = this.pendingJobs; - // } // #endregion pagination + + //#region querying + + jobQuery(params, options = {}) { + this.watchList.jobsIndexIDsController.abort(); + this.watchList.jobsIndexIDsController = new AbortController(); + + return this.store + .query('job', params, { + adapterOptions: { + method: 'GET', // TODO: default + queryType: options.queryType, + abortController: this.watchList.jobsIndexIDsController, + }, + }) + .catch((e) => { + console.error('error fetching job ids', e); + }); + } + + jobAllocsQuery(jobIDs) { + this.watchList.jobsIndexDetailsController.abort(); + this.watchList.jobsIndexDetailsController = new AbortController(); + return this.store + .query( + 'job', + { + jobs: jobIDs, + }, + { + adapterOptions: { + method: 'POST', + queryType: 'update', + abortController: this.watchList.jobsIndexDetailsController, + }, + } + ) + .catch((e) => { + console.error('error fetching job allocs', e); + }); + } + + perPage = 3; + defaultParams = { + meta: true, + per_page: this.perPage, + }; + + // TODO: this is a pretty hacky way of handling params-grabbing. Can probably iterate over this.queryParams instead. + getCurrentParams() { + let currentRouteName = this.router.currentRouteName; + let currentRoute = this.router.currentRoute; + let params = currentRoute.params[currentRouteName] || {}; + console.log('GCP', params, currentRoute, currentRouteName); + return { ...this.defaultParams, ...params }; + } + + @restartableTask *watchJobIDs(params, throttle = 2000) { + while (true) { + let currentParams = this.getCurrentParams(); + console.log('xxx watchJobIDs', this.queryParams); + const newJobs = yield this.jobQuery(currentParams, { + queryType: 'update_ids', + }); + if (newJobs.meta.nextToken) { + this.nextToken = newJobs.meta.nextToken; + } + + const jobIDs = newJobs.map((job) => ({ + id: job.plainId, + namespace: job.belongsTo('namespace').id(), + })); + + const okayToJostle = this.liveUpdatesEnabled; + console.log('okay to jostle?', okayToJostle); + if (okayToJostle) { + this.jobIDs = jobIDs; + this.watchList.jobsIndexDetailsController.abort(); + console.log( + 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', + jobIDs + ); + this.watchList.jobsIndexDetailsController = new AbortController(); + this.watchJobs.perform(jobIDs, 500); + } else { + // this.controller.set('pendingJobIDs', jobIDs); + // this.controller.set('pendingJobs', newJobs); + this.pendingJobIDs = jobIDs; + this.pendingJobs = newJobs; + } + yield timeout(throttle); // Moved to the end of the loop + } + } + + @restartableTask *watchJobs(jobIDs, throttle = 2000) { + while (true) { + // let jobIDs = this.controller.jobIDs; + if (jobIDs && jobIDs.length > 0) { + let jobDetails = yield this.jobAllocsQuery(jobIDs); + + // // Just a sec: what if the user doesnt want their list jostled? + // console.log('xxx jobIds and jobDetails', jobIDs, jobDetails); + + // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); + // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); + // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); + // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { + // this.controller.set('jobListChangePending', true); + // this.controller.set('pendingJobs', jobDetails); + // } else { + // this.controller.set('jobListChangePending', false); + // this.controller.set('jobs', jobDetails); + this.jobs = jobDetails; + // } + } + // TODO: might need an else condition here for if there are no jobIDs, + // which would indicate no jobs, but the updater above might not fire. + yield timeout(throttle); + } + } + + //#endregion querying } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index eb499bfb361..90748f1d4fa 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -13,9 +13,7 @@ import { watchAll } from 'nomad-ui/utils/properties/watch'; import WithWatchers from 'nomad-ui/mixins/with-watchers'; import notifyForbidden from 'nomad-ui/utils/notify-forbidden'; import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; -import { task, restartableTask, timeout } from 'ember-concurrency'; import { action } from '@ember/object'; -import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; export default class IndexRoute extends Route.extend( WithWatchers, @@ -47,130 +45,19 @@ export default class IndexRoute extends Route.extend( return { ...this.defaultParams, ...queryParams }; } - jobQuery(params, options = {}) { - this.watchList.jobsIndexIDsController.abort(); - this.watchList.jobsIndexIDsController = new AbortController(); - - return this.store - .query('job', params, { + async model(params) { + let currentParams = this.getCurrentParams(); // TODO: how do these differ from passed params? + let jobs = await this.store + .query('job', currentParams, { adapterOptions: { method: 'GET', // TODO: default - queryType: options.queryType, + queryType: 'initialize', abortController: this.watchList.jobsIndexIDsController, }, }) .catch(notifyForbidden(this)); - } - - jobAllocsQuery(jobIDs) { - this.watchList.jobsIndexDetailsController.abort(); - this.watchList.jobsIndexDetailsController = new AbortController(); - return this.store - .query( - 'job', - { - jobs: jobIDs, - }, - { - adapterOptions: { - method: 'POST', - queryType: 'update', - abortController: this.watchList.jobsIndexDetailsController, - }, - } - ) - .catch(notifyForbidden(this)); - } - - @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; - - @restartableTask *watchJobIDs(params, throttle = 2000) { - while (true) { - let currentParams = this.getCurrentParams(); - const newJobs = yield this.jobQuery(currentParams, { - queryType: 'update_ids', - }); - if (newJobs.meta.nextToken) { - this.controller.set('nextToken', newJobs.meta.nextToken); - } - - const jobIDs = newJobs.map((job) => ({ - id: job.plainId, - namespace: job.belongsTo('namespace').id(), - })); - - const okayToJostle = this.controller.get('liveUpdatesEnabled'); - console.log('okay to jostle?', okayToJostle); - if (okayToJostle) { - // this.controller.set('jobs', newJobs); - this.controller.set('jobIDs', jobIDs); - this.watchList.jobsIndexDetailsController.abort(); - console.log( - 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', - jobIDs - ); - this.watchList.jobsIndexDetailsController = new AbortController(); - this.watchJobs.perform(jobIDs, 500); - } else { - this.controller.set('pendingJobIDs', jobIDs); - this.controller.set('pendingJobs', newJobs); - } - - // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); - // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); - // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); - // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { - // this.controller.set('jobListChangePending', true); - // this.controller.set('pendingJobs', jobDetails); - // } else { - // this.controller.set('jobListChangePending', false); - // this.controller.set('jobs', jobDetails); - // } - - // this.watchList.jobsIndexDetailsController.abort(); - // console.log( - // 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', - // jobIDs - // ); - // // ^--- TODO: bad assumption! - // this.watchList.jobsIndexDetailsController = new AbortController(); - // this.watchJobs.perform(jobIDs, 500); - - yield timeout(throttle); // Moved to the end of the loop - } - } - - @restartableTask *watchJobs(jobIDs, throttle = 2000) { - while (true) { - // let jobIDs = this.controller.jobIDs; - if (jobIDs && jobIDs.length > 0) { - let jobDetails = yield this.jobAllocsQuery(jobIDs); - - // // Just a sec: what if the user doesnt want their list jostled? - // console.log('xxx jobIds and jobDetails', jobIDs, jobDetails); - - // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); - // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); - // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); - // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { - // this.controller.set('jobListChangePending', true); - // this.controller.set('pendingJobs', jobDetails); - // } else { - // this.controller.set('jobListChangePending', false); - this.controller.set('jobs', jobDetails); - // } - } - // TODO: might need an else condition here for if there are no jobIDs, - // which would indicate no jobs, but the updater above might not fire. - yield timeout(throttle); - } - } - - async model(params) { - let currentParams = this.getCurrentParams(); - return RSVP.hash({ - jobs: await this.jobQuery(currentParams, { queryType: 'initialize' }), + jobs, namespaces: this.store.findAll('namespace'), nodePools: this.store.findAll('node-pool'), }); @@ -190,11 +77,11 @@ export default class IndexRoute extends Route.extend( }) ); + // TODO: maybe do these in controller constructor? // Now that we've set the jobIDs, immediately start watching them - this.watchJobs.perform(controller.jobIDs, 500); - + this.controller.watchJobs.perform(controller.jobIDs, 500); // And also watch for any changes to the jobIDs list - this.watchJobIDs.perform({}, 2000); + this.controller.watchJobIDs.perform({}, 2000); } startWatchers(controller, model) { @@ -203,10 +90,14 @@ export default class IndexRoute extends Route.extend( @action willTransition(transition) { - if (transition.intent.name.startsWith(this.routeName)) { + console.log('WILLTRA', transition, transition.intent.name, this.routeName); + // TODO: Something is preventing jobs -> job -> jobs -> job. + if (!transition.intent.name.startsWith(this.routeName)) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexIDsController.abort(); } + this.cancelAllWatchers(); + return true; } @watchAll('namespace') watchNamespaces; From d96f5a2f147974828e7bc8efef37c3ae49fd907f Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 1 Feb 2024 15:56:31 -0500 Subject: [PATCH 64/98] Smarter cancellations --- ui/app/adapters/job.js | 4 ++- ui/app/components/job-status/panel/steady.js | 3 ++- ui/app/controllers/jobs/index.js | 26 +++++--------------- ui/app/models/task-event.js | 1 + ui/app/routes/jobs/index.js | 3 ++- ui/app/routes/jobs/job.js | 2 +- 6 files changed, 15 insertions(+), 24 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index c35676ddbb2..77b72cb9888 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -276,7 +276,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { } urlForQuery(query, modelName, method) { - let baseUrl = `/${this.namespace}/jobs/statuses3`; + let baseUrl = `/${this.namespace}/jobs/statuses`; if (method === 'POST') { // Setting a base64 hash to represent the body of the POST request // (which is otherwise not represented in the URL) @@ -298,3 +298,5 @@ export default class JobAdapter extends WatchableNamespaceIDs { return hash; } } + +// TODO: First query (0 jobs to 1 job) doesnt seem to kick off POST diff --git a/ui/app/components/job-status/panel/steady.js b/ui/app/components/job-status/panel/steady.js index 048db7ac92f..842514531ad 100644 --- a/ui/app/components/job-status/panel/steady.js +++ b/ui/app/components/job-status/panel/steady.js @@ -144,7 +144,8 @@ export default class JobStatusPanelSteadyComponent extends Component { if (this.args.job.type === 'service' || this.args.job.type === 'batch') { return this.args.job.taskGroups.reduce((sum, tg) => sum + tg.count, 0); } else if (this.atMostOneAllocPerNode) { - return this.args.job.allocations.uniqBy('nodeID').length; + return this.args.job.allocations.filterBy('nodeID').uniqBy('nodeID') + .length; } else { return this.args.job.count; // TODO: this is probably not the correct totalAllocs count for any type. } diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index ee6b85c7d9f..9082bafaa5c 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -92,7 +92,6 @@ export default class JobsIndexController extends Controller { } @restartableTask *updateJobList() { - console.log('updating jobs list'); this.jobs = this.pendingJobs; this.pendingJobs = null; this.jobIDs = this.pendingJobIDs; @@ -119,7 +118,8 @@ export default class JobsIndexController extends Controller { }, }) .catch((e) => { - console.error('error fetching job ids', e); + console.log('error fetching job ids', e); + return; }); } @@ -141,7 +141,8 @@ export default class JobsIndexController extends Controller { } ) .catch((e) => { - console.error('error fetching job allocs', e); + console.log('error fetching job allocs', e); + return; }); } @@ -151,6 +152,8 @@ export default class JobsIndexController extends Controller { per_page: this.perPage, }; + // TODO: set up isEnabled to check blockingQueries rather than just use while (true) + // TODO: this is a pretty hacky way of handling params-grabbing. Can probably iterate over this.queryParams instead. getCurrentParams() { let currentRouteName = this.router.currentRouteName; @@ -163,7 +166,6 @@ export default class JobsIndexController extends Controller { @restartableTask *watchJobIDs(params, throttle = 2000) { while (true) { let currentParams = this.getCurrentParams(); - console.log('xxx watchJobIDs', this.queryParams); const newJobs = yield this.jobQuery(currentParams, { queryType: 'update_ids', }); @@ -202,24 +204,8 @@ export default class JobsIndexController extends Controller { // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); - - // // Just a sec: what if the user doesnt want their list jostled? - // console.log('xxx jobIds and jobDetails', jobIDs, jobDetails); - - // const stringifiedJobsEntries = JSON.stringify(jobDetails.map(j => j.id)); - // const stringifiedJobIDsEntries = JSON.stringify(jobIDs.map(j => JSON.stringify(Object.values(j)))); - // console.log('checking jobs list pending', this.jobs, this.jobIDs, stringifiedJobsEntries, stringifiedJobIDsEntries); - // if (stringifiedJobsEntries !== stringifiedJobIDsEntries) { - // this.controller.set('jobListChangePending', true); - // this.controller.set('pendingJobs', jobDetails); - // } else { - // this.controller.set('jobListChangePending', false); - // this.controller.set('jobs', jobDetails); this.jobs = jobDetails; - // } } - // TODO: might need an else condition here for if there are no jobIDs, - // which would indicate no jobs, but the updater above might not fire. yield timeout(throttle); } } diff --git a/ui/app/models/task-event.js b/ui/app/models/task-event.js index 56d879b8d46..ecc33acd8d9 100644 --- a/ui/app/models/task-event.js +++ b/ui/app/models/task-event.js @@ -17,6 +17,7 @@ export default class TaskEvent extends Fragment { @attr('date') time; @attr('number') timeNanos; @attr('string') displayMessage; + @attr() message; get message() { let message = simplifyTimeMessage(this.displayMessage); diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 90748f1d4fa..74679118669 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -90,11 +90,12 @@ export default class IndexRoute extends Route.extend( @action willTransition(transition) { - console.log('WILLTRA', transition, transition.intent.name, this.routeName); // TODO: Something is preventing jobs -> job -> jobs -> job. if (!transition.intent.name.startsWith(this.routeName)) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexIDsController.abort(); + this.controller.watchJobs.cancelAll(); + this.controller.watchJobIDs.cancelAll(); } this.cancelAllWatchers(); return true; diff --git a/ui/app/routes/jobs/job.js b/ui/app/routes/jobs/job.js index e6b2eeb8921..59656ef3a9d 100644 --- a/ui/app/routes/jobs/job.js +++ b/ui/app/routes/jobs/job.js @@ -44,7 +44,7 @@ export default class JobRoute extends Route.extend(WithWatchers) { const relatedModelsQueries = [ job.get('allocations'), job.get('evaluations'), - this.store.query('job', { namespace, meta: true }), + // this.store.query('job', { namespace, meta: true }), // TODO: I think I am probably nuking the ability to get meta:pack info here. See https://github.com/hashicorp/nomad/pull/14833 this.store.findAll('namespace'), ]; From bd7ae5e2c6ec39c1983720bc7d8e677a51214ec7 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 1 Feb 2024 17:00:26 -0500 Subject: [PATCH 65/98] Reset abortController on index models run, and system/sysbatch jobs now have an improved groupCountSum computed property --- ui/app/models/job.js | 21 ++++++++++++++++----- ui/app/routes/jobs/index.js | 1 + ui/app/serializers/job.js | 1 + ui/app/templates/jobs/index.hbs | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 4ff5d17e571..174dbadf408 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -34,6 +34,15 @@ export default class Job extends Model { @attr('string') nodePool; // Jobs are related to Node Pools either directly or via its Namespace, but no relationship. @attr('number') groupCountSum; + // if it's a system/sysbatch job, groupCountSum is allocs uniqued by nodeID + get expectedRunningAllocCount() { + if (this.type === 'system' || this.type === 'sysbatch') { + return this.allocations.filterBy('nodeID').uniqBy('nodeID').length; + } else { + return this.groupCountSum; + } + } + @attr('string') deploymentID; @attr() childStatuses; @@ -103,7 +112,7 @@ export default class Job extends Model { * for each clientStatus. */ get allocBlocks() { - let availableSlotsToFill = this.groupCountSum; + let availableSlotsToFill = this.expectedRunningAllocCount; // Initialize allocationsOfShowableType with empty arrays for each clientStatus /** @@ -163,11 +172,11 @@ export default class Job extends Model { const status = alloc.clientStatus; // If the alloc has another clientStatus, add it to the corresponding list - // as long as we haven't reached the groupCountSum limit for that clientStatus + // as long as we haven't reached the expectedRunningAllocCount limit for that clientStatus if ( this.allocTypes.map(({ label }) => label).includes(status) && allocationsOfShowableType[status].healthy.nonCanary.length < - this.groupCountSum + this.expectedRunningAllocCount ) { allocationsOfShowableType[status].healthy.nonCanary.push(alloc); availableSlotsToFill--; @@ -210,8 +219,10 @@ export default class Job extends Model { */ get aggregateAllocStatus() { // If all allocs are running, the job is Healthy - const totalAllocs = this.groupCountSum; - // console.log('groupCountSum is', totalAllocs); + + let totalAllocs = this.expectedRunningAllocCount; + + // console.log('expectedRunningAllocCount is', totalAllocs); // console.log('ablocks are', this.allocBlocks); // If deploying: diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 74679118669..a27d230aef0 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -47,6 +47,7 @@ export default class IndexRoute extends Route.extend( async model(params) { let currentParams = this.getCurrentParams(); // TODO: how do these differ from passed params? + this.watchList.jobsIndexIDsController = new AbortController(); let jobs = await this.store .query('job', currentParams, { adapterOptions: { diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index a3a327d96fb..7c4c7b8b754 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -164,6 +164,7 @@ export default class JobSerializer extends ApplicationSerializer { Healthy: alloc.DeploymentStatus.Healthy, Canary: alloc.DeploymentStatus.Canary, }, + nodeID: alloc.NodeID, }, }, }); diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index a500fd62ec9..da656e71c90 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -125,7 +125,7 @@ @compact={{true}} {{!-- @runningAllocs={{B.data.runningAllocs}} --}} @runningAllocs={{B.data.allocBlocks.running.healthy.nonCanary.length}} - @groupCountSum={{B.data.groupCountSum}} + @groupCountSum={{B.data.expectedRunningAllocCount}} /> {{/if}} {{/unless}} From f37c22bbcf2efc93b6e97a8619468b42367b716c Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 2 Feb 2024 16:34:57 -0500 Subject: [PATCH 66/98] Prev Page reverse querying --- ui/app/adapters/job.js | 2 +- ui/app/controllers/jobs/index.js | 60 +++++++++++++++++++++----------- ui/app/routes/jobs/index.js | 18 +++++++++- ui/app/templates/jobs/index.hbs | 4 +-- 4 files changed, 60 insertions(+), 24 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 77b72cb9888..2ffc209ba71 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -283,7 +283,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { // because the watchList uses the URL as a key for index lookups. return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; } else { - return `${baseUrl}?${queryString.stringify(query)}`; + return `${baseUrl}?${queryString.stringify(query)}`; // TODO: maybe nix this, it's doubling up QPs } } diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 9082bafaa5c..d32631374a7 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -20,7 +20,7 @@ export default class JobsIndexController extends Controller { queryParams = [ 'cursorAt', - 'pageSize', + 'perPage', // 'status', { qpNamespace: 'namespace' }, // 'type', @@ -61,22 +61,34 @@ export default class JobsIndexController extends Controller { // #region pagination @tracked cursorAt; @tracked nextToken; // route sets this when new data is fetched - @tracked previousTokens = []; /** * * @param {"prev"|"next"} page */ - @action handlePageChange(page, event, c) { - console.log('hPC', page, event, c); - // event.preventDefault(); + @action async handlePageChange(page, event, c) { if (page === 'prev') { - console.log('prev page'); - this.cursorAt = this.previousTokens.pop(); - this.previousTokens = [...this.previousTokens]; + // Note (and TODO:) this isn't particularly efficient! + // We're making an extra full request to get the nextToken we need, + // but actually the results of that request are the reverse order, plus one job, + // of what we actually want to show on the page! + // I should investigate whether I can use the results of this query to + // overwrite this controller's jobIDs, leverage its index, and + // restart a blocking watchJobIDs here. + let prevPageToken = await this.loadPreviousPageToken(); + if (prevPageToken.length > 1) { + // if there's only one result, it'd be the job you passed into it as your nextToken (and the first shown on your current page) + const [id, namespace] = JSON.parse(prevPageToken.lastObject.id); + // If there's no nextToken, we're at the "start" of our list and can drop the cursorAt + if (!prevPageToken.meta.nextToken) { + this.cursorAt = null; + } else { + this.cursorAt = `${namespace}.${id}`; + } + } } else if (page === 'next') { console.log('next page', this.nextToken); - this.previousTokens = [...this.previousTokens, this.cursorAt]; + // this.previousTokens = [...this.previousTokens, this.cursorAt]; this.cursorAt = this.nextToken; } } @@ -123,6 +135,24 @@ export default class JobsIndexController extends Controller { }); } + async loadPreviousPageToken() { + let prevPageToken = await this.store.query( + 'job', + { + next_token: this.cursorAt, + per_page: this.perPage + 1, + reverse: true, + }, + { + adapterOptions: { + method: 'GET', + queryType: 'initialize', + }, + } + ); + return prevPageToken; + } + jobAllocsQuery(jobIDs) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexDetailsController = new AbortController(); @@ -153,19 +183,9 @@ export default class JobsIndexController extends Controller { }; // TODO: set up isEnabled to check blockingQueries rather than just use while (true) - - // TODO: this is a pretty hacky way of handling params-grabbing. Can probably iterate over this.queryParams instead. - getCurrentParams() { - let currentRouteName = this.router.currentRouteName; - let currentRoute = this.router.currentRoute; - let params = currentRoute.params[currentRouteName] || {}; - console.log('GCP', params, currentRoute, currentRouteName); - return { ...this.defaultParams, ...params }; - } - @restartableTask *watchJobIDs(params, throttle = 2000) { while (true) { - let currentParams = this.getCurrentParams(); + let currentParams = params; const newJobs = yield this.jobQuery(currentParams, { queryType: 'update_ids', }); diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index a27d230aef0..ced44ec3788 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -31,6 +31,9 @@ export default class IndexRoute extends Route.extend( cursorAt: { refreshModel: true, }, + reverse: { + refreshModel: true, + }, }; defaultParams = { @@ -38,6 +41,8 @@ export default class IndexRoute extends Route.extend( per_page: this.perPage, }; + hasBeenInitialized = false; + getCurrentParams() { let queryParams = this.paramsFor(this.routeName); // Get current query params queryParams.next_token = queryParams.cursorAt; @@ -47,6 +52,7 @@ export default class IndexRoute extends Route.extend( async model(params) { let currentParams = this.getCurrentParams(); // TODO: how do these differ from passed params? + this.watchList.jobsIndexIDsController.abort(); this.watchList.jobsIndexIDsController = new AbortController(); let jobs = await this.store .query('job', currentParams, { @@ -57,6 +63,7 @@ export default class IndexRoute extends Route.extend( }, }) .catch(notifyForbidden(this)); + this.hasBeenInitialized = true; return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -82,7 +89,7 @@ export default class IndexRoute extends Route.extend( // Now that we've set the jobIDs, immediately start watching them this.controller.watchJobs.perform(controller.jobIDs, 500); // And also watch for any changes to the jobIDs list - this.controller.watchJobIDs.perform({}, 2000); + this.controller.watchJobIDs.perform(this.getCurrentParams(), 2000); } startWatchers(controller, model) { @@ -102,6 +109,15 @@ export default class IndexRoute extends Route.extend( return true; } + // Determines if we should be put into a loading state (jobs/loading.hbs) + // This is a useful page for when you're first initializing your jobs list, + // but overkill when we paginate / change our queryParams. We should handle that + // with in-compnent loading/skeleton states instead. + @action + loading() { + return !this.hasBeenInitialized; // allows the loading template to be shown + } + @watchAll('namespace') watchNamespaces; @collect('watchNamespaces') watchers; } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index da656e71c90..cfcd2a9a25f 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -137,13 +137,13 @@
    Next token is {{this.nextToken}}
    - Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    + {{!-- Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    --}} Model.jobs length: {{this.model.jobs.length}}
    Controller.jobs length: {{this.jobs.length}}
    Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
    From 746f48f10cf666403c5e3687cd866fbb19f0acd0 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 12 Feb 2024 17:00:41 -0500 Subject: [PATCH 67/98] n+1th jobs existing will trigger nextToken/pagination display --- ui/app/controllers/jobs/index.js | 2 ++ ui/app/templates/jobs/index.hbs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index d32631374a7..d45b46c544e 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -191,6 +191,8 @@ export default class JobsIndexController extends Controller { }); if (newJobs.meta.nextToken) { this.nextToken = newJobs.meta.nextToken; + } else { + this.nextToken = null; } const jobIDs = newJobs.map((job) => ({ diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index cfcd2a9a25f..9d4999c5360 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -138,7 +138,7 @@
    From 51064e94ece9e71b8f5b0a342759b88c9b543f45 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 4 Mar 2024 10:04:31 -0500 Subject: [PATCH 68/98] Start of a GET/POST statuses return --- ui/app/controllers/jobs/index.js | 5 +-- ui/mirage/config.js | 61 ++++++++++++++++++++++++++++++++ ui/mirage/scenarios/default.js | 12 +++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index d45b46c544e..2eb9a6a5160 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -108,7 +108,7 @@ export default class JobsIndexController extends Controller { this.pendingJobs = null; this.jobIDs = this.pendingJobIDs; this.pendingJobIDs = null; - this.watchJobs.perform(this.jobIDs, 500); + this.watchJobs.perform(this.jobIDs, 2000); } @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; @@ -210,7 +210,8 @@ export default class JobsIndexController extends Controller { jobIDs ); this.watchList.jobsIndexDetailsController = new AbortController(); - this.watchJobs.perform(jobIDs, 500); + // make sure throttle has taken place! + this.watchJobs.perform(jobIDs, throttle); } else { // this.controller.set('pendingJobIDs', jobIDs); // this.controller.set('pendingJobs', newJobs); diff --git a/ui/mirage/config.js b/ui/mirage/config.js index fa49903778e..9f8dbd06b18 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -74,6 +74,67 @@ export default function () { }) ); + this.get( + '/jobs/statuses', + withBlockingSupport(function ({ jobs }, req) { + const json = this.serialize(jobs.all()); + const namespace = req.queryParams.namespace || 'default'; + return json + .filter((job) => { + if (namespace === '*') return true; + return namespace === 'default' + ? !job.NamespaceID || job.NamespaceID === 'default' + : job.NamespaceID === namespace; + }) + .map((job) => filterKeys(job, 'TaskGroups', 'NamespaceID')); + }) + ); + + this.post( + '/jobs/statuses', + withBlockingSupport(function ({ jobs }, req) { + // console.log('postbody', req, server.db, server.schema); + + let returnedJobs = this.serialize(jobs.all()).map((j) => { + let job = {}; + // job.priority = Math.floor(Math.random() * 100); + job.ID = j.ID; + job.Name = j.Name; + job.Allocs = server.db.allocations + .where({ jobId: j.ID, namespace: j.Namespace }) + .map((alloc) => { + return { + ClientStatus: alloc.clientStatus, + DeploymentStatus: { + Canary: false, + Healthy: true, + }, + Group: alloc.taskGroup, + JobVersion: alloc.jobVersion, + NodeID: alloc.nodeId, + ID: alloc.id, + }; + }); + job.ChildStatuses = null; // TODO: handle parent job here + job.Datacenters = ['*']; // TODO + job.DeploymentID = j.DeploymentID; + job.GroupCountSum = j.TaskGroups.mapBy('Count').reduce( + (a, b) => a + b, + 0 + ); + job.Namespace = j.NamespaceID; + job.NodePool = j.NodePool; + job.Type = j.Type; + job.Priority = j.Priority; + job.Version = j.Version; + job.SmartAllocs = {}; // TODO + return job; + }); + // console.log('returned', returnedJobs); + return returnedJobs; + }) + ); + this.post('/jobs', function (schema, req) { const body = JSON.parse(req.requestBody); diff --git a/ui/mirage/scenarios/default.js b/ui/mirage/scenarios/default.js index fc70d71542c..52963215690 100644 --- a/ui/mirage/scenarios/default.js +++ b/ui/mirage/scenarios/default.js @@ -28,6 +28,7 @@ export const allScenarios = { policiesTestCluster, rolesTestCluster, namespacesTestCluster, + jobsIndexTestCluster, ...topoScenarios, ...sysbatchScenarios, }; @@ -57,6 +58,17 @@ export default function (server) { // Scenarios +function jobsIndexTestCluster(server) { + faker.seed(1); + server.createList('agent', 1, 'withConsulLink', 'withVaultLink'); + server.createList('node', 1); + server.createList('job', 1, { + namespaceId: 'default', + resourceSpec: Array(1).fill('M: 256, C: 500'), + groupAllocCount: 1, + }); +} + function smallCluster(server) { faker.seed(1); server.create('feature', { name: 'Dynamic Application Sizing' }); From 964e38b91f3a1ccc0d1118d4e30749441b3ab65c Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 4 Mar 2024 11:28:30 -0500 Subject: [PATCH 69/98] Namespace fix --- ui/app/templates/jobs/index.hbs | 2 +- ui/mirage/config.js | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 9d4999c5360..daf6b3fad11 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -90,7 +90,7 @@ {{/if}} {{#if this.system.shouldShowNamespaces}} - {{B.data.namespace}} + {{B.data.namespace.id}} {{/if}} {{#if this.system.shouldShowNodepools}} {{B.data.nodepool}} diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 9f8dbd06b18..904ac1c0daf 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -30,7 +30,7 @@ export function filesForPath(allocFiles, filterPath) { export default function () { this.timing = 0; // delay for each request, automatically set to 0 during testing - this.logging = window.location.search.includes('mirage-logging=true'); + this.logging = true; // TODO: window.location.search.includes('mirage-logging=true'); this.namespace = 'v1'; this.trackRequests = Ember.testing; @@ -97,7 +97,6 @@ export default function () { let returnedJobs = this.serialize(jobs.all()).map((j) => { let job = {}; - // job.priority = Math.floor(Math.random() * 100); job.ID = j.ID; job.Name = j.Name; job.Allocs = server.db.allocations @@ -116,7 +115,7 @@ export default function () { }; }); job.ChildStatuses = null; // TODO: handle parent job here - job.Datacenters = ['*']; // TODO + job.Datacenters = j.Datacenters; job.DeploymentID = j.DeploymentID; job.GroupCountSum = j.TaskGroups.mapBy('Count').reduce( (a, b) => a + b, @@ -130,7 +129,6 @@ export default function () { job.SmartAllocs = {}; // TODO return job; }); - // console.log('returned', returnedJobs); return returnedJobs; }) ); From bfb6496b5c861a298f2c27a66d0341232beaffa7 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Mon, 4 Mar 2024 16:53:20 -0500 Subject: [PATCH 70/98] Unblock tests --- ui/app/controllers/jobs/index.js | 5 +++-- ui/app/routes/jobs/index.js | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 2eb9a6a5160..0e375fd64da 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -11,6 +11,7 @@ import { action, computed } from '@ember/object'; import { tracked } from '@glimmer/tracking'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; import { restartableTask, timeout } from 'ember-concurrency'; +import Ember from 'ember'; export default class JobsIndexController extends Controller { @service router; @@ -184,7 +185,7 @@ export default class JobsIndexController extends Controller { // TODO: set up isEnabled to check blockingQueries rather than just use while (true) @restartableTask *watchJobIDs(params, throttle = 2000) { - while (true) { + while (true && !Ember.testing) { let currentParams = params; const newJobs = yield this.jobQuery(currentParams, { queryType: 'update_ids', @@ -223,7 +224,7 @@ export default class JobsIndexController extends Controller { } @restartableTask *watchJobs(jobIDs, throttle = 2000) { - while (true) { + while (true && !Ember.testing) { // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index ced44ec3788..0d00daa05c6 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -22,7 +22,7 @@ export default class IndexRoute extends Route.extend( @service store; @service watchList; - perPage = 3; + perPage = 10; queryParams = { qpNamespace: { @@ -87,7 +87,7 @@ export default class IndexRoute extends Route.extend( // TODO: maybe do these in controller constructor? // Now that we've set the jobIDs, immediately start watching them - this.controller.watchJobs.perform(controller.jobIDs, 500); + this.controller.watchJobs.perform(controller.jobIDs, 2000); // And also watch for any changes to the jobIDs list this.controller.watchJobIDs.perform(this.getCurrentParams(), 2000); } @@ -99,7 +99,7 @@ export default class IndexRoute extends Route.extend( @action willTransition(transition) { // TODO: Something is preventing jobs -> job -> jobs -> job. - if (!transition.intent.name.startsWith(this.routeName)) { + if (!transition.intent.name?.startsWith(this.routeName)) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexIDsController.abort(); this.controller.watchJobs.cancelAll(); From 3720513459356d4f1858fb163c2d70802e0769ae Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 5 Mar 2024 17:02:29 -0500 Subject: [PATCH 71/98] Realizing to my small horror that this skipURLModification flag may be too heavy handed --- ui/app/adapters/node.js | 3 +++ ui/app/routes/jobs/index.js | 3 ++- ui/mirage/config.js | 2 +- ui/tests/acceptance/application-errors-test.js | 6 +++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/app/adapters/node.js b/ui/app/adapters/node.js index 70cf51b4f14..0cd1b04df16 100644 --- a/ui/app/adapters/node.js +++ b/ui/app/adapters/node.js @@ -27,6 +27,7 @@ export default class NodeAdapter extends Watchable { NodeID: node.id, Eligibility: isEligible ? 'eligible' : 'ineligible', }, + skipURLModification: true, }); } @@ -45,6 +46,7 @@ export default class NodeAdapter extends Watchable { drainSpec ), }, + skipURLModification: true, }); } @@ -71,6 +73,7 @@ export default class NodeAdapter extends Watchable { const url = `/v1/client/metadata?node_id=${node.id}`; return this.ajax(url, 'POST', { data: { Meta: newMeta }, + skipURLModification: true, }); } } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 0d00daa05c6..f4390350eeb 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -73,7 +73,8 @@ export default class IndexRoute extends Route.extend( setupController(controller, model) { super.setupController(controller, model); - controller.set('jobs', model.jobs); + // TODO: consider re-instating this. This is setting them and then their order gets shuffled. + // controller.set('jobs', model.jobs); controller.set('nextToken', model.jobs.meta.nextToken); controller.set( 'jobIDs', diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 904ac1c0daf..4bc64069e6d 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -126,7 +126,7 @@ export default function () { job.Type = j.Type; job.Priority = j.Priority; job.Version = j.Version; - job.SmartAllocs = {}; // TODO + job.SmartAlloc = {}; // TODO return job; }); return returnedJobs; diff --git a/ui/tests/acceptance/application-errors-test.js b/ui/tests/acceptance/application-errors-test.js index 52d9c8a2463..e74f7746820 100644 --- a/ui/tests/acceptance/application-errors-test.js +++ b/ui/tests/acceptance/application-errors-test.js @@ -70,7 +70,11 @@ module('Acceptance | application errors ', function (hooks) { test('the no leader error state gets its own error message', async function (assert) { assert.expect(2); - server.pretender.get('/v1/jobs', () => [500, {}, 'No cluster leader']); + server.pretender.get('/v1/jobs/statuses', () => [ + 500, + {}, + 'No cluster leader', + ]); await JobsList.visit(); From a4221eb4da29ee73809b3057c6a453648d3c57c6 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 6 Mar 2024 09:21:14 -0500 Subject: [PATCH 72/98] Lintfix --- ui/app/adapters/job.js | 3 +-- ui/app/controllers/jobs/index.js | 8 ++++---- ui/app/routes/jobs/index.js | 8 ++++++-- ui/app/services/watch-list.js | 6 +++--- ui/app/utils/properties/watch.js | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 2ffc209ba71..f381b074dc1 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -249,7 +249,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { data: query, skipURLModification: true, }).then((payload) => { - console.log('thenner', payload, query); // If there was a request body, append it to my payload if (query.jobs) { payload._requestBody = query; @@ -258,7 +257,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { }); } - handleResponse(status, headers, payload, requestData) { + handleResponse(status, headers) { // watchList.setIndexFor() happens in the watchable adapter, super'd here /** diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 0e375fd64da..9a8593e944a 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -5,9 +5,9 @@ // @ts-check -import Controller, { inject as controller } from '@ember/controller'; +import Controller from '@ember/controller'; import { inject as service } from '@ember/service'; -import { action, computed } from '@ember/object'; +import { action } from '@ember/object'; import { tracked } from '@glimmer/tracking'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; import { restartableTask, timeout } from 'ember-concurrency'; @@ -67,7 +67,7 @@ export default class JobsIndexController extends Controller { * * @param {"prev"|"next"} page */ - @action async handlePageChange(page, event, c) { + @action async handlePageChange(page) { if (page === 'prev') { // Note (and TODO:) this isn't particularly efficient! // We're making an extra full request to get the nextToken we need, @@ -109,7 +109,7 @@ export default class JobsIndexController extends Controller { this.pendingJobs = null; this.jobIDs = this.pendingJobIDs; this.pendingJobIDs = null; - this.watchJobs.perform(this.jobIDs, 2000); + yield this.watchJobs.perform(this.jobIDs, 2000); } @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index f4390350eeb..dc071975368 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -50,7 +50,7 @@ export default class IndexRoute extends Route.extend( return { ...this.defaultParams, ...queryParams }; } - async model(params) { + async model(/*params*/) { let currentParams = this.getCurrentParams(); // TODO: how do these differ from passed params? this.watchList.jobsIndexIDsController.abort(); this.watchList.jobsIndexIDsController = new AbortController(); @@ -88,12 +88,14 @@ export default class IndexRoute extends Route.extend( // TODO: maybe do these in controller constructor? // Now that we've set the jobIDs, immediately start watching them + // eslint-disable-next-line this.controller.watchJobs.perform(controller.jobIDs, 2000); // And also watch for any changes to the jobIDs list + // eslint-disable-next-line this.controller.watchJobIDs.perform(this.getCurrentParams(), 2000); } - startWatchers(controller, model) { + startWatchers(controller) { controller.set('namespacesWatch', this.watchNamespaces.perform()); } @@ -103,7 +105,9 @@ export default class IndexRoute extends Route.extend( if (!transition.intent.name?.startsWith(this.routeName)) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexIDsController.abort(); + // eslint-disable-next-line this.controller.watchJobs.cancelAll(); + // eslint-disable-next-line this.controller.watchJobIDs.cancelAll(); } this.cancelAllWatchers(); diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 9b140a3012d..19b3e6925c5 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -3,9 +3,9 @@ * SPDX-License-Identifier: BUSL-1.1 */ -import { computed } from '@ember/object'; -import { readOnly } from '@ember/object/computed'; -import { copy } from 'ember-copy'; +// import { computed } from '@ember/object'; +// import { readOnly } from '@ember/object/computed'; +// import { copy } from 'ember-copy'; import Service from '@ember/service'; import { tracked } from '@glimmer/tracking'; diff --git a/ui/app/utils/properties/watch.js b/ui/app/utils/properties/watch.js index 7bb376b9a99..3935f83f5bf 100644 --- a/ui/app/utils/properties/watch.js +++ b/ui/app/utils/properties/watch.js @@ -135,7 +135,7 @@ export function watchAll(modelName) { } export function watchQuery(modelName) { - return task(function* (params, throttle = 2000, options = {}) { + return task(function* (params, throttle = 2000 /*options = {}*/) { assert( 'To watch a query, the adapter for the type being queried MUST extend Watchable', this.store.adapterFor(modelName) instanceof Watchable From 57998e9bc04a3e4d32d478236e720a8c2d5601bc Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 6 Mar 2024 12:56:16 -0500 Subject: [PATCH 73/98] Default liveupdates localStorage setting to true --- ui/app/controllers/jobs/index.js | 2 +- ui/app/controllers/settings/tokens.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 9a8593e944a..c0f106f995c 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -112,7 +112,7 @@ export default class JobsIndexController extends Controller { yield this.watchJobs.perform(this.jobIDs, 2000); } - @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdatesEnabled; + @localStorageProperty('nomadLiveUpdateJobsIndex', true) liveUpdatesEnabled; // #endregion pagination diff --git a/ui/app/controllers/settings/tokens.js b/ui/app/controllers/settings/tokens.js index 5cbfc3d3741..7d679606e28 100644 --- a/ui/app/controllers/settings/tokens.js +++ b/ui/app/controllers/settings/tokens.js @@ -283,6 +283,6 @@ export default class Tokens extends Controller { // #region settings @localStorageProperty('nomadShouldWrapCode', false) wordWrap; - @localStorageProperty('nomadLiveUpdateJobsIndex', false) liveUpdateJobsIndex; + @localStorageProperty('nomadLiveUpdateJobsIndex', true) liveUpdateJobsIndex; // #endregion settings } From 28df4f4b97b8ef913184164699518a079a16c560 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 12 Mar 2024 15:29:31 -0400 Subject: [PATCH 74/98] Pagination and index rethink --- ui/app/adapters/job.js | 4 +- ui/app/adapters/node.js | 3 - ui/app/adapters/watchable.js | 7 +- ui/app/controllers/jobs/index.js | 121 +++++++++--- ui/app/models/job.js | 3 +- ui/app/routes/jobs/index.js | 22 +-- ui/app/serializers/allocation.js | 2 + ui/app/serializers/job.js | 3 +- ui/app/services/watch-list.js | 17 +- ui/app/templates/jobs/index.hbs | 305 +++++++++++++++++++++++++++---- ui/mirage/scenarios/default.js | 2 +- 11 files changed, 393 insertions(+), 96 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index f381b074dc1..64854e9ba57 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -236,8 +236,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { // TODO: adding a new job hash will not necessarily cancel the old one. // You could be holding open a POST on jobs AB and ABC at the same time. - console.log('index for', url, 'is', index); - if (index && index > 1) { query.index = index; } @@ -247,7 +245,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { return this.ajax(url, method, { signal, data: query, - skipURLModification: true, + modifyURL: options.adapterOptions.modifyURL, }).then((payload) => { // If there was a request body, append it to my payload if (query.jobs) { diff --git a/ui/app/adapters/node.js b/ui/app/adapters/node.js index 0cd1b04df16..70cf51b4f14 100644 --- a/ui/app/adapters/node.js +++ b/ui/app/adapters/node.js @@ -27,7 +27,6 @@ export default class NodeAdapter extends Watchable { NodeID: node.id, Eligibility: isEligible ? 'eligible' : 'ineligible', }, - skipURLModification: true, }); } @@ -46,7 +45,6 @@ export default class NodeAdapter extends Watchable { drainSpec ), }, - skipURLModification: true, }); } @@ -73,7 +71,6 @@ export default class NodeAdapter extends Watchable { const url = `/v1/client/metadata?node_id=${node.id}`; return this.ajax(url, 'POST', { data: { Meta: newMeta }, - skipURLModification: true, }); } } diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index 468e5f73829..eb5e7e1a4d2 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -24,10 +24,13 @@ export default class Watchable extends ApplicationAdapter { // // It's either this weird side-effecting thing that also requires a change // to ajaxOptions or overriding ajax completely. - ajax(url, type, options) { + ajax(url, type, options = {}) { const hasParams = hasNonBlockingQueryParams(options); - if (!hasParams || options.skipURLModification) + // if (!hasParams || options.skipURLModification) + // return super.ajax(url, type, options); + if (!hasParams || type !== 'GET' || options.modifyURL === false) return super.ajax(url, type, options); + // ^-- TODO: this means that all non-GETs dont get their URLs modified. Make sure this is what we want. let params = { ...options?.data }; delete params.index; return super.ajax(`${url}?${queryString.stringify(params)}`, type, options); diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index c0f106f995c..e6c3dade9da 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -11,6 +11,12 @@ import { action } from '@ember/object'; import { tracked } from '@glimmer/tracking'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; import { restartableTask, timeout } from 'ember-concurrency'; +import { + serialize, + deserializedQueryParam as selection, +} from 'nomad-ui/utils/qp-serialize'; +// import { scheduleOnce } from '@ember/runloop'; + import Ember from 'ember'; export default class JobsIndexController extends Controller { @@ -19,9 +25,13 @@ export default class JobsIndexController extends Controller { @service store; @service watchList; // TODO: temp + // qpNamespace = '*'; + per_page = 10; + reverse = false; + queryParams = [ 'cursorAt', - 'perPage', + 'per_page', // 'status', { qpNamespace: 'namespace' }, // 'type', @@ -30,6 +40,43 @@ export default class JobsIndexController extends Controller { isForbidden = false; + // #region filtering and sorting + + @selection('qpNamespace') selectionNamespace; + // @computed('qpNamespace', 'model.namespaces.[]') + get optionsNamespace() { + const availableNamespaces = this.model.namespaces.map((namespace) => ({ + key: namespace.name, + label: namespace.name, + })); + + availableNamespaces.unshift({ + key: '*', + label: 'All (*)', + }); + + // // Unset the namespace selection if it was server-side deleted + // if (!availableNamespaces.mapBy('key').includes(this.qpNamespace)) { + // scheduleOnce('actions', () => { + // this.set('qpNamespace', '*'); + // }); + // } + + return availableNamespaces; + } + + @action + handleFilterChange(queryParamValue, option, queryParamLabel) { + if (queryParamValue.includes(option)) { + queryParamValue.removeObject(option); + } else { + queryParamValue.addObject(option); + } + this[queryParamLabel] = serialize(queryParamValue); + } + + // #endregion filtering and sorting + get tableColumns() { return [ 'name', @@ -59,6 +106,11 @@ export default class JobsIndexController extends Controller { this.router.transitionTo('jobs.job.index', job.idWithNamespace); } + @action + goToRun() { + this.router.transitionTo('jobs.run'); + } + // #region pagination @tracked cursorAt; @tracked nextToken; // route sets this when new data is fetched @@ -88,14 +140,11 @@ export default class JobsIndexController extends Controller { } } } else if (page === 'next') { - console.log('next page', this.nextToken); - // this.previousTokens = [...this.previousTokens, this.cursorAt]; this.cursorAt = this.nextToken; } } get pendingJobIDDiff() { - console.log('pending job IDs', this.pendingJobIDs, this.jobIDs); return ( this.pendingJobIDs && JSON.stringify( @@ -104,6 +153,10 @@ export default class JobsIndexController extends Controller { ); } + /** + * Manually, on click, update jobs from pendingJobs + * when live updates are disabled (via nomadLiveUpdateJobsIndex) + */ @restartableTask *updateJobList() { this.jobs = this.pendingJobs; this.pendingJobs = null; @@ -128,32 +181,17 @@ export default class JobsIndexController extends Controller { method: 'GET', // TODO: default queryType: options.queryType, abortController: this.watchList.jobsIndexIDsController, + modifyURL: false, }, }) .catch((e) => { - console.log('error fetching job ids', e); + if (e.name !== 'AbortError') { + console.log('error fetching job ids', e); + } return; }); } - async loadPreviousPageToken() { - let prevPageToken = await this.store.query( - 'job', - { - next_token: this.cursorAt, - per_page: this.perPage + 1, - reverse: true, - }, - { - adapterOptions: { - method: 'GET', - queryType: 'initialize', - }, - } - ); - return prevPageToken; - } - jobAllocsQuery(jobIDs) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexDetailsController = new AbortController(); @@ -168,20 +206,36 @@ export default class JobsIndexController extends Controller { method: 'POST', queryType: 'update', abortController: this.watchList.jobsIndexDetailsController, + // modifyURL: false, }, } ) .catch((e) => { - console.log('error fetching job allocs', e); + if (e.name !== 'AbortError') { + console.log('error fetching job allocs', e); + } return; }); } - perPage = 3; - defaultParams = { - meta: true, - per_page: this.perPage, - }; + async loadPreviousPageToken() { + let prevPageToken = await this.store.query( + 'job', + { + next_token: this.cursorAt, + per_page: this.per_page + 1, + reverse: true, + }, + { + adapterOptions: { + method: 'GET', + queryType: 'initialize', + modifyURL: false, + }, + } + ); + return prevPageToken; + } // TODO: set up isEnabled to check blockingQueries rather than just use while (true) @restartableTask *watchJobIDs(params, throttle = 2000) { @@ -190,6 +244,9 @@ export default class JobsIndexController extends Controller { const newJobs = yield this.jobQuery(currentParams, { queryType: 'update_ids', }); + if (!newJobs) { + return; + } if (newJobs.meta.nextToken) { this.nextToken = newJobs.meta.nextToken; } else { @@ -202,7 +259,6 @@ export default class JobsIndexController extends Controller { })); const okayToJostle = this.liveUpdatesEnabled; - console.log('okay to jostle?', okayToJostle); if (okayToJostle) { this.jobIDs = jobIDs; this.watchList.jobsIndexDetailsController.abort(); @@ -223,6 +279,11 @@ export default class JobsIndexController extends Controller { } } + // Called in 3 ways: + // 1. via the setupController of the jobs index route's model + // (which can happen both on initial load, and should the queryParams change) + // 2. via the watchJobIDs task seeing new jobIDs + // 3. via the user manually clicking to updateJobList() @restartableTask *watchJobs(jobIDs, throttle = 2000) { while (true && !Ember.testing) { // let jobIDs = this.controller.jobIDs; diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 174dbadf408..176ece19306 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -250,8 +250,7 @@ export default class Job extends Model { } const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; - // console.log('healthyAllocs', this.name, healthyAllocs, totalAllocs); - if (healthyAllocs?.length === totalAllocs) { + if (totalAllocs && healthyAllocs?.length === totalAllocs) { return { label: 'Healthy', state: 'success' }; } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index dc071975368..b96fcf8ea6c 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -22,7 +22,7 @@ export default class IndexRoute extends Route.extend( @service store; @service watchList; - perPage = 10; + // perPage = 10; queryParams = { qpNamespace: { @@ -31,14 +31,6 @@ export default class IndexRoute extends Route.extend( cursorAt: { refreshModel: true, }, - reverse: { - refreshModel: true, - }, - }; - - defaultParams = { - meta: true, - per_page: this.perPage, }; hasBeenInitialized = false; @@ -47,7 +39,7 @@ export default class IndexRoute extends Route.extend( let queryParams = this.paramsFor(this.routeName); // Get current query params queryParams.next_token = queryParams.cursorAt; delete queryParams.cursorAt; // TODO: hacky, should be done in the serializer/adapter? - return { ...this.defaultParams, ...queryParams }; + return { ...queryParams }; } async model(/*params*/) { @@ -60,10 +52,10 @@ export default class IndexRoute extends Route.extend( method: 'GET', // TODO: default queryType: 'initialize', abortController: this.watchList.jobsIndexIDsController, + modifyURL: false, }, }) .catch(notifyForbidden(this)); - this.hasBeenInitialized = true; return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -86,13 +78,19 @@ export default class IndexRoute extends Route.extend( }) ); + // Note: we should remove the indexes from the watch-list for jobs index queries if we've already initialized, since + // if we explicitly change our queryParams we want to start from scratch, unindexed + this.watchList.clearJobsIndexIndexes(); + // TODO: maybe do these in controller constructor? // Now that we've set the jobIDs, immediately start watching them // eslint-disable-next-line - this.controller.watchJobs.perform(controller.jobIDs, 2000); + this.controller.watchJobs.perform(controller.jobIDs, 2000, 'update'); // And also watch for any changes to the jobIDs list // eslint-disable-next-line this.controller.watchJobIDs.perform(this.getCurrentParams(), 2000); + + this.hasBeenInitialized = true; } startWatchers(controller) { diff --git a/ui/app/serializers/allocation.js b/ui/app/serializers/allocation.js index 276be41114d..3bc595e792a 100644 --- a/ui/app/serializers/allocation.js +++ b/ui/app/serializers/allocation.js @@ -49,6 +49,8 @@ export default class AllocationSerializer extends ApplicationSerializer { .sort() .map((key) => { const state = states[key] || {}; + // make sure events, if null, is an empty array + state.Events = state.Events || []; const summary = { Name: key }; Object.keys(state).forEach( (stateKey) => (summary[stateKey] = state[stateKey]) diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index 7c4c7b8b754..02735229cb6 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -62,7 +62,6 @@ export default class JobSerializer extends ApplicationSerializer { normalizeQueryResponse(store, primaryModelClass, payload, id, requestType) { // What jobs did we ask for? - console.log('normalizeQueryResponse', payload, id, requestType); if (payload._requestBody?.jobs) { let requestedJobIDs = payload._requestBody.jobs; // If they dont match the jobIDs we got back, we need to create an empty one @@ -88,9 +87,9 @@ export default class JobSerializer extends ApplicationSerializer { }, }; }); - console.log('missingJobIDs', missingJobIDs); // If any were missing, sort them in the order they were requested + // TODO: document why if (missingJobIDs.length > 0) { payload.sort((a, b) => { return ( diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index 19b3e6925c5..e71e9df35bd 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -35,7 +35,20 @@ export default class WatchListService extends Service { setIndexFor(url, value) { this.list[url] = +value; this.list = { ...this.list }; - console.log('total list is now', this.list); - console.log('==================='); + } + + /** + * When we paginate or otherwise manually change queryParams for our jobs index, + * we want our requests to return immediately. This means we need to clear out + * any previous indexes that are associated with the jobs index. + */ + clearJobsIndexIndexes() { + // If it starts with /v1/jobs/statuses, remove it + let keys = Object.keys(this.list); + keys.forEach((key) => { + if (key.startsWith('/v1/jobs/statuses')) { + delete this.list[key]; + } + }); } } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index daf6b3fad11..ed47744d2ba 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -6,53 +6,280 @@ {{page-title "Jobs"}}
    - + {{!-- Jobs (soft J, like "yobs") - + --}} + {{!-- + + + + {{#each this.clientFilterToggles.state as |option|}} + + {{capitalize option.label}} + + {{/each}} + + + {{#each this.clientFilterToggles.eligibility as |option|}} + + {{capitalize option.label}} + + {{/each}} + + + {{#each this.clientFilterToggles.drainStatus as |option|}} + + {{capitalize option.label}} + + {{/each}} + + + + + {{#each this.optionsNodePool key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Node Pool filters + + {{/each}} + + + + + {{#each this.optionsClass key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Class filters + + {{/each}} + + + + + {{#each this.optionsDatacenter key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Datacenter filters + + {{/each}} + + + + + + {{#each this.optionsVersion key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Version filters + + {{/each}} + + + + + {{#each this.optionsVolume key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Volume filters + + {{/each}} + + + --}} + + + + + + + + + {{#if this.system.shouldShowNamespaces}} + {{!-- --}} - - -{{!-- - Search Jobs: - - --}} - {{#if this.pendingJobIDDiff}} - - + + {{#each this.optionsNamespace key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Namespaces + + {{/each}} - {{!-- - Changes pending - There are changes to yer list of jobs! Click the button below to update yer list. - - --}} - {{/if}} - - {{!-- {{#unless this.tokenRecord.isExpired}} - - {{/unless}} --}} - + + + {{/if}} + +
    + +
    + + + + {{#if this.pendingJobIDDiff}} + + {{/if}} +     + Date: Mon, 18 Mar 2024 17:04:43 -0400 Subject: [PATCH 75/98] Big uncoupling of watchable and url-append stuff --- ui/app/adapters/job.js | 63 ++++++++--------- ui/app/adapters/watchable.js | 12 ++-- ui/app/controllers/jobs/index.js | 113 ++++++++++++++++++++----------- ui/app/routes/jobs/index.js | 19 ++---- ui/app/templates/jobs/index.hbs | 33 +++++++-- 5 files changed, 139 insertions(+), 101 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 64854e9ba57..1872e531566 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -11,7 +11,6 @@ import classic from 'ember-classic-decorator'; import { inject as service } from '@ember/service'; import { getOwner } from '@ember/application'; import { get } from '@ember/object'; -import queryString from 'query-string'; @classic export default class JobAdapter extends WatchableNamespaceIDs { @@ -214,24 +213,8 @@ export default class JobAdapter extends WatchableNamespaceIDs { const url = this.urlForQuery(query, type.modelName, method); // Let's establish the index, via watchList.getIndexFor. - let index = this.watchList.getIndexFor(url); - - // In the case that this is of queryType update, - // and its index is found to be 1, - // check for the initialize query's index and use that instead - // if (options.adapterOptions.queryType === 'update' && index === 1) { - // let initializeQueryIndex = this.watchList.getIndexFor( - // '/v1/jobs/statuses3?meta=true&per_page=10' - // ); // TODO: fickle! - // if (initializeQueryIndex) { - // index = initializeQueryIndex; - // } - // } - - // Disregard the index if this is an initialize query - if (options.adapterOptions.queryType === 'initialize') { - index = null; - } + // let index = this.watchList.getIndexFor(url); + let index = query.index || 1; // TODO: adding a new job hash will not necessarily cancel the old one. // You could be holding open a POST on jobs AB and ABC at the same time. @@ -245,7 +228,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { return this.ajax(url, method, { signal, data: query, - modifyURL: options.adapterOptions.modifyURL, }).then((payload) => { // If there was a request body, append it to my payload if (query.jobs) { @@ -267,6 +249,10 @@ export default class JobAdapter extends WatchableNamespaceIDs { if (headers['x-nomad-nexttoken']) { result.meta.nextToken = headers['x-nomad-nexttoken']; } + if (headers['x-nomad-index']) { + // this.watchList.setIndexFor(result.url, headers['x-nomad-index']); + result.meta.index = headers['x-nomad-index']; + } } return result; @@ -274,26 +260,31 @@ export default class JobAdapter extends WatchableNamespaceIDs { urlForQuery(query, modelName, method) { let baseUrl = `/${this.namespace}/jobs/statuses`; - if (method === 'POST') { - // Setting a base64 hash to represent the body of the POST request - // (which is otherwise not represented in the URL) - // because the watchList uses the URL as a key for index lookups. - return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; - } else { - return `${baseUrl}?${queryString.stringify(query)}`; // TODO: maybe nix this, it's doubling up QPs + if (method === 'POST' && query.index) { + return `${baseUrl}?index=${query.index}`; } + return baseUrl; + // if (method === 'POST') { + // // Setting a base64 hash to represent the body of the POST request + // // (which is otherwise not represented in the URL) + // // because the watchList uses the URL as a key for index lookups. + // return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; + // } else { + // // return `${baseUrl}?${queryString.stringify(query)}`; // TODO: maybe nix this, it's doubling up QPs + // return `${baseUrl}`; + // } } - ajaxOptions(url, type, options) { - let hash = super.ajaxOptions(url, type, options); - // Custom handling for POST requests to append 'index' as a query parameter - if (type === 'POST' && options.data && options.data.index) { - let index = encodeURIComponent(options.data.index); - hash.url = `${hash.url}&index=${index}`; - } + // ajaxOptions(url, type, options) { + // let hash = super.ajaxOptions(url, type, options); + // // Custom handling for POST requests to append 'index' as a query parameter + // if (type === 'POST' && options.data && options.data.index) { + // let index = encodeURIComponent(options.data.index); + // hash.url = `${hash.url}&index=${index}`; + // } - return hash; - } + // return hash; + // } } // TODO: First query (0 jobs to 1 job) doesnt seem to kick off POST diff --git a/ui/app/adapters/watchable.js b/ui/app/adapters/watchable.js index eb5e7e1a4d2..4f5ee74ce22 100644 --- a/ui/app/adapters/watchable.js +++ b/ui/app/adapters/watchable.js @@ -26,13 +26,15 @@ export default class Watchable extends ApplicationAdapter { // to ajaxOptions or overriding ajax completely. ajax(url, type, options = {}) { const hasParams = hasNonBlockingQueryParams(options); - // if (!hasParams || options.skipURLModification) - // return super.ajax(url, type, options); - if (!hasParams || type !== 'GET' || options.modifyURL === false) - return super.ajax(url, type, options); - // ^-- TODO: this means that all non-GETs dont get their URLs modified. Make sure this is what we want. + if (!hasParams || type !== 'GET') return super.ajax(url, type, options); let params = { ...options?.data }; delete params.index; + + // Options data gets appended as query params as part of ajaxOptions. + // In order to prevent doubling params, data should only include index + // at this point since everything else is added to the URL in advance. + options.data = options.data.index ? { index: options.data.index } : {}; + return super.ajax(`${url}?${queryString.stringify(params)}`, type, options); } diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index e6c3dade9da..fffa8be5800 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -26,7 +26,7 @@ export default class JobsIndexController extends Controller { @service watchList; // TODO: temp // qpNamespace = '*'; - per_page = 10; + per_page = 3; reverse = false; queryParams = [ @@ -42,6 +42,9 @@ export default class JobsIndexController extends Controller { // #region filtering and sorting + @tracked jobQueryIndex = 0; + @tracked jobAllocsQueryIndex = 0; + @selection('qpNamespace') selectionNamespace; // @computed('qpNamespace', 'model.namespaces.[]') get optionsNamespace() { @@ -120,6 +123,10 @@ export default class JobsIndexController extends Controller { * @param {"prev"|"next"} page */ @action async handlePageChange(page) { + // reset indexes + this.jobQueryIndex = 0; + this.jobAllocsQueryIndex = 0; + if (page === 'prev') { // Note (and TODO:) this isn't particularly efficient! // We're making an extra full request to get the nextToken we need, @@ -171,7 +178,7 @@ export default class JobsIndexController extends Controller { //#region querying - jobQuery(params, options = {}) { + jobQuery(params) { this.watchList.jobsIndexIDsController.abort(); this.watchList.jobsIndexIDsController = new AbortController(); @@ -179,9 +186,7 @@ export default class JobsIndexController extends Controller { .query('job', params, { adapterOptions: { method: 'GET', // TODO: default - queryType: options.queryType, abortController: this.watchList.jobsIndexIDsController, - modifyURL: false, }, }) .catch((e) => { @@ -200,19 +205,20 @@ export default class JobsIndexController extends Controller { 'job', { jobs: jobIDs, + index: this.jobAllocsQueryIndex, // TODO: consider using a passed params object like jobQuery uses, rather than just passing jobIDs }, { adapterOptions: { method: 'POST', - queryType: 'update', abortController: this.watchList.jobsIndexDetailsController, - // modifyURL: false, }, } ) .catch((e) => { if (e.name !== 'AbortError') { console.log('error fetching job allocs', e); + } else { + console.log('|> jobAllocsQuery aborted'); } return; }); @@ -222,6 +228,7 @@ export default class JobsIndexController extends Controller { let prevPageToken = await this.store.query( 'job', { + prev_page_query: true, // TODO: debugging only! next_token: this.cursorAt, per_page: this.per_page + 1, reverse: true, @@ -229,8 +236,6 @@ export default class JobsIndexController extends Controller { { adapterOptions: { method: 'GET', - queryType: 'initialize', - modifyURL: false, }, } ); @@ -240,42 +245,57 @@ export default class JobsIndexController extends Controller { // TODO: set up isEnabled to check blockingQueries rather than just use while (true) @restartableTask *watchJobIDs(params, throttle = 2000) { while (true && !Ember.testing) { + // let watchlistIndex = this.watchList.getIndexFor( + // '/v1/jobs/statuses?per_page=3' + // ); + // console.log('> watchJobIDs', params); let currentParams = params; - const newJobs = yield this.jobQuery(currentParams, { - queryType: 'update_ids', - }); - if (!newJobs) { - return; - } - if (newJobs.meta.nextToken) { - this.nextToken = newJobs.meta.nextToken; - } else { - this.nextToken = null; - } + // currentParams.index = watchlistIndex; + currentParams.index = this.jobQueryIndex; + const newJobs = yield this.jobQuery(currentParams, {}); + if (newJobs) { + // console.log('|> watchJobIDs returned new job IDs', newJobs.length); + if (newJobs.meta.index) { + this.jobQueryIndex = newJobs.meta.index; + } + if (newJobs.meta.nextToken) { + this.nextToken = newJobs.meta.nextToken; + } else { + this.nextToken = null; + } - const jobIDs = newJobs.map((job) => ({ - id: job.plainId, - namespace: job.belongsTo('namespace').id(), - })); - - const okayToJostle = this.liveUpdatesEnabled; - if (okayToJostle) { - this.jobIDs = jobIDs; - this.watchList.jobsIndexDetailsController.abort(); - console.log( - 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', - jobIDs - ); - this.watchList.jobsIndexDetailsController = new AbortController(); - // make sure throttle has taken place! - this.watchJobs.perform(jobIDs, throttle); + const jobIDs = newJobs.map((job) => ({ + id: job.plainId, + namespace: job.belongsTo('namespace').id(), + })); + + const okayToJostle = this.liveUpdatesEnabled; + if (okayToJostle) { + this.jobIDs = jobIDs; + this.watchList.jobsIndexDetailsController.abort(); + console.log( + 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', + jobIDs + ); + // Let's also reset the index for the job details query + this.jobAllocsQueryIndex = 0; + this.watchList.jobsIndexDetailsController = new AbortController(); + // make sure throttle has taken place! + this.watchJobs.perform(jobIDs, throttle); + } else { + // this.controller.set('pendingJobIDs', jobIDs); + // this.controller.set('pendingJobs', newJobs); + this.pendingJobIDs = jobIDs; + this.pendingJobs = newJobs; + } + yield timeout(throttle); // Moved to the end of the loop } else { - // this.controller.set('pendingJobIDs', jobIDs); - // this.controller.set('pendingJobs', newJobs); - this.pendingJobIDs = jobIDs; - this.pendingJobs = newJobs; + // This returns undefined on page change / cursorAt change, resulting from the aborting of the old query. + // console.log('|> watchJobIDs aborted'); + yield timeout(throttle); + this.watchJobs.perform(this.jobIDs, throttle); + continue; } - yield timeout(throttle); // Moved to the end of the loop } } @@ -286,9 +306,22 @@ export default class JobsIndexController extends Controller { // 3. via the user manually clicking to updateJobList() @restartableTask *watchJobs(jobIDs, throttle = 2000) { while (true && !Ember.testing) { + console.log( + '> watchJobs of IDs', + jobIDs.map((j) => j.id) + ); // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { let jobDetails = yield this.jobAllocsQuery(jobIDs); + if (jobDetails) { + if (jobDetails.meta.index) { + this.jobAllocsQueryIndex = jobDetails.meta.index; + } + console.log( + '|> watchJobs returned with', + jobDetails.map((j) => j.id) + ); + } this.jobs = jobDetails; } yield timeout(throttle); diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index b96fcf8ea6c..7ebf6910078 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -50,12 +50,11 @@ export default class IndexRoute extends Route.extend( .query('job', currentParams, { adapterOptions: { method: 'GET', // TODO: default - queryType: 'initialize', abortController: this.watchList.jobsIndexIDsController, - modifyURL: false, }, }) .catch(notifyForbidden(this)); + console.log('model jobs', jobs); return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -64,10 +63,11 @@ export default class IndexRoute extends Route.extend( } setupController(controller, model) { + console.log('== setupController'); super.setupController(controller, model); - // TODO: consider re-instating this. This is setting them and then their order gets shuffled. - // controller.set('jobs', model.jobs); controller.set('nextToken', model.jobs.meta.nextToken); + controller.set('jobQueryIndex', model.jobs.meta.index); + controller.set('jobAllocsQueryIndex', model.jobs.meta.allocsIndex); // Assuming allocsIndex is your meta key for job allocations. controller.set( 'jobIDs', model.jobs.map((job) => { @@ -78,17 +78,10 @@ export default class IndexRoute extends Route.extend( }) ); - // Note: we should remove the indexes from the watch-list for jobs index queries if we've already initialized, since - // if we explicitly change our queryParams we want to start from scratch, unindexed - this.watchList.clearJobsIndexIndexes(); - - // TODO: maybe do these in controller constructor? // Now that we've set the jobIDs, immediately start watching them - // eslint-disable-next-line - this.controller.watchJobs.perform(controller.jobIDs, 2000, 'update'); + controller.watchJobs.perform(controller.jobIDs, 2000, 'update'); // And also watch for any changes to the jobIDs list - // eslint-disable-next-line - this.controller.watchJobIDs.perform(this.getCurrentParams(), 2000); + controller.watchJobIDs.perform(this.getCurrentParams(), 2000); this.hasBeenInitialized = true; } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index ed47744d2ba..e6ccd0b62dc 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -368,6 +368,28 @@ @isDisabledNext={{not this.nextToken}} /> +
    + {{!-- TODO: Temporary keyboard-shortcut tester buttons while I think about nixing the pagination component above --}} + +
    Next token is {{this.nextToken}}
    {{!-- Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    --}} @@ -384,14 +406,11 @@ Live update new/removed jobs? {{this.liveUpdatesEnabled}}
    -
    - Watchlist: -
      - {{#each-in this.watchList.list as |key value|}} -
    • {{key}}: {{value}}
      • - {{/each-in}} -
    + Local watchlist
    + jobQueryIndex: {{this.jobQueryIndex}}
    + jobAllocsQueryIndex: {{this.jobAllocsQueryIndex}}
    +
    From 2058c73fd18ebd3da712d4e73cdf0aa5318be16a Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 20 Mar 2024 11:46:14 -0400 Subject: [PATCH 76/98] Testfixes for region, search, and keyboard --- ui/app/controllers/jobs/index.js | 28 ++++++++++++++++++++++------ ui/app/routes/jobs/index.js | 14 ++++++++++++-- ui/app/templates/jobs/index.hbs | 20 ++++++++++++-------- ui/tests/acceptance/keyboard-test.js | 15 ++++++++------- 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index fffa8be5800..80548add276 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -16,9 +16,10 @@ import { deserializedQueryParam as selection, } from 'nomad-ui/utils/qp-serialize'; // import { scheduleOnce } from '@ember/runloop'; - import Ember from 'ember'; +const DEFAULT_THROTTLE = 2000; + export default class JobsIndexController extends Controller { @service router; @service system; @@ -169,7 +170,10 @@ export default class JobsIndexController extends Controller { this.pendingJobs = null; this.jobIDs = this.pendingJobIDs; this.pendingJobIDs = null; - yield this.watchJobs.perform(this.jobIDs, 2000); + yield this.watchJobs.perform( + this.jobIDs, + Ember.testing ? 0 : DEFAULT_THROTTLE + ); } @localStorageProperty('nomadLiveUpdateJobsIndex', true) liveUpdatesEnabled; @@ -243,8 +247,11 @@ export default class JobsIndexController extends Controller { } // TODO: set up isEnabled to check blockingQueries rather than just use while (true) - @restartableTask *watchJobIDs(params, throttle = 2000) { - while (true && !Ember.testing) { + @restartableTask *watchJobIDs( + params, + throttle = Ember.testing ? 0 : DEFAULT_THROTTLE + ) { + while (true) { // let watchlistIndex = this.watchList.getIndexFor( // '/v1/jobs/statuses?per_page=3' // ); @@ -296,6 +303,9 @@ export default class JobsIndexController extends Controller { this.watchJobs.perform(this.jobIDs, throttle); continue; } + if (Ember.testing) { + break; + } } } @@ -304,8 +314,11 @@ export default class JobsIndexController extends Controller { // (which can happen both on initial load, and should the queryParams change) // 2. via the watchJobIDs task seeing new jobIDs // 3. via the user manually clicking to updateJobList() - @restartableTask *watchJobs(jobIDs, throttle = 2000) { - while (true && !Ember.testing) { + @restartableTask *watchJobs( + jobIDs, + throttle = Ember.testing ? 0 : DEFAULT_THROTTLE + ) { + while (true) { console.log( '> watchJobs of IDs', jobIDs.map((j) => j.id) @@ -325,6 +338,9 @@ export default class JobsIndexController extends Controller { this.jobs = jobDetails; } yield timeout(throttle); + if (Ember.testing) { + break; + } } } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 7ebf6910078..40b44029edf 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -14,6 +14,9 @@ import WithWatchers from 'nomad-ui/mixins/with-watchers'; import notifyForbidden from 'nomad-ui/utils/notify-forbidden'; import WithForbiddenState from 'nomad-ui/mixins/with-forbidden-state'; import { action } from '@ember/object'; +import Ember from 'ember'; + +const DEFAULT_THROTTLE = 2000; export default class IndexRoute extends Route.extend( WithWatchers, @@ -79,9 +82,16 @@ export default class IndexRoute extends Route.extend( ); // Now that we've set the jobIDs, immediately start watching them - controller.watchJobs.perform(controller.jobIDs, 2000, 'update'); + controller.watchJobs.perform( + controller.jobIDs, + Ember.testing ? 0 : DEFAULT_THROTTLE, + 'update' + ); // And also watch for any changes to the jobIDs list - controller.watchJobIDs.perform(this.getCurrentParams(), 2000); + controller.watchJobIDs.perform( + this.getCurrentParams(), + Ember.testing ? 0 : DEFAULT_THROTTLE + ); this.hasBeenInitialized = true; } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index e6ccd0b62dc..8b5b9759f98 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -259,13 +259,15 @@ />
    - +
    + +
    {{#if this.pendingJobIDDiff}} {{!-- {{#each this.tableColumns as |column|}} {{get B.data (lowercase column.label)}} {{/each}} --}} - + {{#if B.data.assumeGC}} {{B.data.name}} {{else}} diff --git a/ui/tests/acceptance/keyboard-test.js b/ui/tests/acceptance/keyboard-test.js index c22d07e9b17..e60ca0b6607 100644 --- a/ui/tests/acceptance/keyboard-test.js +++ b/ui/tests/acceptance/keyboard-test.js @@ -255,10 +255,13 @@ module('Acceptance | keyboard', function (hooks) { await visit('/'); await triggerEvent('.page-layout', 'keydown', { key: 'Shift' }); + + let keyboardService = this.owner.lookup('service:keyboard'); + let hints = keyboardService.keyCommands.filter((c) => c.element); assert.equal( document.querySelectorAll('[data-test-keyboard-hint]').length, - 7, - 'Shows 7 hints by default' + hints.length, + 'Shows correct number of hints by default' ); await triggerEvent('.page-layout', 'keyup', { key: 'Shift' }); @@ -301,7 +304,7 @@ module('Acceptance | keyboard', function (hooks) { triggerEvent('.page-layout', 'keydown', { key: '0' }); await triggerEvent('.page-layout', 'keydown', { key: '1' }); - const clickedJob = server.db.jobs.sortBy('modifyIndex').reverse()[0].id; + const clickedJob = server.db.jobs[0].id; assert.equal( currentURL(), `/jobs/${clickedJob}@default`, @@ -310,9 +313,7 @@ module('Acceptance | keyboard', function (hooks) { }); test('Multi-Table Nav', async function (assert) { server.createList('job', 3, { createRecommendations: true }); - await visit( - `/jobs/${server.db.jobs.sortBy('modifyIndex').reverse()[0].id}@default` - ); + await visit(`/jobs/${server.db.jobs[0].id}@default`); const numberOfGroups = findAll('.task-group-row').length; const numberOfAllocs = findAll('.allocation-row').length; @@ -333,7 +334,7 @@ module('Acceptance | keyboard', function (hooks) { let token = server.create('token', { type: 'management' }); window.localStorage.nomadTokenSecret = token.secretId; server.createList('job', 3, { createAllocations: true, type: 'system' }); - const jobID = server.db.jobs.sortBy('modifyIndex').reverse()[0].id; + const jobID = server.db.jobs[0].id; await visit(`/jobs/${jobID}@default`); await triggerKeyEvent('.page-layout', 'keydown', 'ArrowRight', { From d8f97bce199cb61f49d36eea6ba59cf532971da7 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 20 Mar 2024 12:35:42 -0400 Subject: [PATCH 77/98] Job row class for test purposes --- ui/app/templates/jobs/index.hbs | 2 +- ui/tests/acceptance/token-test.js | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 8b5b9759f98..7b7901b04fa 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -295,7 +295,7 @@ action=(action "gotoJob" B.data) }} {{on "click" (action this.gotoJob B.data)}} - class="{{if B.data.assumeGC "assume-gc"}}" + class="job-row is-interactive {{if B.data.assumeGC "assume-gc"}}" data-test-job-row={{B.data.plainId}} > {{!-- {{#each this.tableColumns as |column|}} diff --git a/ui/tests/acceptance/token-test.js b/ui/tests/acceptance/token-test.js index e73a444dd42..ed0aaa9aaa0 100644 --- a/ui/tests/acceptance/token-test.js +++ b/ui/tests/acceptance/token-test.js @@ -194,12 +194,11 @@ module('Acceptance | tokens', function (hooks) { await Tokens.visit(); await Tokens.secret(secretId).submit(); - server.pretender.get('/v1/jobs', function () { + server.pretender.get('/v1/jobs/statuses', function () { return [200, {}, '[]']; }); await Jobs.visit(); - // If jobs are lingering in the store, they would show up assert.notOk(find('[data-test-job-row]'), 'No jobs found'); }); @@ -272,7 +271,7 @@ module('Acceptance | tokens', function (hooks) { }, ], }; - server.pretender.get('/v1/jobs', function () { + server.pretender.get('/v1/jobs/statuses', function () { return [500, {}, JSON.stringify(expiredServerError)]; }); @@ -298,7 +297,7 @@ module('Acceptance | tokens', function (hooks) { }, ], }; - server.pretender.get('/v1/jobs', function () { + server.pretender.get('/v1/jobs/statuses', function () { return [500, {}, JSON.stringify(notFoundServerError)]; }); @@ -843,8 +842,7 @@ module('Acceptance | tokens', function (hooks) { // Pop over to the jobs page and make sure the Run button is disabled await visit('/jobs'); - assert.dom('[data-test-run-job]').hasTagName('button'); - assert.dom('[data-test-run-job]').isDisabled(); + assert.dom('[data-test-run-job]').hasAttribute('disabled'); // Sign out, and sign back in as a high-level role token await Tokens.visit(); From 52fcead28a412a49feb6c9f2158d855610a01b51 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 20 Mar 2024 13:53:04 -0400 Subject: [PATCH 78/98] Allocations in test now contain events --- ui/tests/unit/serializers/allocation-test.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ui/tests/unit/serializers/allocation-test.js b/ui/tests/unit/serializers/allocation-test.js index 111966566ef..dbe8dc35a90 100644 --- a/ui/tests/unit/serializers/allocation-test.js +++ b/ui/tests/unit/serializers/allocation-test.js @@ -48,6 +48,7 @@ module('Unit | Serializer | Allocation', function (hooks) { name: 'testTask', state: 'running', failed: false, + events: [], }, ], wasPreempted: false, @@ -116,11 +117,13 @@ module('Unit | Serializer | Allocation', function (hooks) { name: 'one.two', state: 'running', failed: false, + events: [], }, { name: 'three.four', state: 'pending', failed: true, + events: [], }, ], wasPreempted: false, @@ -190,6 +193,7 @@ module('Unit | Serializer | Allocation', function (hooks) { name: 'task', state: 'running', failed: false, + events: [], }, ], wasPreempted: true, @@ -278,6 +282,7 @@ module('Unit | Serializer | Allocation', function (hooks) { name: 'task', state: 'running', failed: false, + events: [], }, ], wasPreempted: false, @@ -352,11 +357,13 @@ module('Unit | Serializer | Allocation', function (hooks) { name: 'abc', state: 'running', failed: false, + events: [], }, { name: 'xyz', state: 'running', failed: false, + events: [], }, ], wasPreempted: false, From e1d6894103fde4ddf5d03a9aa1e3c60316dbdac3 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 20 Mar 2024 20:57:53 -0400 Subject: [PATCH 79/98] Starting on the jobs list tests in earnest --- ui/app/controllers/jobs/index.js | 2 +- ui/app/services/system.js | 4 ++ ui/app/templates/jobs/index.hbs | 12 ++-- ui/mirage/config.js | 94 ++++++++++++++++----------- ui/mirage/factories/job.js | 2 + ui/tests/acceptance/jobs-list-test.js | 60 ++++++++++------- ui/tests/pages/jobs/list.js | 2 +- 7 files changed, 106 insertions(+), 70 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 80548add276..80a041ba48f 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -85,9 +85,9 @@ export default class JobsIndexController extends Controller { return [ 'name', this.system.shouldShowNamespaces ? 'namespace' : null, - this.system.shouldShowNodepools ? 'node pools' : null, // TODO: implement on system service 'status', 'type', + this.system.shouldShowNodepools ? 'node pool' : null, // TODO: implement on system service 'priority', 'running allocations', ] diff --git a/ui/app/services/system.js b/ui/app/services/system.js index f5bede53d3d..d1408a3c4f2 100644 --- a/ui/app/services/system.js +++ b/ui/app/services/system.js @@ -136,6 +136,10 @@ export default class SystemService extends Service { ); } + get shouldShowNodepools() { + return true; // TODO: make this dependent on there being at least one non-default node pool + } + @task(function* () { const emptyLicense = { License: { Features: [] } }; diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7b7901b04fa..e4a3d091c67 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -323,18 +323,18 @@ {{#if this.system.shouldShowNamespaces}} {{B.data.namespace.id}} {{/if}} - {{#if this.system.shouldShowNodepools}} - {{B.data.nodepool}} - {{/if}} - + {{#unless B.data.childStatuses}} {{/unless}} - + {{B.data.type}} - + {{#if this.system.shouldShowNodepools}} + {{B.data.nodePool}} + {{/if}} + {{B.data.priority}} diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 4bc64069e6d..1785da8dff7 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -77,58 +77,78 @@ export default function () { this.get( '/jobs/statuses', withBlockingSupport(function ({ jobs }, req) { - const json = this.serialize(jobs.all()); + let per_page = req.queryParams.per_page || 20; const namespace = req.queryParams.namespace || 'default'; + + const json = this.serialize(jobs.all()); return json + .sort((a, b) => b.ModifyIndex - a.ModifyIndex) .filter((job) => { if (namespace === '*') return true; return namespace === 'default' ? !job.NamespaceID || job.NamespaceID === 'default' : job.NamespaceID === namespace; }) - .map((job) => filterKeys(job, 'TaskGroups', 'NamespaceID')); + .map((job) => filterKeys(job, 'TaskGroups', 'NamespaceID')) + .slice(0, per_page); }) ); this.post( '/jobs/statuses', withBlockingSupport(function ({ jobs }, req) { - // console.log('postbody', req, server.db, server.schema); - - let returnedJobs = this.serialize(jobs.all()).map((j) => { - let job = {}; - job.ID = j.ID; - job.Name = j.Name; - job.Allocs = server.db.allocations - .where({ jobId: j.ID, namespace: j.Namespace }) - .map((alloc) => { - return { - ClientStatus: alloc.clientStatus, - DeploymentStatus: { - Canary: false, - Healthy: true, - }, - Group: alloc.taskGroup, - JobVersion: alloc.jobVersion, - NodeID: alloc.nodeId, - ID: alloc.id, - }; + const body = JSON.parse(req.requestBody); + const requestedJobs = body.jobs || []; + const allJobs = this.serialize(jobs.all()); + + let returnedJobs = allJobs + .filter((job) => { + return requestedJobs.some((requestedJob) => { + return ( + job.ID === requestedJob.id && + (requestedJob.namespace === 'default' || + job.NamespaceID === requestedJob.namespace) + ); }); - job.ChildStatuses = null; // TODO: handle parent job here - job.Datacenters = j.Datacenters; - job.DeploymentID = j.DeploymentID; - job.GroupCountSum = j.TaskGroups.mapBy('Count').reduce( - (a, b) => a + b, - 0 - ); - job.Namespace = j.NamespaceID; - job.NodePool = j.NodePool; - job.Type = j.Type; - job.Priority = j.Priority; - job.Version = j.Version; - job.SmartAlloc = {}; // TODO - return job; - }); + }) + .map((j) => { + let job = {}; + job.ID = j.ID; + job.Name = j.Name; + job.ModifyIndex = j.ModifyIndex; + job.Allocs = server.db.allocations + .where({ jobId: j.ID, namespace: j.Namespace }) + .map((alloc) => { + return { + ClientStatus: alloc.clientStatus, + DeploymentStatus: { + Canary: false, + Healthy: true, + }, + Group: alloc.taskGroup, + JobVersion: alloc.jobVersion, + NodeID: alloc.nodeId, + ID: alloc.id, + }; + }); + job.ChildStatuses = null; // TODO: handle parent job here + job.Datacenters = j.Datacenters; + job.DeploymentID = j.DeploymentID; + job.GroupCountSum = j.TaskGroups.mapBy('Count').reduce( + (a, b) => a + b, + 0 + ); + job.Namespace = j.NamespaceID; + job.NodePool = j.NodePool; + job.Type = j.Type; + job.Priority = j.Priority; + job.Version = j.Version; + job.SmartAlloc = {}; // TODO + return job; + }); + // sort by modifyIndex, descending + returnedJobs.sort((a, b) => b.ModifyIndex - a.ModifyIndex); + return returnedJobs; }) ); diff --git a/ui/mirage/factories/job.js b/ui/mirage/factories/job.js index b5c4d82766a..149baa4557c 100644 --- a/ui/mirage/factories/job.js +++ b/ui/mirage/factories/job.js @@ -63,6 +63,8 @@ export default Factory.extend({ childrenCount: () => faker.random.number({ min: 1, max: 2 }), + // TODO: Use the model's aggregateAllocStatus as a property here + meta: null, periodic: trait({ diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index 820a2ad7955..a27194b26eb 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -66,13 +66,23 @@ module('Acceptance | jobs list', function (hooks) { await JobsList.visit(); + const store = this.owner.lookup('service:store'); + const jobInStore = await store.peekRecord( + 'job', + `["${job.id}","${job.namespace}"]` + ); + const jobRow = JobsList.jobs.objectAt(0); assert.equal(jobRow.name, job.name, 'Name'); assert.notOk(jobRow.hasNamespace); assert.equal(jobRow.nodePool, job.nodePool, 'Node Pool'); assert.equal(jobRow.link, `/ui/jobs/${job.id}@default`, 'Detail Link'); - assert.equal(jobRow.status, job.status, 'Status'); + assert.equal( + jobRow.status, + jobInStore.aggregateAllocStatus.label, + 'Status' + ); assert.equal(jobRow.type, typeForJob(job), 'Type'); assert.equal(jobRow.priority, job.priority, 'Priority'); }); @@ -346,30 +356,30 @@ module('Acceptance | jobs list', function (hooks) { }, }); - testFacet('Status', { - facet: JobsList.facets.status, - paramName: 'status', - expectedOptions: ['Pending', 'Running', 'Dead'], - async beforeEach() { - server.createList('job', 2, { - status: 'pending', - createAllocations: false, - childrenCount: 0, - }); - server.createList('job', 2, { - status: 'running', - createAllocations: false, - childrenCount: 0, - }); - server.createList('job', 2, { - status: 'dead', - createAllocations: false, - childrenCount: 0, - }); - await JobsList.visit(); - }, - filter: (job, selection) => selection.includes(job.status), - }); + // testFacet('Status', { + // facet: JobsList.facets.status, + // paramName: 'status', + // expectedOptions: ['Pending', 'Running', 'Dead'], + // async beforeEach() { + // server.createList('job', 2, { + // status: 'pending', + // createAllocations: false, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // status: 'running', + // createAllocations: false, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // status: 'dead', + // createAllocations: false, + // childrenCount: 0, + // }); + // await JobsList.visit(); + // }, + // filter: (job, selection) => selection.includes(job.status), + // }); testFacet('Datacenter', { facet: JobsList.facets.datacenter, diff --git a/ui/tests/pages/jobs/list.js b/ui/tests/pages/jobs/list.js index eca2ac765c0..718b2b08a5b 100644 --- a/ui/tests/pages/jobs/list.js +++ b/ui/tests/pages/jobs/list.js @@ -19,7 +19,7 @@ import { multiFacet, singleFacet } from 'nomad-ui/tests/pages/components/facet'; import pageSizeSelect from 'nomad-ui/tests/pages/components/page-size-select'; export default create({ - pageSize: 25, + pageSize: 3, visit: visitable('/jobs'), From 415642efb42f883f410849a8f36cd0f0be49a092 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 21 Mar 2024 11:16:53 -0400 Subject: [PATCH 80/98] Forbidden state de-bubbling cleanup --- ui/app/routes/jobs/index.js | 5 + ui/app/templates/jobs/index.hbs | 189 ++++++++++++++------------ ui/tests/acceptance/jobs-list-test.js | 51 +++---- ui/tests/pages/jobs/list.js | 2 +- 4 files changed, 135 insertions(+), 112 deletions(-) diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 40b44029edf..6c13ded64e1 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -68,6 +68,11 @@ export default class IndexRoute extends Route.extend( setupController(controller, model) { console.log('== setupController'); super.setupController(controller, model); + + if (!model.jobs) { + return; + } + controller.set('nextToken', model.jobs.meta.nextToken); controller.set('jobQueryIndex', model.jobs.meta.index); controller.set('jobAllocsQueryIndex', model.jobs.meta.allocsIndex); // Assuming allocsIndex is your meta key for job allocations. diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index e4a3d091c67..cd087c763c8 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -282,96 +282,113 @@ - - <:body as |B|> - {{!-- TODO: use --}} - - {{!-- {{#each this.tableColumns as |column|}} - {{get B.data (lowercase column.label)}} - {{/each}} --}} - - {{#if B.data.assumeGC}} - {{B.data.name}} - {{else}} - - {{B.data.name}} - {{#if B.data.meta.structured.pack}} - - {{x-icon "box" class= "test"}} - Pack - + {{#if this.isForbidden}} + + {{else if this.jobs.length}} + + <:body as |B|> + {{!-- TODO: use --}} + + {{!-- {{#each this.tableColumns as |column|}} + {{get B.data (lowercase column.label)}} + {{/each}} --}} + + {{#if B.data.assumeGC}} + {{B.data.name}} + {{else}} + + {{B.data.name}} + {{#if B.data.meta.structured.pack}} + + {{x-icon "box" class= "test"}} + Pack + + {{/if}} + {{/if}} - + + {{#if this.system.shouldShowNamespaces}} + {{B.data.namespace.id}} {{/if}} - - {{#if this.system.shouldShowNamespaces}} - {{B.data.namespace.id}} - {{/if}} - - {{#unless B.data.childStatuses}} - - {{/unless}} - - - {{B.data.type}} - - {{#if this.system.shouldShowNodepools}} - {{B.data.nodePool}} - {{/if}} - - {{B.data.priority}} - - - {{!-- {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
    - {{B.data.allocations.length}} total
    - {{B.data.groupCountSum}} desired -
    --}} -
    - {{#unless B.data.assumeGC}} - {{#if B.data.childStatuses}} - {{B.data.childStatuses.length}} child jobs;
    - {{#each-in B.data.childStatusBreakdown as |status count|}} - {{count}} {{status}}
    - {{/each-in}} - {{else}} - - {{/if}} + + {{#unless B.data.childStatuses}} + {{/unless}} -
    -     - - - - - +     + + {{B.data.type}} + + {{#if this.system.shouldShowNodepools}} + {{B.data.nodePool}} + {{/if}} + + {{B.data.priority}} + + + {{!-- {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
    + {{B.data.allocations.length}} total
    + {{B.data.groupCountSum}} desired +
    --}} +
    + {{#unless B.data.assumeGC}} + {{#if B.data.childStatuses}} + {{B.data.childStatuses.length}} child jobs;
    + {{#each-in B.data.childStatusBreakdown as |status count|}} + {{count}} {{status}}
    + {{/each-in}} + {{else}} + + {{/if}} + {{/unless}} +
    +     + + + + + {{else}} + + {{!-- TODO: differentiate between "empty because there's nothing" and "empty because you have filtered/searched" --}} + + + + {{!-- TODO: HDS4.0, convert to F.LinkStandalone --}} + + + + + {{/if}}
    {{!-- TODO: Temporary keyboard-shortcut tester buttons while I think about nixing the pagination component above --}} [403, {}, null]); + server.pretender.get('/v1/jobs/statuses', () => [403, {}, null]); await JobsList.visit(); assert.equal(JobsList.error.title, 'Not Authorized'); + await percySnapshot(assert); await JobsList.error.seekHelp(); assert.equal(currentURL(), '/settings/tokens'); @@ -356,30 +357,30 @@ module('Acceptance | jobs list', function (hooks) { }, }); - // testFacet('Status', { - // facet: JobsList.facets.status, - // paramName: 'status', - // expectedOptions: ['Pending', 'Running', 'Dead'], - // async beforeEach() { - // server.createList('job', 2, { - // status: 'pending', - // createAllocations: false, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // status: 'running', - // createAllocations: false, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // status: 'dead', - // createAllocations: false, - // childrenCount: 0, - // }); - // await JobsList.visit(); - // }, - // filter: (job, selection) => selection.includes(job.status), - // }); + testFacet('Status', { + facet: JobsList.facets.status, + paramName: 'status', + expectedOptions: ['Pending', 'Running', 'Dead'], + async beforeEach() { + server.createList('job', 2, { + status: 'pending', + createAllocations: false, + childrenCount: 0, + }); + server.createList('job', 2, { + status: 'running', + createAllocations: false, + childrenCount: 0, + }); + server.createList('job', 2, { + status: 'dead', + createAllocations: false, + childrenCount: 0, + }); + await JobsList.visit(); + }, + filter: (job, selection) => selection.includes(job.status), + }); testFacet('Datacenter', { facet: JobsList.facets.datacenter, diff --git a/ui/tests/pages/jobs/list.js b/ui/tests/pages/jobs/list.js index 718b2b08a5b..e0ebe0d7bec 100644 --- a/ui/tests/pages/jobs/list.js +++ b/ui/tests/pages/jobs/list.js @@ -30,7 +30,7 @@ export default create({ runJobButton: { scope: '[data-test-run-job]', - isDisabled: property('disabled'), + isDisabled: attribute('disabled'), }, jobs: collection('[data-test-job-row]', { From eb695f0c0a62bc5261a71a7b51efca24c4c5c0c1 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 21 Mar 2024 15:28:14 -0400 Subject: [PATCH 81/98] Job list page size fixes --- ui/app/controllers/jobs/index.js | 16 ++++++--- ui/app/routes/jobs/index.js | 5 +++ ui/app/templates/jobs/index.hbs | 56 ++++++++++++++++++-------------- ui/tests/pages/jobs/list.js | 3 +- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 80a041ba48f..c80f179de74 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -24,15 +24,19 @@ export default class JobsIndexController extends Controller { @service router; @service system; @service store; + @service userSettings; @service watchList; // TODO: temp - // qpNamespace = '*'; - per_page = 3; + @tracked pageSize; + constructor() { + super(...arguments); + this.pageSize = this.userSettings.pageSize; + } reverse = false; queryParams = [ 'cursorAt', - 'per_page', + 'pageSize', // 'status', { qpNamespace: 'namespace' }, // 'type', @@ -152,6 +156,10 @@ export default class JobsIndexController extends Controller { } } + @action handlePageSizeChange(size) { + this.pageSize = size; + } + get pendingJobIDDiff() { return ( this.pendingJobIDs && @@ -234,7 +242,7 @@ export default class JobsIndexController extends Controller { { prev_page_query: true, // TODO: debugging only! next_token: this.cursorAt, - per_page: this.per_page + 1, + per_page: this.pageSize + 1, reverse: true, }, { diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 6c13ded64e1..33359c89c04 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -34,6 +34,9 @@ export default class IndexRoute extends Route.extend( cursorAt: { refreshModel: true, }, + pageSize: { + refreshModel: true, + }, }; hasBeenInitialized = false; @@ -41,6 +44,8 @@ export default class IndexRoute extends Route.extend( getCurrentParams() { let queryParams = this.paramsFor(this.routeName); // Get current query params queryParams.next_token = queryParams.cursorAt; + queryParams.per_page = queryParams.pageSize; + delete queryParams.pageSize; delete queryParams.cursorAt; // TODO: hacky, should be done in the serializer/adapter? return { ...queryParams }; } diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index cd087c763c8..7807a582182 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -369,11 +369,41 @@ - + {{!-- TODO: Temporary keyboard-shortcut tester buttons while I think about nixing the pagination component --}} + + + + + + {{!-- + @sizeSelectorLabel="Per page" + @showSizeSelector={{true}} + @onPageSizeChange={{this.handlePageSizeChange}} + @pageSizes={{array 10 25 50}} + @currentPageSize={{this.pageSize}} + /> --}} {{else}} {{!-- TODO: differentiate between "empty because there's nothing" and "empty because you have filtered/searched" --}} @@ -390,28 +420,6 @@ {{/if}}
    - {{!-- TODO: Temporary keyboard-shortcut tester buttons while I think about nixing the pagination component above --}} - - -
    Next token is {{this.nextToken}}
    {{!-- Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
    --}} Model.jobs length: {{this.model.jobs.length}}
    diff --git a/ui/tests/pages/jobs/list.js b/ui/tests/pages/jobs/list.js index e0ebe0d7bec..223da271ca8 100644 --- a/ui/tests/pages/jobs/list.js +++ b/ui/tests/pages/jobs/list.js @@ -9,7 +9,6 @@ import { collection, clickable, isPresent, - property, text, triggerable, visitable, @@ -19,7 +18,7 @@ import { multiFacet, singleFacet } from 'nomad-ui/tests/pages/components/facet'; import pageSizeSelect from 'nomad-ui/tests/pages/components/page-size-select'; export default create({ - pageSize: 3, + pageSize: 25, visit: visitable('/jobs'), From c19a422c8bca9d8fd40fcb8866453473af02eaf1 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 21 Mar 2024 15:33:52 -0400 Subject: [PATCH 82/98] Facet/Search/Filter jobs list tests skipped --- ui/tests/acceptance/jobs-list-test.js | 646 +++++++++++++------------- 1 file changed, 328 insertions(+), 318 deletions(-) diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index b0171d092de..cb4d6079d60 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -146,7 +146,8 @@ module('Acceptance | jobs list', function (hooks) { ); }); - test('when there are jobs, but no matches for a search result, there is an empty message', async function (assert) { + // TODO: Jobs list search + test.skip('when there are jobs, but no matches for a search result, there is an empty message', async function (assert) { server.create('job', { name: 'cat 1' }); server.create('job', { name: 'cat 2' }); @@ -161,7 +162,8 @@ module('Acceptance | jobs list', function (hooks) { ); }); - test('searching resets the current page', async function (assert) { + // TODO: Jobs list search + test.skip('searching resets the current page', async function (assert) { server.createList('job', JobsList.pageSize + 1, { createAllocations: false, }); @@ -180,7 +182,8 @@ module('Acceptance | jobs list', function (hooks) { assert.equal(currentURL(), '/jobs?search=foobar', 'No page query param'); }); - test('Search order overrides Sort order', async function (assert) { + // TODO: Jobs list search + test.skip('Search order overrides Sort order', async function (assert) { server.create('job', { name: 'car', modifyIndex: 1, priority: 200 }); server.create('job', { name: 'cat', modifyIndex: 2, priority: 150 }); server.create('job', { name: 'dog', modifyIndex: 3, priority: 100 }); @@ -221,7 +224,8 @@ module('Acceptance | jobs list', function (hooks) { assert.equal(JobsList.jobs.objectAt(1).name, 'car'); }); - test('when a cluster has namespaces, each job row includes the job namespace', async function (assert) { + // TODO: Jobs list search + test.skip('when a cluster has namespaces, each job row includes the job namespace', async function (assert) { server.createList('namespace', 2); server.createList('job', 2); const job = server.db.jobs.sortBy('modifyIndex').reverse()[0]; @@ -232,7 +236,8 @@ module('Acceptance | jobs list', function (hooks) { assert.equal(jobRow.namespace, job.namespaceId); }); - test('when the namespace query param is set, only matching jobs are shown', async function (assert) { + // TODO: Jobs list filter + test.skip('when the namespace query param is set, only matching jobs are shown', async function (assert) { server.createList('namespace', 2); const job1 = server.create('job', { namespaceId: server.db.namespaces[0].id, @@ -287,7 +292,8 @@ module('Acceptance | jobs list', function (hooks) { : job.type; } - test('the jobs list page has appropriate faceted search options', async function (assert) { + // TODO: Jobs list filter + test.skip('the jobs list page has appropriate faceted search options', async function (assert) { await JobsList.visit(); assert.ok( @@ -300,158 +306,161 @@ module('Acceptance | jobs list', function (hooks) { assert.ok(JobsList.facets.prefix.isPresent, 'Prefix facet found'); }); - testSingleSelectFacet('Namespace', { - facet: JobsList.facets.namespace, - paramName: 'namespace', - expectedOptions: ['All (*)', 'default', 'namespace-2'], - optionToSelect: 'namespace-2', - async beforeEach() { - server.create('namespace', { id: 'default' }); - server.create('namespace', { id: 'namespace-2' }); - server.createList('job', 2, { namespaceId: 'default' }); - server.createList('job', 2, { namespaceId: 'namespace-2' }); - await JobsList.visit(); - }, - filter(job, selection) { - return job.namespaceId === selection; - }, - }); - - testFacet('Type', { - facet: JobsList.facets.type, - paramName: 'type', - expectedOptions: [ - 'Batch', - 'Pack', - 'Parameterized', - 'Periodic', - 'Service', - 'System', - 'System Batch', - ], - async beforeEach() { - server.createList('job', 2, { createAllocations: false, type: 'batch' }); - server.createList('job', 2, { - createAllocations: false, - type: 'batch', - periodic: true, - childrenCount: 0, - }); - server.createList('job', 2, { - createAllocations: false, - type: 'batch', - parameterized: true, - childrenCount: 0, - }); - server.createList('job', 2, { - createAllocations: false, - type: 'service', - }); - await JobsList.visit(); - }, - filter(job, selection) { - let displayType = job.type; - if (job.parameterized) displayType = 'parameterized'; - if (job.periodic) displayType = 'periodic'; - return selection.includes(displayType); - }, - }); - - testFacet('Status', { - facet: JobsList.facets.status, - paramName: 'status', - expectedOptions: ['Pending', 'Running', 'Dead'], - async beforeEach() { - server.createList('job', 2, { - status: 'pending', - createAllocations: false, - childrenCount: 0, - }); - server.createList('job', 2, { - status: 'running', - createAllocations: false, - childrenCount: 0, - }); - server.createList('job', 2, { - status: 'dead', - createAllocations: false, - childrenCount: 0, - }); - await JobsList.visit(); - }, - filter: (job, selection) => selection.includes(job.status), - }); - - testFacet('Datacenter', { - facet: JobsList.facets.datacenter, - paramName: 'dc', - expectedOptions(jobs) { - const allDatacenters = new Set( - jobs.mapBy('datacenters').reduce((acc, val) => acc.concat(val), []) - ); - return Array.from(allDatacenters).sort(); - }, - async beforeEach() { - server.create('job', { - datacenters: ['pdx', 'lax'], - createAllocations: false, - childrenCount: 0, - }); - server.create('job', { - datacenters: ['pdx', 'ord'], - createAllocations: false, - childrenCount: 0, - }); - server.create('job', { - datacenters: ['lax', 'jfk'], - createAllocations: false, - childrenCount: 0, - }); - server.create('job', { - datacenters: ['jfk', 'dfw'], - createAllocations: false, - childrenCount: 0, - }); - server.create('job', { - datacenters: ['pdx'], - createAllocations: false, - childrenCount: 0, - }); - await JobsList.visit(); - }, - filter: (job, selection) => - job.datacenters.find((dc) => selection.includes(dc)), - }); - - testFacet('Prefix', { - facet: JobsList.facets.prefix, - paramName: 'prefix', - expectedOptions: ['hashi (3)', 'nmd (2)', 'pre (2)'], - async beforeEach() { - [ - 'pre-one', - 'hashi_one', - 'nmd.one', - 'one-alone', - 'pre_two', - 'hashi.two', - 'hashi-three', - 'nmd_two', - 'noprefix', - ].forEach((name) => { - server.create('job', { - name, - createAllocations: false, - childrenCount: 0, - }); - }); - await JobsList.visit(); - }, - filter: (job, selection) => - selection.find((prefix) => job.name.startsWith(prefix)), - }); - - test('when the facet selections result in no matches, the empty state states why', async function (assert) { + // TODO: Jobs list filter + + // testSingleSelectFacet('Namespace', { + // facet: JobsList.facets.namespace, + // paramName: 'namespace', + // expectedOptions: ['All (*)', 'default', 'namespace-2'], + // optionToSelect: 'namespace-2', + // async beforeEach() { + // server.create('namespace', { id: 'default' }); + // server.create('namespace', { id: 'namespace-2' }); + // server.createList('job', 2, { namespaceId: 'default' }); + // server.createList('job', 2, { namespaceId: 'namespace-2' }); + // await JobsList.visit(); + // }, + // filter(job, selection) { + // return job.namespaceId === selection; + // }, + // }); + + // testFacet('Type', { + // facet: JobsList.facets.type, + // paramName: 'type', + // expectedOptions: [ + // 'Batch', + // 'Pack', + // 'Parameterized', + // 'Periodic', + // 'Service', + // 'System', + // 'System Batch', + // ], + // async beforeEach() { + // server.createList('job', 2, { createAllocations: false, type: 'batch' }); + // server.createList('job', 2, { + // createAllocations: false, + // type: 'batch', + // periodic: true, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // createAllocations: false, + // type: 'batch', + // parameterized: true, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // createAllocations: false, + // type: 'service', + // }); + // await JobsList.visit(); + // }, + // filter(job, selection) { + // let displayType = job.type; + // if (job.parameterized) displayType = 'parameterized'; + // if (job.periodic) displayType = 'periodic'; + // return selection.includes(displayType); + // }, + // }); + + // testFacet('Status', { + // facet: JobsList.facets.status, + // paramName: 'status', + // expectedOptions: ['Pending', 'Running', 'Dead'], + // async beforeEach() { + // server.createList('job', 2, { + // status: 'pending', + // createAllocations: false, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // status: 'running', + // createAllocations: false, + // childrenCount: 0, + // }); + // server.createList('job', 2, { + // status: 'dead', + // createAllocations: false, + // childrenCount: 0, + // }); + // await JobsList.visit(); + // }, + // filter: (job, selection) => selection.includes(job.status), + // }); + + // testFacet('Datacenter', { + // facet: JobsList.facets.datacenter, + // paramName: 'dc', + // expectedOptions(jobs) { + // const allDatacenters = new Set( + // jobs.mapBy('datacenters').reduce((acc, val) => acc.concat(val), []) + // ); + // return Array.from(allDatacenters).sort(); + // }, + // async beforeEach() { + // server.create('job', { + // datacenters: ['pdx', 'lax'], + // createAllocations: false, + // childrenCount: 0, + // }); + // server.create('job', { + // datacenters: ['pdx', 'ord'], + // createAllocations: false, + // childrenCount: 0, + // }); + // server.create('job', { + // datacenters: ['lax', 'jfk'], + // createAllocations: false, + // childrenCount: 0, + // }); + // server.create('job', { + // datacenters: ['jfk', 'dfw'], + // createAllocations: false, + // childrenCount: 0, + // }); + // server.create('job', { + // datacenters: ['pdx'], + // createAllocations: false, + // childrenCount: 0, + // }); + // await JobsList.visit(); + // }, + // filter: (job, selection) => + // job.datacenters.find((dc) => selection.includes(dc)), + // }); + + // testFacet('Prefix', { + // facet: JobsList.facets.prefix, + // paramName: 'prefix', + // expectedOptions: ['hashi (3)', 'nmd (2)', 'pre (2)'], + // async beforeEach() { + // [ + // 'pre-one', + // 'hashi_one', + // 'nmd.one', + // 'one-alone', + // 'pre_two', + // 'hashi.two', + // 'hashi-three', + // 'nmd_two', + // 'noprefix', + // ].forEach((name) => { + // server.create('job', { + // name, + // createAllocations: false, + // childrenCount: 0, + // }); + // }); + // await JobsList.visit(); + // }, + // filter: (job, selection) => + // selection.find((prefix) => job.name.startsWith(prefix)), + // }); + + // TODO: Jobs list filter + test.skip('when the facet selections result in no matches, the empty state states why', async function (assert) { server.createList('job', 2, { status: 'pending', createAllocations: false, @@ -470,7 +479,8 @@ module('Acceptance | jobs list', function (hooks) { ); }); - test('the jobs list is immediately filtered based on query params', async function (assert) { + // TODO: Jobs list filter + test.skip('the jobs list is immediately filtered based on query params', async function (assert) { server.create('job', { type: 'batch', createAllocations: false }); server.create('job', { type: 'service', createAllocations: false }); @@ -563,163 +573,163 @@ module('Acceptance | jobs list', function (hooks) { }, }); - async function facetOptions(assert, beforeEach, facet, expectedOptions) { - await beforeEach(); - await facet.toggle(); - - let expectation; - if (typeof expectedOptions === 'function') { - expectation = expectedOptions(server.db.jobs); - } else { - expectation = expectedOptions; - } - - assert.deepEqual( - facet.options.map((option) => option.label.trim()), - expectation, - 'Options for facet are as expected' - ); - } - - function testSingleSelectFacet( - label, - { facet, paramName, beforeEach, filter, expectedOptions, optionToSelect } - ) { - test(`the ${label} facet has the correct options`, async function (assert) { - await facetOptions(assert, beforeEach, facet, expectedOptions); - }); - - test(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { - await beforeEach(); - await facet.toggle(); - - const option = facet.options.findOneBy('label', optionToSelect); - const selection = option.key; - await option.select(); - - const expectedJobs = server.db.jobs - .filter((job) => filter(job, selection)) - .sortBy('modifyIndex') - .reverse(); - - JobsList.jobs.forEach((job, index) => { - assert.equal( - job.id, - expectedJobs[index].id, - `Job at ${index} is ${expectedJobs[index].id}` - ); - }); - }); - - test(`selecting an option in the ${label} facet updates the ${paramName} query param`, async function (assert) { - await beforeEach(); - await facet.toggle(); - - const option = facet.options.objectAt(1); - const selection = option.key; - await option.select(); - - assert.ok( - currentURL().includes(`${paramName}=${selection}`), - 'URL has the correct query param key and value' - ); - }); - } - - function testFacet( - label, - { facet, paramName, beforeEach, filter, expectedOptions } - ) { - test(`the ${label} facet has the correct options`, async function (assert) { - await facetOptions(assert, beforeEach, facet, expectedOptions); - }); - - test(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { - let option; - - await beforeEach(); - await facet.toggle(); - - option = facet.options.objectAt(0); - await option.toggle(); - - const selection = [option.key]; - const expectedJobs = server.db.jobs - .filter((job) => filter(job, selection)) - .sortBy('modifyIndex') - .reverse(); - - JobsList.jobs.forEach((job, index) => { - assert.equal( - job.id, - expectedJobs[index].id, - `Job at ${index} is ${expectedJobs[index].id}` - ); - }); - }); - - test(`selecting multiple options in the ${label} facet results in a broader search`, async function (assert) { - const selection = []; - - await beforeEach(); - await facet.toggle(); - - const option1 = facet.options.objectAt(0); - const option2 = facet.options.objectAt(1); - await option1.toggle(); - selection.push(option1.key); - await option2.toggle(); - selection.push(option2.key); - - const expectedJobs = server.db.jobs - .filter((job) => filter(job, selection)) - .sortBy('modifyIndex') - .reverse(); - - JobsList.jobs.forEach((job, index) => { - assert.equal( - job.id, - expectedJobs[index].id, - `Job at ${index} is ${expectedJobs[index].id}` - ); - }); - }); - - test(`selecting options in the ${label} facet updates the ${paramName} query param`, async function (assert) { - const selection = []; - - await beforeEach(); - await facet.toggle(); - - const option1 = facet.options.objectAt(0); - const option2 = facet.options.objectAt(1); - await option1.toggle(); - selection.push(option1.key); - await option2.toggle(); - selection.push(option2.key); - - assert.ok( - currentURL().includes(encodeURIComponent(JSON.stringify(selection))), - 'URL has the correct query param key and value' - ); - }); - - test('the run job button works when filters are set', async function (assert) { - ['pre-one', 'pre-two', 'pre-three'].forEach((name) => { - server.create('job', { - name, - createAllocations: false, - childrenCount: 0, - }); - }); - - await JobsList.visit(); - - await JobsList.facets.prefix.toggle(); - await JobsList.facets.prefix.options[0].toggle(); - - await JobsList.runJobButton.click(); - assert.equal(currentURL(), '/jobs/run'); - }); - } + // async function facetOptions(assert, beforeEach, facet, expectedOptions) { + // await beforeEach(); + // await facet.toggle(); + + // let expectation; + // if (typeof expectedOptions === 'function') { + // expectation = expectedOptions(server.db.jobs); + // } else { + // expectation = expectedOptions; + // } + + // assert.deepEqual( + // facet.options.map((option) => option.label.trim()), + // expectation, + // 'Options for facet are as expected' + // ); + // } + + // function testSingleSelectFacet( + // label, + // { facet, paramName, beforeEach, filter, expectedOptions, optionToSelect } + // ) { + // test.skip(`the ${label} facet has the correct options`, async function (assert) { + // await facetOptions(assert, beforeEach, facet, expectedOptions); + // }); + + // test.skip(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { + // await beforeEach(); + // await facet.toggle(); + + // const option = facet.options.findOneBy('label', optionToSelect); + // const selection = option.key; + // await option.select(); + + // const expectedJobs = server.db.jobs + // .filter((job) => filter(job, selection)) + // .sortBy('modifyIndex') + // .reverse(); + + // JobsList.jobs.forEach((job, index) => { + // assert.equal( + // job.id, + // expectedJobs[index].id, + // `Job at ${index} is ${expectedJobs[index].id}` + // ); + // }); + // }); + + // test.skip(`selecting an option in the ${label} facet updates the ${paramName} query param`, async function (assert) { + // await beforeEach(); + // await facet.toggle(); + + // const option = facet.options.objectAt(1); + // const selection = option.key; + // await option.select(); + + // assert.ok( + // currentURL().includes(`${paramName}=${selection}`), + // 'URL has the correct query param key and value' + // ); + // }); + // } + + // function testFacet( + // label, + // { facet, paramName, beforeEach, filter, expectedOptions } + // ) { + // test.skip(`the ${label} facet has the correct options`, async function (assert) { + // await facetOptions(assert, beforeEach, facet, expectedOptions); + // }); + + // test.skip(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { + // let option; + + // await beforeEach(); + // await facet.toggle(); + + // option = facet.options.objectAt(0); + // await option.toggle(); + + // const selection = [option.key]; + // const expectedJobs = server.db.jobs + // .filter((job) => filter(job, selection)) + // .sortBy('modifyIndex') + // .reverse(); + + // JobsList.jobs.forEach((job, index) => { + // assert.equal( + // job.id, + // expectedJobs[index].id, + // `Job at ${index} is ${expectedJobs[index].id}` + // ); + // }); + // }); + + // test.skip(`selecting multiple options in the ${label} facet results in a broader search`, async function (assert) { + // const selection = []; + + // await beforeEach(); + // await facet.toggle(); + + // const option1 = facet.options.objectAt(0); + // const option2 = facet.options.objectAt(1); + // await option1.toggle(); + // selection.push(option1.key); + // await option2.toggle(); + // selection.push(option2.key); + + // const expectedJobs = server.db.jobs + // .filter((job) => filter(job, selection)) + // .sortBy('modifyIndex') + // .reverse(); + + // JobsList.jobs.forEach((job, index) => { + // assert.equal( + // job.id, + // expectedJobs[index].id, + // `Job at ${index} is ${expectedJobs[index].id}` + // ); + // }); + // }); + + // test.skip(`selecting options in the ${label} facet updates the ${paramName} query param`, async function (assert) { + // const selection = []; + + // await beforeEach(); + // await facet.toggle(); + + // const option1 = facet.options.objectAt(0); + // const option2 = facet.options.objectAt(1); + // await option1.toggle(); + // selection.push(option1.key); + // await option2.toggle(); + // selection.push(option2.key); + + // assert.ok( + // currentURL().includes(encodeURIComponent(JSON.stringify(selection))), + // 'URL has the correct query param key and value' + // ); + // }); + + // test.skip('the run job button works when filters are set', async function (assert) { + // ['pre-one', 'pre-two', 'pre-three'].forEach((name) => { + // server.create('job', { + // name, + // createAllocations: false, + // childrenCount: 0, + // }); + // }); + + // await JobsList.visit(); + + // await JobsList.facets.prefix.toggle(); + // await JobsList.facets.prefix.options[0].toggle(); + + // await JobsList.runJobButton.click(); + // assert.equal(currentURL(), '/jobs/run'); + // }); + // } }); From d1201af995a5e3ce62597976561942eebf799d92 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 22 Mar 2024 12:01:05 -0400 Subject: [PATCH 83/98] Maybe it's the automatic mirage logging --- ui/mirage/config.js | 2 +- ui/tests/acceptance/keyboard-test.js | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 1785da8dff7..7bf21a62973 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -30,7 +30,7 @@ export function filesForPath(allocFiles, filterPath) { export default function () { this.timing = 0; // delay for each request, automatically set to 0 during testing - this.logging = true; // TODO: window.location.search.includes('mirage-logging=true'); + this.logging = window.location.search.includes('mirage-logging=true'); this.namespace = 'v1'; this.trackRequests = Ember.testing; diff --git a/ui/tests/acceptance/keyboard-test.js b/ui/tests/acceptance/keyboard-test.js index e60ca0b6607..77653e70877 100644 --- a/ui/tests/acceptance/keyboard-test.js +++ b/ui/tests/acceptance/keyboard-test.js @@ -304,7 +304,7 @@ module('Acceptance | keyboard', function (hooks) { triggerEvent('.page-layout', 'keydown', { key: '0' }); await triggerEvent('.page-layout', 'keydown', { key: '1' }); - const clickedJob = server.db.jobs[0].id; + const clickedJob = server.db.jobs.sortBy('modifyIndex').reverse()[0].id; assert.equal( currentURL(), `/jobs/${clickedJob}@default`, @@ -313,7 +313,9 @@ module('Acceptance | keyboard', function (hooks) { }); test('Multi-Table Nav', async function (assert) { server.createList('job', 3, { createRecommendations: true }); - await visit(`/jobs/${server.db.jobs[0].id}@default`); + await visit( + `/jobs/${server.db.jobs.sortBy('modifyIndex').reverse()[0].id}@default` + ); const numberOfGroups = findAll('.task-group-row').length; const numberOfAllocs = findAll('.allocation-row').length; From 2b8dea280062c0ce532c57f314b355a4721bb9b0 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 22 Mar 2024 15:43:50 -0400 Subject: [PATCH 84/98] Unbreak task unit test --- ui/app/models/task-event.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/app/models/task-event.js b/ui/app/models/task-event.js index ecc33acd8d9..56d879b8d46 100644 --- a/ui/app/models/task-event.js +++ b/ui/app/models/task-event.js @@ -17,7 +17,6 @@ export default class TaskEvent extends Fragment { @attr('date') time; @attr('number') timeNanos; @attr('string') displayMessage; - @attr() message; get message() { let message = simplifyTimeMessage(this.displayMessage); From 69198f784768852cf681a8731e6de429ab92a65a Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 22 Mar 2024 16:45:31 -0400 Subject: [PATCH 85/98] Pre-sort sort --- ui/mirage/config.js | 5 ++++- ui/tests/acceptance/jobs-list-test.js | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 7bf21a62973..6e4d2bc9489 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -82,6 +82,7 @@ export default function () { const json = this.serialize(jobs.all()); return json + .sort((a, b) => b.ID.localeCompare(a.ID)) .sort((a, b) => b.ModifyIndex - a.ModifyIndex) .filter((job) => { if (namespace === '*') return true; @@ -147,7 +148,9 @@ export default function () { return job; }); // sort by modifyIndex, descending - returnedJobs.sort((a, b) => b.ModifyIndex - a.ModifyIndex); + returnedJobs + .sort((a, b) => b.ID.localeCompare(a.ID)) + .sort((a, b) => b.ModifyIndex - a.ModifyIndex); return returnedJobs; }) diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index cb4d6079d60..e3db91e125f 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -53,7 +53,10 @@ module('Acceptance | jobs list', function (hooks) { await percySnapshot(assert); - const sortedJobs = server.db.jobs.sortBy('modifyIndex').reverse(); + const sortedJobs = server.db.jobs + .sortBy('id') + .sortBy('modifyIndex') + .reverse(); assert.equal(JobsList.jobs.length, JobsList.pageSize); JobsList.jobs.forEach((job, index) => { assert.equal(job.name, sortedJobs[index].name, 'Jobs are ordered'); From b02d33b9c1e528d0c9ac64f6221dee9533c10cbe Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Tue, 26 Mar 2024 16:56:28 -0400 Subject: [PATCH 86/98] styling for jobs list pagination and general PR cleanup --- ui/app/adapters/job.js | 44 +--- .../job-status/allocation-status-row.hbs | 1 - ui/app/controllers/jobs/index.js | 75 +++---- ui/app/models/job.js | 33 ++- ui/app/routes/jobs/index.js | 2 - ui/app/serializers/job.js | 28 +-- ui/app/services/watch-list.js | 52 ++--- ui/app/styles/components.scss | 1 + ui/app/styles/components/jobs-list.scss | 34 +++ ui/app/templates/jobs/index.hbs | 195 +++++++++--------- ui/mirage/config.js | 1 - 11 files changed, 214 insertions(+), 252 deletions(-) create mode 100644 ui/app/styles/components/jobs-list.scss diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 1872e531566..fcee0c75399 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -15,7 +15,6 @@ import { get } from '@ember/object'; @classic export default class JobAdapter extends WatchableNamespaceIDs { @service system; - @service watchList; relationshipFallbackLinks = { summary: '/summary', @@ -205,6 +204,7 @@ export default class JobAdapter extends WatchableNamespaceIDs { return wsUrl; } + // TODO: Handle the in-job-page query for pack meta per https://github.com/hashicorp/nomad/pull/14833 query(store, type, query, snapshotRecordArray, options) { options = options || {}; options.adapterOptions = options.adapterOptions || {}; @@ -212,24 +212,28 @@ export default class JobAdapter extends WatchableNamespaceIDs { const method = get(options, 'adapterOptions.method') || 'GET'; const url = this.urlForQuery(query, type.modelName, method); - // Let's establish the index, via watchList.getIndexFor. - // let index = this.watchList.getIndexFor(url); let index = query.index || 1; - // TODO: adding a new job hash will not necessarily cancel the old one. - // You could be holding open a POST on jobs AB and ABC at the same time. - if (index && index > 1) { query.index = index; } const signal = get(options, 'adapterOptions.abortController.signal'); + // when GETting our jobs list, we want to sort in reverse order, because + // the sort property is ModifyIndex and we want the most recent jobs first. + if (method === 'GET') { + query.reverse = true; + } + return this.ajax(url, method, { signal, data: query, }).then((payload) => { - // If there was a request body, append it to my payload + // If there was a request body, append it to the payload + // We can use this in our serializer to maintain returned job order, + // even if one of the requested jobs is not found (has been GC'd) so as + // not to jostle the user's view. if (query.jobs) { payload._requestBody = query; } @@ -238,8 +242,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { } handleResponse(status, headers) { - // watchList.setIndexFor() happens in the watchable adapter, super'd here - /** * @type {Object} */ @@ -250,11 +252,9 @@ export default class JobAdapter extends WatchableNamespaceIDs { result.meta.nextToken = headers['x-nomad-nexttoken']; } if (headers['x-nomad-index']) { - // this.watchList.setIndexFor(result.url, headers['x-nomad-index']); result.meta.index = headers['x-nomad-index']; } } - return result; } @@ -264,27 +264,5 @@ export default class JobAdapter extends WatchableNamespaceIDs { return `${baseUrl}?index=${query.index}`; } return baseUrl; - // if (method === 'POST') { - // // Setting a base64 hash to represent the body of the POST request - // // (which is otherwise not represented in the URL) - // // because the watchList uses the URL as a key for index lookups. - // return `${baseUrl}?hash=${btoa(JSON.stringify(query))}`; - // } else { - // // return `${baseUrl}?${queryString.stringify(query)}`; // TODO: maybe nix this, it's doubling up QPs - // return `${baseUrl}`; - // } } - - // ajaxOptions(url, type, options) { - // let hash = super.ajaxOptions(url, type, options); - // // Custom handling for POST requests to append 'index' as a query parameter - // if (type === 'POST' && options.data && options.data.index) { - // let index = encodeURIComponent(options.data.index); - // hash.url = `${hash.url}&index=${index}`; - // } - - // return hash; - // } } - -// TODO: First query (0 jobs to 1 job) doesnt seem to kick off POST diff --git a/ui/app/components/job-status/allocation-status-row.hbs b/ui/app/components/job-status/allocation-status-row.hbs index 45dd528967f..bb08b2f42f3 100644 --- a/ui/app/components/job-status/allocation-status-row.hbs +++ b/ui/app/components/job-status/allocation-status-row.hbs @@ -55,7 +55,6 @@
{{/if}} {{#if @compact}} - {{!-- TODO: @runningAllocs using the wrong thing --}} {{@runningAllocs}}/{{@groupCountSum}} {{/if}} diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index c80f179de74..2814c3be009 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -7,7 +7,7 @@ import Controller from '@ember/controller'; import { inject as service } from '@ember/service'; -import { action } from '@ember/object'; +import { action, computed } from '@ember/object'; import { tracked } from '@glimmer/tracking'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; import { restartableTask, timeout } from 'ember-concurrency'; @@ -15,10 +15,10 @@ import { serialize, deserializedQueryParam as selection, } from 'nomad-ui/utils/qp-serialize'; -// import { scheduleOnce } from '@ember/runloop'; import Ember from 'ember'; -const DEFAULT_THROTTLE = 2000; +const JOB_LIST_THROTTLE = 5000; +const JOB_DETAILS_THROTTLE = 1000; export default class JobsIndexController extends Controller { @service router; @@ -28,6 +28,7 @@ export default class JobsIndexController extends Controller { @service watchList; // TODO: temp @tracked pageSize; + constructor() { super(...arguments); this.pageSize = this.userSettings.pageSize; @@ -51,7 +52,8 @@ export default class JobsIndexController extends Controller { @tracked jobAllocsQueryIndex = 0; @selection('qpNamespace') selectionNamespace; - // @computed('qpNamespace', 'model.namespaces.[]') + + @computed('qpNamespace', 'model.namespaces.[]') get optionsNamespace() { const availableNamespaces = this.model.namespaces.map((namespace) => ({ key: namespace.name, @@ -133,6 +135,9 @@ export default class JobsIndexController extends Controller { this.jobAllocsQueryIndex = 0; if (page === 'prev') { + if (!this.cursorAt) { + return; + } // Note (and TODO:) this isn't particularly efficient! // We're making an extra full request to get the nextToken we need, // but actually the results of that request are the reverse order, plus one job, @@ -152,6 +157,9 @@ export default class JobsIndexController extends Controller { } } } else if (page === 'next') { + if (!this.nextToken) { + return; + } this.cursorAt = this.nextToken; } } @@ -180,7 +188,7 @@ export default class JobsIndexController extends Controller { this.pendingJobIDs = null; yield this.watchJobs.perform( this.jobIDs, - Ember.testing ? 0 : DEFAULT_THROTTLE + Ember.testing ? 0 : JOB_DETAILS_THROTTLE ); } @@ -209,28 +217,19 @@ export default class JobsIndexController extends Controller { }); } - jobAllocsQuery(jobIDs) { + jobAllocsQuery(params) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexDetailsController = new AbortController(); return this.store - .query( - 'job', - { - jobs: jobIDs, - index: this.jobAllocsQueryIndex, // TODO: consider using a passed params object like jobQuery uses, rather than just passing jobIDs + .query('job', params, { + adapterOptions: { + method: 'POST', + abortController: this.watchList.jobsIndexDetailsController, }, - { - adapterOptions: { - method: 'POST', - abortController: this.watchList.jobsIndexDetailsController, - }, - } - ) + }) .catch((e) => { if (e.name !== 'AbortError') { console.log('error fetching job allocs', e); - } else { - console.log('|> jobAllocsQuery aborted'); } return; }); @@ -257,19 +256,13 @@ export default class JobsIndexController extends Controller { // TODO: set up isEnabled to check blockingQueries rather than just use while (true) @restartableTask *watchJobIDs( params, - throttle = Ember.testing ? 0 : DEFAULT_THROTTLE + throttle = Ember.testing ? 0 : JOB_LIST_THROTTLE ) { while (true) { - // let watchlistIndex = this.watchList.getIndexFor( - // '/v1/jobs/statuses?per_page=3' - // ); - // console.log('> watchJobIDs', params); let currentParams = params; - // currentParams.index = watchlistIndex; currentParams.index = this.jobQueryIndex; const newJobs = yield this.jobQuery(currentParams, {}); if (newJobs) { - // console.log('|> watchJobIDs returned new job IDs', newJobs.length); if (newJobs.meta.index) { this.jobQueryIndex = newJobs.meta.index; } @@ -288,25 +281,16 @@ export default class JobsIndexController extends Controller { if (okayToJostle) { this.jobIDs = jobIDs; this.watchList.jobsIndexDetailsController.abort(); - console.log( - 'new jobIDs have appeared, we should now watch them. We have cancelled the old hash req.', - jobIDs - ); - // Let's also reset the index for the job details query this.jobAllocsQueryIndex = 0; this.watchList.jobsIndexDetailsController = new AbortController(); - // make sure throttle has taken place! this.watchJobs.perform(jobIDs, throttle); } else { - // this.controller.set('pendingJobIDs', jobIDs); - // this.controller.set('pendingJobs', newJobs); this.pendingJobIDs = jobIDs; this.pendingJobs = newJobs; } - yield timeout(throttle); // Moved to the end of the loop + yield timeout(throttle); } else { // This returns undefined on page change / cursorAt change, resulting from the aborting of the old query. - // console.log('|> watchJobIDs aborted'); yield timeout(throttle); this.watchJobs.perform(this.jobIDs, throttle); continue; @@ -324,24 +308,18 @@ export default class JobsIndexController extends Controller { // 3. via the user manually clicking to updateJobList() @restartableTask *watchJobs( jobIDs, - throttle = Ember.testing ? 0 : DEFAULT_THROTTLE + throttle = Ember.testing ? 0 : JOB_DETAILS_THROTTLE ) { while (true) { - console.log( - '> watchJobs of IDs', - jobIDs.map((j) => j.id) - ); - // let jobIDs = this.controller.jobIDs; if (jobIDs && jobIDs.length > 0) { - let jobDetails = yield this.jobAllocsQuery(jobIDs); + let jobDetails = yield this.jobAllocsQuery({ + jobs: jobIDs, + index: this.jobAllocsQueryIndex, + }); if (jobDetails) { if (jobDetails.meta.index) { this.jobAllocsQueryIndex = jobDetails.meta.index; } - console.log( - '|> watchJobs returned with', - jobDetails.map((j) => j.id) - ); } this.jobs = jobDetails; } @@ -351,6 +329,5 @@ export default class JobsIndexController extends Controller { } } } - //#endregion querying } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 176ece19306..bff58fed4ff 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -76,8 +76,8 @@ export default class Job extends Model { /** * @typedef {Object} CurrentStatus - * @property {"Healthy"|"Failed"|"Degraded"|"Recovering"|"Complete"|"Running"} label - The current status of the job - * @property {"highlight"|"success"|"warning"|"critical"} state - + * @property {"Healthy"|"Failed"|"Deploying"|"Degraded"|"Recovering"|"Complete"|"Running"|"Removed"} label - The current status of the job + * @property {"highlight"|"success"|"warning"|"critical"|"neutral"} state - */ /** @@ -185,6 +185,7 @@ export default class Job extends Model { // Handle unplaced allocs if (availableSlotsToFill > 0) { + // TODO: JSDoc types for unhealty and health unknown aren't optional, but should be. allocationsOfShowableType['unplaced'] = { healthy: { nonCanary: Array(availableSlotsToFill) @@ -211,6 +212,7 @@ export default class Job extends Model { * - Recovering: Some allocations are pending * - Degraded: A deployment is not taking place, and some allocations are failed, lost, or unplaced * - Failed: All allocations are failed, lost, or unplaced + * - Removed: The job appeared in our initial query, but has since been garbage collected * @returns {CurrentStatus} */ /** @@ -218,18 +220,16 @@ export default class Job extends Model { * @returns {CurrentStatus} */ get aggregateAllocStatus() { - // If all allocs are running, the job is Healthy - let totalAllocs = this.expectedRunningAllocCount; - // console.log('expectedRunningAllocCount is', totalAllocs); - // console.log('ablocks are', this.allocBlocks); - // If deploying: if (this.deploymentID) { return { label: 'Deploying', state: 'highlight' }; } + // If the job was requested initially, but a subsequent request for it was + // not found, we can remove links to it but maintain its presence in the list + // until the user specifies they want a refresh if (this.assumeGC) { return { label: 'Removed', state: 'neutral' }; } @@ -249,35 +249,34 @@ export default class Job extends Model { } } + // All the exepected allocs are running and healthy? Congratulations! const healthyAllocs = this.allocBlocks.running?.healthy?.nonCanary; if (totalAllocs && healthyAllocs?.length === totalAllocs) { return { label: 'Healthy', state: 'success' }; } // If any allocations are pending the job is "Recovering" - // TODO: weird, but batch jobs (which do not have deployments!) go into "recovering" right away, since some of their statuses are "pending" as they come online. - // This feels a little wrong. + // Note: Batch/System jobs (which do not have deployments) + // go into "recovering" right away, since some of their statuses are + // "pending" as they come online. This feels a little wrong but it's kind + // of correct? const pendingAllocs = this.allocBlocks.pending?.healthy?.nonCanary; if (pendingAllocs?.length > 0) { return { label: 'Recovering', state: 'highlight' }; } - // If any allocations are failed, lost, or unplaced in a steady state, the job is "Degraded" + // If any allocations are failed, lost, or unplaced in a steady state, + // the job is "Degraded" const failedOrLostAllocs = [ ...this.allocBlocks.failed?.healthy?.nonCanary, ...this.allocBlocks.lost?.healthy?.nonCanary, ...this.allocBlocks.unplaced?.healthy?.nonCanary, ]; + // TODO: GroupCountSum for a parameterized parent job is the count present at group level, but that's not quite true, as the parent job isn't expecting any allocs, its children are. Chat with BFF about this. + // TODO: handle garbage collected cases not showing "failed" for batch jobs here maybe? - // console.log( - // 'numFailedAllocs', - // failedOrLostAllocs.length, - // failedOrLostAllocs, - // totalAllocs - // ); - // if (failedOrLostAllocs.length === totalAllocs) { if (failedOrLostAllocs.length >= totalAllocs) { // TODO: when totalAllocs only cares about latest version, change back to === return { label: 'Failed', state: 'critical' }; diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 33359c89c04..bb5fdc1f9ea 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -62,7 +62,6 @@ export default class IndexRoute extends Route.extend( }, }) .catch(notifyForbidden(this)); - console.log('model jobs', jobs); return RSVP.hash({ jobs, namespaces: this.store.findAll('namespace'), @@ -71,7 +70,6 @@ export default class IndexRoute extends Route.extend( } setupController(controller, model) { - console.log('== setupController'); super.setupController(controller, model); if (!model.jobs) { diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index 02735229cb6..7854dd61d2b 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -88,26 +88,26 @@ export default class JobSerializer extends ApplicationSerializer { }; }); - // If any were missing, sort them in the order they were requested - // TODO: document why - if (missingJobIDs.length > 0) { - payload.sort((a, b) => { - return ( - requestedJobIDs.findIndex( - (j) => j.id === a.ID && j.namespace === a.Namespace - ) - - requestedJobIDs.findIndex( - (j) => j.id === b.ID && j.namespace === b.Namespace - ) - ); - }); - } + // Note: we want our returned jobs to come back in the order we requested them, + // including jobs that were missing from the initial request. + payload.sort((a, b) => { + return ( + requestedJobIDs.findIndex( + (j) => j.id === a.ID && j.namespace === a.Namespace + ) - + requestedJobIDs.findIndex( + (j) => j.id === b.ID && j.namespace === b.Namespace + ) + ); + }); delete payload._requestBody; } const jobs = payload; // Signal that it's a query response at individual normalization level for allocation placement + // Sort by ModifyIndex, reverse + jobs.sort((a, b) => b.ModifyIndex - a.ModifyIndex); jobs.forEach((job) => { if (job.Allocs) { job.relationships = { diff --git a/ui/app/services/watch-list.js b/ui/app/services/watch-list.js index e71e9df35bd..873d9ae2fc3 100644 --- a/ui/app/services/watch-list.js +++ b/ui/app/services/watch-list.js @@ -3,52 +3,34 @@ * SPDX-License-Identifier: BUSL-1.1 */ -// import { computed } from '@ember/object'; -// import { readOnly } from '@ember/object/computed'; -// import { copy } from 'ember-copy'; +import { computed } from '@ember/object'; +import { readOnly } from '@ember/object/computed'; +import { copy } from 'ember-copy'; import Service from '@ember/service'; -import { tracked } from '@glimmer/tracking'; -// let list = {}; +let list = {}; export default class WatchListService extends Service { - // @computed - // get _list() { - // return copy(list, true); - // } - - jobsIndexIDsController = new AbortController(); - jobsIndexDetailsController = new AbortController(); + @computed + get _list() { + return copy(list, true); + } - // @readOnly('_list') list; - @tracked list = {}; + @readOnly('_list') list; - // constructor() { - // super(...arguments); - // list = {}; - // } + constructor() { + super(...arguments); + list = {}; + } getIndexFor(url) { - return this.list[url] || 1; + return list[url] || 1; } setIndexFor(url, value) { - this.list[url] = +value; - this.list = { ...this.list }; + list[url] = +value; } - /** - * When we paginate or otherwise manually change queryParams for our jobs index, - * we want our requests to return immediately. This means we need to clear out - * any previous indexes that are associated with the jobs index. - */ - clearJobsIndexIndexes() { - // If it starts with /v1/jobs/statuses, remove it - let keys = Object.keys(this.list); - keys.forEach((key) => { - if (key.startsWith('/v1/jobs/statuses')) { - delete this.list[key]; - } - }); - } + jobsIndexIDsController = new AbortController(); + jobsIndexDetailsController = new AbortController(); } diff --git a/ui/app/styles/components.scss b/ui/app/styles/components.scss index 3321362e3e4..a1cb9dcdf97 100644 --- a/ui/app/styles/components.scss +++ b/ui/app/styles/components.scss @@ -60,3 +60,4 @@ @import './components/job-status-panel'; @import './components/access-control'; @import './components/actions'; +@import './components/jobs-list'; diff --git a/ui/app/styles/components/jobs-list.scss b/ui/app/styles/components/jobs-list.scss new file mode 100644 index 00000000000..5e69b3c3d93 --- /dev/null +++ b/ui/app/styles/components/jobs-list.scss @@ -0,0 +1,34 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +// Styling for compnents around the redesigned Jobs Index page. +// Over time, we can phase most of these custom styles out as Helios components +// adapt to more use-cases (like custom footer, etc). + +#jobs-list-actions { + margin-bottom: 1rem; +} + +#jobs-list-pagination { + display: grid; + grid-template-columns: 1fr auto 1fr; + grid-template-areas: 'info nav-buttons page-size'; + align-items: center; + justify-items: start; + padding: 1rem 0; + gap: 1rem; + + .nav-buttons { + grid-area: nav-buttons; + display: flex; + justify-content: center; + gap: 1rem; + } + + .page-size { + grid-area: page-size; + justify-self: end; + } +} diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7807a582182..8c1d738f1a3 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -6,10 +6,7 @@ {{page-title "Jobs"}}
- {{!-- - Jobs (soft J, like "yobs") - --}} - + {{!-- --}} - + + {{#if this.system.shouldShowNamespaces}} + {{!-- --}} - - - - - - - {{#if this.system.shouldShowNamespaces}} - {{!-- --}} - - - - {{#each this.optionsNamespace key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Namespaces - - {{/each}} - - - - {{/if}} - -
- + -
+ {{#each this.optionsNamespace key="label" as |option|}} + + {{option.label}} + + {{else}} + + No Namespaces + + {{/each}} + + {{/if}} -
- -
+
+ +
+ +
+ +
- {{#if this.pendingJobIDDiff}} + {{#if this.pendingJobIDDiff}} {{B.data.name}} + {{!-- TODO: going to lose .meta with statuses endpoint! --}} {{#if B.data.meta.structured.pack}} {{x-icon "box" class= "test"}} @@ -369,31 +358,37 @@ -
- {{!-- TODO: Temporary keyboard-shortcut tester buttons while I think about nixing the pagination component --}} - - - - - +
+
+ + + + + + +
+
+ +
+
{{!-- Date: Thu, 18 Apr 2024 12:39:44 -0400 Subject: [PATCH 87/98] moving from Job.ActiveDeploymentID to Job.LatestDeployment.ID --- ui/app/models/job.js | 4 ++-- ui/app/serializers/job.js | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ui/app/models/job.js b/ui/app/models/job.js index bff58fed4ff..7506fd77901 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -43,7 +43,7 @@ export default class Job extends Model { } } - @attr('string') deploymentID; + @attr() latestDeploymentSummary; // TODO: model this out @attr() childStatuses; @@ -223,7 +223,7 @@ export default class Job extends Model { let totalAllocs = this.expectedRunningAllocCount; // If deploying: - if (this.deploymentID) { + if (this.latestDeploymentSummary?.IsActive) { return { label: 'Deploying', state: 'highlight' }; } diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index 7854dd61d2b..7046006edb0 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -119,6 +119,10 @@ export default class JobSerializer extends ApplicationSerializer { }, }; } + if (job.LatestDeployment) { + job.LatestDeploymentSummary = job.LatestDeployment; + delete job.LatestDeployment; + } job._aggregate = true; }); return super.normalizeQueryResponse( From 3c5a77a5ae72f4847fbcc4191124fea4c5bd4dc3 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 18 Apr 2024 15:07:36 -0400 Subject: [PATCH 88/98] modifyIndex-based pagination (#20350) * modifyIndex-based pagination * modifyIndex gets its own column and pagination compacted with icons * A generic withPagination handler for mirage * Some live-PR changes * Pagination and button disabled tests * Job update handling tests for jobs index * assertion timeout in case of long setTimeouts * assert.timeouts down to 500ms * de-to-do * Clarifying comment and test descriptions --- ui/app/adapters/job.js | 6 - ui/app/controllers/jobs/index.js | 40 ++- ui/app/templates/jobs/index.hbs | 58 +++- ui/mirage/config.js | 87 +++-- ui/mirage/scenarios/default.js | 15 +- ui/tests/acceptance/jobs-list-test.js | 475 +++++++++++++++++++++++++- ui/tests/pages/jobs/list.js | 3 - 7 files changed, 623 insertions(+), 61 deletions(-) diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index fcee0c75399..09fcbe0e3d9 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -220,12 +220,6 @@ export default class JobAdapter extends WatchableNamespaceIDs { const signal = get(options, 'adapterOptions.abortController.signal'); - // when GETting our jobs list, we want to sort in reverse order, because - // the sort property is ModifyIndex and we want the most recent jobs first. - if (method === 'GET') { - query.reverse = true; - } - return this.ajax(url, method, { signal, data: query, diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 2814c3be009..8d358ad9e6f 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -33,7 +33,6 @@ export default class JobsIndexController extends Controller { super(...arguments); this.pageSize = this.userSettings.pageSize; } - reverse = false; queryParams = [ 'cursorAt', @@ -146,21 +145,28 @@ export default class JobsIndexController extends Controller { // overwrite this controller's jobIDs, leverage its index, and // restart a blocking watchJobIDs here. let prevPageToken = await this.loadPreviousPageToken(); - if (prevPageToken.length > 1) { - // if there's only one result, it'd be the job you passed into it as your nextToken (and the first shown on your current page) - const [id, namespace] = JSON.parse(prevPageToken.lastObject.id); - // If there's no nextToken, we're at the "start" of our list and can drop the cursorAt - if (!prevPageToken.meta.nextToken) { - this.cursorAt = null; - } else { - this.cursorAt = `${namespace}.${id}`; - } + // If there's no nextToken, we're at the "start" of our list and can drop the cursorAt + if (!prevPageToken.meta.nextToken) { + this.cursorAt = undefined; + } else { + // cursorAt should be the highest modifyIndex from the previous query. + // This will immediately fire the route model hook with the new cursorAt + this.cursorAt = prevPageToken + .sortBy('modifyIndex') + .get('lastObject').modifyIndex; } } else if (page === 'next') { if (!this.nextToken) { return; } this.cursorAt = this.nextToken; + } else if (page === 'first') { + this.cursorAt = undefined; + } else if (page === 'last') { + let prevPageToken = await this.loadPreviousPageToken({ last: true }); + this.cursorAt = prevPageToken + .sortBy('modifyIndex') + .get('lastObject').modifyIndex; } } @@ -235,13 +241,19 @@ export default class JobsIndexController extends Controller { }); } - async loadPreviousPageToken() { + // Ask for the previous #page_size jobs, starting at the first job that's currently shown + // on our page, and the last one in our list should be the one we use for our + // subsequent nextToken. + async loadPreviousPageToken({ last = false } = {}) { + let next_token = +this.cursorAt + 1; + if (last) { + next_token = undefined; + } let prevPageToken = await this.store.query( 'job', { - prev_page_query: true, // TODO: debugging only! - next_token: this.cursorAt, - per_page: this.pageSize + 1, + next_token, + per_page: this.pageSize, reverse: true, }, { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 8c1d738f1a3..6f614a3df94 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -264,6 +264,7 @@ @color="primary" @icon="sync" {{on "click" (perform this.updateJobList)}} + data-test-updates-pending-button /> {{/if}} @@ -288,6 +289,7 @@ {{on "click" (action this.gotoJob B.data)}} class="job-row is-interactive {{if B.data.assumeGC "assume-gc"}}" data-test-job-row={{B.data.plainId}} + data-test-modify-index={{B.data.modifyIndex}} > {{!-- {{#each this.tableColumns as |column|}} {{get B.data (lowercase column.label)}} @@ -360,15 +362,35 @@
+ + + @@ -379,26 +401,36 @@ > + + +
- {{!-- --}} {{else}} {{!-- TODO: differentiate between "empty because there's nothing" and "empty because you have filtered/searched" --}} diff --git a/ui/mirage/config.js b/ui/mirage/config.js index edf94ba1fa3..c33f209c7b9 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -37,12 +37,20 @@ export default function () { const nomadIndices = {}; // used for tracking blocking queries const server = this; - const withBlockingSupport = function (fn) { + const withBlockingSupport = function ( + fn, + { pagination = false, tokenProperty = 'ModifyIndex' } = {} + ) { return function (schema, request) { + let handler = fn; + if (pagination) { + handler = withPagination(handler, tokenProperty); + } + // Get the original response let { url } = request; url = url.replace(/index=\d+[&;]?/, ''); - const response = fn.apply(this, arguments); + let response = handler.apply(this, arguments); // Get and increment the appropriate index nomadIndices[url] || (nomadIndices[url] = 2); @@ -58,6 +66,34 @@ export default function () { }; }; + const withPagination = function (fn, tokenProperty = 'ModifyIndex') { + return function (schema, request) { + let response = fn.apply(this, arguments); + let perPage = parseInt(request.queryParams.per_page || 25); + let page = parseInt(request.queryParams.page || 1); + let totalItems = response.length; + let totalPages = Math.ceil(totalItems / perPage); + let hasMore = page < totalPages; + + let paginatedItems = response.slice((page - 1) * perPage, page * perPage); + + let nextToken = null; + if (hasMore) { + nextToken = response[page * perPage][tokenProperty]; + } + + if (nextToken) { + return new Response( + 200, + { 'x-nomad-nexttoken': nextToken }, + paginatedItems + ); + } else { + return new Response(200, {}, paginatedItems); + } + }; + }; + this.get( '/jobs', withBlockingSupport(function ({ jobs }, { queryParams }) { @@ -76,23 +112,36 @@ export default function () { this.get( '/jobs/statuses', - withBlockingSupport(function ({ jobs }, req) { - let per_page = req.queryParams.per_page || 20; - const namespace = req.queryParams.namespace || 'default'; - - const json = this.serialize(jobs.all()); - return json - .sort((a, b) => b.ID.localeCompare(a.ID)) - .sort((a, b) => b.ModifyIndex - a.ModifyIndex) - .filter((job) => { - if (namespace === '*') return true; - return namespace === 'default' - ? !job.NamespaceID || job.NamespaceID === 'default' - : job.NamespaceID === namespace; - }) - .map((job) => filterKeys(job, 'TaskGroups', 'NamespaceID')) - .slice(0, per_page); - }) + withBlockingSupport( + function ({ jobs }, req) { + const namespace = req.queryParams.namespace || 'default'; + let nextToken = req.queryParams.next_token || 0; + let reverse = req.queryParams.reverse === 'true'; + const json = this.serialize(jobs.all()); + let sortedJson = json + .sort((a, b) => + reverse + ? a.ModifyIndex - b.ModifyIndex + : b.ModifyIndex - a.ModifyIndex + ) + .filter((job) => { + if (namespace === '*') return true; + return namespace === 'default' + ? !job.NamespaceID || job.NamespaceID === 'default' + : job.NamespaceID === namespace; + }) + .map((job) => filterKeys(job, 'TaskGroups', 'NamespaceID')); + if (nextToken) { + sortedJson = sortedJson.filter((job) => + reverse + ? job.ModifyIndex >= nextToken + : job.ModifyIndex <= nextToken + ); + } + return sortedJson; + }, + { pagination: true, tokenProperty: 'ModifyIndex' } + ) ); this.post( diff --git a/ui/mirage/scenarios/default.js b/ui/mirage/scenarios/default.js index 54a9ae3a756..896c5f26eb8 100644 --- a/ui/mirage/scenarios/default.js +++ b/ui/mirage/scenarios/default.js @@ -62,11 +62,16 @@ function jobsIndexTestCluster(server) { faker.seed(1); server.createList('agent', 1, 'withConsulLink', 'withVaultLink'); server.createList('node', 1); - server.createList('job', 1, { - namespaceId: 'default', - resourceSpec: Array(1).fill('M: 256, C: 500'), - groupAllocCount: 1, - }); + + const jobsToCreate = 55; + for (let i = 0; i < jobsToCreate; i++) { + server.create('job', { + namespaceId: 'default', + resourceSpec: Array(1).fill('M: 256, C: 500'), + groupAllocCount: 1, + modifyIndex: i + 1, + }); + } } function smallCluster(server) { diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index e3db91e125f..02aa6d52ebc 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -4,7 +4,7 @@ */ /* eslint-disable qunit/require-expect */ -import { currentURL, click } from '@ember/test-helpers'; +import { currentURL, click, triggerKeyEvent } from '@ember/test-helpers'; import { module, test } from 'qunit'; import { setupApplicationTest } from 'ember-qunit'; import { setupMirage } from 'ember-cli-mirage/test-support'; @@ -735,4 +735,477 @@ module('Acceptance | jobs list', function (hooks) { // assert.equal(currentURL(), '/jobs/run'); // }); // } + + module('Pagination', function () { + module('Buttons are appropriately disabled', function () { + test('when there are no jobs', async function (assert) { + await JobsList.visit(); + assert.dom('[data-test-pager="first"]').doesNotExist(); + assert.dom('[data-test-pager="previous"]').doesNotExist(); + assert.dom('[data-test-pager="next"]').doesNotExist(); + assert.dom('[data-test-pager="last"]').doesNotExist(); + await percySnapshot(assert); + }); + test('when there are fewer jobs than your page size setting', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 5); + await JobsList.visit(); + assert.dom('[data-test-pager="first"]').isDisabled(); + assert.dom('[data-test-pager="previous"]').isDisabled(); + assert.dom('[data-test-pager="next"]').isDisabled(); + assert.dom('[data-test-pager="last"]').isDisabled(); + await percySnapshot(assert); + localStorage.removeItem('nomadPageSize'); + }); + test('when you have plenty of jobs', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 25); + await JobsList.visit(); + assert.dom('.job-row').exists({ count: 10 }); + assert.dom('[data-test-pager="first"]').isDisabled(); + assert.dom('[data-test-pager="previous"]').isDisabled(); + assert.dom('[data-test-pager="next"]').isNotDisabled(); + assert.dom('[data-test-pager="last"]').isNotDisabled(); + // Clicking next brings me to another full page + await click('[data-test-pager="next"]'); + assert.dom('.job-row').exists({ count: 10 }); + assert.dom('[data-test-pager="first"]').isNotDisabled(); + assert.dom('[data-test-pager="previous"]').isNotDisabled(); + assert.dom('[data-test-pager="next"]').isNotDisabled(); + assert.dom('[data-test-pager="last"]').isNotDisabled(); + // clicking next again brings me to the last page, showing jobs 20-25 + await click('[data-test-pager="next"]'); + assert.dom('.job-row').exists({ count: 5 }); + assert.dom('[data-test-pager="first"]').isNotDisabled(); + assert.dom('[data-test-pager="previous"]').isNotDisabled(); + assert.dom('[data-test-pager="next"]').isDisabled(); + assert.dom('[data-test-pager="last"]').isDisabled(); + await percySnapshot(assert); + localStorage.removeItem('nomadPageSize'); + }); + }); + module('Jobs are appropriately sorted by modify index', function () { + test('on a single long page', async function (assert) { + const jobsToCreate = 25; + localStorage.setItem('nomadPageSize', '25'); + createJobs(server, jobsToCreate); + await JobsList.visit(); + assert.dom('.job-row').exists({ count: 25 }); + // Check the data-test-modify-index attribute on each row + let rows = document.querySelectorAll('.job-row'); + let modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse(), + 'Jobs are sorted by modify index' + ); + localStorage.removeItem('nomadPageSize'); + }); + test('across multiple pages', async function (assert) { + const jobsToCreate = 90; + const pageSize = 25; + localStorage.setItem('nomadPageSize', pageSize.toString()); + createJobs(server, jobsToCreate); + await JobsList.visit(); + let rows = document.querySelectorAll('.job-row'); + let modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(0, pageSize), + 'First page is sorted by modify index' + ); + // Click next + await click('[data-test-pager="next"]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(pageSize, pageSize * 2), + 'Second page is sorted by modify index' + ); + + // Click next again + await click('[data-test-pager="next"]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(pageSize * 2, pageSize * 3), + 'Third page is sorted by modify index' + ); + + // Click previous + await click('[data-test-pager="previous"]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(pageSize, pageSize * 2), + 'Second page is sorted by modify index' + ); + + // Click next twice, should be the last page, and therefore fewer than pageSize jobs + await click('[data-test-pager="next"]'); + await click('[data-test-pager="next"]'); + + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(pageSize * 3), + 'Fourth page is sorted by modify index' + ); + assert.equal( + rows.length, + jobsToCreate - pageSize * 3, + 'Last page has fewer jobs' + ); + + // Go back to the first page + await click('[data-test-pager="first"]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(0, pageSize), + 'First page is sorted by modify index' + ); + + // Click "last" to get an even number of jobs at the end of the list + await click('[data-test-pager="last"]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(-pageSize), + 'Last page is sorted by modify index' + ); + assert.equal( + rows.length, + pageSize, + 'Last page has the correct number of jobs' + ); + + // type "{{" to go to the beginning + triggerKeyEvent('.page-layout', 'keydown', '{'); + await triggerKeyEvent('.page-layout', 'keydown', '{'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(0, pageSize), + 'Keynav takes me back to the starting page' + ); + + // type "]]" to go forward a page + triggerKeyEvent('.page-layout', 'keydown', ']'); + await triggerKeyEvent('.page-layout', 'keydown', ']'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(jobsToCreate) + .fill() + .map((_, i) => i + 1) + .reverse() + .slice(pageSize, pageSize * 2), + 'Keynav takes me forward a page' + ); + + localStorage.removeItem('nomadPageSize'); + }); + }); + module('Live updates are reflected in the list', function () { + test('When you have live updates enabled, the list updates when new jobs are created', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 10); + await JobsList.visit(); + assert.dom('.job-row').exists({ count: 10 }); + let rows = document.querySelectorAll('.job-row'); + assert.equal(rows.length, 10, 'List is still 10 rows'); + let modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(10) + .fill() + .map((_, i) => i + 1) + .reverse(), + 'Jobs are sorted by modify index' + ); + assert.dom('[data-test-pager="next"]').isDisabled(); + + // Create a new job + server.create('job', { + namespaceId: 'default', + resourceSpec: Array(1).fill('M: 256, C: 500'), + groupAllocCount: 1, + modifyIndex: 11, + createAllocations: false, + shallow: true, + name: 'new-job', + }); + + const controller = this.owner.lookup('controller:jobs.index'); + + let currentParams = { + per_page: 10, + }; + + // We have to wait for watchJobIDs to trigger the "dueling query" with watchJobs. + // Since we can't await the watchJobs promise, we set a reasonably short timeout + // to check the state of the list after the dueling query has completed. + await controller.watchJobIDs.perform(currentParams, 0); + + let updatedJob = assert.async(); // watch for this to say "My tests oughta be passing by now" + const duelingQueryUpdateTime = 200; + + assert.timeout(500); + + setTimeout(async () => { + // Order should now be 11-2 + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(10) + .fill() + .map((_, i) => i + 2) + .reverse(), + 'Jobs are sorted by modify index' + ); + + // Simulate one of the on-page jobs getting its modify-index bumped. It should bump to the top of the list. + let existingJobToUpdate = server.db.jobs.findBy( + (job) => job.modifyIndex === 5 + ); + server.db.jobs.update(existingJobToUpdate.id, { modifyIndex: 12 }); + await controller.watchJobIDs.perform(currentParams, 0); + let updatedOnPageJob = assert.async(); + + setTimeout(async () => { + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + [12, 11, 10, 9, 8, 7, 6, 4, 3, 2], + 'Jobs are sorted by modify index, on-page job moves up to the top, and off-page pending' + ); + updatedOnPageJob(); + + assert.dom('[data-test-pager="next"]').isNotDisabled(); + + await click('[data-test-pager="next"]'); + + rows = document.querySelectorAll('.job-row'); + assert.equal(rows.length, 1, 'List is now 1 row'); + assert.equal( + rows[0].getAttribute('data-test-modify-index'), + '1', + 'Job is the first job, now pushed to the second page' + ); + }, duelingQueryUpdateTime); + updatedJob(); + }, duelingQueryUpdateTime); + + localStorage.removeItem('nomadPageSize'); + }); + test('When you have live updates disabled, the list does not update, but prompts you to refresh', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + localStorage.setItem('nomadLiveUpdateJobsIndex', 'false'); + createJobs(server, 10); + await JobsList.visit(); + assert.dom('[data-test-updates-pending-button]').doesNotExist(); + + let rows = document.querySelectorAll('.job-row'); + assert.equal(rows.length, 10, 'List is still 10 rows'); + let modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(10) + .fill() + .map((_, i) => i + 1) + .reverse(), + 'Jobs are sorted by modify index' + ); + + // Create a new job + server.create('job', { + namespaceId: 'default', + resourceSpec: Array(1).fill('M: 256, C: 500'), + groupAllocCount: 1, + modifyIndex: 11, + createAllocations: false, + shallow: true, + name: 'new-job', + }); + + const controller = this.owner.lookup('controller:jobs.index'); + + let currentParams = { + per_page: 10, + }; + + // We have to wait for watchJobIDs to trigger the "dueling query" with watchJobs. + // Since we can't await the watchJobs promise, we set a reasonably short timeout + // to check the state of the list after the dueling query has completed. + await controller.watchJobIDs.perform(currentParams, 0); + + let updatedUnshownJob = assert.async(); // watch for this to say "My tests oughta be passing by now" + const duelingQueryUpdateTime = 200; + + assert.timeout(500); + + setTimeout(async () => { + // Order should still be be 10-1 + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + Array(10) + .fill() + .map((_, i) => i + 1) + .reverse(), + 'Jobs are sorted by modify index, off-page job not showing up yet' + ); + assert + .dom('[data-test-updates-pending-button]') + .exists('The refresh button is present'); + assert + .dom('[data-test-pager="next"]') + .isNotDisabled( + 'Next button is enabled in spite of the new job not showing up yet' + ); + + // Simulate one of the on-page jobs getting its modify-index bumped. It should remain in place. + let existingJobToUpdate = server.db.jobs.findBy( + (job) => job.modifyIndex === 5 + ); + server.db.jobs.update(existingJobToUpdate.id, { modifyIndex: 12 }); + await controller.watchJobIDs.perform(currentParams, 0); + let updatedShownJob = assert.async(); + + setTimeout(async () => { + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + [10, 9, 8, 7, 6, 12, 4, 3, 2, 1], + 'Jobs are sorted by modify index, on-page job remains in-place, and off-page pending' + ); + assert + .dom('[data-test-updates-pending-button]') + .exists('The refresh button is still present'); + assert + .dom('[data-test-pager="next"]') + .isNotDisabled('Next button is still enabled'); + + // Click the refresh button + await click('[data-test-updates-pending-button]'); + rows = document.querySelectorAll('.job-row'); + modifyIndexes = Array.from(rows).map((row) => + parseInt(row.getAttribute('data-test-modify-index')) + ); + assert.deepEqual( + modifyIndexes, + [12, 11, 10, 9, 8, 7, 6, 4, 3, 2], + 'Jobs are sorted by modify index, after refresh' + ); + assert + .dom('[data-test-updates-pending-button]') + .doesNotExist('The refresh button is gone'); + updatedShownJob(); + }, duelingQueryUpdateTime); + updatedUnshownJob(); + }, duelingQueryUpdateTime); + + localStorage.removeItem('nomadPageSize'); + localStorage.removeItem('nomadLiveUpdateJobsIndex'); + }); + }); + }); }); + +/** + * + * @param {*} server + * @param {number} jobsToCreate + */ +function createJobs(server, jobsToCreate) { + for (let i = 0; i < jobsToCreate; i++) { + server.create('job', { + namespaceId: 'default', + resourceSpec: Array(1).fill('M: 256, C: 500'), + groupAllocCount: 1, + modifyIndex: i + 1, + createAllocations: false, + shallow: true, + }); + } +} diff --git a/ui/tests/pages/jobs/list.js b/ui/tests/pages/jobs/list.js index 223da271ca8..c330c59f068 100644 --- a/ui/tests/pages/jobs/list.js +++ b/ui/tests/pages/jobs/list.js @@ -48,9 +48,6 @@ export default create({ clickName: clickable('[data-test-job-name] a'), }), - nextPage: clickable('[data-test-pager="next"]'), - prevPage: clickable('[data-test-pager="prev"]'), - isEmpty: isPresent('[data-test-empty-jobs-list]'), emptyState: { headline: text('[data-test-empty-jobs-list-headline]'), From da6fa95d59be4747a794a4c91c0346c7e130e658 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 19 Apr 2024 10:37:34 -0400 Subject: [PATCH 89/98] Bugfix: resizing your browser on the new jobs index page would make the viz grow forever (#20458) --- .../job-status/allocation-status-row.hbs | 2 +- .../job-status/allocation-status-row.js | 24 +++++++++++++++++-- ui/app/templates/jobs/index.hbs | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ui/app/components/job-status/allocation-status-row.hbs b/ui/app/components/job-status/allocation-status-row.hbs index bb08b2f42f3..1de96bb18c7 100644 --- a/ui/app/components/job-status/allocation-status-row.hbs +++ b/ui/app/components/job-status/allocation-status-row.hbs @@ -4,7 +4,7 @@ ~}}
- {{#if (or this.showSummaries @compact)}} + {{#if this.showSummaries}}
- this.width + this.width ); } + // When we calculate how much width to give to a row in our viz, + // we want to also offset the gap BETWEEN summaries. The way that css grid + // works, a gap only appears between 2 elements, not at the start or end of a row. + // Thus, we need to calculate total gap space using the number of summaries shown. + get numberOfSummariesShown() { + return Object.values(this.args.allocBlocks) + .flatMap((statusObj) => Object.values(statusObj)) + .flatMap((healthObj) => Object.values(healthObj)) + .filter((allocs) => allocs.length > 0).length; + } + calcPerc(count) { - return (count / this.allocBlockSlots) * this.width; + if (this.args.compact) { + const totalGaps = + (this.numberOfSummariesShown - 1) * COMPACT_INTER_SUMMARY_GAP; + const usableWidth = this.width - totalGaps; + return (count / this.allocBlockSlots) * usableWidth; + } else { + return (count / this.allocBlockSlots) * this.width; + } } @action reflow(element) { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 6f614a3df94..7f8e176cfe0 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -336,7 +336,7 @@ {{B.data.allocations.length}} total
{{B.data.groupCountSum}} desired
--}} -
+
{{#unless B.data.assumeGC}} {{#if B.data.childStatuses}} {{B.data.childStatuses.length}} child jobs;
From 3350ebc07af8808bc0283b080715a445263aaac9 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Fri, 26 Apr 2024 23:34:01 -0400 Subject: [PATCH 90/98] [ui] Searching and filtering options (#20459) * Beginnings of a search box for filter expressions * jobSearchBox integration test * jobs list updateFilter initial test * Basic jobs list filtering tests * First attempt at side-by-side facets and search with a computed filter * Weirdly close to an iterative approach but checked isnt tracked properly * Big rework to make filter composition and decomposition work nicely with the url * Namespace facet dropdown added * NodePool facet dropdown added * hdsFacet for future testing and basic namespace filtering test * Namespace filter existence test * Status filtering * Node pool/dynamic facet test * Test patchups * Attempt at optimize test fix * Allocation re-load on optimize page explainer * The Big Un-Skip * Post-PR-review cleanup --- ui/app/adapters/job.js | 7 +- ui/app/components/job-search-box.hbs | 20 + ui/app/components/job-search-box.js | 39 + ui/app/controllers/jobs/index.js | 303 +++- ui/app/routes/jobs/index.js | 92 +- ui/app/routes/optimize.js | 9 + ui/app/serializers/job.js | 4 + ui/app/styles/components/jobs-list.scss | 10 + ui/app/templates/jobs/index.hbs | 392 ++--- ui/mirage/config.js | 62 +- ui/mirage/scenarios/default.js | 7 +- ui/tests/acceptance/jobs-list-test.js | 1276 ++++++++++++----- .../components/job-search-box-test.js | 63 + ui/tests/pages/components/facet.js | 13 + ui/tests/pages/jobs/list.js | 15 +- 15 files changed, 1575 insertions(+), 737 deletions(-) create mode 100644 ui/app/components/job-search-box.hbs create mode 100644 ui/app/components/job-search-box.js create mode 100644 ui/tests/integration/components/job-search-box-test.js diff --git a/ui/app/adapters/job.js b/ui/app/adapters/job.js index 09fcbe0e3d9..076603b1122 100644 --- a/ui/app/adapters/job.js +++ b/ui/app/adapters/job.js @@ -255,7 +255,12 @@ export default class JobAdapter extends WatchableNamespaceIDs { urlForQuery(query, modelName, method) { let baseUrl = `/${this.namespace}/jobs/statuses`; if (method === 'POST' && query.index) { - return `${baseUrl}?index=${query.index}`; + baseUrl += baseUrl.includes('?') ? '&' : '?'; + baseUrl += `index=${query.index}`; + } + if (method === 'POST' && query.jobs) { + baseUrl += baseUrl.includes('?') ? '&' : '?'; + baseUrl += 'namespace=*'; } return baseUrl; } diff --git a/ui/app/components/job-search-box.hbs b/ui/app/components/job-search-box.hbs new file mode 100644 index 00000000000..f6b20c2bf73 --- /dev/null +++ b/ui/app/components/job-search-box.hbs @@ -0,0 +1,20 @@ +{{! + Copyright (c) HashiCorp, Inc. + SPDX-License-Identifier: BUSL-1.1 +~}} + +<@S.TextInput + @type="search" + @value={{@searchText}} + aria-label="Job Search" + placeholder="Name contains myJob" + @icon="search" + @width="300px" + {{on "input" (action this.updateSearchText)}} + {{keyboard-shortcut + label="Search Jobs" + pattern=(array "Shift+F") + action=(action this.focus) + }} + data-test-jobs-search +/> diff --git a/ui/app/components/job-search-box.js b/ui/app/components/job-search-box.js new file mode 100644 index 00000000000..886f492aa74 --- /dev/null +++ b/ui/app/components/job-search-box.js @@ -0,0 +1,39 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +// @ts-check + +import Component from '@glimmer/component'; +import { action } from '@ember/object'; +import { inject as service } from '@ember/service'; +import { debounce } from '@ember/runloop'; + +const DEBOUNCE_MS = 500; + +export default class JobSearchBoxComponent extends Component { + @service keyboard; + + element = null; + + @action + updateSearchText(event) { + debounce(this, this.sendUpdate, event.target.value, DEBOUNCE_MS); + } + + sendUpdate(value) { + this.args.onSearchTextChange(value); + } + + @action + focus(element) { + element.focus(); + // Because the element is an input, + // and the "hide hints" part of our keynav implementation is on keyUp, + // but the focus action happens on keyDown, + // and the keynav explicitly ignores key input while focused in a text input, + // we need to manually hide the hints here. + this.keyboard.displayHints = false; + } +} diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 8d358ad9e6f..cfb7506b740 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -7,14 +7,10 @@ import Controller from '@ember/controller'; import { inject as service } from '@ember/service'; -import { action, computed } from '@ember/object'; +import { action, computed, set } from '@ember/object'; import { tracked } from '@glimmer/tracking'; import localStorageProperty from 'nomad-ui/utils/properties/local-storage'; import { restartableTask, timeout } from 'ember-concurrency'; -import { - serialize, - deserializedQueryParam as selection, -} from 'nomad-ui/utils/qp-serialize'; import Ember from 'ember'; const JOB_LIST_THROTTLE = 5000; @@ -37,54 +33,16 @@ export default class JobsIndexController extends Controller { queryParams = [ 'cursorAt', 'pageSize', - // 'status', { qpNamespace: 'namespace' }, - // 'type', - // 'searchTerm', + 'filter', ]; isForbidden = false; - // #region filtering and sorting - @tracked jobQueryIndex = 0; @tracked jobAllocsQueryIndex = 0; - @selection('qpNamespace') selectionNamespace; - - @computed('qpNamespace', 'model.namespaces.[]') - get optionsNamespace() { - const availableNamespaces = this.model.namespaces.map((namespace) => ({ - key: namespace.name, - label: namespace.name, - })); - - availableNamespaces.unshift({ - key: '*', - label: 'All (*)', - }); - - // // Unset the namespace selection if it was server-side deleted - // if (!availableNamespaces.mapBy('key').includes(this.qpNamespace)) { - // scheduleOnce('actions', () => { - // this.set('qpNamespace', '*'); - // }); - // } - - return availableNamespaces; - } - - @action - handleFilterChange(queryParamValue, option, queryParamLabel) { - if (queryParamValue.includes(option)) { - queryParamValue.removeObject(option); - } else { - queryParamValue.addObject(option); - } - this[queryParamLabel] = serialize(queryParamValue); - } - - // #endregion filtering and sorting + @tracked qpNamespace = '*'; get tableColumns() { return [ @@ -93,7 +51,6 @@ export default class JobsIndexController extends Controller { 'status', 'type', this.system.shouldShowNodepools ? 'node pool' : null, // TODO: implement on system service - 'priority', 'running allocations', ] .filter((c) => !!c) @@ -226,6 +183,7 @@ export default class JobsIndexController extends Controller { jobAllocsQuery(params) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexDetailsController = new AbortController(); + params.namespace = '*'; return this.store .query('job', params, { adapterOptions: { @@ -334,6 +292,9 @@ export default class JobsIndexController extends Controller { } } this.jobs = jobDetails; + } else { + // No jobs have returned, so clear the list + this.jobs = []; } yield timeout(throttle); if (Ember.testing) { @@ -342,4 +303,254 @@ export default class JobsIndexController extends Controller { } } //#endregion querying + + //#region filtering and searching + + @tracked statusFacet = { + label: 'Status', + options: [ + { + key: 'pending', + string: 'Status == pending', + checked: false, + }, + { + key: 'running', + string: 'Status == running', + checked: false, + }, + { + key: 'dead', + string: 'Status == dead', + checked: false, + }, + ], + }; + + @tracked typeFacet = { + label: 'Type', + options: [ + { + key: 'batch', + string: 'Type == batch', + checked: false, + }, + { + key: 'service', + string: 'Type == service', + checked: false, + }, + { + key: 'system', + string: 'Type == system', + checked: false, + }, + { + key: 'sysbatch', + string: 'Type == sysbatch', + checked: false, + }, + ], + }; + + @tracked nodePoolFacet = { + label: 'NodePool', + options: (this.model.nodePools || []).map((nodePool) => ({ + key: nodePool.name, + string: `NodePool == ${nodePool.name}`, + checked: false, + })), + }; + + @computed('system.shouldShowNamespaces', 'model.namespaces.[]', 'qpNamespace') + get namespaceFacet() { + if (!this.system.shouldShowNamespaces) { + return null; + } + + const availableNamespaces = (this.model.namespaces || []).map( + (namespace) => ({ + key: namespace.name, + label: namespace.name, + }) + ); + + availableNamespaces.unshift({ + key: '*', + label: 'All', + }); + + let selectedNamespaces = this.qpNamespace || '*'; + availableNamespaces.forEach((opt) => { + if (selectedNamespaces.includes(opt.key)) { + opt.checked = true; + } + }); + + return { + label: 'Namespace', + options: availableNamespaces, + }; + } + + get filterFacets() { + let facets = [this.statusFacet, this.typeFacet]; + if (this.system.shouldShowNodepools) { + facets.push(this.nodePoolFacet); + } + return facets; + } + + /** + * On page load, takes the ?filter queryParam, and extracts it into those + * properties used by the dropdown filter toggles, and the search text. + */ + parseFilter() { + let filterString = this.filter; + if (!filterString) { + return; + } + + const filterParts = filterString.split(' and '); + + let unmatchedFilters = []; + + // For each of those splits, if it starts and ends with (), and if all entries within it have thes ame Propname and operator of ==, populate them into the appropriate dropdown + // If it doesnt start with and end with (), or if it does but not all entries are the same propname, or not all entries have == operators, populate them into the searchbox + + filterParts.forEach((part) => { + let matched = false; + if (part.startsWith('(') && part.endsWith(')')) { + part = part.slice(1, -1); // trim the parens + // Check to see if the property name (first word) is one of the ones for which we have a dropdown + let propName = part.split(' ')[0]; + if (this.filterFacets.find((facet) => facet.label === propName)) { + // Split along "or" and check that all parts have the same propName + let facetParts = part.split(' or '); + let allMatch = facetParts.every((facetPart) => + facetPart.startsWith(propName) + ); + let allEqualityOperators = facetParts.every((facetPart) => + facetPart.includes('==') + ); + if (allMatch && allEqualityOperators) { + // Set all the options in the dropdown to checked + this.filterFacets.forEach((group) => { + if (group.label === propName) { + group.options.forEach((option) => { + set(option, 'checked', facetParts.includes(option.string)); + }); + } + }); + matched = true; + } + } + } + if (!matched) { + unmatchedFilters.push(part); + } + }); + + // Combine all unmatched filter parts into the searchText + this.searchText = unmatchedFilters.join(' and '); + } + + @computed( + 'filterFacets', + 'nodePoolFacet.options.@each.checked', + 'searchText', + 'statusFacet.options.@each.checked', + 'typeFacet.options.@each.checked' + ) + get computedFilter() { + let parts = this.searchText ? [this.searchText] : []; + this.filterFacets.forEach((group) => { + let groupParts = []; + group.options.forEach((option) => { + if (option.checked) { + groupParts.push(option.string); + } + }); + if (groupParts.length) { + parts.push(`(${groupParts.join(' or ')})`); + } + }); + return parts.join(' and '); + } + + @action + toggleOption(option) { + set(option, 'checked', !option.checked); + this.updateFilter(); + } + + // Radio button set + @action + toggleNamespaceOption(option, dropdown) { + this.qpNamespace = option.key; + dropdown.close(); + } + + @action + updateFilter() { + this.cursorAt = null; + this.filter = this.computedFilter; + } + + @tracked filter = ''; + @tracked searchText = ''; + + @action resetFilters() { + this.searchText = ''; + this.filterFacets.forEach((group) => { + group.options.forEach((option) => { + set(option, 'checked', false); + }); + }); + this.qpNamespace = '*'; + this.updateFilter(); + } + + /** + * Updates the filter based on the input, distinguishing between simple job names and filter expressions. + * A simple check for operators with surrounding spaces is used to identify filter expressions. + * + * @param {string} newFilter + */ + @action + updateSearchText(newFilter) { + if (!newFilter.trim()) { + this.searchText = ''; + return; + } + + newFilter = newFilter.trim(); + + const operators = [ + '==', + '!=', + 'contains', + 'not contains', + 'is empty', + 'is not empty', + 'matches', + 'not matches', + 'in', + 'not in', + ]; + + // Check for any operator surrounded by spaces + let isFilterExpression = operators.some((op) => + newFilter.includes(` ${op} `) + ); + + if (isFilterExpression) { + this.searchText = newFilter; + } else { + // If it's a string without a filter operator, assume the user is trying to look up a job name + this.searchText = `Name contains ${newFilter}`; + } + } + + //#endregion filtering and searching } diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index bb5fdc1f9ea..fdb0def121c 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -24,8 +24,7 @@ export default class IndexRoute extends Route.extend( ) { @service store; @service watchList; - - // perPage = 10; + @service notifications; queryParams = { qpNamespace: { @@ -37,36 +36,101 @@ export default class IndexRoute extends Route.extend( pageSize: { refreshModel: true, }, + filter: { + refreshModel: true, + }, }; hasBeenInitialized = false; getCurrentParams() { let queryParams = this.paramsFor(this.routeName); // Get current query params - queryParams.next_token = queryParams.cursorAt; + if (queryParams.cursorAt) { + queryParams.next_token = queryParams.cursorAt; + } queryParams.per_page = queryParams.pageSize; + + /* eslint-disable ember/no-controller-access-in-routes */ + let filter = this.controllerFor('jobs.index').filter; + if (filter) { + queryParams.filter = filter; + } + // namespace + queryParams.namespace = queryParams.qpNamespace; + delete queryParams.qpNamespace; delete queryParams.pageSize; - delete queryParams.cursorAt; // TODO: hacky, should be done in the serializer/adapter? + delete queryParams.cursorAt; + return { ...queryParams }; } async model(/*params*/) { - let currentParams = this.getCurrentParams(); // TODO: how do these differ from passed params? + let currentParams = this.getCurrentParams(); this.watchList.jobsIndexIDsController.abort(); this.watchList.jobsIndexIDsController = new AbortController(); - let jobs = await this.store - .query('job', currentParams, { + try { + let jobs = await this.store.query('job', currentParams, { adapterOptions: { method: 'GET', // TODO: default abortController: this.watchList.jobsIndexIDsController, }, - }) - .catch(notifyForbidden(this)); - return RSVP.hash({ - jobs, - namespaces: this.store.findAll('namespace'), - nodePools: this.store.findAll('node-pool'), + }); + return RSVP.hash({ + jobs, + namespaces: this.store.findAll('namespace'), + nodePools: this.store.findAll('node-pool'), + }); + } catch (error) { + try { + notifyForbidden(this)(error); + } catch (secondaryError) { + return this.handleErrors(error); + } + } + return {}; + } + + /** + * @typedef {Object} HTTPError + * @property {string} stack + * @property {string} message + * @property {string} name + * @property {HTTPErrorDetail[]} errors + */ + + /** + * @typedef {Object} HTTPErrorDetail + * @property {string} status - HTTP status code + * @property {string} title + * @property {string} detail + */ + + /** + * Handles HTTP errors by returning an appropriate message based on the HTTP status code and details in the error object. + * + * @param {HTTPError} error + * @returns {Object} + */ + handleErrors(error) { + error.errors.forEach((err) => { + this.notifications.add({ + title: err.title, + message: err.detail, + color: 'critical', + timeout: 8000, + }); }); + + // if it's an innocuous-enough seeming "You mistyped something while searching" error, + // handle it with a notification and don't throw. Otherwise, throw. + if ( + error.errors[0].detail.includes("couldn't find key") || + error.errors[0].detail.includes('failed to read filter expression') + ) { + return error; + } else { + throw error; + } } setupController(controller, model) { @@ -101,6 +165,8 @@ export default class IndexRoute extends Route.extend( Ember.testing ? 0 : DEFAULT_THROTTLE ); + controller.parseFilter(); + this.hasBeenInitialized = true; } diff --git a/ui/app/routes/optimize.js b/ui/app/routes/optimize.js index 6918f78bbe6..ca1a921ce6d 100644 --- a/ui/app/routes/optimize.js +++ b/ui/app/routes/optimize.js @@ -32,6 +32,15 @@ export default class OptimizeRoute extends Route { .map((j) => j.reload()), ]); + // reload the /allocations for each job, + // as the jobs-index-provided ones are less detailed than what + // the optimize/recommendation components require + await RSVP.all( + jobs + .filter((job) => job) + .map((j) => this.store.query('allocation', { job_id: j.id })) + ); + return { summaries: summaries.sortBy('submitTime').reverse(), namespaces, diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index 7046006edb0..f2b782949ea 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -156,6 +156,10 @@ export default class JobSerializer extends ApplicationSerializer { if (hash._aggregate && hash.Allocs) { // Manually push allocations to store + // These allocations have enough information to be useful on a jobs index page, + // but less than the /allocations endpoint for an individual job might give us. + // As such, pages like /optimize require a specific call to the endpoint + // of any jobs' allocations to get more detailed information. hash.Allocs.forEach((alloc) => { this.store.push({ data: { diff --git a/ui/app/styles/components/jobs-list.scss b/ui/app/styles/components/jobs-list.scss index 5e69b3c3d93..731f0c753e7 100644 --- a/ui/app/styles/components/jobs-list.scss +++ b/ui/app/styles/components/jobs-list.scss @@ -7,8 +7,18 @@ // Over time, we can phase most of these custom styles out as Helios components // adapt to more use-cases (like custom footer, etc). +#jobs-list-header { + z-index: $z-base; +} + #jobs-list-actions { margin-bottom: 1rem; + // If the screen is made very small, don't try to multi-line the text, + // instead wrap the flex and let dropdowns/buttons go on a new line. + .hds-segmented-group { + flex-wrap: wrap; + gap: 0.5rem 0; + } } #jobs-list-pagination { diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7f8e176cfe0..7fdffa1d944 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -5,269 +5,105 @@ {{page-title "Jobs"}}
- + - {{!-- - - - - {{#each this.clientFilterToggles.state as |option|}} - - {{capitalize option.label}} - - {{/each}} - - - {{#each this.clientFilterToggles.eligibility as |option|}} - - {{capitalize option.label}} - - {{/each}} - - - {{#each this.clientFilterToggles.drainStatus as |option|}} - - {{capitalize option.label}} - - {{/each}} - - - - - {{#each this.optionsNodePool key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Node Pool filters - - {{/each}} - - - - {{#each this.optionsClass key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Class filters - - {{/each}} - + - - - {{#each this.optionsDatacenter key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Datacenter filters - - {{/each}} - - - - - - {{#each this.optionsVersion key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Version filters - - {{/each}} - - - - - {{#each this.optionsVolume key="label" as |option|}} - - {{option.label}} - - {{else}} - - No Volume filters - - {{/each}} - - - --}} - - - {{#if this.system.shouldShowNamespaces}} - {{!-- --}} + - + {{#each this.filterFacets as |group|}} + - {{#each this.optionsNamespace key="label" as |option|}} + {{#each group.options as |option|}} - {{option.label}} + {{option.key}} {{else}} - No Namespaces + No {{group.label}} filters {{/each}} - - {{/if}} + + {{/each}} -
- -
- -
- -
+ {{#if this.system.shouldShowNamespaces}} + + + {{#each this.namespaceFacet.options as |option|}} + + {{option.label}} + + {{/each}} + + {{/if}} - {{#if this.pendingJobIDDiff}} - {{/if}} - + + + + {{#if this.pendingJobIDDiff}} + + {{/if}} + +
+ +
+ @@ -315,27 +151,20 @@ {{/if}} {{#if this.system.shouldShowNamespaces}} - {{B.data.namespace.id}} + {{B.data.namespace.id}} {{/if}} {{#unless B.data.childStatuses}} {{/unless}} - + {{B.data.type}} {{#if this.system.shouldShowNodepools}} {{B.data.nodePool}} {{/if}} - - {{B.data.priority}} - - {{!-- {{get (filter-by 'clientStatus' 'running' B.data.allocations) "length"}} running
- {{B.data.allocations.length}} total
- {{B.data.groupCountSum}} desired -
--}}
{{#unless B.data.assumeGC}} {{#if B.data.childStatuses}} @@ -348,7 +177,6 @@ @allocBlocks={{B.data.allocBlocks}} @steady={{true}} @compact={{true}} - {{!-- @runningAllocs={{B.data.runningAllocs}} --}} @runningAllocs={{B.data.allocBlocks.running.healthy.nonCanary.length}} @groupCountSum={{B.data.expectedRunningAllocCount}} /> @@ -433,41 +261,35 @@
{{else}} - {{!-- TODO: differentiate between "empty because there's nothing" and "empty because you have filtered/searched" --}} - - + {{#if this.filter}} + + + + {{!-- TODO: HDS4.0, convert to F.LinkStandalone --}} - + {{else}} + + + + {{!-- TODO: HDS4.0, convert to F.LinkStandalone --}} + + + {{/if}} {{/if}} -
- Next token is {{this.nextToken}}
- {{!-- Previous tokens ({{this.previousTokens.length}}) are {{this.previousTokens}}
--}} - Model.jobs length: {{this.model.jobs.length}}
- Controller.jobs length: {{this.jobs.length}}
- Job IDs to watch for ({{this.jobIDs.length}}): {{#each this.jobIDs as |id|}}{{id.id}} | {{/each}}
- Pending Job IDs to watch for ({{this.pendingJobIDs.length}}): {{#each this.pendingJobIDs as |id|}}{{id.id}} | {{/each}}
- - Live update new/removed jobs? - - {{this.liveUpdatesEnabled}}
-
- Local watchlist
- jobQueryIndex: {{this.jobQueryIndex}}
- jobAllocsQueryIndex: {{this.jobAllocsQueryIndex}}
- -
- - -
\ No newline at end of file + diff --git a/ui/mirage/config.js b/ui/mirage/config.js index c33f209c7b9..fdccbc18f72 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -118,7 +118,67 @@ export default function () { let nextToken = req.queryParams.next_token || 0; let reverse = req.queryParams.reverse === 'true'; const json = this.serialize(jobs.all()); - let sortedJson = json + + // Let's implement a very basic handling of ?filter here. + // We'll assume at most "and" combinations, and only positive filters + // (no "not Type contains sys" or similar) + let filteredJson = json; + if (req.queryParams.filter) { + // Format will be something like "Name contains NAME" or "Type == sysbatch" or combinations thereof + const filterConditions = req.queryParams.filter + .split(' and ') + .map((condition) => { + // Dropdowns user parenthesis wrapping; remove them for mock/test purposes + // We want to test multiple conditions within parens, like "(Type == system or Type == sysbatch)" + // So if we see parenthesis, we should re-split on "or" and treat them as separate conditions. + if (condition.startsWith('(') && condition.endsWith(')')) { + condition = condition.slice(1, -1); + } + if (condition.includes(' or ')) { + // multiple or condition + return { + field: condition.split(' ')[0], + operator: '==', + parts: condition.split(' or ').map((part) => { + return part.split(' ')[2]; + }), + }; + } else { + const parts = condition.split(' '); + + return { + field: parts[0], + operator: parts[1], + value: parts.slice(2).join(' ').replace(/['"]+/g, ''), + }; + } + }); + + filteredJson = filteredJson.filter((job) => { + return filterConditions.every((condition) => { + if (condition.parts) { + // Making a shortcut assumption that any condition.parts situations + // will be == as operator for testing sake. + return condition.parts.some((part) => { + return job[condition.field] === part; + }); + } + if (condition.operator === 'contains') { + return ( + job[condition.field] && + job[condition.field].includes(condition.value) + ); + } else if (condition.operator === '==') { + return job[condition.field] === condition.value; + } else if (condition.operator === '!=') { + return job[condition.field] !== condition.value; + } + return true; + }); + }); + } + + let sortedJson = filteredJson .sort((a, b) => reverse ? a.ModifyIndex - b.ModifyIndex diff --git a/ui/mirage/scenarios/default.js b/ui/mirage/scenarios/default.js index 896c5f26eb8..d4828d5ae6d 100644 --- a/ui/mirage/scenarios/default.js +++ b/ui/mirage/scenarios/default.js @@ -62,13 +62,14 @@ function jobsIndexTestCluster(server) { faker.seed(1); server.createList('agent', 1, 'withConsulLink', 'withVaultLink'); server.createList('node', 1); + server.create('node-pool'); const jobsToCreate = 55; for (let i = 0; i < jobsToCreate; i++) { + let groupCount = Math.floor(Math.random() * 2) + 1; server.create('job', { - namespaceId: 'default', - resourceSpec: Array(1).fill('M: 256, C: 500'), - groupAllocCount: 1, + resourceSpec: Array(groupCount).fill('M: 256, C: 500'), + groupAllocCount: Math.floor(Math.random() * 3) + 1, modifyIndex: i + 1, }); } diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index 02aa6d52ebc..1613035d407 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -4,7 +4,12 @@ */ /* eslint-disable qunit/require-expect */ -import { currentURL, click, triggerKeyEvent } from '@ember/test-helpers'; +import { + currentURL, + settled, + click, + triggerKeyEvent, +} from '@ember/test-helpers'; import { module, test } from 'qunit'; import { setupApplicationTest } from 'ember-qunit'; import { setupMirage } from 'ember-cli-mirage/test-support'; @@ -87,7 +92,6 @@ module('Acceptance | jobs list', function (hooks) { 'Status' ); assert.equal(jobRow.type, typeForJob(job), 'Type'); - assert.equal(jobRow.priority, job.priority, 'Priority'); }); test('each job row should link to the corresponding job', async function (assert) { @@ -149,14 +153,14 @@ module('Acceptance | jobs list', function (hooks) { ); }); - // TODO: Jobs list search - test.skip('when there are jobs, but no matches for a search result, there is an empty message', async function (assert) { + test('when there are jobs, but no matches for a search result, there is an empty message', async function (assert) { server.create('job', { name: 'cat 1' }); server.create('job', { name: 'cat 2' }); await JobsList.visit(); await JobsList.search.fillIn('dog'); + assert.ok(JobsList.isEmpty, 'The empty message is shown'); assert.equal( JobsList.emptyState.headline, @@ -165,70 +169,29 @@ module('Acceptance | jobs list', function (hooks) { ); }); - // TODO: Jobs list search - test.skip('searching resets the current page', async function (assert) { + test('searching resets the current page', async function (assert) { server.createList('job', JobsList.pageSize + 1, { createAllocations: false, }); await JobsList.visit(); - await JobsList.nextPage(); + await click('[data-test-pager="next"]'); - assert.equal( - currentURL(), - '/jobs?page=2', - 'Page query param captures page=2' + assert.ok( + currentURL().includes('cursorAt'), + 'Page query param contains cursorAt' ); await JobsList.search.fillIn('foobar'); - assert.equal(currentURL(), '/jobs?search=foobar', 'No page query param'); - }); - - // TODO: Jobs list search - test.skip('Search order overrides Sort order', async function (assert) { - server.create('job', { name: 'car', modifyIndex: 1, priority: 200 }); - server.create('job', { name: 'cat', modifyIndex: 2, priority: 150 }); - server.create('job', { name: 'dog', modifyIndex: 3, priority: 100 }); - server.create('job', { name: 'dot', modifyIndex: 4, priority: 50 }); - - await JobsList.visit(); - - // Expect list to be in reverse modifyIndex order by default - assert.equal(JobsList.jobs.objectAt(0).name, 'dot'); - assert.equal(JobsList.jobs.objectAt(1).name, 'dog'); - assert.equal(JobsList.jobs.objectAt(2).name, 'cat'); - assert.equal(JobsList.jobs.objectAt(3).name, 'car'); - - // When sorting by name, expect list to be in alphabetical order - await click('[data-test-sort-by="name"]'); // sorts desc - await click('[data-test-sort-by="name"]'); // sorts asc - - assert.equal(JobsList.jobs.objectAt(0).name, 'car'); - assert.equal(JobsList.jobs.objectAt(1).name, 'cat'); - assert.equal(JobsList.jobs.objectAt(2).name, 'dog'); - assert.equal(JobsList.jobs.objectAt(3).name, 'dot'); - - // When searching, the "name" sort is locked in. Fuzzy results for cat return both car and cat, but cat first. - await JobsList.search.fillIn('cat'); - assert.equal(JobsList.jobs.length, 2); - assert.equal(JobsList.jobs.objectAt(0).name, 'cat'); // higher fuzzy - assert.equal(JobsList.jobs.objectAt(1).name, 'car'); - - // Clicking priority sorter will maintain the search filter, but change the order - await click('[data-test-sort-by="priority"]'); // sorts desc - assert.equal(JobsList.jobs.objectAt(0).name, 'car'); // higher priority first - assert.equal(JobsList.jobs.objectAt(1).name, 'cat'); - - // Modifying search again will prioritize search "fuzzy" order - await JobsList.search.fillIn(''); // trigger search reset - await JobsList.search.fillIn('cat'); - assert.equal(JobsList.jobs.objectAt(0).name, 'cat'); // higher fuzzy - assert.equal(JobsList.jobs.objectAt(1).name, 'car'); + assert.equal( + currentURL(), + '/jobs?filter=Name%20contains%20foobar', + 'No page query param' + ); }); - // TODO: Jobs list search - test.skip('when a cluster has namespaces, each job row includes the job namespace', async function (assert) { + test('when a cluster has namespaces, each job row includes the job namespace', async function (assert) { server.createList('namespace', 2); server.createList('job', 2); const job = server.db.jobs.sortBy('modifyIndex').reverse()[0]; @@ -239,8 +202,7 @@ module('Acceptance | jobs list', function (hooks) { assert.equal(jobRow.namespace, job.namespaceId); }); - // TODO: Jobs list filter - test.skip('when the namespace query param is set, only matching jobs are shown', async function (assert) { + test('when the namespace query param is set, only matching jobs are shown', async function (assert) { server.createList('namespace', 2); const job1 = server.create('job', { namespaceId: server.db.namespaces[0].id, @@ -295,8 +257,7 @@ module('Acceptance | jobs list', function (hooks) { : job.type; } - // TODO: Jobs list filter - test.skip('the jobs list page has appropriate faceted search options', async function (assert) { + test('the jobs list page has appropriate faceted search options', async function (assert) { await JobsList.visit(); assert.ok( @@ -305,165 +266,95 @@ module('Acceptance | jobs list', function (hooks) { ); assert.ok(JobsList.facets.type.isPresent, 'Type facet found'); assert.ok(JobsList.facets.status.isPresent, 'Status facet found'); - assert.ok(JobsList.facets.datacenter.isPresent, 'Datacenter facet found'); - assert.ok(JobsList.facets.prefix.isPresent, 'Prefix facet found'); + assert.ok(JobsList.facets.nodePool.isPresent, 'Node Pools facet found'); + assert.notOk( + JobsList.facets.namespace.isPresent, + 'Namespace facet not found by default' + ); + }); + + testSingleSelectFacet('Namespace', { + facet: JobsList.facets.namespace, + paramName: 'namespace', + expectedOptions: ['All', 'default', 'namespace-2'], + optionToSelect: 'namespace-2', + async beforeEach() { + server.create('namespace', { id: 'default' }); + server.create('namespace', { id: 'namespace-2' }); + server.createList('job', 2, { namespaceId: 'default' }); + server.createList('job', 2, { namespaceId: 'namespace-2' }); + await JobsList.visit(); + }, + filter(job, selection) { + return job.namespaceId === selection; + }, + }); + + testFacet('Type', { + facet: JobsList.facets.type, + paramName: 'type', + expectedOptions: [ + 'batch', + 'service', + 'system', + 'sysbatch', + // TODO: add Parameterized and Periodic + ], + async beforeEach() { + server.createList('job', 2, { createAllocations: false, type: 'batch' }); + server.createList('job', 2, { + createAllocations: false, + type: 'batch', + periodic: true, + childrenCount: 0, + }); + server.createList('job', 2, { + createAllocations: false, + type: 'batch', + parameterized: true, + childrenCount: 0, + }); + server.createList('job', 2, { + createAllocations: false, + type: 'service', + }); + await JobsList.visit(); + }, + filter(job, selection) { + let displayType = job.type; + // TODO: if/when we allow for parameterized/batch filtering, uncomment these. + // if (job.parameterized) displayType = 'parameterized'; + // if (job.periodic) displayType = 'periodic'; + return selection.includes(displayType); + }, + }); + + testFacet('Status', { + facet: JobsList.facets.status, + paramName: 'status', + expectedOptions: ['pending', 'running', 'dead'], + async beforeEach() { + server.createList('job', 2, { + status: 'pending', + createAllocations: false, + childrenCount: 0, + }); + server.createList('job', 2, { + status: 'running', + createAllocations: false, + childrenCount: 0, + }); + server.createList('job', 2, { + status: 'dead', + createAllocations: false, + childrenCount: 0, + }); + await JobsList.visit(); + }, + filter: (job, selection) => selection.includes(job.status), }); - // TODO: Jobs list filter - - // testSingleSelectFacet('Namespace', { - // facet: JobsList.facets.namespace, - // paramName: 'namespace', - // expectedOptions: ['All (*)', 'default', 'namespace-2'], - // optionToSelect: 'namespace-2', - // async beforeEach() { - // server.create('namespace', { id: 'default' }); - // server.create('namespace', { id: 'namespace-2' }); - // server.createList('job', 2, { namespaceId: 'default' }); - // server.createList('job', 2, { namespaceId: 'namespace-2' }); - // await JobsList.visit(); - // }, - // filter(job, selection) { - // return job.namespaceId === selection; - // }, - // }); - - // testFacet('Type', { - // facet: JobsList.facets.type, - // paramName: 'type', - // expectedOptions: [ - // 'Batch', - // 'Pack', - // 'Parameterized', - // 'Periodic', - // 'Service', - // 'System', - // 'System Batch', - // ], - // async beforeEach() { - // server.createList('job', 2, { createAllocations: false, type: 'batch' }); - // server.createList('job', 2, { - // createAllocations: false, - // type: 'batch', - // periodic: true, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // createAllocations: false, - // type: 'batch', - // parameterized: true, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // createAllocations: false, - // type: 'service', - // }); - // await JobsList.visit(); - // }, - // filter(job, selection) { - // let displayType = job.type; - // if (job.parameterized) displayType = 'parameterized'; - // if (job.periodic) displayType = 'periodic'; - // return selection.includes(displayType); - // }, - // }); - - // testFacet('Status', { - // facet: JobsList.facets.status, - // paramName: 'status', - // expectedOptions: ['Pending', 'Running', 'Dead'], - // async beforeEach() { - // server.createList('job', 2, { - // status: 'pending', - // createAllocations: false, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // status: 'running', - // createAllocations: false, - // childrenCount: 0, - // }); - // server.createList('job', 2, { - // status: 'dead', - // createAllocations: false, - // childrenCount: 0, - // }); - // await JobsList.visit(); - // }, - // filter: (job, selection) => selection.includes(job.status), - // }); - - // testFacet('Datacenter', { - // facet: JobsList.facets.datacenter, - // paramName: 'dc', - // expectedOptions(jobs) { - // const allDatacenters = new Set( - // jobs.mapBy('datacenters').reduce((acc, val) => acc.concat(val), []) - // ); - // return Array.from(allDatacenters).sort(); - // }, - // async beforeEach() { - // server.create('job', { - // datacenters: ['pdx', 'lax'], - // createAllocations: false, - // childrenCount: 0, - // }); - // server.create('job', { - // datacenters: ['pdx', 'ord'], - // createAllocations: false, - // childrenCount: 0, - // }); - // server.create('job', { - // datacenters: ['lax', 'jfk'], - // createAllocations: false, - // childrenCount: 0, - // }); - // server.create('job', { - // datacenters: ['jfk', 'dfw'], - // createAllocations: false, - // childrenCount: 0, - // }); - // server.create('job', { - // datacenters: ['pdx'], - // createAllocations: false, - // childrenCount: 0, - // }); - // await JobsList.visit(); - // }, - // filter: (job, selection) => - // job.datacenters.find((dc) => selection.includes(dc)), - // }); - - // testFacet('Prefix', { - // facet: JobsList.facets.prefix, - // paramName: 'prefix', - // expectedOptions: ['hashi (3)', 'nmd (2)', 'pre (2)'], - // async beforeEach() { - // [ - // 'pre-one', - // 'hashi_one', - // 'nmd.one', - // 'one-alone', - // 'pre_two', - // 'hashi.two', - // 'hashi-three', - // 'nmd_two', - // 'noprefix', - // ].forEach((name) => { - // server.create('job', { - // name, - // createAllocations: false, - // childrenCount: 0, - // }); - // }); - // await JobsList.visit(); - // }, - // filter: (job, selection) => - // selection.find((prefix) => job.name.startsWith(prefix)), - // }); - - // TODO: Jobs list filter - test.skip('when the facet selections result in no matches, the empty state states why', async function (assert) { + test('when the facet selections result in no matches, the empty state states why', async function (assert) { server.createList('job', 2, { status: 'pending', createAllocations: false, @@ -482,12 +373,11 @@ module('Acceptance | jobs list', function (hooks) { ); }); - // TODO: Jobs list filter - test.skip('the jobs list is immediately filtered based on query params', async function (assert) { + test('the jobs list is immediately filtered based on query params', async function (assert) { server.create('job', { type: 'batch', createAllocations: false }); server.create('job', { type: 'service', createAllocations: false }); - await JobsList.visit({ type: JSON.stringify(['batch']) }); + await JobsList.visit({ filter: 'Type == batch' }); assert.equal( JobsList.jobs.length, @@ -576,165 +466,29 @@ module('Acceptance | jobs list', function (hooks) { }, }); - // async function facetOptions(assert, beforeEach, facet, expectedOptions) { - // await beforeEach(); - // await facet.toggle(); - - // let expectation; - // if (typeof expectedOptions === 'function') { - // expectation = expectedOptions(server.db.jobs); - // } else { - // expectation = expectedOptions; - // } - - // assert.deepEqual( - // facet.options.map((option) => option.label.trim()), - // expectation, - // 'Options for facet are as expected' - // ); - // } - - // function testSingleSelectFacet( - // label, - // { facet, paramName, beforeEach, filter, expectedOptions, optionToSelect } - // ) { - // test.skip(`the ${label} facet has the correct options`, async function (assert) { - // await facetOptions(assert, beforeEach, facet, expectedOptions); - // }); - - // test.skip(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { - // await beforeEach(); - // await facet.toggle(); - - // const option = facet.options.findOneBy('label', optionToSelect); - // const selection = option.key; - // await option.select(); - - // const expectedJobs = server.db.jobs - // .filter((job) => filter(job, selection)) - // .sortBy('modifyIndex') - // .reverse(); - - // JobsList.jobs.forEach((job, index) => { - // assert.equal( - // job.id, - // expectedJobs[index].id, - // `Job at ${index} is ${expectedJobs[index].id}` - // ); - // }); - // }); - - // test.skip(`selecting an option in the ${label} facet updates the ${paramName} query param`, async function (assert) { - // await beforeEach(); - // await facet.toggle(); - - // const option = facet.options.objectAt(1); - // const selection = option.key; - // await option.select(); - - // assert.ok( - // currentURL().includes(`${paramName}=${selection}`), - // 'URL has the correct query param key and value' - // ); - // }); - // } - - // function testFacet( - // label, - // { facet, paramName, beforeEach, filter, expectedOptions } - // ) { - // test.skip(`the ${label} facet has the correct options`, async function (assert) { - // await facetOptions(assert, beforeEach, facet, expectedOptions); - // }); - - // test.skip(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { - // let option; - - // await beforeEach(); - // await facet.toggle(); - - // option = facet.options.objectAt(0); - // await option.toggle(); - - // const selection = [option.key]; - // const expectedJobs = server.db.jobs - // .filter((job) => filter(job, selection)) - // .sortBy('modifyIndex') - // .reverse(); - - // JobsList.jobs.forEach((job, index) => { - // assert.equal( - // job.id, - // expectedJobs[index].id, - // `Job at ${index} is ${expectedJobs[index].id}` - // ); - // }); - // }); - - // test.skip(`selecting multiple options in the ${label} facet results in a broader search`, async function (assert) { - // const selection = []; - - // await beforeEach(); - // await facet.toggle(); - - // const option1 = facet.options.objectAt(0); - // const option2 = facet.options.objectAt(1); - // await option1.toggle(); - // selection.push(option1.key); - // await option2.toggle(); - // selection.push(option2.key); - - // const expectedJobs = server.db.jobs - // .filter((job) => filter(job, selection)) - // .sortBy('modifyIndex') - // .reverse(); - - // JobsList.jobs.forEach((job, index) => { - // assert.equal( - // job.id, - // expectedJobs[index].id, - // `Job at ${index} is ${expectedJobs[index].id}` - // ); - // }); - // }); - - // test.skip(`selecting options in the ${label} facet updates the ${paramName} query param`, async function (assert) { - // const selection = []; - - // await beforeEach(); - // await facet.toggle(); - - // const option1 = facet.options.objectAt(0); - // const option2 = facet.options.objectAt(1); - // await option1.toggle(); - // selection.push(option1.key); - // await option2.toggle(); - // selection.push(option2.key); - - // assert.ok( - // currentURL().includes(encodeURIComponent(JSON.stringify(selection))), - // 'URL has the correct query param key and value' - // ); - // }); - - // test.skip('the run job button works when filters are set', async function (assert) { - // ['pre-one', 'pre-two', 'pre-three'].forEach((name) => { - // server.create('job', { - // name, - // createAllocations: false, - // childrenCount: 0, - // }); - // }); - - // await JobsList.visit(); - - // await JobsList.facets.prefix.toggle(); - // await JobsList.facets.prefix.options[0].toggle(); - - // await JobsList.runJobButton.click(); - // assert.equal(currentURL(), '/jobs/run'); - // }); - // } + test('the run job button works when filters are set', async function (assert) { + server.create('job', { + name: 'un', + createAllocations: false, + childrenCount: 0, + type: 'batch', + }); + + server.create('job', { + name: 'deux', + createAllocations: false, + childrenCount: 0, + type: 'system', + }); + + await JobsList.visit(); + + await JobsList.facets.type.toggle(); + await JobsList.facets.type.options[0].toggle(); + + await JobsList.runJobButton.click(); + assert.equal(currentURL(), '/jobs/run'); + }); module('Pagination', function () { module('Buttons are appropriately disabled', function () { @@ -1190,6 +944,622 @@ module('Acceptance | jobs list', function (hooks) { }); }); }); + + module('Searching and Filtering', function () { + module('Search', function () { + test('Searching reasons about whether you intended a job name or a filter expression', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 10); + await JobsList.visit(); + + await JobsList.search.fillIn('something-that-surely-doesnt-exist'); + // check to see that we fired off a request; check handledRequests to find one with a ?filter in it + assert.ok( + server.pretender.handledRequests.find((req) => + decodeURIComponent(req.url).includes( + '?filter=Name contains something-that-surely-doesnt-exist' + ) + ), + 'A request was made with a filter query param that assumed job name' + ); + + await JobsList.search.fillIn('Namespace == ns-2'); + + assert.ok( + server.pretender.handledRequests.find((req) => + decodeURIComponent(req.url).includes('?filter=Namespace == ns-2') + ), + 'A request was made with a filter query param for a filter expression as typed' + ); + + localStorage.removeItem('nomadPageSize'); + }); + + test('Searching by name filters the list', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + createJobs(server, 10); + server.create('job', { + name: 'hashi-one', + id: 'hashi-one', + modifyIndex: 0, + }); + server.create('job', { + name: 'hashi-two', + id: 'hashi-two', + modifyIndex: 0, + }); + await JobsList.visit(); + + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'Initially, 10 jobs are listed without any filters.' + ); + assert + .dom('[data-test-job-row="hashi-one"]') + .doesNotExist( + 'The specific job hashi-one should not appear without filtering.' + ); + assert + .dom('[data-test-job-row="hashi-two"]') + .doesNotExist( + 'The specific job hashi-two should also not appear without filtering.' + ); + + await JobsList.search.fillIn('hashi-one'); + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Only one job should be visible when filtering by the name "hashi-one".' + ); + assert + .dom('[data-test-job-row="hashi-one"]') + .exists( + 'The job hashi-one appears as expected when filtered by name.' + ); + assert + .dom('[data-test-job-row="hashi-two"]') + .doesNotExist( + 'The job hashi-two should not appear when filtering by "hashi-one".' + ); + + await JobsList.search.fillIn('hashi'); + assert + .dom('.job-row') + .exists( + { count: 2 }, + 'Two jobs should appear when the filter "hashi" matches both job names.' + ); + assert + .dom('[data-test-job-row="hashi-one"]') + .exists( + 'Job hashi-one is correctly displayed under the "hashi" filter.' + ); + assert + .dom('[data-test-job-row="hashi-two"]') + .exists( + 'Job hashi-two is correctly displayed under the "hashi" filter.' + ); + + await JobsList.search.fillIn('Name == hashi'); + assert + .dom('.job-row') + .exists( + { count: 0 }, + 'No jobs should appear when an incorrect filter format "Name == hashi" is used.' + ); + + await JobsList.search.fillIn(''); + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'All jobs reappear when the search filter is cleared.' + ); + assert + .dom('[data-test-job-row="hashi-one"]') + .doesNotExist( + 'The job hashi-one should disappear again when the filter is cleared.' + ); + assert + .dom('[data-test-job-row="hashi-two"]') + .doesNotExist( + 'The job hashi-two should disappear again when the filter is cleared.' + ); + + localStorage.removeItem('nomadPageSize'); + }); + + test('Searching by type filters the list', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + server.createList('job', 10, { + createAllocations: false, + type: 'service', + modifyIndex: 10, + }); + + server.create('job', { + id: 'batch-job', + type: 'batch', + createAllocations: false, + modifyIndex: 9, + }); + server.create('job', { + id: 'system-job', + type: 'system', + createAllocations: false, + modifyIndex: 9, + }); + server.create('job', { + id: 'sysbatch-job', + type: 'sysbatch', + createAllocations: false, + modifyIndex: 9, + }); + server.create('job', { + id: 'sysbatch-job-2', + type: 'sysbatch', + createAllocations: false, + modifyIndex: 9, + }); + + await JobsList.visit(); + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'Initial setup should show 10 jobs of type "service".' + ); + assert + .dom('[data-test-job-type="service"]') + .exists( + { count: 10 }, + 'All initial jobs are confirmed to be of type "service".' + ); + + await JobsList.search.fillIn('Type == batch'); + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Filtering by "Type == batch" should show exactly one job.' + ); + assert + .dom('[data-test-job-type="batch"]') + .exists( + { count: 1 }, + 'The single job of type "batch" is displayed as expected.' + ); + + await JobsList.search.fillIn('Type == system'); + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Only one job should be displayed when filtering by "Type == system".' + ); + assert + .dom('[data-test-job-type="system"]') + .exists( + { count: 1 }, + 'The job of type "system" appears as expected.' + ); + + await JobsList.search.fillIn('Type == sysbatch'); + assert + .dom('.job-row') + .exists( + { count: 2 }, + 'Two jobs should be visible under the filter "Type == sysbatch".' + ); + assert + .dom('[data-test-job-type="sysbatch"]') + .exists( + { count: 2 }, + 'Both jobs of type "sysbatch" are correctly displayed.' + ); + + await JobsList.search.fillIn('Type contains sys'); + assert + .dom('.job-row') + .exists( + { count: 3 }, + 'Filter "Type contains sys" should show three jobs.' + ); + assert + .dom('[data-test-job-type="sysbatch"]') + .exists( + { count: 2 }, + 'Two jobs of type "sysbatch" match the "sys" substring.' + ); + assert + .dom('[data-test-job-type="system"]') + .exists( + { count: 1 }, + 'One job of type "system" matches the "sys" substring.' + ); + + await JobsList.search.fillIn('Type != service'); + assert + .dom('.job-row') + .exists( + { count: 4 }, + 'Four jobs should be visible when excluding type "service".' + ); + assert + .dom('[data-test-job-type="batch"]') + .exists({ count: 1 }, 'One batch job is visible.'); + assert + .dom('[data-test-job-type="system"]') + .exists({ count: 1 }, 'One system job is visible.'); + assert + .dom('[data-test-job-type="sysbatch"]') + .exists({ count: 2 }, 'Two sysbatch jobs are visible.'); + + // Next/Last buttons are disabled when searching for the 10 services bc there's just 10 + await JobsList.search.fillIn('Type == service'); + assert.dom('.job-row').exists({ count: 10 }); + assert.dom('[data-test-job-type="service"]').exists({ count: 10 }); + assert + .dom('[data-test-pager="next"]') + .isDisabled( + 'The next page button should be disabled when all jobs fit on one page.' + ); + assert + .dom('[data-test-pager="last"]') + .isDisabled( + 'The last page button should also be disabled under the same conditions.' + ); + + // But if we disinclude sysbatch we'll have 12, so next/last should be clickable + await JobsList.search.fillIn('Type != sysbatch'); + assert.dom('.job-row').exists({ count: 10 }); + assert + .dom('[data-test-pager="next"]') + .isNotDisabled( + 'The next page button should be enabled when not all jobs are shown on one page.' + ); + assert + .dom('[data-test-pager="last"]') + .isNotDisabled('The last page button should be enabled as well.'); + + localStorage.removeItem('nomadPageSize'); + }); + }); + module('Filtering', function () { + test('Filtering by namespace filters the list', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + + server.create('namespace', { + id: 'default', + name: 'default', + }); + + server.create('namespace', { + id: 'ns-2', + name: 'ns-2', + }); + + server.createList('job', 10, { + createAllocations: false, + namespaceId: 'default', + modifyIndex: 10, + }); + + server.create('job', { + id: 'ns-2-job', + namespaceId: 'ns-2', + createAllocations: false, + modifyIndex: 9, + }); + + // By default, start on "All" namespace + await JobsList.visit(); + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'Initial setup should show 10 jobs in the default namespace.' + ); + assert + .dom('[data-test-job-row="ns-2-job"]') + .doesNotExist( + 'The job in the ns-2 namespace should not appear without filtering.' + ); + + assert + .dom('[data-test-pager="next"]') + .isNotDisabled( + '11 jobs on "All" namespace, so second page is available' + ); + + // Toggle ns-2 namespace + await JobsList.facets.namespace.toggle(); + await JobsList.facets.namespace.options[2].toggle(); + + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Only one job should be visible when filtering by the ns-2 namespace.' + ); + assert + .dom('[data-test-job-row="ns-2-job"]') + .exists( + 'The job in the ns-2 namespace appears as expected when filtered.' + ); + + // Switch to default namespace + await JobsList.facets.namespace.toggle(); + await JobsList.facets.namespace.options[1].toggle(); + + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'All jobs reappear when the search filter is cleared.' + ); + assert + .dom('[data-test-job-row="ns-2-job"]') + .doesNotExist( + 'The job in the ns-2 namespace should disappear when the filter is cleared.' + ); + + assert + .dom('[data-test-pager="next"]') + .isDisabled( + '10 jobs in "Default" namespace, so second page is not available' + ); + + localStorage.removeItem('nomadPageSize'); + }); + test('Namespace filter only shows up if the server has more than one namespace', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + + server.create('namespace', { + id: 'default', + name: 'default', + }); + + server.createList('job', 10, { + createAllocations: false, + namespaceId: 'default', + modifyIndex: 10, + }); + + await JobsList.visit(); + assert + .dom('[data-test-facet="Namespace"]') + .doesNotExist( + 'Namespace filter should not appear with only one namespace.' + ); + + let system = this.owner.lookup('service:system'); + system.shouldShowNamespaces = true; + + await settled(); + + assert + .dom('[data-test-facet="Namespace"]') + .exists( + 'Namespace filter should appear with more than one namespace.' + ); + + localStorage.removeItem('nomadPageSize'); + }); + test('Filtering by status filters the list', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + server.createList('job', 10, { + createAllocations: false, + status: 'running', + modifyIndex: 10, + }); + + server.create('job', { + id: 'pending-job', + status: 'pending', + createAllocations: false, + modifyIndex: 9, + }); + + server.create('job', { + id: 'dead-job', + status: 'dead', + createAllocations: false, + modifyIndex: 8, + }); + + await JobsList.visit(); + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'Initial setup should show 10 jobs in the "running" status.' + ); + assert + .dom('[data-test-job-row="pending-job"]') + .doesNotExist( + 'The job in the "pending" status should not appear without filtering.' + ); + assert + .dom('[data-test-pager="next"]') + .isNotDisabled( + '10 jobs in "running" status, so second page is available' + ); + + await JobsList.facets.status.toggle(); + await JobsList.facets.status.options[0].toggle(); // pending + + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Only one job should be visible when filtering by the "pending" status.' + ); + assert + .dom('[data-test-job-row="pending-job"]') + .exists( + 'The job in the "pending" status appears as expected when filtered.' + ); + + assert + .dom('[data-test-pager="next"]') + .isDisabled( + '1 job in "pending" status, so second page is not available' + ); + + await JobsList.facets.status.options[2].toggle(); // dead + assert + .dom('.job-row') + .exists( + { count: 2 }, + 'Two jobs should be visible when the "dead" filter is added' + ); + assert + .dom('[data-test-job-row="dead-job"]') + .exists( + { count: 1 }, + 'The job in the "dead" status appears as expected when filtered.' + ); + + localStorage.removeItem('nomadPageSize'); + }); + + test('Filtering by a dynamically-generated facet: data-test-facet="Node Pool"', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + + server.create('node-pool', { + id: 'pool-1', + name: 'pool-1', + }); + server.create('node-pool', { + id: 'pool-2', + name: 'pool-2', + }); + + server.createList('job', 10, { + createAllocations: false, + nodePool: 'pool-1', + modifyIndex: 10, + }); + + server.create('job', { + id: 'pool-2-job', + nodePool: 'pool-2', + createAllocations: false, + modifyIndex: 9, + }); + + await JobsList.visit(); + assert + .dom('.job-row') + .exists( + { count: 10 }, + 'Initial setup should show 10 jobs in the "pool-1" node pool.' + ); + assert + .dom('[data-test-job-row="pool-2-job"]') + .doesNotExist( + 'The job in the "pool-2" node pool should not appear without filtering.' + ); + await JobsList.facets.nodePool.toggle(); + + await JobsList.facets.nodePool.options[2].toggle(); // pool-2 + assert + .dom('.job-row') + .exists( + { count: 1 }, + 'Only one job should be visible when filtering by the "pool-2" node pool.' + ); + assert + .dom('[data-test-job-row="pool-2-job"]') + .exists( + 'The job in the "pool-2" node pool appears as expected when filtered.' + ); + + localStorage.removeItem('nomadPageSize'); + }); + + test('Combined Filtering and Searching', async function (assert) { + localStorage.setItem('nomadPageSize', '10'); + // 2 service, 1 batch, 1 system, 1 sysbatch + // 3 running, 1 dead, 1 pending + server.create('job', { + id: 'job1', + name: 'Alpha Processing', + type: 'batch', + status: 'running', + }); + server.create('job', { + id: 'job2', + name: 'Beta Calculation', + type: 'service', + status: 'dead', + }); + server.create('job', { + id: 'job3', + name: 'Gamma Analysis', + type: 'sysbatch', + status: 'pending', + }); + server.create('job', { + id: 'job4', + name: 'Delta Research', + type: 'system', + status: 'running', + }); + server.create('job', { + id: 'job5', + name: 'Epsilon Development', + type: 'service', + status: 'running', + }); + + // All 5 jobs show up by default + await JobsList.visit(); + assert.dom('.job-row').exists({ count: 5 }, 'All 5 jobs are visible'); + + // Toggle type to "service", should see 2 jobs + await JobsList.facets.type.toggle(); + await JobsList.facets.type.options[1].toggle(); + assert + .dom('.job-row') + .exists({ count: 2 }, 'Two service jobs are visible'); + + // additionally, enable "batch" type + await JobsList.facets.type.options[0].toggle(); + assert + .dom('.job-row') + .exists( + { count: 3 }, + 'Three jobs are visible with service and batch types' + ); + assert.dom('[data-test-job-row="job1"]').exists(); + assert.dom('[data-test-job-row="job2"]').exists(); + assert.dom('[data-test-job-row="job5"]').exists(); + + // additionally, enable "running" status to filter down to just the running ones + await JobsList.facets.status.toggle(); + await JobsList.facets.status.options[1].toggle(); + assert + .dom('.job-row') + .exists({ count: 2 }, 'Two running service/batch jobs are visible'); + assert.dom('[data-test-job-row="job1"]').exists(); + assert.dom('[data-test-job-row="job5"]').exists(); + assert.dom('[data-test-job-row="job2"]').doesNotExist(); + + // additionally, perform a search for Name != "Alpha Processing" + await JobsList.search.fillIn('Name != "Alpha Processing"'); + assert + .dom('.job-row') + .exists({ count: 1 }, 'One running service job is visible'); + assert.dom('[data-test-job-row="job5"]').exists(); + assert.dom('[data-test-job-row="job1"]').doesNotExist(); + }); + }); + }); }); /** @@ -1209,3 +1579,151 @@ function createJobs(server, jobsToCreate) { }); } } + +async function facetOptions(assert, beforeEach, facet, expectedOptions) { + await beforeEach(); + await facet.toggle(); + + let expectation; + if (typeof expectedOptions === 'function') { + expectation = expectedOptions(server.db.jobs); + } else { + expectation = expectedOptions; + } + + assert.deepEqual( + facet.options.map((option) => option.label.trim()), + expectation, + 'Options for facet are as expected' + ); +} + +function testFacet( + label, + { facet, paramName, beforeEach, filter, expectedOptions } +) { + test(`the ${label} facet has the correct options`, async function (assert) { + await facetOptions(assert, beforeEach, facet, expectedOptions); + }); + + test(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { + let option; + + await beforeEach(); + await facet.toggle(); + + option = facet.options.objectAt(0); + await option.toggle(); + + const selection = [option.label]; + const expectedJobs = server.db.jobs + .filter((job) => filter(job, selection)) + .sortBy('modifyIndex') + .reverse(); + + JobsList.jobs.forEach((job, index) => { + assert.equal( + job.id, + expectedJobs[index].id, + `Job at ${index} is ${expectedJobs[index].id}` + ); + }); + }); + + test(`selecting multiple options in the ${label} facet results in a broader search`, async function (assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.label); + await option2.toggle(); + selection.push(option2.label); + + const expectedJobs = server.db.jobs + .filter((job) => filter(job, selection)) + .sortBy('modifyIndex') + .reverse(); + + JobsList.jobs.forEach((job, index) => { + assert.equal( + job.id, + expectedJobs[index].id, + `Job at ${index} is ${expectedJobs[index].id}` + ); + }); + }); + + test(`selecting options in the ${label} facet updates the ${paramName} query param`, async function (assert) { + const selection = []; + + await beforeEach(); + await facet.toggle(); + + const option1 = facet.options.objectAt(0); + const option2 = facet.options.objectAt(1); + await option1.toggle(); + selection.push(option1.label); + await option2.toggle(); + selection.push(option2.label); + + selection.forEach((selection) => { + let capitalizedParamName = + paramName.charAt(0).toUpperCase() + paramName.slice(1); + assert.ok( + currentURL().includes( + encodeURIComponent(`${capitalizedParamName} == ${selection}`) + ), + `URL has the correct query param key and value for ${selection}` + ); + }); + }); +} + +function testSingleSelectFacet( + label, + { facet, paramName, beforeEach, filter, expectedOptions, optionToSelect } +) { + test(`the ${label} facet has the correct options`, async function (assert) { + await facetOptions(assert, beforeEach, facet, expectedOptions); + }); + + test(`the ${label} facet filters the jobs list by ${label}`, async function (assert) { + await beforeEach(); + await facet.toggle(); + + const option = facet.options.findOneBy('label', optionToSelect); + const selection = option.label; + await option.toggle(); + + const expectedJobs = server.db.jobs + .filter((job) => filter(job, selection)) + .sortBy('modifyIndex') + .reverse(); + + JobsList.jobs.forEach((job, index) => { + assert.equal( + job.id, + expectedJobs[index].id, + `Job at ${index} is ${expectedJobs[index].id}` + ); + }); + }); + + test(`selecting an option in the ${label} facet updates the ${paramName} query param`, async function (assert) { + await beforeEach(); + await facet.toggle(); + + const option = facet.options.objectAt(1); + const selection = option.label; + await option.toggle(); + + assert.ok( + currentURL().includes(`${paramName}=${selection}`), + 'URL has the correct query param key and value' + ); + }); +} diff --git a/ui/tests/integration/components/job-search-box-test.js b/ui/tests/integration/components/job-search-box-test.js new file mode 100644 index 00000000000..c1b0bb60a8c --- /dev/null +++ b/ui/tests/integration/components/job-search-box-test.js @@ -0,0 +1,63 @@ +/** + * Copyright (c) HashiCorp, Inc. + * SPDX-License-Identifier: BUSL-1.1 + */ + +import { module, test } from 'qunit'; +import { setupRenderingTest } from 'ember-qunit'; +import { render } from '@ember/test-helpers'; +import { hbs } from 'ember-cli-htmlbars'; +import { fillIn, find, triggerEvent } from '@ember/test-helpers'; +import { componentA11yAudit } from 'nomad-ui/tests/helpers/a11y-audit'; + +const DEBOUNCE_MS = 500; + +module('Integration | Component | job-search-box', function (hooks) { + setupRenderingTest(hooks); + + test('debouncer debounces appropriately', async function (assert) { + assert.expect(5); + + let message = ''; + + this.set('externalAction', (value) => { + message = value; + }); + + await render( + hbs`` + ); + await componentA11yAudit(this.element, assert); + + const element = find('input'); + await fillIn('input', 'test1'); + assert.equal(message, 'test1', 'Initial typing'); + element.value += ' wont be '; + triggerEvent('input', 'input'); + assert.equal( + message, + 'test1', + 'Typing has happened within debounce window' + ); + element.value += 'seen '; + triggerEvent('input', 'input'); + await delay(DEBOUNCE_MS - 100); + assert.equal( + message, + 'test1', + 'Typing has happened within debounce window, albeit a little slower' + ); + element.value += 'until now.'; + triggerEvent('input', 'input'); + await delay(DEBOUNCE_MS + 100); + assert.equal( + message, + 'test1 wont be seen until now.', + 'debounce window has closed' + ); + }); +}); + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/ui/tests/pages/components/facet.js b/ui/tests/pages/components/facet.js index a77c2eae1fb..1583c8965d9 100644 --- a/ui/tests/pages/components/facet.js +++ b/ui/tests/pages/components/facet.js @@ -44,3 +44,16 @@ export const singleFacet = (scope) => ({ }, }), }); + +export const hdsFacet = (scope) => ({ + scope, + + toggle: clickable('.hds-dropdown-toggle-button'), + + options: collection('.hds-dropdown-list-item', { + resetScope: true, + label: text(), + key: attribute('data-test-hds-facet-option'), + toggle: clickable('.hds-dropdown-list-item__label'), + }), +}); diff --git a/ui/tests/pages/jobs/list.js b/ui/tests/pages/jobs/list.js index c330c59f068..eb5a071c54a 100644 --- a/ui/tests/pages/jobs/list.js +++ b/ui/tests/pages/jobs/list.js @@ -14,7 +14,7 @@ import { visitable, } from 'ember-cli-page-object'; -import { multiFacet, singleFacet } from 'nomad-ui/tests/pages/components/facet'; +import { hdsFacet } from 'nomad-ui/tests/pages/components/facet'; import pageSizeSelect from 'nomad-ui/tests/pages/components/page-size-select'; export default create({ @@ -23,7 +23,7 @@ export default create({ visit: visitable('/jobs'), search: { - scope: '[data-test-jobs-search] input', + scope: '[data-test-jobs-search]', keydown: triggerable('keydown'), }, @@ -40,8 +40,6 @@ export default create({ nodePool: text('[data-test-job-node-pool]'), status: text('[data-test-job-status]'), type: text('[data-test-job-type]'), - priority: text('[data-test-job-priority]'), - taskGroups: text('[data-test-job-task-groups]'), hasNamespace: isPresent('[data-test-job-namespace]'), clickRow: clickable(), @@ -66,10 +64,9 @@ export default create({ pageSizeSelect: pageSizeSelect(), facets: { - namespace: singleFacet('[data-test-namespace-facet]'), - type: multiFacet('[data-test-type-facet]'), - status: multiFacet('[data-test-status-facet]'), - datacenter: multiFacet('[data-test-datacenter-facet]'), - prefix: multiFacet('[data-test-prefix-facet]'), + namespace: hdsFacet('[data-test-facet="Namespace"]'), + type: hdsFacet('[data-test-facet="Type"]'), + status: hdsFacet('[data-test-facet="Status"]'), + nodePool: hdsFacet('[data-test-facet="NodePool"]'), }, }); From 02dcfe92ebf4b0598a318c6e66afa17b398c05d4 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Fri, 3 May 2024 17:03:04 -0400 Subject: [PATCH 91/98] drop .go changes; TODO: rebase cleanly from main --- api/jobs.go | 1 - command/agent/job_endpoint.go | 56 ----------------------------------- 2 files changed, 57 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 737c1691d6c..407b13568e0 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -1545,7 +1545,6 @@ func (j *Jobs) ActionExec(ctx context.Context, return s.run(ctx) } - // JobStatusesRequest is used to get statuses for jobs, // their allocations and deployments. type JobStatusesRequest struct { diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 49419d5cc13..05f1f04a162 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -2108,59 +2108,3 @@ func validateEvalPriorityOpt(priority int) HTTPCodedError { } return nil } - -func (s *HTTPServer) JobStatusesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { - args := structs.JobStatusesRequest{} - if s.parse(resp, req, &args.Region, &args.QueryOptions) { - return nil, nil // seems whack - } - - switch req.Method { - case http.MethodGet, http.MethodPost: - break - default: - return nil, CodedError(405, ErrInvalidMethod) - } - - if includeChildren, err := parseBool(req, "include_children"); err != nil { - return nil, err - } else if includeChildren != nil { - args.IncludeChildren = *includeChildren - } - - // ostensibly GETs should not accept structured body, but the HTTP spec - // on this is more what you'd call "guidelines" than actual rules. - if req.Body != http.NoBody { - var in api.JobStatusesRequest - if err := decodeBody(req, &in); err != nil { - return nil, err - } - if len(in.Jobs) == 0 { - return nil, CodedError(http.StatusBadRequest, "no jobs in request") - } - - // each job has a separate namespace, so default to wildcard - // in case the NSes are mixed. - if args.QueryOptions.Namespace == "" { - args.QueryOptions.Namespace = "*" - } - - for _, j := range in.Jobs { - if j.Namespace == "" { - j.Namespace = "default" - } - args.Jobs = append(args.Jobs, structs.NamespacedID{ - ID: j.ID, - Namespace: j.Namespace, - }) - } - } - - var out structs.JobStatusesResponse - if err := s.agent.RPC("Job.Statuses", &args, &out); err != nil { - return nil, err - } - - setMeta(resp, &out.QueryMeta) - return out.Jobs, nil -} From 243d45c194d5138731550df5521c55c7c737af81 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Sun, 5 May 2024 23:04:33 -0400 Subject: [PATCH 92/98] todo-squashing --- ui/app/components/job-status/panel/steady.js | 7 +- ui/app/controllers/jobs/index.js | 4 +- ui/app/controllers/jobs/index_old.js | 352 ------------------- ui/app/models/job.js | 16 +- ui/app/routes/jobs/index.js | 2 - ui/app/templates/jobs/index_old.hbs | 222 ------------ ui/mirage/config.js | 2 +- ui/mirage/factories/job.js | 2 - ui/tests/acceptance/jobs-list-test.js | 11 +- 9 files changed, 14 insertions(+), 604 deletions(-) delete mode 100644 ui/app/controllers/jobs/index_old.js delete mode 100644 ui/app/templates/jobs/index_old.hbs diff --git a/ui/app/components/job-status/panel/steady.js b/ui/app/components/job-status/panel/steady.js index 842514531ad..ac03654b7dd 100644 --- a/ui/app/components/job-status/panel/steady.js +++ b/ui/app/components/job-status/panel/steady.js @@ -11,13 +11,8 @@ import { jobAllocStatuses } from '../../../utils/allocation-client-statuses'; export default class JobStatusPanelSteadyComponent extends Component { @alias('args.job') job; - // TODO: use the job model for this get allocTypes() { - return jobAllocStatuses[this.args.job.type].map((type) => { - return { - label: type, - }; - }); + return this.args.job.allocTypes; } /** diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index cfb7506b740..720ca4592c0 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -21,7 +21,7 @@ export default class JobsIndexController extends Controller { @service system; @service store; @service userSettings; - @service watchList; // TODO: temp + @service watchList; @tracked pageSize; @@ -168,7 +168,6 @@ export default class JobsIndexController extends Controller { return this.store .query('job', params, { adapterOptions: { - method: 'GET', // TODO: default abortController: this.watchList.jobsIndexIDsController, }, }) @@ -223,7 +222,6 @@ export default class JobsIndexController extends Controller { return prevPageToken; } - // TODO: set up isEnabled to check blockingQueries rather than just use while (true) @restartableTask *watchJobIDs( params, throttle = Ember.testing ? 0 : JOB_LIST_THROTTLE diff --git a/ui/app/controllers/jobs/index_old.js b/ui/app/controllers/jobs/index_old.js deleted file mode 100644 index 86ca64abed7..00000000000 --- a/ui/app/controllers/jobs/index_old.js +++ /dev/null @@ -1,352 +0,0 @@ -/** - * Copyright (c) HashiCorp, Inc. - * SPDX-License-Identifier: BUSL-1.1 - */ - -//@ts-check - -/* eslint-disable ember/no-incorrect-calls-with-inline-anonymous-functions */ -import { inject as service } from '@ember/service'; -import { alias, readOnly } from '@ember/object/computed'; -import Controller from '@ember/controller'; -import { computed, action } from '@ember/object'; -import { scheduleOnce } from '@ember/runloop'; -import intersection from 'lodash.intersection'; -import Sortable from 'nomad-ui/mixins/sortable'; -import Searchable from 'nomad-ui/mixins/searchable'; -import { - serialize, - deserializedQueryParam as selection, -} from 'nomad-ui/utils/qp-serialize'; -import classic from 'ember-classic-decorator'; - -const DEFAULT_SORT_PROPERTY = 'modifyIndex'; -const DEFAULT_SORT_DESCENDING = true; - -@classic -export default class IndexController extends Controller.extend( - Sortable, - Searchable -) { - @service system; - @service userSettings; - @service router; - - isForbidden = false; - - queryParams = [ - { - currentPage: 'page', - }, - { - searchTerm: 'search', - }, - { - sortProperty: 'sort', - }, - { - sortDescending: 'desc', - }, - { - qpType: 'type', - }, - { - qpStatus: 'status', - }, - { - qpDatacenter: 'dc', - }, - { - qpPrefix: 'prefix', - }, - { - qpNamespace: 'namespace', - }, - { - qpNodePool: 'nodePool', - }, - ]; - - currentPage = 1; - @readOnly('userSettings.pageSize') pageSize; - - sortProperty = DEFAULT_SORT_PROPERTY; - sortDescending = DEFAULT_SORT_DESCENDING; - - @computed - get searchProps() { - return ['id', 'name']; - } - - @computed - get fuzzySearchProps() { - return ['name']; - } - - fuzzySearchEnabled = true; - - qpType = ''; - qpStatus = ''; - qpDatacenter = ''; - qpPrefix = ''; - qpNodePool = ''; - - @selection('qpType') selectionType; - @selection('qpStatus') selectionStatus; - @selection('qpDatacenter') selectionDatacenter; - @selection('qpPrefix') selectionPrefix; - @selection('qpNodePool') selectionNodePool; - - @computed - get optionsType() { - return [ - { key: 'batch', label: 'Batch' }, - { key: 'pack', label: 'Pack' }, - { key: 'parameterized', label: 'Parameterized' }, - { key: 'periodic', label: 'Periodic' }, - { key: 'service', label: 'Service' }, - { key: 'system', label: 'System' }, - { key: 'sysbatch', label: 'System Batch' }, - ]; - } - - @computed - get optionsStatus() { - return [ - { key: 'pending', label: 'Pending' }, - { key: 'running', label: 'Running' }, - { key: 'dead', label: 'Dead' }, - ]; - } - - @computed('selectionDatacenter', 'visibleJobs.[]') - get optionsDatacenter() { - const flatten = (acc, val) => acc.concat(val); - const allDatacenters = new Set( - this.visibleJobs.mapBy('datacenters').reduce(flatten, []) - ); - - // Remove any invalid datacenters from the query param/selection - const availableDatacenters = Array.from(allDatacenters).compact(); - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpDatacenter', - serialize(intersection(availableDatacenters, this.selectionDatacenter)) - ); - }); - - return availableDatacenters.sort().map((dc) => ({ key: dc, label: dc })); - } - - @computed('selectionPrefix', 'visibleJobs.[]') - get optionsPrefix() { - // A prefix is defined as the start of a job name up to the first - or . - // ex: mktg-analytics -> mktg, ds.supermodel.classifier -> ds - const hasPrefix = /.[-._]/; - - // Collect and count all the prefixes - const allNames = this.visibleJobs.mapBy('name'); - const nameHistogram = allNames.reduce((hist, name) => { - if (hasPrefix.test(name)) { - const prefix = name.match(/(.+?)[-._]/)[1]; - hist[prefix] = hist[prefix] ? hist[prefix] + 1 : 1; - } - return hist; - }, {}); - - // Convert to an array - const nameTable = Object.keys(nameHistogram).map((key) => ({ - prefix: key, - count: nameHistogram[key], - })); - - // Only consider prefixes that match more than one name - const prefixes = nameTable.filter((name) => name.count > 1); - - // Remove any invalid prefixes from the query param/selection - const availablePrefixes = prefixes.mapBy('prefix'); - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpPrefix', - serialize(intersection(availablePrefixes, this.selectionPrefix)) - ); - }); - - // Sort, format, and include the count in the label - return prefixes.sortBy('prefix').map((name) => ({ - key: name.prefix, - label: `${name.prefix} (${name.count})`, - })); - } - - @computed('qpNamespace', 'model.namespaces.[]') - get optionsNamespaces() { - const availableNamespaces = this.model.namespaces.map((namespace) => ({ - key: namespace.name, - label: namespace.name, - })); - - availableNamespaces.unshift({ - key: '*', - label: 'All (*)', - }); - - // Unset the namespace selection if it was server-side deleted - if (!availableNamespaces.mapBy('key').includes(this.qpNamespace)) { - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set('qpNamespace', '*'); - }); - } - - return availableNamespaces; - } - - @computed('selectionNodePool', 'model.nodePools.[]') - get optionsNodePool() { - const availableNodePools = this.model.nodePools; - - scheduleOnce('actions', () => { - // eslint-disable-next-line ember/no-side-effects - this.set( - 'qpNodePool', - serialize( - intersection( - availableNodePools.map(({ name }) => name), - this.selectionNodePool - ) - ) - ); - }); - - return availableNodePools.map((nodePool) => ({ - key: nodePool.name, - label: nodePool.name, - })); - } - - /** - Visible jobs are those that match the selected namespace and aren't children - of periodic or parameterized jobs. - */ - @computed('model.jobs.@each.parent') - get visibleJobs() { - if (!this.model || !this.model.jobs) return []; - return this.model.jobs - .compact() - .filter((job) => !job.isNew) - .filter((job) => !job.get('parent.content')); - } - - @computed( - 'visibleJobs.[]', - 'selectionType', - 'selectionStatus', - 'selectionDatacenter', - 'selectionNodePool', - 'selectionPrefix' - ) - get filteredJobs() { - const { - selectionType: types, - selectionStatus: statuses, - selectionDatacenter: datacenters, - selectionPrefix: prefixes, - selectionNodePool: nodePools, - } = this; - - // A job must match ALL filter facets, but it can match ANY selection within a facet - // Always return early to prevent unnecessary facet predicates. - return this.visibleJobs.filter((job) => { - const shouldShowPack = types.includes('pack') && job.displayType.isPack; - - if (types.length && shouldShowPack) { - return true; - } - - if (types.length && !types.includes(job.get('displayType.type'))) { - return false; - } - - if (statuses.length && !statuses.includes(job.get('status'))) { - return false; - } - - if ( - datacenters.length && - !job.get('datacenters').find((dc) => datacenters.includes(dc)) - ) { - return false; - } - - if (nodePools.length && !nodePools.includes(job.get('nodePool'))) { - return false; - } - - const name = job.get('name'); - if ( - prefixes.length && - !prefixes.find((prefix) => name.startsWith(prefix)) - ) { - return false; - } - - return true; - }); - } - - // eslint-disable-next-line ember/require-computed-property-dependencies - @computed('searchTerm') - get sortAtLastSearch() { - return { - sortProperty: this.sortProperty, - sortDescending: this.sortDescending, - searchTerm: this.searchTerm, - }; - } - - @computed( - 'searchTerm', - 'sortAtLastSearch.{sortDescending,sortProperty}', - 'sortDescending', - 'sortProperty' - ) - get prioritizeSearchOrder() { - let shouldPrioritizeSearchOrder = - !!this.searchTerm && - this.sortAtLastSearch.sortProperty === this.sortProperty && - this.sortAtLastSearch.sortDescending === this.sortDescending; - if (shouldPrioritizeSearchOrder) { - /* eslint-disable ember/no-side-effects */ - this.set('sortDescending', DEFAULT_SORT_DESCENDING); - this.set('sortProperty', DEFAULT_SORT_PROPERTY); - this.set('sortAtLastSearch.sortProperty', DEFAULT_SORT_PROPERTY); - this.set('sortAtLastSearch.sortDescending', DEFAULT_SORT_DESCENDING); - } - /* eslint-enable ember/no-side-effects */ - return shouldPrioritizeSearchOrder; - } - - @alias('filteredJobs') listToSearch; - @alias('listSearched') listToSort; - - // sortedJobs is what we use to populate the table; - // If the user has searched but not sorted, we return the (fuzzy) searched list verbatim - // If the user has sorted, we allow the fuzzy search to filter down the list, but return it in a sorted order. - get sortedJobs() { - return this.prioritizeSearchOrder ? this.listSearched : this.listSorted; - } - - isShowingDeploymentDetails = false; - - setFacetQueryParam(queryParam, selection) { - this.set(queryParam, serialize(selection)); - } - - @action - goToRun() { - this.router.transitionTo('jobs.run'); - } -} diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 7506fd77901..4492c663dea 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -43,7 +43,16 @@ export default class Job extends Model { } } - @attr() latestDeploymentSummary; // TODO: model this out + /** + * @typedef {Object} LatestDeploymentSummary + * @property {boolean} IsActive - Whether the deployment is currently active + * @property {number} JobVersion - The version of the job that was deployed + * @property {string} Status - The status of the deployment + * @property {string} StatusDescription - A description of the deployment status + * @property {boolean} AllAutoPromote - Whether all allocations were auto-promoted + * @property {boolean} RequiresPromotion - Whether the deployment requires promotion + */ + @attr() latestDeploymentSummary; @attr() childStatuses; @@ -273,12 +282,7 @@ export default class Job extends Model { ...this.allocBlocks.unplaced?.healthy?.nonCanary, ]; - // TODO: GroupCountSum for a parameterized parent job is the count present at group level, but that's not quite true, as the parent job isn't expecting any allocs, its children are. Chat with BFF about this. - - // TODO: handle garbage collected cases not showing "failed" for batch jobs here maybe? - if (failedOrLostAllocs.length >= totalAllocs) { - // TODO: when totalAllocs only cares about latest version, change back to === return { label: 'Failed', state: 'critical' }; } else { return { label: 'Degraded', state: 'warning' }; diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index fdb0def121c..29152d5eadc 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -71,7 +71,6 @@ export default class IndexRoute extends Route.extend( try { let jobs = await this.store.query('job', currentParams, { adapterOptions: { - method: 'GET', // TODO: default abortController: this.watchList.jobsIndexIDsController, }, }); @@ -176,7 +175,6 @@ export default class IndexRoute extends Route.extend( @action willTransition(transition) { - // TODO: Something is preventing jobs -> job -> jobs -> job. if (!transition.intent.name?.startsWith(this.routeName)) { this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexIDsController.abort(); diff --git a/ui/app/templates/jobs/index_old.hbs b/ui/app/templates/jobs/index_old.hbs deleted file mode 100644 index 4ed62d2b1b2..00000000000 --- a/ui/app/templates/jobs/index_old.hbs +++ /dev/null @@ -1,222 +0,0 @@ -{{! - Copyright (c) HashiCorp, Inc. - SPDX-License-Identifier: BUSL-1.1 -~}} - -{{page-title "Jobs"}} -
-
-
- {{#if this.visibleJobs.length}} - - {{/if}} -
- {{#if (media "isMobile")}} -
- {{#if (can "run job" namespace=this.qpNamespace)}} - - Run Job - - {{else}} - - {{/if}} -
- {{/if}} -
-
- {{#if this.system.shouldShowNamespaces}} - - {{/if}} - - - - - -
-
- {{#if (not (media "isMobile"))}} -
- {{#if (can "run job" namespace=this.qpNamespace)}} - - Run Job - - {{else}} - - {{/if}} -
- {{/if}} -
- {{#if this.isForbidden}} - - {{else if this.sortedJobs}} - - - - - Name - - {{#if this.system.shouldShowNamespaces}} - - Namespace - - {{/if}} - - Status - - - Type - - - Node Pool - - - Priority - - - Summary - - - - - - -
- - -
- - {{else}} -
- {{#if (eq this.visibleJobs.length 0)}} -

- No Jobs -

-

- The cluster is currently empty. -

- {{else if (eq this.filteredJobs.length 0)}} -

- No Matches -

-

- No jobs match your current filter selection. -

- {{else if this.searchTerm}} -

- No Matches -

-

- No jobs match the term - - {{this.searchTerm}} - -

- {{/if}} -
- {{/if}} -
diff --git a/ui/mirage/config.js b/ui/mirage/config.js index fdccbc18f72..4435ca40ec2 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -241,7 +241,7 @@ export default function () { ID: alloc.id, }; }); - job.ChildStatuses = null; // TODO: handle parent job here + job.ChildStatuses = null; job.Datacenters = j.Datacenters; job.DeploymentID = j.DeploymentID; job.GroupCountSum = j.TaskGroups.mapBy('Count').reduce( diff --git a/ui/mirage/factories/job.js b/ui/mirage/factories/job.js index 149baa4557c..b5c4d82766a 100644 --- a/ui/mirage/factories/job.js +++ b/ui/mirage/factories/job.js @@ -63,8 +63,6 @@ export default Factory.extend({ childrenCount: () => faker.random.number({ min: 1, max: 2 }), - // TODO: Use the model's aggregateAllocStatus as a property here - meta: null, periodic: trait({ diff --git a/ui/tests/acceptance/jobs-list-test.js b/ui/tests/acceptance/jobs-list-test.js index 1613035d407..309584b8392 100644 --- a/ui/tests/acceptance/jobs-list-test.js +++ b/ui/tests/acceptance/jobs-list-test.js @@ -293,13 +293,7 @@ module('Acceptance | jobs list', function (hooks) { testFacet('Type', { facet: JobsList.facets.type, paramName: 'type', - expectedOptions: [ - 'batch', - 'service', - 'system', - 'sysbatch', - // TODO: add Parameterized and Periodic - ], + expectedOptions: ['batch', 'service', 'system', 'sysbatch'], async beforeEach() { server.createList('job', 2, { createAllocations: false, type: 'batch' }); server.createList('job', 2, { @@ -322,9 +316,6 @@ module('Acceptance | jobs list', function (hooks) { }, filter(job, selection) { let displayType = job.type; - // TODO: if/when we allow for parameterized/batch filtering, uncomment these. - // if (job.parameterized) displayType = 'parameterized'; - // if (job.periodic) displayType = 'periodic'; return selection.includes(displayType); }, }); From 7a63e483fded2a2e6844cfffc7db609095e3dce7 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 3 Apr 2024 17:09:54 -0400 Subject: [PATCH 93/98] Beginnings of deployment-aware alerting on the jobs index page --- ui/app/components/job-row.js | 172 ++++++++++++-- ui/app/controllers/jobs/index.js | 2 + ui/app/models/job.js | 223 +++++++++++++----- ui/app/serializers/job.js | 2 + .../styles/components/job-status-panel.scss | 1 + ui/app/styles/components/jobs-list.scss | 7 + ui/app/templates/components/job-row.hbs | 163 ++++++++----- ui/app/templates/jobs/index.hbs | 70 +----- 8 files changed, 434 insertions(+), 206 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 9cc7fe65506..1c237f8583a 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -3,39 +3,171 @@ * SPDX-License-Identifier: BUSL-1.1 */ -import Component from '@ember/component'; +// @ts-check + +import Component from '@glimmer/component'; import { action } from '@ember/object'; import { inject as service } from '@ember/service'; -import { lazyClick } from '../helpers/lazy-click'; -import { - classNames, - tagName, - attributeBindings, -} from '@ember-decorators/component'; -import classic from 'ember-classic-decorator'; - -@classic -@tagName('tr') -@classNames('job-row', 'is-interactive') -@attributeBindings('data-test-job-row') +import { tracked } from '@glimmer/tracking'; +import { task } from 'ember-concurrency'; + export default class JobRow extends Component { @service router; @service store; @service system; - job = null; + @tracked activeDeployment = null; + + // /** + // * If our job has an activeDeploymentID, as determined by the statuses endpoint, + // * we check if this component's activeDeployment has the same ID. + // * If it does, we don't need to do any fetching: we can simply check this.activeDeployment.requiresPromotion + // * If it doesn't, we need to fetch the deployment with the activeDeploymentID + // * and set it to this.activeDeployment, then check this.activeDeployment.requiresPromotion. + // */ + // get requiresPromotion() { + // if (!this.args.job.hasActiveCanaries || !this.args.job.activeDeploymentID) { + // return false; + // } + + // if (this.activeDeployment && this.activeDeployment.id === this.args.job.activeDeploymentID) { + // return this.activeDeployment.requiresPromotion; + // } + + // this.fetchActiveDeployment(); + // return false; + // } + + // @action + // async fetchActiveDeployment() { + // if (this.args.job.hasActiveCanaries && this.args.job.activeDeploymentID) { + // let deployment = await this.store.findRecord('deployment', this.args.job.activeDeploymentID); + // this.activeDeployment = deployment; + // } + // } + + /** + * Promotion of a deployment will error if the canary allocations are not of status "Healthy"; + * this function will check for that and disable the promote button if necessary. + * @returns {boolean} + */ + get canariesHealthy() { + const relevantAllocs = this.args.job.allocations.filter( + (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled + ); + return ( + relevantAllocs.length && + relevantAllocs.every((a) => a.clientStatus === 'running' && a.isHealthy) + ); + } + + get someCanariesHaveFailed() { + const relevantAllocs = this.args.job.allocations.filter( + (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled + ); + console.log( + 'relevantAllocs', + relevantAllocs.map((a) => a.clientStatus), + relevantAllocs.map((a) => a.isUnhealthy) + ); + return relevantAllocs.some( + (a) => + a.clientStatus === 'failed' || + a.clientStatus === 'lost' || + a.isUnhealthy + ); + } + + @task(function* () { + console.log( + 'checking if requries promotion', + this.args.job.activeDeploymentID, + this.args.job.hasActiveCanaries + ); + if (!this.args.job.hasActiveCanaries || !this.args.job.activeDeploymentID) { + return false; + } + + if ( + !this.activeDeployment || + this.activeDeployment.id !== this.args.job.activeDeploymentID + ) { + this.activeDeployment = yield this.store.findRecord( + 'deployment', + this.args.job.activeDeploymentID + ); + } + + if (this.activeDeployment.requiresPromotion) { + if (this.canariesHealthy) { + return 'canary-promote'; + } + if (this.someCanariesHaveFailed) { + return 'canary-failure'; + } + if (this.activeDeployment.isAutoPromoted) { + // return "This deployment is set to auto-promote; canaries are being checked now"; + return false; + } else { + // return "This deployment requires manual promotion and things are being checked now"; + return false; + } + } + return false; + }) + requiresPromotionTask; + + @task(function* () { + try { + yield this.args.job.latestDeployment.content.promote(); + // dont bubble up + return false; + } catch (err) { + this.handleError({ + title: 'Could Not Promote Deployment', + // description: messageFromAdapterError(err, 'promote deployments'), + }); + } + }) + promote; - // One of independent, parent, or child. Used to customize the template - // based on the relationship of this job to others. - context = 'independent'; + /** + * If there is not a deployment happening, + * and the running allocations have a jobVersion that differs from the job's version, + * we can assume a failed latest deployment. + */ + get latestDeploymentFailed() { + /** + * Import from app/models/job.js + * @type {import('../models/job').default} + */ + const job = this.args.job; + if (job.activeDeploymentID) { + return false; + } - click(event) { - lazyClick([this.gotoJob, event]); + // We only want to show this status if the job is running, to indicate to + // the user that the job is not running the version they expect given their + // latest deployment. + if ( + !( + job.aggregateAllocStatus.label === 'Healthy' || + job.aggregateAllocStatus.label === 'Degraded' || + job.aggregateAllocStatus.label === 'Recovering' + ) + ) { + return false; + } + const runningAllocs = job.allocations.filter( + (a) => a.clientStatus === 'running' + ); + const jobVersion = job.version; + return runningAllocs.some((a) => a.jobVersion !== jobVersion); } @action gotoJob() { - const { job } = this; + const { job } = this.args; this.router.transitionTo('jobs.job.index', job.idWithNamespace); } } diff --git a/ui/app/controllers/jobs/index.js b/ui/app/controllers/jobs/index.js index 720ca4592c0..106a6e94bd4 100644 --- a/ui/app/controllers/jobs/index.js +++ b/ui/app/controllers/jobs/index.js @@ -180,6 +180,8 @@ export default class JobsIndexController extends Controller { } jobAllocsQuery(params) { + // TODO: Noticing a pattern with long-running jobs with alloc changes, where there are multiple POST statuses blocking queries being held open at once. + // This is a problem and I should get to the bottom of it. this.watchList.jobsIndexDetailsController.abort(); this.watchList.jobsIndexDetailsController = new AbortController(); params.namespace = '*'; diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 4492c663dea..dccaf554dc0 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -54,6 +54,31 @@ export default class Job extends Model { */ @attr() latestDeploymentSummary; + get hasActiveCanaries() { + // console.log('tell me about ur active canaries plz', this.allocBlocks, this.allocations, this.activeDeploymentID); + // TODO: Monday/Tuesday: go over AllocBlocks.{all}.canary and if there are any? make the latestDeployment lookup, + // and check to see if it requires promotion / isnt yet promoted. + if (!this.latestDeploymentSummary?.IsActive) { + return false; + } + return Object.keys(this.allocBlocks) + .map((status) => { + return Object.keys(this.allocBlocks[status]) + .map((health) => { + return this.allocBlocks[status][health].canary.length; + }) + .flat(); + }) + .flat() + .any((n) => !!n); + // return this.activeDeploymentID; + } + // TODO: moved to job-row + // get requiresPromotion() { + // console.log('getting requiresPromotion', this.activeDeploymentID, this.runningDeployment); + // return this.runningDeployment; + // } + @attr() childStatuses; get childStatusBreakdown() { @@ -75,6 +100,9 @@ export default class Job extends Model { // We set this flag to true to let the user know that the job has been removed without simply nixing it from view. @attr('boolean', { defaultValue: false }) assumeGC; + /** + * @returns {Array<{label: string}>} + */ get allocTypes() { return jobAllocStatuses[this.type].map((type) => { return { @@ -123,87 +151,164 @@ export default class Job extends Model { get allocBlocks() { let availableSlotsToFill = this.expectedRunningAllocCount; + let isDeploying = this.latestDeploymentSummary.IsActive; // Initialize allocationsOfShowableType with empty arrays for each clientStatus /** * @type {AllocationBlock} */ let allocationsOfShowableType = this.allocTypes.reduce( - (accumulator, type) => { - accumulator[type.label] = { healthy: { nonCanary: [] } }; - return accumulator; + (categories, type) => { + categories[type.label] = { + healthy: { canary: [], nonCanary: [] }, + unhealthy: { canary: [], nonCanary: [] }, + health_unknown: { canary: [], nonCanary: [] }, + }; + return categories; }, {} ); - // First accumulate the Running/Pending allocations - for (const alloc of this.allocations.filter( - (a) => a.clientStatus === 'running' || a.clientStatus === 'pending' - )) { - if (availableSlotsToFill === 0) { - break; - } - - const status = alloc.clientStatus; - allocationsOfShowableType[status].healthy.nonCanary.push(alloc); - availableSlotsToFill--; - } - // TODO: return early here if !availableSlotsToFill - // Sort all allocs by jobVersion in descending order - const sortedAllocs = this.allocations - .filter( - (a) => a.clientStatus !== 'running' && a.clientStatus !== 'pending' - ) - .sort((a, b) => { - // First sort by jobVersion - if (a.jobVersion > b.jobVersion) return 1; - if (a.jobVersion < b.jobVersion) return -1; - - // If jobVersion is the same, sort by status order - // For example, we may have some allocBlock slots to fill, and need to determine - // if the user expects to see, from non-running/non-pending allocs, some old "failed" ones - // or "lost" or "complete" ones, etc. jobAllocStatuses give us this order. - if (a.jobVersion === b.jobVersion) { - return ( - jobAllocStatuses[this.type].indexOf(b.clientStatus) - - jobAllocStatuses[this.type].indexOf(a.clientStatus) - ); + if (isDeploying) { + // Start with just the new-version allocs + let allocationsOfDeploymentVersion = this.allocations.filter( + (a) => !a.isOld + ); + // For each of them, check to see if we still have slots to fill, based on our desired Count + for (let alloc of allocationsOfDeploymentVersion) { + if (availableSlotsToFill <= 0) { + break; + } + let status = alloc.clientStatus; + let canary = alloc.isCanary ? 'canary' : 'nonCanary'; + // TODO: do I need to dig into alloc.DeploymentStatus for these? + + // Health status only matters in the context of a "running" allocation. + // However, healthy/unhealthy is never purged when an allocation moves to a different clientStatus + // Thus, we should only show something as "healthy" in the event that it is running. + // Otherwise, we'd have arbitrary groupings based on previous health status. + let health; + + if (status === 'running') { + if (alloc.isHealthy) { + health = 'healthy'; + } else if (alloc.isUnhealthy) { + health = 'unhealthy'; + } else { + health = 'health_unknown'; + } } else { - return 0; + health = 'health_unknown'; } - }) - .reverse(); - // Iterate over the sorted allocs - for (const alloc of sortedAllocs) { - if (availableSlotsToFill === 0) { - break; + if (allocationsOfShowableType[status]) { + // If status is failed or lost, we only want to show it IF it's used up its restarts/rescheds. + // Otherwise, we'd be showing an alloc that had been replaced. + if (alloc.willNotRestart) { + if (!alloc.willNotReschedule) { + // Dont count it + continue; + } + } + allocationsOfShowableType[status][health][canary].push(alloc); + availableSlotsToFill--; + } } + } else { + // First accumulate the Running/Pending allocations + for (const alloc of this.allocations.filter( + (a) => a.clientStatus === 'running' || a.clientStatus === 'pending' + )) { + if (availableSlotsToFill === 0) { + break; + } - const status = alloc.clientStatus; - // If the alloc has another clientStatus, add it to the corresponding list - // as long as we haven't reached the expectedRunningAllocCount limit for that clientStatus - if ( - this.allocTypes.map(({ label }) => label).includes(status) && - allocationsOfShowableType[status].healthy.nonCanary.length < - this.expectedRunningAllocCount - ) { + const status = alloc.clientStatus; + // console.log('else and pushing with', status, 'and', alloc); + // We are not actively deploying in this condition, + // so we can assume Healthy and Non-Canary allocationsOfShowableType[status].healthy.nonCanary.push(alloc); availableSlotsToFill--; } + // TODO: return early here if !availableSlotsToFill + + // So, we've tried filling our desired Count with running/pending allocs. + // If we still have some slots remaining, we should sort our other allocations + // by version number, descending, and then by status order (arbitrary, via allocation-client-statuses.js). + let sortedAllocs; + + // Sort all allocs by jobVersion in descending order + sortedAllocs = this.allocations + .filter( + (a) => a.clientStatus !== 'running' && a.clientStatus !== 'pending' + ) + .sort((a, b) => { + // First sort by jobVersion + if (a.jobVersion > b.jobVersion) return 1; + if (a.jobVersion < b.jobVersion) return -1; + + // If jobVersion is the same, sort by status order + // For example, we may have some allocBlock slots to fill, and need to determine + // if the user expects to see, from non-running/non-pending allocs, some old "failed" ones + // or "lost" or "complete" ones, etc. jobAllocStatuses give us this order. + if (a.jobVersion === b.jobVersion) { + return ( + jobAllocStatuses[this.type].indexOf(b.clientStatus) - + jobAllocStatuses[this.type].indexOf(a.clientStatus) + ); + } else { + return 0; + } + }) + .reverse(); + + // Iterate over the sorted allocs + for (const alloc of sortedAllocs) { + if (availableSlotsToFill === 0) { + break; + } + + const status = alloc.clientStatus; + // If the alloc has another clientStatus, add it to the corresponding list + // as long as we haven't reached the expectedRunningAllocCount limit for that clientStatus + if ( + this.allocTypes.map(({ label }) => label).includes(status) && + allocationsOfShowableType[status].healthy.nonCanary.length < + this.expectedRunningAllocCount + ) { + allocationsOfShowableType[status].healthy.nonCanary.push(alloc); + availableSlotsToFill--; + } + } } - // Handle unplaced allocs + // // Handle unplaced allocs + // if (availableSlotsToFill > 0) { + // // TODO: JSDoc types for unhealty and health unknown aren't optional, but should be. + // allocationsOfShowableType['unplaced'] = { + // healthy: { + // nonCanary: Array(availableSlotsToFill) + // .fill() + // .map(() => { + // return { clientStatus: 'unplaced' }; + // }), + // }, + // }; + // } + + // Fill unplaced slots if availableSlotsToFill > 0 if (availableSlotsToFill > 0) { - // TODO: JSDoc types for unhealty and health unknown aren't optional, but should be. allocationsOfShowableType['unplaced'] = { - healthy: { - nonCanary: Array(availableSlotsToFill) - .fill() - .map(() => { - return { clientStatus: 'unplaced' }; - }), - }, + healthy: { canary: [], nonCanary: [] }, + unhealthy: { canary: [], nonCanary: [] }, + health_unknown: { canary: [], nonCanary: [] }, }; + allocationsOfShowableType['unplaced']['healthy']['nonCanary'] = Array( + availableSlotsToFill + ) + .fill() + .map(() => { + return { clientStatus: 'unplaced' }; + }); } // console.log('allocBlocks for', this.name, 'is', allocationsOfShowableType); diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index f2b782949ea..d7ecefea436 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -165,12 +165,14 @@ export default class JobSerializer extends ApplicationSerializer { data: { id: alloc.ID, type: 'allocation', + // TODO: This is too much manual pushing! I should take attributes as they come in. attributes: { clientStatus: alloc.ClientStatus, deploymentStatus: { Healthy: alloc.DeploymentStatus.Healthy, Canary: alloc.DeploymentStatus.Canary, }, + jobVersion: alloc.JobVersion, nodeID: alloc.NodeID, }, }, diff --git a/ui/app/styles/components/job-status-panel.scss b/ui/app/styles/components/job-status-panel.scss index 917277ad595..e689ce16a66 100644 --- a/ui/app/styles/components/job-status-panel.scss +++ b/ui/app/styles/components/job-status-panel.scss @@ -411,6 +411,7 @@ align-items: center; gap: 1rem; max-width: 400px; + min-width: 200px; .alloc-status-summaries { height: 6px; gap: 6px; diff --git a/ui/app/styles/components/jobs-list.scss b/ui/app/styles/components/jobs-list.scss index 731f0c753e7..fd969074df4 100644 --- a/ui/app/styles/components/jobs-list.scss +++ b/ui/app/styles/components/jobs-list.scss @@ -42,3 +42,10 @@ justify-self: end; } } + +// TODO: make this a little cleaner. +.status-cell { + display: flex; + gap: 0.5rem; + align-items: center; +} diff --git a/ui/app/templates/components/job-row.hbs b/ui/app/templates/components/job-row.hbs index 3f062378b5d..fd11b3a8417 100644 --- a/ui/app/templates/components/job-row.hbs +++ b/ui/app/templates/components/job-row.hbs @@ -3,71 +3,118 @@ SPDX-License-Identifier: BUSL-1.1 ~}} - - - {{this.job.name}} - {{#if this.job.meta.structured.pack}} - - {{x-icon "box" class= "test"}} - Pack - + <@tableBody.Td data-test-job-name> + {{#if @job.assumeGC}} + {{@job.name}} + {{else}} + + {{@job.name}} + {{!-- TODO: going to lose .meta with statuses endpoint! --}} + {{#if @job.meta.structured.pack}} + + {{x-icon "box" class= "test"}} + Pack + + {{/if}} + {{/if}} + - - -{{#if (not (eq @context "child"))}} {{#if this.system.shouldShowNamespaces}} - - {{this.job.namespace.name}} - + <@tableBody.Td data-test-job-namespace>{{B.data.namespace.id}} {{/if}} -{{/if}} -{{#if (eq @context "child")}} - - {{format-month-ts this.job.submitTime}} - -{{/if}} - - - {{this.job.status}} - - -{{#if (not (eq @context "child"))}} - - {{this.job.displayType.type}} - - - {{#if this.job.nodePool}}{{this.job.nodePool}}{{else}}-{{/if}} - - - {{this.job.priority}} - -{{/if}} - -
- {{#if this.job.hasChildren}} - {{#if (gt this.job.totalChildren 0)}} - - {{else}} - - No Children - - {{/if}} - {{else}} - - {{/if}} -
- \ No newline at end of file + <@tableBody.Td data-test-job-status> + {{#unless @job.childStatuses}} +
+ + {{#if this.latestDeploymentFailed}} + + {{/if}} +
+ {{/unless}} + + <@tableBody.Td data-test-job-type={{B.data.type}}> + {{B.data.type}} + + + + {{#if this.system.shouldShowNodepools}} + <@tableBody.Td data-test-job-node-pool>{{@job.nodePool}} + {{/if}} + <@tableBody.Td data-test-job-priority> + {{@job.priority}} + + + <@tableBody.Td> + {{!-- {{get (filter-by 'clientStatus' 'running' @job.allocations) "length"}} running
+ {{@job.allocations.length}} total
+ {{@job.groupCountSum}} desired +
--}} +
+ {{#unless @job.assumeGC}} + {{#if @job.childStatuses}} + {{@job.childStatuses.length}} child jobs;
+ {{#each-in @job.childStatusBreakdown as |status count|}} + {{count}} {{status}}
+ {{/each-in}} + {{else}} +
+ + {{#if this.requiresPromotionTask.isRunning}} + Loading... + {{else if this.requiresPromotionTask.lastSuccessful.value}} + {{#if (eq this.requiresPromotionTask.lastSuccessful.value "canary-promote")}} + + {{else if (eq this.requiresPromotionTask.lastSuccessful.value "canary-failure")}} + + {{/if}} + {{/if}} +
+ {{/if}} + {{/unless}} +
+ + diff --git a/ui/app/templates/jobs/index.hbs b/ui/app/templates/jobs/index.hbs index 7fdffa1d944..629e09b7074 100644 --- a/ui/app/templates/jobs/index.hbs +++ b/ui/app/templates/jobs/index.hbs @@ -116,75 +116,7 @@ @valign="middle" > <:body as |B|> - {{!-- TODO: use --}} - - {{!-- {{#each this.tableColumns as |column|}} - {{get B.data (lowercase column.label)}} - {{/each}} --}} - - {{#if B.data.assumeGC}} - {{B.data.name}} - {{else}} - - {{B.data.name}} - {{!-- TODO: going to lose .meta with statuses endpoint! --}} - {{#if B.data.meta.structured.pack}} - - {{x-icon "box" class= "test"}} - Pack - - {{/if}} - - {{/if}} - - {{#if this.system.shouldShowNamespaces}} - {{B.data.namespace.id}} - {{/if}} - - {{#unless B.data.childStatuses}} - - {{/unless}} - - - {{B.data.type}} - - {{#if this.system.shouldShowNodepools}} - {{B.data.nodePool}} - {{/if}} - -
- {{#unless B.data.assumeGC}} - {{#if B.data.childStatuses}} - {{B.data.childStatuses.length}} child jobs;
- {{#each-in B.data.childStatusBreakdown as |status count|}} - {{count}} {{status}}
- {{/each-in}} - {{else}} - - {{/if}} - {{/unless}} -
- - + From 09b80ebf443772a14bbb1ab2166d663a3b78fbeb Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 1 May 2024 01:00:19 -0400 Subject: [PATCH 94/98] ActiveDeployment to LateestDeploymentSummary --- ui/app/components/job-row.js | 16 ++++++++++------ ui/app/models/job.js | 2 +- ui/app/templates/components/job-row.hbs | 16 +++++----------- ui/mirage/config.js | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 1c237f8583a..6eb88f48b6b 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -81,20 +81,24 @@ export default class JobRow extends Component { @task(function* () { console.log( 'checking if requries promotion', - this.args.job.activeDeploymentID, + this.args.job.name, + this.args.job.latestDeploymentSummary, this.args.job.hasActiveCanaries ); - if (!this.args.job.hasActiveCanaries || !this.args.job.activeDeploymentID) { + if ( + !this.args.job.hasActiveCanaries || + !this.args.job.latestDeploymentSummary?.IsActive + ) { return false; } if ( - !this.activeDeployment || - this.activeDeployment.id !== this.args.job.activeDeploymentID + !this.latestDeploymentSummary?.IsActive || + this.activeDeployment.id !== this.args.job?.latestDeploymentSummary.ID ) { this.activeDeployment = yield this.store.findRecord( 'deployment', - this.args.job.activeDeploymentID + this.args.job.latestDeploymentSummary.ID ); } @@ -142,7 +146,7 @@ export default class JobRow extends Component { * @type {import('../models/job').default} */ const job = this.args.job; - if (job.activeDeploymentID) { + if (job.latestDeploymentSummary?.IsActive) { return false; } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index dccaf554dc0..1aab59fe86a 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -151,7 +151,7 @@ export default class Job extends Model { get allocBlocks() { let availableSlotsToFill = this.expectedRunningAllocCount; - let isDeploying = this.latestDeploymentSummary.IsActive; + let isDeploying = this.latestDeploymentSummary?.IsActive; // Initialize allocationsOfShowableType with empty arrays for each clientStatus /** * @type {AllocationBlock} diff --git a/ui/app/templates/components/job-row.hbs b/ui/app/templates/components/job-row.hbs index fd11b3a8417..34696f1adde 100644 --- a/ui/app/templates/components/job-row.hbs +++ b/ui/app/templates/components/job-row.hbs @@ -9,6 +9,8 @@ action=(action "gotoJob" @job) }} {{on "click" (action this.gotoJob @job)}} + {{did-insert this.requiresPromotionTask.perform}} + {{did-update this.requiresPromotionTask.perform @job.activeDeploymentID @job.hasActiveCanaries}} class="job-row is-interactive {{if @job.assumeGC "assume-gc"}}" data-test-job-row={{@job.plainId}} data-test-modify-index={{@job.modifyIndex}} @@ -36,7 +38,7 @@ {{#if this.system.shouldShowNamespaces}} - <@tableBody.Td data-test-job-namespace>{{B.data.namespace.id}} + <@tableBody.Td data-test-job-namespace>{{@job.namespace.id}} {{/if}} <@tableBody.Td data-test-job-status> {{#unless @job.childStatuses}} @@ -58,23 +60,15 @@ {{/unless}} - <@tableBody.Td data-test-job-type={{B.data.type}}> - {{B.data.type}} + <@tableBody.Td data-test-job-type={{@job.type}}> + {{@job.type}} - {{#if this.system.shouldShowNodepools}} <@tableBody.Td data-test-job-node-pool>{{@job.nodePool}} {{/if}} - <@tableBody.Td data-test-job-priority> - {{@job.priority}} - <@tableBody.Td> - {{!-- {{get (filter-by 'clientStatus' 'running' @job.allocations) "length"}} running
- {{@job.allocations.length}} total
- {{@job.groupCountSum}} desired -
--}}
{{#unless @job.assumeGC}} {{#if @job.childStatuses}} diff --git a/ui/mirage/config.js b/ui/mirage/config.js index 4435ca40ec2..64977ba06ef 100644 --- a/ui/mirage/config.js +++ b/ui/mirage/config.js @@ -222,10 +222,24 @@ export default function () { }); }) .map((j) => { + let jobDeployments = server.db.deployments.where({ + jobId: j.ID, + namespace: j.Namespace, + }); let job = {}; job.ID = j.ID; job.Name = j.Name; job.ModifyIndex = j.ModifyIndex; + job.LatestDeployment = { + ID: jobDeployments[0]?.id, + IsActive: jobDeployments[0]?.status === 'running', + // IsActive: true, + JobVersion: jobDeployments[0]?.versionNumber, + Status: jobDeployments[0]?.status, + StatusDescription: jobDeployments[0]?.statusDescription, + AllAutoPromote: false, + RequiresPromotion: true, // TODO: lever + }; job.Allocs = server.db.allocations .where({ jobId: j.ID, namespace: j.Namespace }) .map((alloc) => { From 07a72cda1b1bb29b98ae1926435fbbaa690b56e0 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Wed, 1 May 2024 11:53:29 -0400 Subject: [PATCH 95/98] Reconfigured to use latestDeploymentSummary in job-row computed properties --- ui/app/components/job-row.js | 94 +++++++++++-------------- ui/app/models/job.js | 8 +-- ui/app/serializers/job.js | 16 ++++- ui/app/templates/components/job-row.hbs | 7 +- 4 files changed, 63 insertions(+), 62 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 6eb88f48b6b..45d6424e481 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -10,28 +10,29 @@ import { action } from '@ember/object'; import { inject as service } from '@ember/service'; import { tracked } from '@glimmer/tracking'; import { task } from 'ember-concurrency'; +import { computed } from '@ember/object'; export default class JobRow extends Component { @service router; @service store; @service system; - @tracked activeDeployment = null; + // @tracked fullActiveDeploymentObject = {}; // /** // * If our job has an activeDeploymentID, as determined by the statuses endpoint, - // * we check if this component's activeDeployment has the same ID. - // * If it does, we don't need to do any fetching: we can simply check this.activeDeployment.requiresPromotion + // * we check if this component's fullActiveDeploymentObject has the same ID. + // * If it does, we don't need to do any fetching: we can simply check this.fullActiveDeploymentObject.requiresPromotion // * If it doesn't, we need to fetch the deployment with the activeDeploymentID - // * and set it to this.activeDeployment, then check this.activeDeployment.requiresPromotion. + // * and set it to this.fullActiveDeploymentObject, then check this.fullActiveDeploymentObject.requiresPromotion. // */ // get requiresPromotion() { // if (!this.args.job.hasActiveCanaries || !this.args.job.activeDeploymentID) { // return false; // } - // if (this.activeDeployment && this.activeDeployment.id === this.args.job.activeDeploymentID) { - // return this.activeDeployment.requiresPromotion; + // if (this.fullActiveDeploymentObject && this.fullActiveDeploymentObject.id === this.args.job.activeDeploymentID) { + // return this.fullActiveDeploymentObject.requiresPromotion; // } // this.fetchActiveDeployment(); @@ -42,7 +43,7 @@ export default class JobRow extends Component { // async fetchActiveDeployment() { // if (this.args.job.hasActiveCanaries && this.args.job.activeDeploymentID) { // let deployment = await this.store.findRecord('deployment', this.args.job.activeDeploymentID); - // this.activeDeployment = deployment; + // this.fullActiveDeploymentObject = deployment; // } // } @@ -79,37 +80,54 @@ export default class JobRow extends Component { } @task(function* () { + // ID: jobDeployments[0]?.id, + // IsActive: jobDeployments[0]?.status === 'running', + // // IsActive: true, + // JobVersion: jobDeployments[0]?.versionNumber, + // Status: jobDeployments[0]?.status, + // StatusDescription: jobDeployments[0]?.statusDescription, + // AllAutoPromote: false, + // RequiresPromotion: true, // TODO: lever + + /** + * @typedef DeploymentSummary + * @property {string} id + * @property {boolean} isActive + * @property {string} jobVersion + * @property {string} status + * @property {string} statusDescription + * @property {boolean} allAutoPromote + * @property {boolean} requiresPromotion + */ + /** + * @type {DeploymentSummary} + */ + let latestDeploymentSummary = this.args.job.latestDeploymentSummary; + console.log( 'checking if requries promotion', this.args.job.name, - this.args.job.latestDeploymentSummary, + latestDeploymentSummary, this.args.job.hasActiveCanaries ); - if ( - !this.args.job.hasActiveCanaries || - !this.args.job.latestDeploymentSummary?.IsActive - ) { + // Early return false if we don't have an active deployment + if (latestDeploymentSummary.isActive) { return false; } - if ( - !this.latestDeploymentSummary?.IsActive || - this.activeDeployment.id !== this.args.job?.latestDeploymentSummary.ID - ) { - this.activeDeployment = yield this.store.findRecord( - 'deployment', - this.args.job.latestDeploymentSummary.ID - ); + // Early return if we our deployment doesn't have any canaries + if (!this.args.job.hasActiveCanaries) { + return false; } - if (this.activeDeployment.requiresPromotion) { + if (latestDeploymentSummary.requiresPromotion) { if (this.canariesHealthy) { return 'canary-promote'; } if (this.someCanariesHaveFailed) { return 'canary-failure'; } - if (this.activeDeployment.isAutoPromoted) { + if (latestDeploymentSummary.allAutoPromote) { // return "This deployment is set to auto-promote; canaries are being checked now"; return false; } else { @@ -135,38 +153,8 @@ export default class JobRow extends Component { }) promote; - /** - * If there is not a deployment happening, - * and the running allocations have a jobVersion that differs from the job's version, - * we can assume a failed latest deployment. - */ get latestDeploymentFailed() { - /** - * Import from app/models/job.js - * @type {import('../models/job').default} - */ - const job = this.args.job; - if (job.latestDeploymentSummary?.IsActive) { - return false; - } - - // We only want to show this status if the job is running, to indicate to - // the user that the job is not running the version they expect given their - // latest deployment. - if ( - !( - job.aggregateAllocStatus.label === 'Healthy' || - job.aggregateAllocStatus.label === 'Degraded' || - job.aggregateAllocStatus.label === 'Recovering' - ) - ) { - return false; - } - const runningAllocs = job.allocations.filter( - (a) => a.clientStatus === 'running' - ); - const jobVersion = job.version; - return runningAllocs.some((a) => a.jobVersion !== jobVersion); + return this.args.job.latestDeploymentSummary.status === 'failed'; } @action diff --git a/ui/app/models/job.js b/ui/app/models/job.js index 1aab59fe86a..ec5eec24ba8 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -52,13 +52,13 @@ export default class Job extends Model { * @property {boolean} AllAutoPromote - Whether all allocations were auto-promoted * @property {boolean} RequiresPromotion - Whether the deployment requires promotion */ - @attr() latestDeploymentSummary; + @attr({ defaultValue: () => ({}) }) latestDeploymentSummary; get hasActiveCanaries() { // console.log('tell me about ur active canaries plz', this.allocBlocks, this.allocations, this.activeDeploymentID); // TODO: Monday/Tuesday: go over AllocBlocks.{all}.canary and if there are any? make the latestDeployment lookup, // and check to see if it requires promotion / isnt yet promoted. - if (!this.latestDeploymentSummary?.IsActive) { + if (!this.latestDeploymentSummary.isActive) { return false; } return Object.keys(this.allocBlocks) @@ -151,7 +151,7 @@ export default class Job extends Model { get allocBlocks() { let availableSlotsToFill = this.expectedRunningAllocCount; - let isDeploying = this.latestDeploymentSummary?.IsActive; + let isDeploying = this.latestDeploymentSummary.isActive; // Initialize allocationsOfShowableType with empty arrays for each clientStatus /** * @type {AllocationBlock} @@ -337,7 +337,7 @@ export default class Job extends Model { let totalAllocs = this.expectedRunningAllocCount; // If deploying: - if (this.latestDeploymentSummary?.IsActive) { + if (this.latestDeploymentSummary.isActive) { return { label: 'Deploying', state: 'highlight' }; } diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index d7ecefea436..a0435641a8b 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -7,6 +7,7 @@ import { assign } from '@ember/polyfills'; import ApplicationSerializer from './application'; import queryString from 'query-string'; import classic from 'ember-classic-decorator'; +import { camelize } from '@ember/string'; @classic export default class JobSerializer extends ApplicationSerializer { @@ -120,8 +121,21 @@ export default class JobSerializer extends ApplicationSerializer { }; } if (job.LatestDeployment) { - job.LatestDeploymentSummary = job.LatestDeployment; + // camelize property names and save it as a non-conflicting name (latestDeployment is already used as a computed property on the job model) + job.LatestDeploymentSummary = Object.keys(job.LatestDeployment).reduce( + (acc, key) => { + if (key === 'ID') { + acc.id = job.LatestDeployment[key]; + } else { + acc[camelize(key)] = job.LatestDeployment[key]; + } + return acc; + }, + {} + ); delete job.LatestDeployment; + } else { + job.LatestDeploymentSummary = {}; } job._aggregate = true; }); diff --git a/ui/app/templates/components/job-row.hbs b/ui/app/templates/components/job-row.hbs index 34696f1adde..e4a217a53fc 100644 --- a/ui/app/templates/components/job-row.hbs +++ b/ui/app/templates/components/job-row.hbs @@ -10,8 +10,8 @@ }} {{on "click" (action this.gotoJob @job)}} {{did-insert this.requiresPromotionTask.perform}} - {{did-update this.requiresPromotionTask.perform @job.activeDeploymentID @job.hasActiveCanaries}} - class="job-row is-interactive {{if @job.assumeGC "assume-gc"}}" + {{did-update this.requiresPromotionTask.perform @job.latestDeploymentSummary.id @job.hasActiveCanaries}} + class="job-row {{if @job.assumeGC "assume-gc"}}" data-test-job-row={{@job.plainId}} data-test-modify-index={{@job.modifyIndex}} > @@ -82,7 +82,6 @@ @allocBlocks={{@job.allocBlocks}} @steady={{true}} @compact={{true}} - {{!-- @runningAllocs={{@job.runningAllocs}} --}} @runningAllocs={{@job.allocBlocks.running.healthy.nonCanary.length}} @groupCountSum={{@job.expectedRunningAllocCount}} /> @@ -104,7 +103,7 @@ {{hds-tooltip "Some canaries have failed; you will have to investigate" options=(hash placement="right")}} @name="alert-triangle" @color="#c84034" /> - {{/if}} + {{/if}} {{/if}}
{{/if}} From fb02dbea036058e836ca9c8abd613fbce1f787d2 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 2 May 2024 00:11:22 -0400 Subject: [PATCH 96/98] A couple findings about wontReschedule and health status noted. Work in progress. --- ui/app/components/job-row.js | 63 +++++++++++++++++++++---- ui/app/models/job.js | 19 ++++---- ui/app/templates/components/job-row.hbs | 3 +- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 45d6424e481..43895ca1c04 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -62,12 +62,43 @@ export default class JobRow extends Component { ); } - get someCanariesHaveFailed() { + /** + * Similar to the below, but cares if any non-old canaries have failed, regardless of their rescheduled status. + */ + get someCanariesHaveRescheduled() { + // TODO: Weird thing where alloc.isUnhealthy right away, because alloc.DeploymentStatus.Healthy is false. + // But that doesn't seem right: health check in that state should be unknown or null, perhaps. + const relevantAllocs = this.args.job.allocations.filter( + (a) => !a.isOld && a.isCanary + ); + console.log( + 'relevantAllocs', + relevantAllocs, + relevantAllocs.map((a) => a.jobVersion), + relevantAllocs.map((a) => a.clientStatus), + relevantAllocs.map((a) => a.isUnhealthy) + ); + + return relevantAllocs.some( + (a) => + a.clientStatus === 'failed' || + a.clientStatus === 'lost' || + a.isUnhealthy + ); + } + + get someCanariesHaveFailedAndWontReschedule() { + let availableSlotsToFill = this.args.job.expectedRunningAllocCount; + let runningOrPendingCanaries = this.args.job.allocations.filter( + (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled + ); const relevantAllocs = this.args.job.allocations.filter( (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled ); console.log( 'relevantAllocs', + relevantAllocs, + relevantAllocs.map((a) => a.jobVersion), relevantAllocs.map((a) => a.clientStatus), relevantAllocs.map((a) => a.isUnhealthy) ); @@ -104,33 +135,47 @@ export default class JobRow extends Component { */ let latestDeploymentSummary = this.args.job.latestDeploymentSummary; - console.log( - 'checking if requries promotion', - this.args.job.name, - latestDeploymentSummary, - this.args.job.hasActiveCanaries - ); + // console.log( + // 'checking if requries promotion', + // this.args.job.name, + // latestDeploymentSummary, + // this.args.job.hasActiveCanaries + // ); // Early return false if we don't have an active deployment - if (latestDeploymentSummary.isActive) { + if (!latestDeploymentSummary.isActive) { return false; } // Early return if we our deployment doesn't have any canaries if (!this.args.job.hasActiveCanaries) { + console.log('!hasActiveCan'); return false; } if (latestDeploymentSummary.requiresPromotion) { + console.log('requires promotion, and...'); if (this.canariesHealthy) { + console.log('canaries are healthy.'); return 'canary-promote'; } - if (this.someCanariesHaveFailed) { + // if (this.someCanariesHaveFailedAndWontReschedule) { + if (this.someCanariesHaveRescheduled) { + // TODO: I'm uncertain about when to alert the user here. It seems like it might be important + // enough to let them know when ANY canary has to be rescheduled, but there's an argument to be + // made that we oughtn't bother them until it's un-reschedulable. + console.log('some canaries have failed.'); return 'canary-failure'; } if (latestDeploymentSummary.allAutoPromote) { + console.log( + 'This deployment is set to auto-promote; canaries are being checked now' + ); // return "This deployment is set to auto-promote; canaries are being checked now"; return false; } else { + console.log( + 'This deployment requires manual promotion and things are being checked now' + ); // return "This deployment requires manual promotion and things are being checked now"; return false; } diff --git a/ui/app/models/job.js b/ui/app/models/job.js index ec5eec24ba8..06eae95950f 100644 --- a/ui/app/models/job.js +++ b/ui/app/models/job.js @@ -55,9 +55,6 @@ export default class Job extends Model { @attr({ defaultValue: () => ({}) }) latestDeploymentSummary; get hasActiveCanaries() { - // console.log('tell me about ur active canaries plz', this.allocBlocks, this.allocations, this.activeDeploymentID); - // TODO: Monday/Tuesday: go over AllocBlocks.{all}.canary and if there are any? make the latestDeployment lookup, - // and check to see if it requires promotion / isnt yet promoted. if (!this.latestDeploymentSummary.isActive) { return false; } @@ -71,13 +68,7 @@ export default class Job extends Model { }) .flat() .any((n) => !!n); - // return this.activeDeploymentID; } - // TODO: moved to job-row - // get requiresPromotion() { - // console.log('getting requiresPromotion', this.activeDeploymentID, this.runningDeployment); - // return this.runningDeployment; - // } @attr() childStatuses; @@ -203,6 +194,16 @@ export default class Job extends Model { if (allocationsOfShowableType[status]) { // If status is failed or lost, we only want to show it IF it's used up its restarts/rescheds. // Otherwise, we'd be showing an alloc that had been replaced. + + // TODO: We can't know about .willNotRestart and .willNotReschedule here, as we don't have access to alloc.followUpEvaluation. + // in deploying.js' newVersionAllocBlocks, we can know to ignore a canary if it has been rescheduled by virtue of seeing its .hasBeenRescheduled, + // which checks allocation.followUpEvaluation. This is not currently possible here. + // As such, we should count our running/pending canaries first, and if our expected count is still not filled, we can look to failed canaries. + // The goal of this is that any failed allocation that gets rescheduled would first have its place in relevantAllocs eaten up by a running/pending allocation, + // leaving it on the outside of what the user sees. + + // ^--- actually, scratch this. We should just get alloc.FollowupEvalID. If it's not null, we can assume it's been rescheduled. + if (alloc.willNotRestart) { if (!alloc.willNotReschedule) { // Dont count it diff --git a/ui/app/templates/components/job-row.hbs b/ui/app/templates/components/job-row.hbs index e4a217a53fc..1dc06efbc7f 100644 --- a/ui/app/templates/components/job-row.hbs +++ b/ui/app/templates/components/job-row.hbs @@ -81,7 +81,8 @@ From b4aa1fe40c9db6e0747149bd2d5c5fcda17aab54 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 2 May 2024 13:56:08 -0400 Subject: [PATCH 97/98] Small note on error handling --- ui/app/components/job-row.js | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 43895ca1c04..716e5fac3fe 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -186,14 +186,25 @@ export default class JobRow extends Component { @task(function* () { try { - yield this.args.job.latestDeployment.content.promote(); + yield this.args.job.latestDeployment.content.promote(); // TODO: need to do a deployment findRecord here first. // dont bubble up return false; } catch (err) { - this.handleError({ - title: 'Could Not Promote Deployment', - // description: messageFromAdapterError(err, 'promote deployments'), - }); + // TODO: handle error. add notifications. + console.log('caught error', err); + // this.handleError({ + // title: 'Could Not Promote Deployment', + // // description: messageFromAdapterError(err, 'promote deployments'), + // }); + + // err.errors.forEach((err) => { + // this.notifications.add({ + // title: "Could not promote deployment", + // message: err.detail, + // color: 'critical', + // timeout: 8000, + // }); + // }); } }) promote; From 7f48efaa13b9a0b19ab2519ff9280b01413ad370 Mon Sep 17 00:00:00 2001 From: Phil Renaud Date: Thu, 2 May 2024 16:32:27 -0400 Subject: [PATCH 98/98] Correctly using followUpEvaluation --- ui/app/components/job-row.js | 92 ++----------------------- ui/app/models/allocation.js | 6 +- ui/app/routes/jobs/index.js | 1 + ui/app/serializers/job.js | 9 ++- ui/app/templates/components/job-row.hbs | 3 +- 5 files changed, 19 insertions(+), 92 deletions(-) diff --git a/ui/app/components/job-row.js b/ui/app/components/job-row.js index 716e5fac3fe..6c5ba2e78c8 100644 --- a/ui/app/components/job-row.js +++ b/ui/app/components/job-row.js @@ -8,45 +8,13 @@ import Component from '@glimmer/component'; import { action } from '@ember/object'; import { inject as service } from '@ember/service'; -import { tracked } from '@glimmer/tracking'; import { task } from 'ember-concurrency'; -import { computed } from '@ember/object'; export default class JobRow extends Component { @service router; @service store; @service system; - // @tracked fullActiveDeploymentObject = {}; - - // /** - // * If our job has an activeDeploymentID, as determined by the statuses endpoint, - // * we check if this component's fullActiveDeploymentObject has the same ID. - // * If it does, we don't need to do any fetching: we can simply check this.fullActiveDeploymentObject.requiresPromotion - // * If it doesn't, we need to fetch the deployment with the activeDeploymentID - // * and set it to this.fullActiveDeploymentObject, then check this.fullActiveDeploymentObject.requiresPromotion. - // */ - // get requiresPromotion() { - // if (!this.args.job.hasActiveCanaries || !this.args.job.activeDeploymentID) { - // return false; - // } - - // if (this.fullActiveDeploymentObject && this.fullActiveDeploymentObject.id === this.args.job.activeDeploymentID) { - // return this.fullActiveDeploymentObject.requiresPromotion; - // } - - // this.fetchActiveDeployment(); - // return false; - // } - - // @action - // async fetchActiveDeployment() { - // if (this.args.job.hasActiveCanaries && this.args.job.activeDeploymentID) { - // let deployment = await this.store.findRecord('deployment', this.args.job.activeDeploymentID); - // this.fullActiveDeploymentObject = deployment; - // } - // } - /** * Promotion of a deployment will error if the canary allocations are not of status "Healthy"; * this function will check for that and disable the promote button if necessary. @@ -63,45 +31,14 @@ export default class JobRow extends Component { } /** - * Similar to the below, but cares if any non-old canaries have failed, regardless of their rescheduled status. + * Used to inform the user that an allocation has entered into a perment state of failure: + * That is, it has exhausted its restarts and its reschedules and is in a terminal state. */ - get someCanariesHaveRescheduled() { - // TODO: Weird thing where alloc.isUnhealthy right away, because alloc.DeploymentStatus.Healthy is false. - // But that doesn't seem right: health check in that state should be unknown or null, perhaps. - const relevantAllocs = this.args.job.allocations.filter( - (a) => !a.isOld && a.isCanary - ); - console.log( - 'relevantAllocs', - relevantAllocs, - relevantAllocs.map((a) => a.jobVersion), - relevantAllocs.map((a) => a.clientStatus), - relevantAllocs.map((a) => a.isUnhealthy) - ); - - return relevantAllocs.some( - (a) => - a.clientStatus === 'failed' || - a.clientStatus === 'lost' || - a.isUnhealthy - ); - } - get someCanariesHaveFailedAndWontReschedule() { - let availableSlotsToFill = this.args.job.expectedRunningAllocCount; - let runningOrPendingCanaries = this.args.job.allocations.filter( - (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled - ); const relevantAllocs = this.args.job.allocations.filter( (a) => !a.isOld && a.isCanary && !a.hasBeenRescheduled ); - console.log( - 'relevantAllocs', - relevantAllocs, - relevantAllocs.map((a) => a.jobVersion), - relevantAllocs.map((a) => a.clientStatus), - relevantAllocs.map((a) => a.isUnhealthy) - ); + return relevantAllocs.some( (a) => a.clientStatus === 'failed' || @@ -110,16 +47,8 @@ export default class JobRow extends Component { ); } + // eslint-disable-next-line require-yield @task(function* () { - // ID: jobDeployments[0]?.id, - // IsActive: jobDeployments[0]?.status === 'running', - // // IsActive: true, - // JobVersion: jobDeployments[0]?.versionNumber, - // Status: jobDeployments[0]?.status, - // StatusDescription: jobDeployments[0]?.statusDescription, - // AllAutoPromote: false, - // RequiresPromotion: true, // TODO: lever - /** * @typedef DeploymentSummary * @property {string} id @@ -135,12 +64,6 @@ export default class JobRow extends Component { */ let latestDeploymentSummary = this.args.job.latestDeploymentSummary; - // console.log( - // 'checking if requries promotion', - // this.args.job.name, - // latestDeploymentSummary, - // this.args.job.hasActiveCanaries - // ); // Early return false if we don't have an active deployment if (!latestDeploymentSummary.isActive) { return false; @@ -158,11 +81,8 @@ export default class JobRow extends Component { console.log('canaries are healthy.'); return 'canary-promote'; } - // if (this.someCanariesHaveFailedAndWontReschedule) { - if (this.someCanariesHaveRescheduled) { - // TODO: I'm uncertain about when to alert the user here. It seems like it might be important - // enough to let them know when ANY canary has to be rescheduled, but there's an argument to be - // made that we oughtn't bother them until it's un-reschedulable. + + if (this.someCanariesHaveFailedAndWontReschedule) { console.log('some canaries have failed.'); return 'canary-failure'; } diff --git a/ui/app/models/allocation.js b/ui/app/models/allocation.js index 4d83507a6ff..d1a0cec7f9d 100644 --- a/ui/app/models/allocation.js +++ b/ui/app/models/allocation.js @@ -71,12 +71,12 @@ export default class Allocation extends Model { return ( this.willNotRestart && !this.get('nextAllocation.content') && - !this.get('followUpEvaluation.content') + !this.belongsTo('followUpEvaluation').id() ); } get hasBeenRescheduled() { - return this.get('followUpEvaluation.content'); + return Boolean(this.belongsTo('followUpEvaluation').id()); } get hasBeenRestarted() { @@ -131,7 +131,7 @@ export default class Allocation extends Model { preemptedByAllocation; @attr('boolean') wasPreempted; - @belongsTo('evaluation') followUpEvaluation; + @belongsTo('evaluation', { async: true }) followUpEvaluation; @computed('clientStatus') get statusClass() { diff --git a/ui/app/routes/jobs/index.js b/ui/app/routes/jobs/index.js index 29152d5eadc..750dbe93dbb 100644 --- a/ui/app/routes/jobs/index.js +++ b/ui/app/routes/jobs/index.js @@ -111,6 +111,7 @@ export default class IndexRoute extends Route.extend( * @returns {Object} */ handleErrors(error) { + console.log('handling error', error); error.errors.forEach((err) => { this.notifications.add({ title: err.title, diff --git a/ui/app/serializers/job.js b/ui/app/serializers/job.js index a0435641a8b..6662d6e7811 100644 --- a/ui/app/serializers/job.js +++ b/ui/app/serializers/job.js @@ -179,7 +179,6 @@ export default class JobSerializer extends ApplicationSerializer { data: { id: alloc.ID, type: 'allocation', - // TODO: This is too much manual pushing! I should take attributes as they come in. attributes: { clientStatus: alloc.ClientStatus, deploymentStatus: { @@ -189,6 +188,14 @@ export default class JobSerializer extends ApplicationSerializer { jobVersion: alloc.JobVersion, nodeID: alloc.NodeID, }, + relationships: { + followUpEvaluation: alloc.FollowupEvalID && { + data: { + id: alloc.FollowupEvalID, + type: 'evaluation', + }, + }, + }, }, }); }); diff --git a/ui/app/templates/components/job-row.hbs b/ui/app/templates/components/job-row.hbs index 1dc06efbc7f..e4a217a53fc 100644 --- a/ui/app/templates/components/job-row.hbs +++ b/ui/app/templates/components/job-row.hbs @@ -81,8 +81,7 @@