From 929465b26bd60a8013f146e5d3cd97b8c5044f1b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Nov 2025 03:18:46 +0000 Subject: [PATCH 1/4] Initial plan From 0248506e4e3019cfcbe5a1b8ad3fefb0290a87ab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Nov 2025 03:38:51 +0000 Subject: [PATCH 2/4] feat: Add domain models, data sources, and JSON parsers Co-authored-by: Antvirf <26534322+Antvirf@users.noreply.github.com> --- internal/datasource/binary.go | 166 +++++++++++++++++++++++++ internal/datasource/file.go | 75 ++++++++++++ internal/datasource/interface.go | 30 +++++ internal/datasource/rest.go | 51 ++++++++ internal/domain/job.go | 66 ++++++++++ internal/domain/node.go | 73 +++++++++++ internal/domain/partition.go | 60 +++++++++ internal/parser/jobs.go | 202 +++++++++++++++++++++++++++++++ internal/parser/nodes.go | 194 +++++++++++++++++++++++++++++ internal/parser/partitions.go | 153 +++++++++++++++++++++++ 10 files changed, 1070 insertions(+) create mode 100644 internal/datasource/binary.go create mode 100644 internal/datasource/file.go create mode 100644 internal/datasource/interface.go create mode 100644 internal/datasource/rest.go create mode 100644 internal/domain/job.go create mode 100644 internal/domain/node.go create mode 100644 internal/domain/partition.go create mode 100644 internal/parser/jobs.go create mode 100644 internal/parser/nodes.go create mode 100644 internal/parser/partitions.go diff --git a/internal/datasource/binary.go b/internal/datasource/binary.go new file mode 100644 index 0000000..4f4a8fe --- /dev/null +++ b/internal/datasource/binary.go @@ -0,0 +1,166 @@ +package datasource + +import ( + "context" + "fmt" + "os/exec" + "path" + "strings" + "time" + + "github.com/antvirf/stui/internal/config" + "github.com/antvirf/stui/internal/logger" +) + +// BinaryJSONSource implements SlurmDataSource by calling Slurm binaries with --json flag +type BinaryJSONSource struct { + timeout time.Duration +} + +// NewBinaryJSONSource creates a new binary-based data source +func NewBinaryJSONSource(timeout time.Duration) *BinaryJSONSource { + return &BinaryJSONSource{ + timeout: timeout, + } +} + +// Name returns the name of this data source +func (b *BinaryJSONSource) Name() string { + return "BinaryJSON" +} + +// FetchJobsJSON fetches jobs using scontrol with JSON output +func (b *BinaryJSONSource) FetchJobsJSON(ctx context.Context) ([]byte, error) { + startTime := time.Now() + + cmdCtx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + + fullCommand := path.Join(config.SlurmBinariesPath, "scontrol") + " show job --detail --all --json" + cmd := b.execStringCommand(cmdCtx, fullCommand) + + rawOut, err := cmd.CombinedOutput() + execTime := time.Since(startTime).Milliseconds() + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + logger.Debugf("scontrol: timed out after %dms: %s", execTime, fullCommand) + return nil, fmt.Errorf("timeout after %v", b.timeout) + } + logger.Debugf("scontrol: failed after %dms: %s (%v)", execTime, fullCommand, err) + return nil, fmt.Errorf("scontrol failed: %v", err) + } + + logger.Debugf("scontrol: completed in %dms: %s", execTime, fullCommand) + return rawOut, nil +} + +// FetchNodesJSON fetches nodes using scontrol with JSON output +func (b *BinaryJSONSource) FetchNodesJSON(ctx context.Context) ([]byte, error) { + startTime := time.Now() + + cmdCtx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + + fullCommand := path.Join(config.SlurmBinariesPath, "scontrol") + " show node --detail --all --json" + cmd := b.execStringCommand(cmdCtx, fullCommand) + + rawOut, err := cmd.CombinedOutput() + execTime := time.Since(startTime).Milliseconds() + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + logger.Debugf("scontrol: timed out after %dms: %s", execTime, fullCommand) + return nil, fmt.Errorf("timeout after %v", b.timeout) + } + logger.Debugf("scontrol: failed after %dms: %s (%v)", execTime, fullCommand, err) + return nil, fmt.Errorf("scontrol failed: %v", err) + } + + logger.Debugf("scontrol: completed in %dms: %s", execTime, fullCommand) + return rawOut, nil +} + +// FetchPartitionsJSON fetches partitions using scontrol with JSON output +func (b *BinaryJSONSource) FetchPartitionsJSON(ctx context.Context) ([]byte, error) { + startTime := time.Now() + + cmdCtx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + + fullCommand := path.Join(config.SlurmBinariesPath, "scontrol") + " show partition --all --json" + cmd := b.execStringCommand(cmdCtx, fullCommand) + + rawOut, err := cmd.CombinedOutput() + execTime := time.Since(startTime).Milliseconds() + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + logger.Debugf("scontrol: timed out after %dms: %s", execTime, fullCommand) + return nil, fmt.Errorf("timeout after %v", b.timeout) + } + logger.Debugf("scontrol: failed after %dms: %s (%v)", execTime, fullCommand, err) + return nil, fmt.Errorf("scontrol failed: %v", err) + } + + logger.Debugf("scontrol: completed in %dms: %s", execTime, fullCommand) + return rawOut, nil +} + +// FetchJobDetailJSON fetches detailed job info using scontrol with JSON output +func (b *BinaryJSONSource) FetchJobDetailJSON(ctx context.Context, jobID string) ([]byte, error) { + startTime := time.Now() + + cmdCtx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + + fullCommand := fmt.Sprintf("%s show job %s --json", path.Join(config.SlurmBinariesPath, "scontrol"), jobID) + cmd := b.execStringCommand(cmdCtx, fullCommand) + + rawOut, err := cmd.CombinedOutput() + execTime := time.Since(startTime).Milliseconds() + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + logger.Debugf("scontrol: timed out after %dms: %s", execTime, fullCommand) + return nil, fmt.Errorf("timeout after %v", b.timeout) + } + logger.Debugf("scontrol: failed after %dms: %s (%v)", execTime, fullCommand, err) + return nil, fmt.Errorf("scontrol failed: %v", err) + } + + logger.Debugf("scontrol: completed in %dms: %s", execTime, fullCommand) + return rawOut, nil +} + +// FetchNodeDetailJSON fetches detailed node info using scontrol with JSON output +func (b *BinaryJSONSource) FetchNodeDetailJSON(ctx context.Context, nodeName string) ([]byte, error) { + startTime := time.Now() + + cmdCtx, cancel := context.WithTimeout(ctx, b.timeout) + defer cancel() + + fullCommand := fmt.Sprintf("%s show node %s --json", path.Join(config.SlurmBinariesPath, "scontrol"), nodeName) + cmd := b.execStringCommand(cmdCtx, fullCommand) + + rawOut, err := cmd.CombinedOutput() + execTime := time.Since(startTime).Milliseconds() + + if err != nil { + if cmdCtx.Err() == context.DeadlineExceeded { + logger.Debugf("scontrol: timed out after %dms: %s", execTime, fullCommand) + return nil, fmt.Errorf("timeout after %v", b.timeout) + } + logger.Debugf("scontrol: failed after %dms: %s (%v)", execTime, fullCommand, err) + return nil, fmt.Errorf("scontrol failed: %v", err) + } + + logger.Debugf("scontrol: completed in %dms: %s", execTime, fullCommand) + return rawOut, nil +} + +// execStringCommand is a helper to execute a command string +func (b *BinaryJSONSource) execStringCommand(ctx context.Context, cmd string) *exec.Cmd { + parts := strings.Split(cmd, " ") + return exec.CommandContext(ctx, parts[0], parts[1:]...) +} diff --git a/internal/datasource/file.go b/internal/datasource/file.go new file mode 100644 index 0000000..86efdc7 --- /dev/null +++ b/internal/datasource/file.go @@ -0,0 +1,75 @@ +package datasource + +import ( + "context" + "fmt" + "os" +) + +// FileSource implements SlurmDataSource by reading JSON from files +// Useful for testing and offline development +type FileSource struct { + jobsFile string + nodesFile string + partitionsFile string +} + +// NewFileSource creates a new file-based data source +func NewFileSource(jobsFile, nodesFile, partitionsFile string) *FileSource { + return &FileSource{ + jobsFile: jobsFile, + nodesFile: nodesFile, + partitionsFile: partitionsFile, + } +} + +// Name returns the name of this data source +func (f *FileSource) Name() string { + return "FileJSON" +} + +// FetchJobsJSON reads jobs JSON from file +func (f *FileSource) FetchJobsJSON(ctx context.Context) ([]byte, error) { + if f.jobsFile == "" { + return nil, fmt.Errorf("jobs file path not configured") + } + data, err := os.ReadFile(f.jobsFile) + if err != nil { + return nil, fmt.Errorf("failed to read jobs file: %w", err) + } + return data, nil +} + +// FetchNodesJSON reads nodes JSON from file +func (f *FileSource) FetchNodesJSON(ctx context.Context) ([]byte, error) { + if f.nodesFile == "" { + return nil, fmt.Errorf("nodes file path not configured") + } + data, err := os.ReadFile(f.nodesFile) + if err != nil { + return nil, fmt.Errorf("failed to read nodes file: %w", err) + } + return data, nil +} + +// FetchPartitionsJSON reads partitions JSON from file +func (f *FileSource) FetchPartitionsJSON(ctx context.Context) ([]byte, error) { + if f.partitionsFile == "" { + return nil, fmt.Errorf("partitions file path not configured") + } + data, err := os.ReadFile(f.partitionsFile) + if err != nil { + return nil, fmt.Errorf("failed to read partitions file: %w", err) + } + return data, nil +} + +// FetchJobDetailJSON reads job detail - for files, just returns same as FetchJobsJSON +func (f *FileSource) FetchJobDetailJSON(ctx context.Context, jobID string) ([]byte, error) { + return f.FetchJobsJSON(ctx) +} + +// FetchNodeDetailJSON reads node detail - for files, just returns same as FetchNodesJSON +func (f *FileSource) FetchNodeDetailJSON(ctx context.Context, nodeName string) ([]byte, error) { + return f.FetchNodesJSON(ctx) +} diff --git a/internal/datasource/interface.go b/internal/datasource/interface.go new file mode 100644 index 0000000..9ef5d08 --- /dev/null +++ b/internal/datasource/interface.go @@ -0,0 +1,30 @@ +package datasource + +import ( + "context" +) + +// SlurmDataSource is the interface for fetching JSON data from various sources +// This abstraction allows for: +// - Binary calls to Slurm commands (scontrol, sacct, etc.) with --json +// - REST API calls to slurmrestd +// - File-based sources for testing/offline mode +type SlurmDataSource interface { + // FetchJobsJSON retrieves jobs data as JSON + FetchJobsJSON(ctx context.Context) ([]byte, error) + + // FetchNodesJSON retrieves nodes data as JSON + FetchNodesJSON(ctx context.Context) ([]byte, error) + + // FetchPartitionsJSON retrieves partitions data as JSON + FetchPartitionsJSON(ctx context.Context) ([]byte, error) + + // FetchJobDetailJSON retrieves detailed info for a specific job as JSON + FetchJobDetailJSON(ctx context.Context, jobID string) ([]byte, error) + + // FetchNodeDetailJSON retrieves detailed info for a specific node as JSON + FetchNodeDetailJSON(ctx context.Context, nodeName string) ([]byte, error) + + // Name returns the name/type of this data source for logging + Name() string +} diff --git a/internal/datasource/rest.go b/internal/datasource/rest.go new file mode 100644 index 0000000..4fae50f --- /dev/null +++ b/internal/datasource/rest.go @@ -0,0 +1,51 @@ +package datasource + +import ( + "context" + "fmt" +) + +// RestAPISource implements SlurmDataSource by calling slurmrestd REST API +// This is a placeholder for future implementation +type RestAPISource struct { + baseURL string + token string +} + +// NewRestAPISource creates a new REST API-based data source +func NewRestAPISource(baseURL, token string) *RestAPISource { + return &RestAPISource{ + baseURL: baseURL, + token: token, + } +} + +// Name returns the name of this data source +func (r *RestAPISource) Name() string { + return "RestAPI" +} + +// FetchJobsJSON fetches jobs via slurmrestd REST API +func (r *RestAPISource) FetchJobsJSON(ctx context.Context) ([]byte, error) { + return nil, fmt.Errorf("RestAPISource not yet implemented") +} + +// FetchNodesJSON fetches nodes via slurmrestd REST API +func (r *RestAPISource) FetchNodesJSON(ctx context.Context) ([]byte, error) { + return nil, fmt.Errorf("RestAPISource not yet implemented") +} + +// FetchPartitionsJSON fetches partitions via slurmrestd REST API +func (r *RestAPISource) FetchPartitionsJSON(ctx context.Context) ([]byte, error) { + return nil, fmt.Errorf("RestAPISource not yet implemented") +} + +// FetchJobDetailJSON fetches job detail via slurmrestd REST API +func (r *RestAPISource) FetchJobDetailJSON(ctx context.Context, jobID string) ([]byte, error) { + return nil, fmt.Errorf("RestAPISource not yet implemented") +} + +// FetchNodeDetailJSON fetches node detail via slurmrestd REST API +func (r *RestAPISource) FetchNodeDetailJSON(ctx context.Context, nodeName string) ([]byte, error) { + return nil, fmt.Errorf("RestAPISource not yet implemented") +} diff --git a/internal/domain/job.go b/internal/domain/job.go new file mode 100644 index 0000000..e778fce --- /dev/null +++ b/internal/domain/job.go @@ -0,0 +1,66 @@ +package domain + +import "time" + +// Job represents a Slurm job in domain terms, independent of display or API format +type Job struct { + // Core identification + JobID string + ArrayJobID string + ArrayTaskID string + Name string + + // Ownership & accounting + User string + Account string + Partition string + QOS string + + // State & lifecycle + State string + StateReason string + ExitCode string + DerivedExitCode string + + // Time tracking + SubmitTime *time.Time + StartTime *time.Time + EndTime *time.Time + EligibleTime *time.Time + RunTime time.Duration + TimeLimit time.Duration + Deadline *time.Time + + // Resources + Nodes string + NodeCount int + CPUs int + Memory string + TRES string + UsedGRES string + + // Execution details + WorkingDir string + Command string + Script string + StdOut string + StdErr string + StdIn string + Comment string + + // Priority & constraints + Priority int + Constraints string + Features string + + // Misc + Licenses string + Cluster string + Reservation string + FailedNode string + Container string + Flags []string +} + +// Jobs is a collection of Job entities +type Jobs []Job diff --git a/internal/domain/node.go b/internal/domain/node.go new file mode 100644 index 0000000..d2a35e2 --- /dev/null +++ b/internal/domain/node.go @@ -0,0 +1,73 @@ +package domain + +import "time" + +// Node represents a Slurm compute node in domain terms +type Node struct { + // Core identification + Name string + Address string + Hostname string + Cluster string + + // State & status + State []string + StateReason string + ReasonSetByUser string + ReasonChangedAt *time.Time + NextStateAfterReboot []string + + // Hardware specs + Architecture string + CPUs int + EffectiveCPUs int + SpecializedCPUs string + Boards int + Cores int + SpecializedCores int + Sockets int + Threads int + + // Memory + RealMemory int64 + AllocMemory int64 + FreeMemory int64 + SpecializedMemory int64 + + // Resources + GRES string + GRESDrained string + GRESUsed string + GPUSpec string + + // Features & constraints + Features []string + ActiveFeatures []string + Extra string + + // Partitions & ownership + Partitions []string + Owner string + + // System info + OperatingSystem string + BootTime *time.Time + LastBusy *time.Time + CPULoad int + + // Network + Port int + BurstBufferNetworkAddress string + + // Cloud + InstanceID string + InstanceType string + + // Misc + Comment string + MCSLabel string + Weight int +} + +// Nodes is a collection of Node entities +type Nodes []Node diff --git a/internal/domain/partition.go b/internal/domain/partition.go new file mode 100644 index 0000000..7577c09 --- /dev/null +++ b/internal/domain/partition.go @@ -0,0 +1,60 @@ +package domain + +import "time" + +// Partition represents a Slurm partition in domain terms +type Partition struct { + // Core identification + Name string + Cluster string + + // State & flags + State string + Flags []string + + // Node allocation + Nodes string + NodeCount int + TotalNodes int + TotalCPUs int + AllocatedNodes int + AllocatedCPUs int + + // Time limits + DefaultTime time.Duration + MaxTime time.Duration + + // Priority & scheduling + Priority int + PriorityTier int + OverSubscribe string + Preempt string + GraceTime int + PreemptMode string + + // Resource limits + MaxCPUsPerNode int + MaxMemPerNode int64 + MaxMemPerCPU int64 + MinNodes int + MaxNodes int + DefaultCPUsPerNode int + DefaultMemPerNode int64 + + // Access control + AllowAccounts []string + DenyAccounts []string + AllowGroups []string + AllowQOS []string + DenyQOS []string + QOS string + + // Misc + TRES string + Billing string + Features string + MaxJobsAccrue int +} + +// Partitions is a collection of Partition entities +type Partitions []Partition diff --git a/internal/parser/jobs.go b/internal/parser/jobs.go new file mode 100644 index 0000000..6310c7d --- /dev/null +++ b/internal/parser/jobs.go @@ -0,0 +1,202 @@ +package parser + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/slurmapi" +) + +// ParseJobsJSON parses JSON from scontrol show job --json into domain.Jobs +func ParseJobsJSON(jsonData []byte) (domain.Jobs, error) { + var resp slurmapi.V0043OpenapiJobInfoResp + if err := json.Unmarshal(jsonData, &resp); err != nil { + return nil, fmt.Errorf("failed to unmarshal jobs JSON: %w", err) + } + + jobs := make(domain.Jobs, 0, len(resp.Jobs)) + for _, apiJob := range resp.Jobs { + job := convertJobInfoToDomain(&apiJob) + jobs = append(jobs, job) + } + + return jobs, nil +} + +// convertJobInfoToDomain converts slurmapi.V0043JobInfo to domain.Job +func convertJobInfoToDomain(apiJob *slurmapi.V0043JobInfo) domain.Job { + job := domain.Job{} + + // Core identification + if apiJob.JobId != nil { + job.JobID = strconv.Itoa(int(*apiJob.JobId)) + } + if apiJob.ArrayJobId != nil && apiJob.ArrayJobId.Number != nil { + job.ArrayJobID = strconv.Itoa(int(*apiJob.ArrayJobId.Number)) + } + if apiJob.ArrayTaskId != nil && apiJob.ArrayTaskId.Number != nil { + job.ArrayTaskID = strconv.Itoa(int(*apiJob.ArrayTaskId.Number)) + } + if apiJob.Name != nil { + job.Name = *apiJob.Name + } + + // Ownership & accounting + if apiJob.Account != nil { + job.Account = *apiJob.Account + } + if apiJob.UserId != nil { + job.User = strconv.Itoa(int(*apiJob.UserId)) + } + if apiJob.UserName != nil { + job.User = *apiJob.UserName + } + if apiJob.Partition != nil { + job.Partition = *apiJob.Partition + } + if apiJob.Qos != nil { + job.QOS = *apiJob.Qos + } + + // State & lifecycle + if apiJob.JobState != nil && len(*apiJob.JobState) > 0 { + job.State = (*apiJob.JobState)[0] + } + if apiJob.StateReason != nil { + job.StateReason = *apiJob.StateReason + } + if apiJob.ExitCode != nil { + job.ExitCode = formatExitCode(apiJob.ExitCode) + } + if apiJob.DerivedExitCode != nil { + job.DerivedExitCode = formatExitCode(apiJob.DerivedExitCode) + } + + // Time tracking + if apiJob.SubmitTime != nil && apiJob.SubmitTime.Number != nil { + t := time.Unix(int64(*apiJob.SubmitTime.Number), 0) + job.SubmitTime = &t + } + if apiJob.StartTime != nil && apiJob.StartTime.Number != nil { + t := time.Unix(int64(*apiJob.StartTime.Number), 0) + job.StartTime = &t + } + if apiJob.EndTime != nil && apiJob.EndTime.Number != nil { + t := time.Unix(int64(*apiJob.EndTime.Number), 0) + job.EndTime = &t + } + if apiJob.EligibleTime != nil && apiJob.EligibleTime.Number != nil { + t := time.Unix(int64(*apiJob.EligibleTime.Number), 0) + job.EligibleTime = &t + } + if apiJob.RunTime != nil && apiJob.RunTime.Number != nil { + job.RunTime = time.Duration(*apiJob.RunTime.Number) * time.Second + } + if apiJob.TimeLimit != nil && apiJob.TimeLimit.Number != nil { + job.TimeLimit = time.Duration(*apiJob.TimeLimit.Number) * time.Minute + } + if apiJob.Deadline != nil && apiJob.Deadline.Number != nil { + t := time.Unix(int64(*apiJob.Deadline.Number), 0) + job.Deadline = &t + } + + // Resources + if apiJob.Nodes != nil { + job.Nodes = *apiJob.Nodes + } + if apiJob.NodeCount != nil { + job.NodeCount = int(*apiJob.NodeCount) + } + if apiJob.Cpus != nil && apiJob.Cpus.Number != nil { + job.CPUs = int(*apiJob.Cpus.Number) + } + if apiJob.Memory != nil { + job.Memory = *apiJob.Memory + } + if apiJob.TresAllocStr != nil { + job.TRES = *apiJob.TresAllocStr + } + if apiJob.UsedGres != nil { + job.UsedGRES = *apiJob.UsedGres + } + + // Execution details + if apiJob.WorkDir != nil { + job.WorkingDir = *apiJob.WorkDir + } + if apiJob.Command != nil { + job.Command = *apiJob.Command + } + if apiJob.BatchScript != nil { + job.Script = *apiJob.BatchScript + } + if apiJob.StandardOutput != nil { + job.StdOut = *apiJob.StandardOutput + } + if apiJob.StandardError != nil { + job.StdErr = *apiJob.StandardError + } + if apiJob.StandardInput != nil { + job.StdIn = *apiJob.StandardInput + } + if apiJob.Comment != nil { + job.Comment = *apiJob.Comment + } + + // Priority & constraints + if apiJob.Priority != nil && apiJob.Priority.Number != nil { + job.Priority = int(*apiJob.Priority.Number) + } + if apiJob.Features != nil { + job.Constraints = *apiJob.Features + job.Features = *apiJob.Features + } + + // Misc + if apiJob.Licenses != nil { + job.Licenses = *apiJob.Licenses + } + if apiJob.Cluster != nil { + job.Cluster = *apiJob.Cluster + } + if apiJob.Reservation != nil { + job.Reservation = *apiJob.Reservation + } + if apiJob.FailedNode != nil { + job.FailedNode = *apiJob.FailedNode + } + if apiJob.Container != nil { + job.Container = *apiJob.Container + } + if apiJob.Flags != nil { + job.Flags = *apiJob.Flags + } + + return job +} + +// formatExitCode converts exit code struct to string representation +func formatExitCode(ec *slurmapi.V0043ProcessExitCodeVerbose) string { + if ec == nil { + return "" + } + if ec.Status != nil && len(*ec.Status) > 0 { + return (*ec.Status)[0] + } + if ec.ReturnCode != nil && ec.ReturnCode.Number != nil { + return strconv.Itoa(int(*ec.ReturnCode.Number)) + } + return "" +} + +// safeStringJoin safely joins string slices, handling nil pointers +func safeStringJoin(slice *[]string, sep string) string { + if slice == nil || len(*slice) == 0 { + return "" + } + return strings.Join(*slice, sep) +} diff --git a/internal/parser/nodes.go b/internal/parser/nodes.go new file mode 100644 index 0000000..24f6fe2 --- /dev/null +++ b/internal/parser/nodes.go @@ -0,0 +1,194 @@ +package parser + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/slurmapi" +) + +// ParseNodesJSON parses JSON from scontrol show node --json into domain.Nodes +func ParseNodesJSON(jsonData []byte) (domain.Nodes, error) { + var resp slurmapi.V0043OpenapiNodesResp + if err := json.Unmarshal(jsonData, &resp); err != nil { + return nil, fmt.Errorf("failed to unmarshal nodes JSON: %w", err) + } + + nodes := make(domain.Nodes, 0, len(resp.Nodes)) + for _, apiNode := range resp.Nodes { + node := convertNodeToDomain(&apiNode) + nodes = append(nodes, node) + } + + return nodes, nil +} + +// convertNodeToDomain converts slurmapi.V0043Node to domain.Node +func convertNodeToDomain(apiNode *slurmapi.V0043Node) domain.Node { + node := domain.Node{} + + // Core identification + if apiNode.Name != nil { + node.Name = *apiNode.Name + } + if apiNode.Address != nil { + node.Address = *apiNode.Address + } + if apiNode.Hostname != nil { + node.Hostname = *apiNode.Hostname + } + if apiNode.ClusterName != nil { + node.Cluster = *apiNode.ClusterName + } + + // State & status + if apiNode.State != nil { + node.State = *apiNode.State + } + if apiNode.Reason != nil { + node.StateReason = *apiNode.Reason + } + if apiNode.ReasonSetByUser != nil { + node.ReasonSetByUser = *apiNode.ReasonSetByUser + } + if apiNode.ReasonChangedAt != nil && apiNode.ReasonChangedAt.Number != nil { + t := time.Unix(int64(*apiNode.ReasonChangedAt.Number), 0) + node.ReasonChangedAt = &t + } + if apiNode.NextStateAfterReboot != nil { + node.NextStateAfterReboot = *apiNode.NextStateAfterReboot + } + + // Hardware specs + if apiNode.Architecture != nil { + node.Architecture = *apiNode.Architecture + } + if apiNode.Cpus != nil { + node.CPUs = int(*apiNode.Cpus) + } + if apiNode.EffectiveCpus != nil { + node.EffectiveCPUs = int(*apiNode.EffectiveCpus) + } + if apiNode.SpecializedCpus != nil { + node.SpecializedCPUs = *apiNode.SpecializedCpus + } + if apiNode.Boards != nil { + node.Boards = int(*apiNode.Boards) + } + if apiNode.Cores != nil { + node.Cores = int(*apiNode.Cores) + } + if apiNode.SpecializedCores != nil { + node.SpecializedCores = int(*apiNode.SpecializedCores) + } + if apiNode.Sockets != nil { + node.Sockets = int(*apiNode.Sockets) + } + if apiNode.Threads != nil { + node.Threads = int(*apiNode.Threads) + } + + // Memory + if apiNode.RealMemory != nil { + node.RealMemory = *apiNode.RealMemory + } + if apiNode.AllocMemory != nil { + node.AllocMemory = *apiNode.AllocMemory + } + if apiNode.FreeMem != nil && apiNode.FreeMem.Number != nil { + node.FreeMemory = int64(*apiNode.FreeMem.Number) + } + if apiNode.SpecializedMemory != nil { + node.SpecializedMemory = *apiNode.SpecializedMemory + } + + // Resources + if apiNode.Gres != nil { + node.GRES = *apiNode.Gres + } + if apiNode.GresDrained != nil { + node.GRESDrained = *apiNode.GresDrained + } + if apiNode.GresUsed != nil { + node.GRESUsed = *apiNode.GresUsed + } + if apiNode.GpuSpec != nil { + node.GPUSpec = *apiNode.GpuSpec + } + + // Features & constraints + if apiNode.Features != nil { + node.Features = *apiNode.Features + } + if apiNode.ActiveFeatures != nil { + node.ActiveFeatures = *apiNode.ActiveFeatures + } + if apiNode.Extra != nil { + node.Extra = *apiNode.Extra + } + + // Partitions & ownership + if apiNode.Partitions != nil { + node.Partitions = *apiNode.Partitions + } + if apiNode.Owner != nil { + node.Owner = *apiNode.Owner + } + + // System info + if apiNode.OperatingSystem != nil { + node.OperatingSystem = *apiNode.OperatingSystem + } + if apiNode.BootTime != nil && apiNode.BootTime.Number != nil { + t := time.Unix(int64(*apiNode.BootTime.Number), 0) + node.BootTime = &t + } + if apiNode.LastBusy != nil && apiNode.LastBusy.Number != nil { + t := time.Unix(int64(*apiNode.LastBusy.Number), 0) + node.LastBusy = &t + } + if apiNode.CpuLoad != nil { + node.CPULoad = int(*apiNode.CpuLoad) + } + + // Network + if apiNode.Port != nil { + node.Port = int(*apiNode.Port) + } + if apiNode.BurstbufferNetworkAddress != nil { + node.BurstBufferNetworkAddress = *apiNode.BurstbufferNetworkAddress + } + + // Cloud + if apiNode.InstanceId != nil { + node.InstanceID = *apiNode.InstanceId + } + if apiNode.InstanceType != nil { + node.InstanceType = *apiNode.InstanceType + } + + // Misc + if apiNode.Comment != nil { + node.Comment = *apiNode.Comment + } + if apiNode.McsLabel != nil { + node.MCSLabel = *apiNode.McsLabel + } + if apiNode.Weight != nil { + node.Weight = int(*apiNode.Weight) + } + + return node +} + +// formatNodeState converts node state array to string +func formatNodeState(states *[]string) string { + if states == nil || len(*states) == 0 { + return "" + } + return strings.Join(*states, "+") +} diff --git a/internal/parser/partitions.go b/internal/parser/partitions.go new file mode 100644 index 0000000..52de100 --- /dev/null +++ b/internal/parser/partitions.go @@ -0,0 +1,153 @@ +package parser + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/slurmapi" +) + +// ParsePartitionsJSON parses JSON from scontrol show partition --json into domain.Partitions +func ParsePartitionsJSON(jsonData []byte) (domain.Partitions, error) { + var resp slurmapi.V0043OpenapiPartitionResp + if err := json.Unmarshal(jsonData, &resp); err != nil { + return nil, fmt.Errorf("failed to unmarshal partitions JSON: %w", err) + } + + partitions := make(domain.Partitions, 0, len(resp.Partitions)) + for _, apiPartition := range resp.Partitions { + partition := convertPartitionToDomain(&apiPartition) + partitions = append(partitions, partition) + } + + return partitions, nil +} + +// convertPartitionToDomain converts slurmapi.V0043PartitionInfo to domain.Partition +func convertPartitionToDomain(apiPart *slurmapi.V0043PartitionInfo) domain.Partition { + part := domain.Partition{} + + // Core identification + if apiPart.Name != nil { + part.Name = *apiPart.Name + } + if apiPart.Cluster != nil { + part.Cluster = *apiPart.Cluster + } + + // State & flags + if apiPart.State != nil && len(*apiPart.State) > 0 { + part.State = (*apiPart.State)[0] + } + if apiPart.Flags != nil { + part.Flags = *apiPart.Flags + } + + // Node allocation + if apiPart.Nodes != nil && apiPart.Nodes.Configured != nil { + part.Nodes = *apiPart.Nodes.Configured + } + if apiPart.NodeSets != nil { + part.NodeCount = len(*apiPart.NodeSets) + } + if apiPart.Nodes != nil && apiPart.Nodes.Total != nil { + part.TotalNodes = int(*apiPart.Nodes.Total) + } + if apiPart.Cpus != nil && apiPart.Cpus.Total != nil { + part.TotalCPUs = int(*apiPart.Cpus.Total) + } + + // Time limits + if apiPart.Timeouts != nil { + if apiPart.Timeouts.DefaultTime != nil && apiPart.Timeouts.DefaultTime.Number != nil { + part.DefaultTime = time.Duration(*apiPart.Timeouts.DefaultTime.Number) * time.Minute + } + if apiPart.Timeouts.MaximumTime != nil && apiPart.Timeouts.MaximumTime.Number != nil { + part.MaxTime = time.Duration(*apiPart.Timeouts.MaximumTime.Number) * time.Minute + } + } + + // Priority & scheduling + if apiPart.Priority != nil && apiPart.Priority.JobFactor != nil { + part.Priority = int(*apiPart.Priority.JobFactor) + } + if apiPart.Priority != nil && apiPart.Priority.Tier != nil { + part.PriorityTier = int(*apiPart.Priority.Tier) + } + if apiPart.Oversubscribe != nil && len(*apiPart.Oversubscribe) > 0 { + part.OverSubscribe = (*apiPart.Oversubscribe)[0] + } + if apiPart.Preemption != nil { + if apiPart.Preemption.Type != nil && len(*apiPart.Preemption.Type) > 0 { + part.Preempt = (*apiPart.Preemption.Type)[0] + } + if apiPart.Preemption.GraceTime != nil { + part.GraceTime = int(*apiPart.Preemption.GraceTime) + } + } + + // Resource limits + if apiPart.Maximums != nil { + if apiPart.Maximums.CpusPerNode != nil && apiPart.Maximums.CpusPerNode.Number != nil { + part.MaxCPUsPerNode = int(*apiPart.Maximums.CpusPerNode.Number) + } + if apiPart.Maximums.MemoryPerNode != nil { + part.MaxMemPerNode = *apiPart.Maximums.MemoryPerNode + } + if apiPart.Maximums.MemoryPerCpu != nil { + part.MaxMemPerCPU = *apiPart.Maximums.MemoryPerCpu + } + if apiPart.Maximums.Nodes != nil && apiPart.Maximums.Nodes.Number != nil { + part.MaxNodes = int(*apiPart.Maximums.Nodes.Number) + } + } + if apiPart.Minimums != nil { + if apiPart.Minimums.Nodes != nil { + part.MinNodes = int(*apiPart.Minimums.Nodes) + } + } + if apiPart.Defaults != nil { + if apiPart.Defaults.CpusPerNode != nil && apiPart.Defaults.CpusPerNode.Number != nil { + part.DefaultCPUsPerNode = int(*apiPart.Defaults.CpusPerNode.Number) + } + if apiPart.Defaults.MemoryPerNode != nil { + part.DefaultMemPerNode = *apiPart.Defaults.MemoryPerNode + } + } + + // Access control + if apiPart.Accounts != nil { + if apiPart.Accounts.Allow != nil { + part.AllowAccounts = *apiPart.Accounts.Allow + } + if apiPart.Accounts.Deny != nil { + part.DenyAccounts = *apiPart.Accounts.Deny + } + } + if apiPart.Groups != nil && apiPart.Groups.Allowed != nil { + part.AllowGroups = *apiPart.Groups.Allowed + } + if apiPart.Qos != nil { + if apiPart.Qos.Allowed != nil { + part.AllowQOS = *apiPart.Qos.Allowed + } + if apiPart.Qos.Denied != nil { + part.DenyQOS = *apiPart.Qos.Denied + } + if apiPart.Qos.Assigned != nil { + part.QOS = *apiPart.Qos.Assigned + } + } + + // Misc + if apiPart.TresBillingWeights != nil { + part.Billing = *apiPart.TresBillingWeights + } + if apiPart.AllowedAllocatingNodes != nil { + part.Features = *apiPart.AllowedAllocatingNodes + } + + return part +} From 0fd3c43d79e2a4ecb46c60698b50ecc8319edb66 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Nov 2025 03:46:45 +0000 Subject: [PATCH 3/4] feat: Implement adapters and V2 providers with JSON-based architecture Co-authored-by: Antvirf <26534322+Antvirf@users.noreply.github.com> --- internal/model/adapters.go | 317 +++++++++++++++++++++++ internal/model/provider_jobs_v2.go | 113 ++++++++ internal/model/provider_nodes_v2.go | 113 ++++++++ internal/model/provider_partitions_v2.go | 108 ++++++++ internal/parser/jobs.go | 31 +-- internal/parser/nodes.go | 1 - internal/parser/partitions.go | 98 +++---- 7 files changed, 699 insertions(+), 82 deletions(-) create mode 100644 internal/model/adapters.go create mode 100644 internal/model/provider_jobs_v2.go create mode 100644 internal/model/provider_nodes_v2.go create mode 100644 internal/model/provider_partitions_v2.go diff --git a/internal/model/adapters.go b/internal/model/adapters.go new file mode 100644 index 0000000..84c0b3e --- /dev/null +++ b/internal/model/adapters.go @@ -0,0 +1,317 @@ +package model + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/antvirf/stui/internal/config" + "github.com/antvirf/stui/internal/domain" +) + +// JobsToTableData converts domain.Jobs to TableData for display +func JobsToTableData(jobs domain.Jobs, columns *[]config.ColumnConfig) *TableData { + rows := make([][]string, 0, len(jobs)) + + for _, job := range jobs { + row := make([]string, len(*columns)) + + for j, col := range *columns { + row[j] = extractJobField(job, col.DisplayName) + } + + rows = append(rows, row) + } + + return &TableData{ + Headers: columns, + Rows: rows, + RowsAsSingleStrings: convertRowsToRowsAsSingleStrings(rows), + } +} + +// extractJobField extracts a field from domain.Job based on field name +func extractJobField(job domain.Job, fieldName string) string { + switch fieldName { + case "JobID": + return job.JobID + case "ArrayJobId": + return job.ArrayJobID + case "ArrayTaskId": + return job.ArrayTaskID + case "Name", "JobName": + return job.Name + case "UserId", "User", "UserName": + return job.User + case "Account": + return job.Account + case "Partition": + return job.Partition + case "QOS": + return job.QOS + case "JobState", "State": + return job.State + case "StateReason", "Reason": + return job.StateReason + case "ExitCode": + return job.ExitCode + case "DerivedExitCode": + return job.DerivedExitCode + case "SubmitTime": + if job.SubmitTime != nil { + return job.SubmitTime.Format("2006-01-02T15:04:05") + } + return "" + case "StartTime": + if job.StartTime != nil { + return job.StartTime.Format("2006-01-02T15:04:05") + } + return "" + case "EndTime": + if job.EndTime != nil { + return job.EndTime.Format("2006-01-02T15:04:05") + } + return "" + case "EligibleTime": + if job.EligibleTime != nil { + return job.EligibleTime.Format("2006-01-02T15:04:05") + } + return "" + case "RunTime": + return formatDuration(job.RunTime) + case "TimeLimit": + return formatDuration(job.TimeLimit) + case "Nodes", "NodeList": + return job.Nodes + case "NodeCount", "NumNodes": + return strconv.Itoa(job.NodeCount) + case "Cpus", "NumCPUs": + return strconv.Itoa(job.CPUs) + case "Memory": + return job.Memory + case "TresAllocStr", "TRES": + return job.TRES + case "UsedGres", "Gres": + return job.UsedGRES + case "WorkDir", "WorkingDirectory": + return job.WorkingDir + case "Command": + return job.Command + case "StdOut", "StandardOutput": + return job.StdOut + case "StdErr", "StandardError": + return job.StdErr + case "StdIn", "StandardInput": + return job.StdIn + case "Comment": + return job.Comment + case "Priority": + return strconv.Itoa(job.Priority) + case "Features", "Constraints": + return job.Constraints + case "Licenses": + return job.Licenses + case "Cluster": + return job.Cluster + case "Reservation": + return job.Reservation + case "FailedNode": + return job.FailedNode + case "Container": + return job.Container + default: + return "" + } +} + +// NodesToTableData converts domain.Nodes to TableData for display +func NodesToTableData(nodes domain.Nodes, columns *[]config.ColumnConfig) *TableData { + rows := make([][]string, 0, len(nodes)) + + for _, node := range nodes { + row := make([]string, len(*columns)) + + for j, col := range *columns { + row[j] = extractNodeField(node, col.DisplayName) + } + + rows = append(rows, row) + } + + return &TableData{ + Headers: columns, + Rows: rows, + RowsAsSingleStrings: convertRowsToRowsAsSingleStrings(rows), + } +} + +// extractNodeField extracts a field from domain.Node based on field name +func extractNodeField(node domain.Node, fieldName string) string { + switch fieldName { + case "NodeName", "Name": + return node.Name + case "NodeAddr", "Address": + return node.Address + case "NodeHostName", "Hostname": + return node.Hostname + case "Cluster": + return node.Cluster + case "State": + return strings.Join(node.State, "+") + case "StateReason", "Reason": + return node.StateReason + case "ReasonSetByUser": + return node.ReasonSetByUser + case "Architecture", "Arch": + return node.Architecture + case "CPUs", "Cpus": + return strconv.Itoa(node.CPUs) + case "EffectiveCPUs": + return strconv.Itoa(node.EffectiveCPUs) + case "SpecializedCPUs": + return node.SpecializedCPUs + case "Boards": + return strconv.Itoa(node.Boards) + case "Cores": + return strconv.Itoa(node.Cores) + case "SpecializedCores": + return strconv.Itoa(node.SpecializedCores) + case "Sockets": + return strconv.Itoa(node.Sockets) + case "Threads": + return strconv.Itoa(node.Threads) + case "RealMemory", "Memory": + return strconv.FormatInt(node.RealMemory, 10) + case "AllocMem": + return strconv.FormatInt(node.AllocMemory, 10) + case "FreeMem": + return strconv.FormatInt(node.FreeMemory, 10) + case "SpecializedMemory": + return strconv.FormatInt(node.SpecializedMemory, 10) + case "Gres", "GRES": + return node.GRES + case "GRESDrained": + return node.GRESDrained + case "GRESUsed": + return node.GRESUsed + case "GPUSpec": + return node.GPUSpec + case "Features", "AvailableFeatures": + return strings.Join(node.Features, ",") + case "ActiveFeatures": + return strings.Join(node.ActiveFeatures, ",") + case "Partitions": + return strings.Join(node.Partitions, ",") + case "Owner": + return node.Owner + case "OS", "OperatingSystem": + return node.OperatingSystem + case "BootTime": + if node.BootTime != nil { + return node.BootTime.Format("2006-01-02T15:04:05") + } + return "" + case "CPULoad": + return strconv.Itoa(node.CPULoad) + case "Port": + return strconv.Itoa(node.Port) + case "Comment": + return node.Comment + case "Weight": + return strconv.Itoa(node.Weight) + default: + return "" + } +} + +// PartitionsToTableData converts domain.Partitions to TableData for display +func PartitionsToTableData(partitions domain.Partitions, columns *[]config.ColumnConfig) *TableData { + rows := make([][]string, 0, len(partitions)) + + for _, partition := range partitions { + row := make([]string, len(*columns)) + + for j, col := range *columns { + row[j] = extractPartitionField(partition, col.DisplayName) + } + + rows = append(rows, row) + } + + return &TableData{ + Headers: columns, + Rows: rows, + RowsAsSingleStrings: convertRowsToRowsAsSingleStrings(rows), + } +} + +// extractPartitionField extracts a field from domain.Partition based on field name +func extractPartitionField(partition domain.Partition, fieldName string) string { + switch fieldName { + case "PartitionName", "Name": + return partition.Name + case "Cluster": + return partition.Cluster + case "State": + return partition.State + case "Nodes": + return partition.Nodes + case "TotalNodes": + return strconv.Itoa(partition.TotalNodes) + case "TotalCPUs": + return strconv.Itoa(partition.TotalCPUs) + case "DefaultTime": + return formatDuration(partition.DefaultTime) + case "MaxTime": + return formatDuration(partition.MaxTime) + case "Priority": + return strconv.Itoa(partition.Priority) + case "PriorityTier": + return strconv.Itoa(partition.PriorityTier) + case "OverSubscribe": + return partition.OverSubscribe + case "Preempt": + return partition.Preempt + case "GraceTime": + return strconv.Itoa(partition.GraceTime) + case "MaxCPUsPerNode": + return strconv.Itoa(partition.MaxCPUsPerNode) + case "MaxMemPerNode": + return strconv.FormatInt(partition.MaxMemPerNode, 10) + case "MinNodes": + return strconv.Itoa(partition.MinNodes) + case "MaxNodes": + return strconv.Itoa(partition.MaxNodes) + case "AllowAccounts": + return strings.Join(partition.AllowAccounts, ",") + case "DenyAccounts": + return strings.Join(partition.DenyAccounts, ",") + case "AllowGroups": + return strings.Join(partition.AllowGroups, ",") + case "AllowQOS": + return strings.Join(partition.AllowQOS, ",") + case "QOS": + return partition.QOS + case "TRES": + return partition.TRES + default: + return "" + } +} + +// formatDuration formats a duration for display +func formatDuration(d time.Duration) string { + if d == 0 { + return "" + } + + hours := int(d.Hours()) + minutes := int(d.Minutes()) % 60 + seconds := int(d.Seconds()) % 60 + + if hours > 0 { + return fmt.Sprintf("%d:%02d:%02d", hours, minutes, seconds) + } + return fmt.Sprintf("%d:%02d", minutes, seconds) +} diff --git a/internal/model/provider_jobs_v2.go b/internal/model/provider_jobs_v2.go new file mode 100644 index 0000000..d8e1fad --- /dev/null +++ b/internal/model/provider_jobs_v2.go @@ -0,0 +1,113 @@ +package model + +import ( + "context" + + "github.com/antvirf/stui/internal/config" + "github.com/antvirf/stui/internal/datasource" + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/parser" +) + +// JobsProviderV2 is the new provider using domain models and data sources +type JobsProviderV2 struct { + BaseProvider[*TableData] + dataSource datasource.SlurmDataSource + domainJobs domain.Jobs +} + +// NewJobsProviderV2 creates a new jobs provider using the new architecture +func NewJobsProviderV2(dataSource datasource.SlurmDataSource) *JobsProviderV2 { + p := &JobsProviderV2{ + BaseProvider: BaseProvider[*TableData]{ + data: EmptyTableData(), + }, + dataSource: dataSource, + } + p.Fetch() + return p +} + +// Fetch retrieves jobs using the data source and parser +func (p *JobsProviderV2) Fetch() error { + FetchCounter.increment() + + // Compute column widths on first fetch only + computeColumnWidths := p.lastUpdated.IsZero() + + // Step 1: Get JSON from data source + ctx := context.Background() + jsonData, err := p.dataSource.FetchJobsJSON(ctx) + if err != nil { + p.updateError(err) + return err + } + + // Step 2: Parse JSON into domain models + jobs, err := parser.ParseJobsJSON(jsonData) + if err != nil { + p.updateError(err) + return err + } + + // Store domain models for potential future use + p.domainJobs = jobs + + // Step 3: Convert domain models to TableData for display + tableData := JobsToTableData(jobs, config.JobViewColumns) + + // Compute column widths if needed + if computeColumnWidths { + p.computeColumnWidths(tableData) + } + + p.updateData(tableData) + return nil +} + +// FilteredData applies filters to the table data +func (p *JobsProviderV2) FilteredData() *TableData { + p.mu.RLock() + defer p.mu.RUnlock() + return p.data.ApplyFilters( + map[int]string{ + config.JobsViewColumnsStateIndex: config.JobStateCurrentChoice, + config.JobsViewColumnsPartitionIndex: config.PartitionFilter, + }, + ) +} + +// GetDomainJobs returns the domain models directly (for advanced use cases) +func (p *JobsProviderV2) GetDomainJobs() domain.Jobs { + p.mu.RLock() + defer p.mu.RUnlock() + return p.domainJobs +} + +// computeColumnWidths updates column widths based on content +func (p *JobsProviderV2) computeColumnWidths(tableData *TableData) { + if tableData.Headers == nil { + return + } + + for i := range *tableData.Headers { + col := &(*tableData.Headers)[i] + maxWidth := len(col.DisplayName) + + for _, row := range tableData.Rows { + if i < len(row) { + cellWidth := len(row[i]) + if cellWidth > maxWidth { + maxWidth = cellWidth + } + } + } + + // Cap at maximum column width + if maxWidth > config.MaximumColumnWidth { + maxWidth = config.MaximumColumnWidth + } + + col.Width = maxWidth + } +} diff --git a/internal/model/provider_nodes_v2.go b/internal/model/provider_nodes_v2.go new file mode 100644 index 0000000..e17647c --- /dev/null +++ b/internal/model/provider_nodes_v2.go @@ -0,0 +1,113 @@ +package model + +import ( + "context" + + "github.com/antvirf/stui/internal/config" + "github.com/antvirf/stui/internal/datasource" + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/parser" +) + +// NodesProviderV2 is the new provider using domain models and data sources +type NodesProviderV2 struct { + BaseProvider[*TableData] + dataSource datasource.SlurmDataSource + domainNodes domain.Nodes +} + +// NewNodesProviderV2 creates a new nodes provider using the new architecture +func NewNodesProviderV2(dataSource datasource.SlurmDataSource) *NodesProviderV2 { + p := &NodesProviderV2{ + BaseProvider: BaseProvider[*TableData]{ + data: EmptyTableData(), + }, + dataSource: dataSource, + } + p.Fetch() + return p +} + +// Fetch retrieves nodes using the data source and parser +func (p *NodesProviderV2) Fetch() error { + FetchCounter.increment() + + // Compute column widths on first fetch only + computeColumnWidths := p.lastUpdated.IsZero() + + // Step 1: Get JSON from data source + ctx := context.Background() + jsonData, err := p.dataSource.FetchNodesJSON(ctx) + if err != nil { + p.updateError(err) + return err + } + + // Step 2: Parse JSON into domain models + nodes, err := parser.ParseNodesJSON(jsonData) + if err != nil { + p.updateError(err) + return err + } + + // Store domain models for potential future use + p.domainNodes = nodes + + // Step 3: Convert domain models to TableData for display + tableData := NodesToTableData(nodes, config.NodeViewColumns) + + // Compute column widths if needed + if computeColumnWidths { + p.computeColumnWidths(tableData) + } + + p.updateData(tableData) + return nil +} + +// FilteredData applies filters to the table data +func (p *NodesProviderV2) FilteredData() *TableData { + p.mu.RLock() + defer p.mu.RUnlock() + return p.data.ApplyFilters( + map[int]string{ + config.NodeViewColumnsStateIndex: config.NodeStateCurrentChoice, + config.NodeViewColumnsPartitionIndex: config.PartitionFilter, + }, + ) +} + +// GetDomainNodes returns the domain models directly (for advanced use cases) +func (p *NodesProviderV2) GetDomainNodes() domain.Nodes { + p.mu.RLock() + defer p.mu.RUnlock() + return p.domainNodes +} + +// computeColumnWidths updates column widths based on content +func (p *NodesProviderV2) computeColumnWidths(tableData *TableData) { + if tableData.Headers == nil { + return + } + + for i := range *tableData.Headers { + col := &(*tableData.Headers)[i] + maxWidth := len(col.DisplayName) + + for _, row := range tableData.Rows { + if i < len(row) { + cellWidth := len(row[i]) + if cellWidth > maxWidth { + maxWidth = cellWidth + } + } + } + + // Cap at maximum column width + if maxWidth > config.MaximumColumnWidth { + maxWidth = config.MaximumColumnWidth + } + + col.Width = maxWidth + } +} diff --git a/internal/model/provider_partitions_v2.go b/internal/model/provider_partitions_v2.go new file mode 100644 index 0000000..15713cc --- /dev/null +++ b/internal/model/provider_partitions_v2.go @@ -0,0 +1,108 @@ +package model + +import ( + "context" + + "github.com/antvirf/stui/internal/config" + "github.com/antvirf/stui/internal/datasource" + "github.com/antvirf/stui/internal/domain" + "github.com/antvirf/stui/internal/parser" +) + +// PartitionsProviderV2 is the new provider using domain models and data sources +type PartitionsProviderV2 struct { + BaseProvider[*TableData] + dataSource datasource.SlurmDataSource + domainPartitions domain.Partitions +} + +// NewPartitionsProviderV2 creates a new partitions provider using the new architecture +func NewPartitionsProviderV2(dataSource datasource.SlurmDataSource) *PartitionsProviderV2 { + p := &PartitionsProviderV2{ + BaseProvider: BaseProvider[*TableData]{ + data: EmptyTableData(), + }, + dataSource: dataSource, + } + p.Fetch() + return p +} + +// Fetch retrieves partitions using the data source and parser +func (p *PartitionsProviderV2) Fetch() error { + FetchCounter.increment() + + // Compute column widths on first fetch only + computeColumnWidths := p.lastUpdated.IsZero() + + // Step 1: Get JSON from data source + ctx := context.Background() + jsonData, err := p.dataSource.FetchPartitionsJSON(ctx) + if err != nil { + p.updateError(err) + return err + } + + // Step 2: Parse JSON into domain models + partitions, err := parser.ParsePartitionsJSON(jsonData) + if err != nil { + p.updateError(err) + return err + } + + // Store domain models for potential future use + p.domainPartitions = partitions + + // Step 3: Convert domain models to TableData for display + // Use a simple column config for partitions + columns := &[]config.ColumnConfig{{RawName: "PartitionName", DisplayName: "PartitionName"}} + tableData := PartitionsToTableData(partitions, columns) + + // Compute column widths if needed + if computeColumnWidths { + p.computeColumnWidths(tableData) + } + + p.updateData(tableData) + return nil +} + +// FilteredData returns the data (partitions typically don't have filters) +func (p *PartitionsProviderV2) FilteredData() *TableData { + return p.Data() +} + +// GetDomainPartitions returns the domain models directly (for advanced use cases) +func (p *PartitionsProviderV2) GetDomainPartitions() domain.Partitions { + p.mu.RLock() + defer p.mu.RUnlock() + return p.domainPartitions +} + +// computeColumnWidths updates column widths based on content +func (p *PartitionsProviderV2) computeColumnWidths(tableData *TableData) { + if tableData.Headers == nil { + return + } + + for i := range *tableData.Headers { + col := &(*tableData.Headers)[i] + maxWidth := len(col.DisplayName) + + for _, row := range tableData.Rows { + if i < len(row) { + cellWidth := len(row[i]) + if cellWidth > maxWidth { + maxWidth = cellWidth + } + } + } + + // Cap at maximum column width + if maxWidth > config.MaximumColumnWidth { + maxWidth = config.MaximumColumnWidth + } + + col.Width = maxWidth + } +} diff --git a/internal/parser/jobs.go b/internal/parser/jobs.go index 6310c7d..781f283 100644 --- a/internal/parser/jobs.go +++ b/internal/parser/jobs.go @@ -93,8 +93,9 @@ func convertJobInfoToDomain(apiJob *slurmapi.V0043JobInfo) domain.Job { t := time.Unix(int64(*apiJob.EligibleTime.Number), 0) job.EligibleTime = &t } - if apiJob.RunTime != nil && apiJob.RunTime.Number != nil { - job.RunTime = time.Duration(*apiJob.RunTime.Number) * time.Second + // RunTime is calculated, not a direct field + if job.StartTime != nil && job.EndTime != nil { + job.RunTime = job.EndTime.Sub(*job.StartTime) } if apiJob.TimeLimit != nil && apiJob.TimeLimit.Number != nil { job.TimeLimit = time.Duration(*apiJob.TimeLimit.Number) * time.Minute @@ -108,32 +109,32 @@ func convertJobInfoToDomain(apiJob *slurmapi.V0043JobInfo) domain.Job { if apiJob.Nodes != nil { job.Nodes = *apiJob.Nodes } - if apiJob.NodeCount != nil { - job.NodeCount = int(*apiJob.NodeCount) + if apiJob.NodeCount != nil && apiJob.NodeCount.Number != nil { + job.NodeCount = int(*apiJob.NodeCount.Number) } if apiJob.Cpus != nil && apiJob.Cpus.Number != nil { job.CPUs = int(*apiJob.Cpus.Number) } - if apiJob.Memory != nil { - job.Memory = *apiJob.Memory + if apiJob.MemoryPerNode != nil && apiJob.MemoryPerNode.Number != nil { + job.Memory = strconv.FormatInt(*apiJob.MemoryPerNode.Number, 10) + } else if apiJob.MemoryPerCpu != nil && apiJob.MemoryPerCpu.Number != nil { + job.Memory = strconv.FormatInt(*apiJob.MemoryPerCpu.Number, 10) } if apiJob.TresAllocStr != nil { job.TRES = *apiJob.TresAllocStr } - if apiJob.UsedGres != nil { - job.UsedGRES = *apiJob.UsedGres + if apiJob.GresDetail != nil && len(*apiJob.GresDetail) > 0 { + job.UsedGRES = strings.Join(*apiJob.GresDetail, ",") } // Execution details - if apiJob.WorkDir != nil { - job.WorkingDir = *apiJob.WorkDir + if apiJob.CurrentWorkingDirectory != nil { + job.WorkingDir = *apiJob.CurrentWorkingDirectory } if apiJob.Command != nil { job.Command = *apiJob.Command } - if apiJob.BatchScript != nil { - job.Script = *apiJob.BatchScript - } + // BatchScript is not available in JobInfo, would need separate call if apiJob.StandardOutput != nil { job.StdOut = *apiJob.StandardOutput } @@ -163,8 +164,8 @@ func convertJobInfoToDomain(apiJob *slurmapi.V0043JobInfo) domain.Job { if apiJob.Cluster != nil { job.Cluster = *apiJob.Cluster } - if apiJob.Reservation != nil { - job.Reservation = *apiJob.Reservation + if apiJob.ResvName != nil { + job.Reservation = *apiJob.ResvName } if apiJob.FailedNode != nil { job.FailedNode = *apiJob.FailedNode diff --git a/internal/parser/nodes.go b/internal/parser/nodes.go index 24f6fe2..43932ec 100644 --- a/internal/parser/nodes.go +++ b/internal/parser/nodes.go @@ -3,7 +3,6 @@ package parser import ( "encoding/json" "fmt" - "strconv" "strings" "time" diff --git a/internal/parser/partitions.go b/internal/parser/partitions.go index 52de100..6bbe106 100644 --- a/internal/parser/partitions.go +++ b/internal/parser/partitions.go @@ -3,7 +3,7 @@ package parser import ( "encoding/json" "fmt" - "time" + "strings" "github.com/antvirf/stui/internal/domain" "github.com/antvirf/stui/internal/slurmapi" @@ -25,7 +25,7 @@ func ParsePartitionsJSON(jsonData []byte) (domain.Partitions, error) { return partitions, nil } -// convertPartitionToDomain converts slurmapi.V0043PartitionInfo to domain.Partition +// convertPartitionToDomain converts slurmapi.V0043PartitionInfo to domain.Partition func convertPartitionToDomain(apiPart *slurmapi.V0043PartitionInfo) domain.Partition { part := domain.Partition{} @@ -37,21 +37,15 @@ func convertPartitionToDomain(apiPart *slurmapi.V0043PartitionInfo) domain.Parti part.Cluster = *apiPart.Cluster } - // State & flags - if apiPart.State != nil && len(*apiPart.State) > 0 { - part.State = (*apiPart.State)[0] - } - if apiPart.Flags != nil { - part.Flags = *apiPart.Flags + // State + if apiPart.Partition != nil && apiPart.Partition.State != nil && len(*apiPart.Partition.State) > 0 { + part.State = (*apiPart.Partition.State)[0] } - // Node allocation + // Nodes if apiPart.Nodes != nil && apiPart.Nodes.Configured != nil { part.Nodes = *apiPart.Nodes.Configured } - if apiPart.NodeSets != nil { - part.NodeCount = len(*apiPart.NodeSets) - } if apiPart.Nodes != nil && apiPart.Nodes.Total != nil { part.TotalNodes = int(*apiPart.Nodes.Total) } @@ -59,43 +53,24 @@ func convertPartitionToDomain(apiPart *slurmapi.V0043PartitionInfo) domain.Parti part.TotalCPUs = int(*apiPart.Cpus.Total) } - // Time limits - if apiPart.Timeouts != nil { - if apiPart.Timeouts.DefaultTime != nil && apiPart.Timeouts.DefaultTime.Number != nil { - part.DefaultTime = time.Duration(*apiPart.Timeouts.DefaultTime.Number) * time.Minute - } - if apiPart.Timeouts.MaximumTime != nil && apiPart.Timeouts.MaximumTime.Number != nil { - part.MaxTime = time.Duration(*apiPart.Timeouts.MaximumTime.Number) * time.Minute - } - } - - // Priority & scheduling + // Priority if apiPart.Priority != nil && apiPart.Priority.JobFactor != nil { part.Priority = int(*apiPart.Priority.JobFactor) } if apiPart.Priority != nil && apiPart.Priority.Tier != nil { part.PriorityTier = int(*apiPart.Priority.Tier) } - if apiPart.Oversubscribe != nil && len(*apiPart.Oversubscribe) > 0 { - part.OverSubscribe = (*apiPart.Oversubscribe)[0] - } - if apiPart.Preemption != nil { - if apiPart.Preemption.Type != nil && len(*apiPart.Preemption.Type) > 0 { - part.Preempt = (*apiPart.Preemption.Type)[0] - } - if apiPart.Preemption.GraceTime != nil { - part.GraceTime = int(*apiPart.Preemption.GraceTime) - } + + // Grace time + if apiPart.GraceTime != nil { + part.GraceTime = int(*apiPart.GraceTime) } - // Resource limits + // Limits if apiPart.Maximums != nil { if apiPart.Maximums.CpusPerNode != nil && apiPart.Maximums.CpusPerNode.Number != nil { part.MaxCPUsPerNode = int(*apiPart.Maximums.CpusPerNode.Number) } - if apiPart.Maximums.MemoryPerNode != nil { - part.MaxMemPerNode = *apiPart.Maximums.MemoryPerNode - } if apiPart.Maximums.MemoryPerCpu != nil { part.MaxMemPerCPU = *apiPart.Maximums.MemoryPerCpu } @@ -103,50 +78,41 @@ func convertPartitionToDomain(apiPart *slurmapi.V0043PartitionInfo) domain.Parti part.MaxNodes = int(*apiPart.Maximums.Nodes.Number) } } - if apiPart.Minimums != nil { - if apiPart.Minimums.Nodes != nil { - part.MinNodes = int(*apiPart.Minimums.Nodes) - } - } - if apiPart.Defaults != nil { - if apiPart.Defaults.CpusPerNode != nil && apiPart.Defaults.CpusPerNode.Number != nil { - part.DefaultCPUsPerNode = int(*apiPart.Defaults.CpusPerNode.Number) - } - if apiPart.Defaults.MemoryPerNode != nil { - part.DefaultMemPerNode = *apiPart.Defaults.MemoryPerNode - } + if apiPart.Minimums != nil && apiPart.Minimums.Nodes != nil { + part.MinNodes = int(*apiPart.Minimums.Nodes) } - // Access control + // Accounts - these are comma-separated strings, not arrays if apiPart.Accounts != nil { - if apiPart.Accounts.Allow != nil { - part.AllowAccounts = *apiPart.Accounts.Allow + if apiPart.Accounts.Allowed != nil && *apiPart.Accounts.Allowed != "" { + part.AllowAccounts = strings.Split(*apiPart.Accounts.Allowed, ",") } - if apiPart.Accounts.Deny != nil { - part.DenyAccounts = *apiPart.Accounts.Deny + if apiPart.Accounts.Deny != nil && *apiPart.Accounts.Deny != "" { + part.DenyAccounts = strings.Split(*apiPart.Accounts.Deny, ",") } } - if apiPart.Groups != nil && apiPart.Groups.Allowed != nil { - part.AllowGroups = *apiPart.Groups.Allowed + + // Groups - also comma-separated string + if apiPart.Groups != nil && apiPart.Groups.Allowed != nil && *apiPart.Groups.Allowed != "" { + part.AllowGroups = strings.Split(*apiPart.Groups.Allowed, ",") } + + // QOS - same pattern if apiPart.Qos != nil { - if apiPart.Qos.Allowed != nil { - part.AllowQOS = *apiPart.Qos.Allowed + if apiPart.Qos.Allowed != nil && *apiPart.Qos.Allowed != "" { + part.AllowQOS = strings.Split(*apiPart.Qos.Allowed, ",") } - if apiPart.Qos.Denied != nil { - part.DenyQOS = *apiPart.Qos.Denied + if apiPart.Qos.Deny != nil && *apiPart.Qos.Deny != "" { + part.DenyQOS = strings.Split(*apiPart.Qos.Deny, ",") } if apiPart.Qos.Assigned != nil { part.QOS = *apiPart.Qos.Assigned } } - // Misc - if apiPart.TresBillingWeights != nil { - part.Billing = *apiPart.TresBillingWeights - } - if apiPart.AllowedAllocatingNodes != nil { - part.Features = *apiPart.AllowedAllocatingNodes + // TRES billing + if apiPart.Tres != nil && apiPart.Tres.BillingWeights != nil { + part.Billing = *apiPart.Tres.BillingWeights } return part From 16b1336685f0f7bb88b21718fc355a016cde6a06 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Nov 2025 03:47:47 +0000 Subject: [PATCH 4/4] docs: Add architecture documentation for JSON migration Co-authored-by: Antvirf <26534322+Antvirf@users.noreply.github.com> --- docs/architecture-json-migration.md | 216 ++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 docs/architecture-json-migration.md diff --git a/docs/architecture-json-migration.md b/docs/architecture-json-migration.md new file mode 100644 index 0000000..76edfc6 --- /dev/null +++ b/docs/architecture-json-migration.md @@ -0,0 +1,216 @@ +# Approach 2 Implementation: Domain Model First Architecture + +## Overview + +This document explains the implementation of Approach 2 for migrating from string parsing to JSON-based Slurm command parsing. + +## Architecture + +### Layer 1: Data Sources (`internal/datasource/`) + +The `SlurmDataSource` interface abstracts where JSON comes from: + +```go +type SlurmDataSource interface { + FetchJobsJSON(ctx context.Context) ([]byte, error) + FetchNodesJSON(ctx context.Context) ([]byte, error) + FetchPartitionsJSON(ctx context.Context) ([]byte, error) + // ... detail methods +} +``` + +**Implementations:** +1. **BinaryJSONSource** - Calls Slurm binaries with `--json` flag + - `scontrol show job --json` + - `scontrol show node --json` + - `scontrol show partition --json` + +2. **FileSource** - Reads JSON from files (testing/offline) + +3. **RestAPISource** - Placeholder for future slurmrestd support + +### Layer 2: JSON Parsers (`internal/parser/`) + +Parsers convert raw JSON (using generated OpenAPI models) into domain models: + +- `ParseJobsJSON()` - Unmarshals `V0043OpenapiJobInfoResp` → `domain.Jobs` +- `ParseNodesJSON()` - Unmarshals `V0043OpenapiNodesResp` → `domain.Nodes` +- `ParsePartitionsJSON()` - Unmarshals `V0043OpenapiPartitionResp` → `domain.Partitions` + +### Layer 3: Domain Models (`internal/domain/`) + +Clean, business-focused data structures independent of: +- Display format (tables, JSON, etc.) +- Data source (binary, REST, file) +- API version + +```go +type Job struct { + JobID string + Name string + User string + State string + Partition string + // ... 40+ fields +} +``` + +### Layer 4: Adapters (`internal/model/adapters.go`) + +Convert domain models to TableData for the existing UI: + +- `JobsToTableData(jobs, columns)` → `*TableData` +- `NodesToTableData(nodes, columns)` → `*TableData` +- `PartitionsToTableData(partitions, columns)` → `*TableData` + +### Layer 5: Providers V2 (`internal/model/provider_*_v2.go`) + +New providers demonstrating the pattern: + +```go +func (p *JobsProviderV2) Fetch() error { + // 1. Get JSON from data source + jsonData, err := p.dataSource.FetchJobsJSON(ctx) + + // 2. Parse into domain models + jobs, err := parser.ParseJobsJSON(jsonData) + + // 3. Convert to TableData for display + tableData := JobsToTableData(jobs, config.JobViewColumns) + + // 4. Store both formats + p.domainJobs = jobs // For advanced use + p.updateData(tableData) // For existing UI +} +``` + +## Benefits + +### 1. Clear Separation of Concerns +- **Data retrieval** → DataSource interface +- **Parsing** → Parser functions +- **Business logic** → Domain models +- **Presentation** → Adapters + UI + +### 2. Multiple Data Sources +Easy to add new sources: +```go +// Current: Binary calls +source := datasource.NewBinaryJSONSource(timeout) + +// Future: REST API +source := datasource.NewRestAPISource(url, token) + +// Testing: Files +source := datasource.NewFileSource("jobs.json", "nodes.json", "partitions.json") +``` + +### 3. API Version Independence +- Generated OpenAPI models handle JSON structure +- Domain models shield business logic from API changes +- Adapters handle field mapping variations + +### 4. Testability +Each layer can be tested independently: +- Mock data sources for parsers +- Mock parsers for providers +- Fixed domain models for adapters + +### 5. Future-Proof +- slurmrestd support: Just implement `RestAPISource` +- New Slurm versions: Update OpenAPI models, adjust parsers +- New features: Extend domain models, update adapters + +## Usage Example + +```go +// Create data source +dataSource := datasource.NewBinaryJSONSource(config.RequestTimeout) + +// Create provider +jobsProvider := model.NewJobsProviderV2(dataSource) + +// Use as before +tableData := jobsProvider.FilteredData() + +// OR access domain models directly +domainJobs := jobsProvider.GetDomainJobs() +for _, job := range domainJobs { + fmt.Printf("Job %s: %s\n", job.JobID, job.State) +} +``` + +## Migration Path + +### Phase 1: ✅ Complete +- Domain models defined +- Data source interface + implementations +- JSON parsers implemented +- Adapters created +- V2 providers as examples + +### Phase 2: Future +- Add unit tests for all layers +- Create integration tests with mock data +- Gradually migrate V1 providers to use new architecture +- Maintain backward compatibility during transition + +### Phase 3: Future +- Implement RestAPISource for slurmrestd +- Add configuration option to choose data source +- Performance optimization +- Full migration complete + +## Files Created + +``` +internal/ +├── domain/ +│ ├── job.go # Job domain model +│ ├── node.go # Node domain model +│ └── partition.go # Partition domain model +├── datasource/ +│ ├── interface.go # SlurmDataSource interface +│ ├── binary.go # Binary JSON source +│ ├── file.go # File source +│ └── rest.go # REST API source (placeholder) +├── parser/ +│ ├── jobs.go # Job JSON parser +│ ├── nodes.go # Node JSON parser +│ └── partitions.go # Partition JSON parser +└── model/ + ├── adapters.go # Domain → TableData converters + ├── provider_jobs_v2.go # Jobs provider example + ├── provider_nodes_v2.go # Nodes provider example + └── provider_partitions_v2.go # Partitions provider example +``` + +## Key Design Decisions + +1. **Why separate parsers from data sources?** + - Data sources focus on retrieval (network, file I/O) + - Parsers focus on transformation (JSON → domain) + - Enables testing each independently + +2. **Why domain models AND TableData?** + - Domain models: Business logic, API-independent + - TableData: Display format, existing UI compatibility + - Adapters bridge the gap + +3. **Why V2 providers instead of modifying V1?** + - Demonstrates pattern without breaking existing code + - Allows gradual migration + - V1 and V2 can coexist during transition + +4. **Why use generated OpenAPI models?** + - Type safety when unmarshaling JSON + - Auto-update when Slurm API changes + - Comprehensive field coverage + +## Next Steps + +1. Add comprehensive unit tests +2. Create integration tests with sample JSON files +3. Add documentation for developers +4. Consider performance benchmarks (string vs JSON parsing) +5. Plan V1 → V2 migration strategy