From 8d8ebe305990d69568cc3a72b401a18254bba3bb Mon Sep 17 00:00:00 2001 From: jwillp Date: Wed, 21 Aug 2024 03:31:50 -0400 Subject: [PATCH] Rename Outputs to Artefacts --- .gitignore | 2 +- README.md | 4 +- artefactregistry.go | 107 ++++++++++++++ artefactregistry_test.go | 303 +++++++++++++++++++++++++++++++++++++++ depresolve.go | 10 +- depresolve_test.go | 14 +- linting.go | 12 +- linting_test.go | 30 ++-- logging.go | 2 +- logging_test.go | 2 +- mkdirartfproc.go | 56 ++++++++ mkdiroutproc.go | 56 -------- outputregistry.go | 108 -------------- outputregistry_test.go | 303 --------------------------------------- processing.go | 95 ++++++------ processing_test.go | 96 ++++++------- specter.go | 102 ++++++------- specter_test.go | 8 +- writefileartfproc.go | 97 +++++++++++++ writefileoutproc.go | 97 ------------- 20 files changed, 755 insertions(+), 749 deletions(-) create mode 100644 artefactregistry.go create mode 100644 artefactregistry_test.go create mode 100644 mkdirartfproc.go delete mode 100644 mkdiroutproc.go delete mode 100644 outputregistry.go delete mode 100644 outputregistry_test.go create mode 100644 writefileartfproc.go delete mode 100644 writefileoutproc.go diff --git a/.gitignore b/.gitignore index bc56e74..526afe4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ # Test binary, built with `go test -c` *.test -# Output of the go coverage tool, specifically when used with LiteIDE +# Artifact of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) diff --git a/README.md b/README.md index c59d7d8..7736138 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Specter is a development toolkit in Go that allows you to develop configuration file processors based on HashiCorp Configuration Language (HCL). With Specter, you can define your own Domain-Specific Language (DSL) -using HCL and create a processing pipeline to validate, lint, resolve dependencies, and generate code or output +using HCL and create a processing pipeline to validate, lint, resolve dependencies, and generate code or artifact files from these DSL configuration files. ## Features @@ -12,7 +12,7 @@ files from these DSL configuration files. - Develop your own DSL using HCL - Validate and lint configuration files - Resolve dependencies between configuration files -- Generate code or output files from configuration files +- Generate code or artifact files from configuration files ## Getting Started diff --git a/artefactregistry.go b/artefactregistry.go new file mode 100644 index 0000000..5e25b00 --- /dev/null +++ b/artefactregistry.go @@ -0,0 +1,107 @@ +package specter + +import ( + "encoding/json" + "github.com/morebec/go-errors/errors" + "os" + "sync" + "time" +) + +// JSONArtifactRegistry implementation of a ArtifactRegistry that is saved as a JSON file. +type JSONArtifactRegistry struct { + GeneratedAt time.Time `json:"generatedAt"` + ArtifactMap map[string]*JSONArtifactRegistryProcessor `json:"files"` + FilePath string + mu sync.RWMutex // Mutex to protect concurrent access +} + +type JSONArtifactRegistryProcessor struct { + Artifacts []string `json:"files"` +} + +// NewJSONArtifactRegistry returns a new artifact file registry. +func NewJSONArtifactRegistry(fileName string) *JSONArtifactRegistry { + return &JSONArtifactRegistry{ + GeneratedAt: time.Now(), + ArtifactMap: nil, + FilePath: fileName, + } +} + +func (r *JSONArtifactRegistry) Load() error { + r.mu.Lock() + defer r.mu.Unlock() + + bytes, err := os.ReadFile(r.FilePath) + + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WrapWithMessage(err, errors.InternalErrorCode, "failed loading artifact file registry") + } + if err := json.Unmarshal(bytes, r); err != nil { + return errors.WrapWithMessage(err, errors.InternalErrorCode, "failed loading artifact file registry") + } + + return nil +} + +func (r *JSONArtifactRegistry) Save() error { + r.mu.RLock() + defer r.mu.RUnlock() + + if r.ArtifactMap == nil { + return nil + } + // Generate a JSON file containing all artifact files for clean up later on + js, err := json.MarshalIndent(r, "", " ") + if err != nil { + return errors.Wrap(err, "failed generating artifact file registry") + } + if err := os.WriteFile(r.FilePath, js, os.ModePerm); err != nil { + return errors.Wrap(err, "failed generating artifact file registry") + } + + return nil +} + +func (r *JSONArtifactRegistry) AddArtifact(processorName string, artifactName string) { + r.mu.Lock() + defer r.mu.Unlock() + + if _, ok := r.ArtifactMap[processorName]; !ok { + r.ArtifactMap[processorName] = &JSONArtifactRegistryProcessor{} + } + r.ArtifactMap[processorName].Artifacts = append(r.ArtifactMap[processorName].Artifacts, artifactName) +} + +func (r *JSONArtifactRegistry) RemoveArtifact(processorName string, artifactName string) { + r.mu.Lock() + defer r.mu.Unlock() + + if _, ok := r.ArtifactMap[processorName]; !ok { + return + } + + var files []string + for _, file := range r.ArtifactMap[processorName].Artifacts { + if file != artifactName { + files = append(files, file) + } + } + + r.ArtifactMap[processorName].Artifacts = files +} + +func (r *JSONArtifactRegistry) Artifacts(processorName string) []string { + r.mu.RLock() + defer r.mu.RUnlock() + + artifacts, ok := r.ArtifactMap[processorName] + if !ok { + return nil + } + return artifacts.Artifacts +} diff --git a/artefactregistry_test.go b/artefactregistry_test.go new file mode 100644 index 0000000..edaa0fc --- /dev/null +++ b/artefactregistry_test.go @@ -0,0 +1,303 @@ +package specter + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestJSONArtifactRegistry_Load(t *testing.T) { + tests := []struct { + name string + fileContent string + expectError bool + expectedValue *JSONArtifactRegistry + }{ + { + name: "Successful Load", + fileContent: `{"generatedAt":"2024-01-01T00:00:00Z","files":{"processor1":{"files":["file1.txt"]}}}`, + expectError: false, + expectedValue: &JSONArtifactRegistry{ + GeneratedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + ArtifactMap: map[string]*JSONArtifactRegistryProcessor{"processor1": {Artifacts: []string{"file1.txt"}}}, + }, + }, + { + name: "File Not Exist", + fileContent: "", + expectError: false, + expectedValue: &JSONArtifactRegistry{ + GeneratedAt: time.Time{}, + ArtifactMap: nil, + }, + }, + { + name: "Malformed JSON", + fileContent: `{"files":{`, + expectError: true, + expectedValue: &JSONArtifactRegistry{ + GeneratedAt: time.Time{}, + ArtifactMap: nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup + filePath := "test_registry.json" + err := os.WriteFile(filePath, []byte(tt.fileContent), 0644) + if err != nil { + t.Fatalf("Failed to write test file: %v", err) + } + defer os.Remove(filePath) + + registry := &JSONArtifactRegistry{ + FilePath: filePath, + } + + // Act + err = registry.Load() + + // Assert + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expectedValue, registry) + } + }) + } +} + +func TestJSONArtifactRegistry_Save(t *testing.T) { + tests := []struct { + name string + initialState *JSONArtifactRegistry + expectedJSON string + }{ + { + name: "Successful Save", + initialState: &JSONArtifactRegistry{ + ArtifactMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + }, + expectedJSON: `{ + "generatedAt": "0001-01-01T00:00:00Z", + "files": { + "processor1": { + "files": [ + "file1.txt" + ] + } + } +}`, + }, + { + name: "Empty Registry", + initialState: &JSONArtifactRegistry{}, + expectedJSON: `{ + "generatedAt": "0001-01-01T00:00:00Z", + "files": {} +}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup + filePath := "test_registry.json" + registry := &JSONArtifactRegistry{ + FilePath: filePath, + } + registry.ArtifactMap = tt.initialState.ArtifactMap + + // Act + err := registry.Save() + + // Assert + assert.NoError(t, err) + + // Read back and verify + data, err := os.ReadFile(filePath) + assert.NoError(t, err) + assert.JSONEq(t, tt.expectedJSON, string(data)) + }) + } +} + +func TestJSONArtifactRegistry_AddArtifact(t *testing.T) { + tests := []struct { + name string + initialMap map[string]*JSONArtifactRegistryProcessor + processorName string + artifactName string + expectedMap map[string]*JSONArtifactRegistryProcessor + }{ + { + name: "Add New Artifact", + initialMap: map[string]*JSONArtifactRegistryProcessor{}, + processorName: "processor1", + artifactName: "file1.txt", + expectedMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + }, + { + name: "Add Artifact to Existing Processor", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file2.txt"}, + }, + }, + processorName: "processor1", + artifactName: "file1.txt", + expectedMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file2.txt", "file1.txt"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry := &JSONArtifactRegistry{ + ArtifactMap: tt.initialMap, + } + + // Act + registry.AddArtifact(tt.processorName, tt.artifactName) + + // Assert + assert.Equal(t, tt.expectedMap, registry.ArtifactMap) + }) + } +} + +func TestJSONArtifactRegistry_RemoveArtifact(t *testing.T) { + tests := []struct { + name string + initialMap map[string]*JSONArtifactRegistryProcessor + processorName string + artifactName string + expectedMap map[string]*JSONArtifactRegistryProcessor + }{ + { + name: "Remove Existing Artifact", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt", "file2.txt"}, + }, + }, + processorName: "processor1", + artifactName: "file1.txt", + expectedMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file2.txt"}, + }, + }, + }, + { + name: "Remove Non-Existing Artifact", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + processorName: "processor1", + artifactName: "file2.txt", + expectedMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + }, + { + name: "Remove From Non-Existing Processor", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + processorName: "processor2", + artifactName: "file1.txt", + expectedMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry := &JSONArtifactRegistry{ + ArtifactMap: tt.initialMap, + } + + // Act + registry.RemoveArtifact(tt.processorName, tt.artifactName) + + // Assert + assert.Equal(t, tt.expectedMap, registry.ArtifactMap) + }) + } +} + +func TestJSONArtifactRegistry_Artifacts(t *testing.T) { + tests := []struct { + name string + initialMap map[string]*JSONArtifactRegistryProcessor + processorName string + expectedArtifacts []string + }{ + { + name: "Get Artifacts for Existing Processor", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt", "file2.txt"}, + }, + }, + processorName: "processor1", + expectedArtifacts: []string{"file1.txt", "file2.txt"}, + }, + { + name: "Get Artifacts for Non-Existing Processor", + initialMap: map[string]*JSONArtifactRegistryProcessor{ + "processor1": { + Artifacts: []string{"file1.txt"}, + }, + }, + processorName: "processor2", + expectedArtifacts: nil, + }, + { + name: "Empty Registry", + initialMap: map[string]*JSONArtifactRegistryProcessor{}, + processorName: "processor1", + expectedArtifacts: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry := &JSONArtifactRegistry{ + ArtifactMap: tt.initialMap, + } + + // Act + artifacts := registry.Artifacts(tt.processorName) + + // Assert + assert.Equal(t, tt.expectedArtifacts, artifacts) + }) + } +} diff --git a/depresolve.go b/depresolve.go index 4f32119..dde7cd7 100644 --- a/depresolve.go +++ b/depresolve.go @@ -20,7 +20,7 @@ import ( "strings" ) -const ResolvedDependencyContextOutputName = "_resolved_dependencies" +const ResolvedDependencyContextArtifactName = "_resolved_dependencies" // ResolvedDependencies represents an ordered list of Specification that should be processed in that specific order to avoid // unresolved types. @@ -45,7 +45,7 @@ func (p DependencyResolutionProcessor) Name() string { return "dependency_resolution_processor" } -func (p DependencyResolutionProcessor) Process(ctx ProcessingContext) ([]ProcessingOutput, error) { +func (p DependencyResolutionProcessor) Process(ctx ProcessingContext) ([]ProcessingArtifact, error) { ctx.Logger.Info("\nResolving dependencies...") var nodes []dependencyNode @@ -68,16 +68,16 @@ func (p DependencyResolutionProcessor) Process(ctx ProcessingContext) ([]Process } ctx.Logger.Success("Dependencies resolved successfully.") - return []ProcessingOutput{ + return []ProcessingArtifact{ { - Name: ResolvedDependencyContextOutputName, + Name: ResolvedDependencyContextArtifactName, Value: deps, }, }, nil } func GetResolvedDependenciesFromContext(ctx ProcessingContext) ResolvedDependencies { - return GetContextOutput[ResolvedDependencies](ctx, ResolvedDependencyContextOutputName) + return GetContextArtifact[ResolvedDependencies](ctx, ResolvedDependencyContextArtifactName) } type dependencySet map[SpecificationName]struct{} diff --git a/depresolve_test.go b/depresolve_test.go index 84ccf6e..7dce7e0 100644 --- a/depresolve_test.go +++ b/depresolve_test.go @@ -175,15 +175,15 @@ func TestDependencyResolutionProcessor_Process(t *testing.T) { } var err error - ctx.Outputs, err = processor.Process(ctx) + ctx.Artifacts, err = processor.Process(ctx) if tt.expectedError != nil { require.Error(t, err) assert.ErrorContains(t, err, tt.expectedError.Error()) return } - output := ctx.Output(ResolvedDependencyContextOutputName).Value - graph := output.(ResolvedDependencies) + artifact := ctx.Artifact(ResolvedDependencyContextArtifactName).Value + graph := artifact.(ResolvedDependencies) require.NoError(t, err) require.Equal(t, tt.then, graph) @@ -200,9 +200,9 @@ func TestGetResolvedDependenciesFromContext(t *testing.T) { { name: "GIVEN a context with resolved dependencies THEN return resolved dependencies", given: ProcessingContext{ - Outputs: []ProcessingOutput{ + Artifacts: []ProcessingArtifact{ { - Name: ResolvedDependencyContextOutputName, + Name: ResolvedDependencyContextArtifactName, Value: ResolvedDependencies{ NewGenericSpecification("name", "type", Source{}), }, @@ -216,9 +216,9 @@ func TestGetResolvedDependenciesFromContext(t *testing.T) { { name: "GIVEN a context with resolved dependencies with wrong type THEN return nil", given: ProcessingContext{ - Outputs: []ProcessingOutput{ + Artifacts: []ProcessingArtifact{ { - Name: ResolvedDependencyContextOutputName, + Name: ResolvedDependencyContextArtifactName, Value: "this is not the right value", }, }, diff --git a/linting.go b/linting.go index d91f969..6a71ae7 100644 --- a/linting.go +++ b/linting.go @@ -7,7 +7,7 @@ import ( "unicode" ) -const LintingProcessingContextOutputName = "_linting_processor_results" +const LintingProcessingContextArtifactName = "_linting_processor_results" // UndefinedSpecificationName constant used to test against undefined SpecificationName. const UndefinedSpecificationName SpecificationName = "" @@ -33,14 +33,14 @@ func (l LintingProcessor) Name() string { return "linting_processor" } -func (l LintingProcessor) Process(ctx ProcessingContext) (outputs []ProcessingOutput, err error) { +func (l LintingProcessor) Process(ctx ProcessingContext) (artifacts []ProcessingArtifact, err error) { linter := CompositeSpecificationLinter(l.linters...) ctx.Logger.Info("\nLinting specifications ...") lr := linter.Lint(ctx.Specifications) - outputs = append(outputs, ProcessingOutput{ - Name: LintingProcessingContextOutputName, + artifacts = append(artifacts, ProcessingArtifact{ + Name: LintingProcessingContextArtifactName, Value: lr, }) @@ -61,12 +61,12 @@ func (l LintingProcessor) Process(ctx ProcessingContext) (outputs []ProcessingOu ctx.Logger.Success("Specifications linted successfully.") } - return outputs, err + return artifacts, err } func GetLintingResultsFromContext(ctx ProcessingContext) LinterResultSet { - return GetContextOutput[LinterResultSet](ctx, LintingProcessingContextOutputName) + return GetContextArtifact[LinterResultSet](ctx, LintingProcessingContextArtifactName) } type LinterResult struct { diff --git a/linting_test.go b/linting_test.go index cf7deba..c349fed 100644 --- a/linting_test.go +++ b/linting_test.go @@ -469,7 +469,7 @@ func TestLintingProcessor_Process(t *testing.T) { tests := []struct { name string given args - then []ProcessingOutput + then []ProcessingArtifact expectedError error }{ { @@ -480,16 +480,16 @@ func TestLintingProcessor_Process(t *testing.T) { Logger: NewDefaultLogger(DefaultLoggerConfig{}), }, }, - then: []ProcessingOutput{ + then: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: LinterResultSet(nil), }, }, expectedError: nil, }, { - name: "GIVEN a processing context with specifications that raise warnings THEN return a processing output with the result set", + name: "GIVEN a processing context with specifications that raise warnings THEN return a processing artifact with the result set", given: args{ linters: []SpecificationLinter{ SpecificationLinterFunc(func(specifications SpecificationGroup) LinterResultSet { @@ -501,9 +501,9 @@ func TestLintingProcessor_Process(t *testing.T) { Logger: NewDefaultLogger(DefaultLoggerConfig{}), }, }, - then: []ProcessingOutput{ + then: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: LinterResultSet{{Severity: WarningSeverity, Message: "a warning"}}, }, }, @@ -521,9 +521,9 @@ func TestLintingProcessor_Process(t *testing.T) { Logger: NewDefaultLogger(DefaultLoggerConfig{}), }, }, - then: []ProcessingOutput{ + then: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: LinterResultSet{{Severity: ErrorSeverity, Message: assert.AnError.Error()}}, }, }, @@ -549,9 +549,9 @@ func TestLintingProcessor_Process(t *testing.T) { Logger: NewDefaultLogger(DefaultLoggerConfig{}), }, }, - then: []ProcessingOutput{ + then: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: LinterResultSet{ { Severity: ErrorSeverity, Message: assert.AnError.Error(), @@ -590,9 +590,9 @@ func TestGetLintingResultsFromContext(t *testing.T) { { name: "GIVEN context with linting results THEN return linting results", given: ProcessingContext{ - Outputs: []ProcessingOutput{ + Artifacts: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: LinterResultSet{{Severity: WarningSeverity, Message: "a warning"}}, }, }, @@ -605,11 +605,11 @@ func TestGetLintingResultsFromContext(t *testing.T) { then: LinterResultSet(nil), }, { - name: "GIVEN a context with wrong value for output name THEN return nil", + name: "GIVEN a context with wrong value for artifact name THEN return nil", given: ProcessingContext{ - Outputs: []ProcessingOutput{ + Artifacts: []ProcessingArtifact{ { - Name: LintingProcessingContextOutputName, + Name: LintingProcessingContextArtifactName, Value: "this is not the right value", }, }, diff --git a/logging.go b/logging.go index f3f71a1..3c48c17 100644 --- a/logging.go +++ b/logging.go @@ -9,7 +9,7 @@ import ( ) // Logger interface to be used by specter and processors to perform logging. -// implementations can be made for different scenarios, such as outputting to a file, stderr, silencing the logger etc. +// implementations can be made for different scenarios, such as artifactting to a file, stderr, silencing the logger etc. // The logger only provides contextual logging. type Logger interface { // Trace should only be used for debugging purposes. diff --git a/logging_test.go b/logging_test.go index 971dae2..a5b512a 100644 --- a/logging_test.go +++ b/logging_test.go @@ -23,7 +23,7 @@ func TestNewDefaultLogger(t *testing.T) { require.NotNil(t, l) require.Equal(t, os.Stdout, l.writer) - // Check colors enabled by capturing output + // Check colors enabled by capturing artifact buffer := bytes.Buffer{} l.writer = &buffer l.Success("hello world") diff --git a/mkdirartfproc.go b/mkdirartfproc.go new file mode 100644 index 0000000..8e8713d --- /dev/null +++ b/mkdirartfproc.go @@ -0,0 +1,56 @@ +package specter + +import ( + "fmt" + "github.com/morebec/go-errors/errors" + "os" +) + +type DirectoryArtifact struct { + Path string + Mode os.FileMode +} + +type WriteDirectoryArtifactsProcessorConfig struct { + // Add any configuration specific to directory processing if needed + UseRegistry bool +} + +type WriteDirectoryArtifactsProcessor struct { + config WriteDirectoryArtifactsProcessorConfig +} + +func NewWriteDirectoryArtifactsProcessor(conf WriteDirectoryArtifactsProcessorConfig) *WriteDirectoryArtifactsProcessor { + return &WriteDirectoryArtifactsProcessor{ + config: conf, + } +} + +func (p WriteDirectoryArtifactsProcessor) Name() string { + return "directory_artifacts_processor" +} + +func (p WriteDirectoryArtifactsProcessor) Process(ctx ArtifactProcessingContext) error { + ctx.Logger.Info("Creating artifact directories ...") + + var directories []DirectoryArtifact + for _, o := range ctx.Artifacts { + dir, ok := o.Value.(DirectoryArtifact) + if !ok { + continue + } + directories = append(directories, dir) + } + + for _, dir := range directories { + ctx.Logger.Info(fmt.Sprintf("Creating directory %s ...", dir.Path)) + err := os.MkdirAll(dir.Path, dir.Mode) + if err != nil { + ctx.Logger.Error(fmt.Sprintf("failed creating directory at %s", dir.Path)) + return errors.Wrap(err, "failed creating artifact directories") + } + } + + ctx.Logger.Success("Artifact directories created successfully.") + return nil +} diff --git a/mkdiroutproc.go b/mkdiroutproc.go deleted file mode 100644 index e7eb705..0000000 --- a/mkdiroutproc.go +++ /dev/null @@ -1,56 +0,0 @@ -package specter - -import ( - "fmt" - "github.com/morebec/go-errors/errors" - "os" -) - -type DirectoryOutput struct { - Path string - Mode os.FileMode -} - -type WriteDirectoryOutputsProcessorConfig struct { - // Add any configuration specific to directory processing if needed - UseRegistry bool -} - -type WriteDirectoryOutputsProcessor struct { - config WriteDirectoryOutputsProcessorConfig -} - -func NewWriteDirectoryOutputsProcessor(conf WriteDirectoryOutputsProcessorConfig) *WriteDirectoryOutputsProcessor { - return &WriteDirectoryOutputsProcessor{ - config: conf, - } -} - -func (p WriteDirectoryOutputsProcessor) Name() string { - return "directory_outputs_processor" -} - -func (p WriteDirectoryOutputsProcessor) Process(ctx OutputProcessingContext) error { - ctx.Logger.Info("Creating output directories ...") - - var directories []DirectoryOutput - for _, o := range ctx.Outputs { - dir, ok := o.Value.(DirectoryOutput) - if !ok { - continue - } - directories = append(directories, dir) - } - - for _, dir := range directories { - ctx.Logger.Info(fmt.Sprintf("Creating directory %s ...", dir.Path)) - err := os.MkdirAll(dir.Path, dir.Mode) - if err != nil { - ctx.Logger.Error(fmt.Sprintf("failed creating directory at %s", dir.Path)) - return errors.Wrap(err, "failed creating output directories") - } - } - - ctx.Logger.Success("Output directories created successfully.") - return nil -} diff --git a/outputregistry.go b/outputregistry.go deleted file mode 100644 index 8058bba..0000000 --- a/outputregistry.go +++ /dev/null @@ -1,108 +0,0 @@ -package specter - -import ( - "encoding/json" - "github.com/morebec/go-errors/errors" - "os" - "sync" - "time" -) - -// JSONOutputRegistry allows tracking on the file system the files that were written by the last execution -// of the WriteFileOutputsProcessor to perform cleaning operations on next executions. -type JSONOutputRegistry struct { - GeneratedAt time.Time `json:"generatedAt"` - OutputMap map[string]*JSONOutputRegistryProcessor `json:"files"` - FilePath string - mu sync.RWMutex // Mutex to protect concurrent access -} - -type JSONOutputRegistryProcessor struct { - Outputs []string `json:"files"` -} - -// NewJSONOutputRegistry returns a new output file registry. -func NewJSONOutputRegistry(fileName string) *JSONOutputRegistry { - return &JSONOutputRegistry{ - GeneratedAt: time.Now(), - OutputMap: nil, - FilePath: fileName, - } -} - -func (r *JSONOutputRegistry) Load() error { - r.mu.Lock() - defer r.mu.Unlock() - - bytes, err := os.ReadFile(r.FilePath) - - if err != nil { - if os.IsNotExist(err) { - return nil - } - return errors.WrapWithMessage(err, errors.InternalErrorCode, "failed loading output file registry") - } - if err := json.Unmarshal(bytes, r); err != nil { - return errors.WrapWithMessage(err, errors.InternalErrorCode, "failed loading output file registry") - } - - return nil -} - -func (r *JSONOutputRegistry) Save() error { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.OutputMap == nil { - return nil - } - // Generate a JSON file containing all output files for clean up later on - js, err := json.MarshalIndent(r, "", " ") - if err != nil { - return errors.Wrap(err, "failed generating output file registry") - } - if err := os.WriteFile(r.FilePath, js, os.ModePerm); err != nil { - return errors.Wrap(err, "failed generating output file registry") - } - - return nil -} - -func (r *JSONOutputRegistry) AddOutput(processorName string, outputName string) { - r.mu.Lock() - defer r.mu.Unlock() - - if _, ok := r.OutputMap[processorName]; !ok { - r.OutputMap[processorName] = &JSONOutputRegistryProcessor{} - } - r.OutputMap[processorName].Outputs = append(r.OutputMap[processorName].Outputs, outputName) -} - -func (r *JSONOutputRegistry) RemoveOutput(processorName string, outputName string) { - r.mu.Lock() - defer r.mu.Unlock() - - if _, ok := r.OutputMap[processorName]; !ok { - return - } - - var files []string - for _, file := range r.OutputMap[processorName].Outputs { - if file != outputName { - files = append(files, file) - } - } - - r.OutputMap[processorName].Outputs = files -} - -func (r *JSONOutputRegistry) Outputs(processorName string) []string { - r.mu.RLock() - defer r.mu.RUnlock() - - outputs, ok := r.OutputMap[processorName] - if !ok { - return nil - } - return outputs.Outputs -} diff --git a/outputregistry_test.go b/outputregistry_test.go deleted file mode 100644 index 28a66b5..0000000 --- a/outputregistry_test.go +++ /dev/null @@ -1,303 +0,0 @@ -package specter - -import ( - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestJSONOutputRegistry_Load(t *testing.T) { - tests := []struct { - name string - fileContent string - expectError bool - expectedValue *JSONOutputRegistry - }{ - { - name: "Successful Load", - fileContent: `{"generatedAt":"2024-01-01T00:00:00Z","files":{"processor1":{"files":["file1.txt"]}}}`, - expectError: false, - expectedValue: &JSONOutputRegistry{ - GeneratedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - OutputMap: map[string]*JSONOutputRegistryProcessor{"processor1": {Outputs: []string{"file1.txt"}}}, - }, - }, - { - name: "File Not Exist", - fileContent: "", - expectError: false, - expectedValue: &JSONOutputRegistry{ - GeneratedAt: time.Time{}, - OutputMap: nil, - }, - }, - { - name: "Malformed JSON", - fileContent: `{"files":{`, - expectError: true, - expectedValue: &JSONOutputRegistry{ - GeneratedAt: time.Time{}, - OutputMap: nil, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup - filePath := "test_registry.json" - err := os.WriteFile(filePath, []byte(tt.fileContent), 0644) - if err != nil { - t.Fatalf("Failed to write test file: %v", err) - } - defer os.Remove(filePath) - - registry := &JSONOutputRegistry{ - FilePath: filePath, - } - - // Act - err = registry.Load() - - // Assert - if tt.expectError { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.expectedValue, registry) - } - }) - } -} - -func TestJSONOutputRegistry_Save(t *testing.T) { - tests := []struct { - name string - initialState *JSONOutputRegistry - expectedJSON string - }{ - { - name: "Successful Save", - initialState: &JSONOutputRegistry{ - OutputMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - }, - expectedJSON: `{ - "generatedAt": "0001-01-01T00:00:00Z", - "files": { - "processor1": { - "files": [ - "file1.txt" - ] - } - } -}`, - }, - { - name: "Empty Registry", - initialState: &JSONOutputRegistry{}, - expectedJSON: `{ - "generatedAt": "0001-01-01T00:00:00Z", - "files": {} -}`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Setup - filePath := "test_registry.json" - registry := &JSONOutputRegistry{ - FilePath: filePath, - } - registry.OutputMap = tt.initialState.OutputMap - - // Act - err := registry.Save() - - // Assert - assert.NoError(t, err) - - // Read back and verify - data, err := os.ReadFile(filePath) - assert.NoError(t, err) - assert.JSONEq(t, tt.expectedJSON, string(data)) - }) - } -} - -func TestJSONOutputRegistry_AddOutput(t *testing.T) { - tests := []struct { - name string - initialMap map[string]*JSONOutputRegistryProcessor - processorName string - outputName string - expectedMap map[string]*JSONOutputRegistryProcessor - }{ - { - name: "Add New Output", - initialMap: map[string]*JSONOutputRegistryProcessor{}, - processorName: "processor1", - outputName: "file1.txt", - expectedMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - }, - { - name: "Add Output to Existing Processor", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file2.txt"}, - }, - }, - processorName: "processor1", - outputName: "file1.txt", - expectedMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file2.txt", "file1.txt"}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - registry := &JSONOutputRegistry{ - OutputMap: tt.initialMap, - } - - // Act - registry.AddOutput(tt.processorName, tt.outputName) - - // Assert - assert.Equal(t, tt.expectedMap, registry.OutputMap) - }) - } -} - -func TestJSONOutputRegistry_RemoveOutput(t *testing.T) { - tests := []struct { - name string - initialMap map[string]*JSONOutputRegistryProcessor - processorName string - outputName string - expectedMap map[string]*JSONOutputRegistryProcessor - }{ - { - name: "Remove Existing Output", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt", "file2.txt"}, - }, - }, - processorName: "processor1", - outputName: "file1.txt", - expectedMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file2.txt"}, - }, - }, - }, - { - name: "Remove Non-Existing Output", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - processorName: "processor1", - outputName: "file2.txt", - expectedMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - }, - { - name: "Remove From Non-Existing Processor", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - processorName: "processor2", - outputName: "file1.txt", - expectedMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - registry := &JSONOutputRegistry{ - OutputMap: tt.initialMap, - } - - // Act - registry.RemoveOutput(tt.processorName, tt.outputName) - - // Assert - assert.Equal(t, tt.expectedMap, registry.OutputMap) - }) - } -} - -func TestJSONOutputRegistry_Outputs(t *testing.T) { - tests := []struct { - name string - initialMap map[string]*JSONOutputRegistryProcessor - processorName string - expectedOutputs []string - }{ - { - name: "Get Outputs for Existing Processor", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt", "file2.txt"}, - }, - }, - processorName: "processor1", - expectedOutputs: []string{"file1.txt", "file2.txt"}, - }, - { - name: "Get Outputs for Non-Existing Processor", - initialMap: map[string]*JSONOutputRegistryProcessor{ - "processor1": { - Outputs: []string{"file1.txt"}, - }, - }, - processorName: "processor2", - expectedOutputs: nil, - }, - { - name: "Empty Registry", - initialMap: map[string]*JSONOutputRegistryProcessor{}, - processorName: "processor1", - expectedOutputs: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - registry := &JSONOutputRegistry{ - OutputMap: tt.initialMap, - } - - // Act - outputs := registry.Outputs(tt.processorName) - - // Assert - assert.Equal(t, tt.expectedOutputs, outputs) - }) - } -} diff --git a/processing.go b/processing.go index 0f3a290..3de5f43 100644 --- a/processing.go +++ b/processing.go @@ -5,48 +5,53 @@ import "context" type ProcessingContext struct { context.Context Specifications SpecificationGroup - Outputs []ProcessingOutput + Artifacts []ProcessingArtifact Logger Logger } -// Output returns the output associated with a given processor. -func (c ProcessingContext) Output(outputName string) ProcessingOutput { - for _, o := range c.Outputs { - if o.Name == outputName { +// Artifact returns the artifact associated with a given processor. +func (c ProcessingContext) Artifact(artifactName string) ProcessingArtifact { + for _, o := range c.Artifacts { + if o.Name == artifactName { return o } } - return ProcessingOutput{Name: outputName, Value: nil} + return ProcessingArtifact{Name: artifactName, Value: nil} } -// ProcessingOutput represents an output generated by a SpecificationProcessor. -type ProcessingOutput struct { - // Name of the Output +// ProcessingArtifact represents a result or output generated by a SpecificationProcessor. +// An artifact is a unit of data or information produced as part of the processing workflow. +// It can be a transient, in-memory object, or it might represent more permanent entities such as +// files on disk, records in a database, deployment units, or other forms of data artifacts. +type ProcessingArtifact struct { + // Name is a unique identifier of the artifact, which helps in distinguishing and referencing + // the artifact within a processing context. Name string - // Value of the output + // Value is the actual content or data of the artifact. This can be any type, including but not + // limited to in-memory structures, file paths, database records, or serialized deployment units. Value any } // SpecificationProcessor are services responsible for performing work using Specifications -// and which can possibly generate outputs. +// and which can possibly generate artifacts. type SpecificationProcessor interface { // Name returns the unique FilePath of this processor. Name() string // Process processes a group of specifications. - Process(ctx ProcessingContext) ([]ProcessingOutput, error) + Process(ctx ProcessingContext) ([]ProcessingArtifact, error) } -// OutputRegistry provides an interface for managing a registry of outputs. This -// registry tracks outputs generated during processing runs, enabling clean-up +// ArtifactRegistry provides an interface for managing a registry of artifacts. This +// registry tracks artifacts generated during processing runs, enabling clean-up // in subsequent runs to avoid residual artifacts and maintain a clean slate. // -// Implementations of the OutputRegistry interface must be thread-safe to handle +// Implementations of the ArtifactRegistry interface must be thread-safe to handle // concurrent calls to TrackFile and UntrackFile methods. Multiple goroutines may // access the registry simultaneously, so appropriate synchronization mechanisms // should be implemented to prevent race conditions and ensure data integrity. -type OutputRegistry interface { +type ArtifactRegistry interface { // Load the registry state from persistent storage. If an error occurs, it // should be returned to indicate the failure of the loading operation. Load() error @@ -55,72 +60,72 @@ type OutputRegistry interface { // error occurs, it should be returned to indicate the failure of the saving operation. Save() error - // AddOutput registers an output name under a specific processor name. This method + // AddArtifact registers an artifact name under a specific processor name. This method // should ensure that the file path is associated with the given processor name // in the registry. - AddOutput(processorName string, outputName string) + AddArtifact(processorName string, artifactName string) - // RemoveOutput removes a given named output registration for a specific processor name. This + // RemoveArtifact removes a given named artifact registration for a specific processor name. This // method should ensure that the file path is disassociated from the given // processor name in the registry. - RemoveOutput(processorName string, outputName string) + RemoveArtifact(processorName string, artifactName string) - // Outputs returns the outputs for a given processor. - Outputs(processorName string) []string + // Artifacts returns the artifacts for a given processor. + Artifacts(processorName string) []string } -type NoopOutputRegistry struct { +type NoopArtifactRegistry struct { } -func (n NoopOutputRegistry) Load() error { +func (n NoopArtifactRegistry) Load() error { return nil } -func (n NoopOutputRegistry) Save() error { +func (n NoopArtifactRegistry) Save() error { return nil } -func (n NoopOutputRegistry) AddOutput(processorName string, outputName string) {} +func (n NoopArtifactRegistry) AddArtifact(processorName string, artifactName string) {} -func (n NoopOutputRegistry) RemoveOutput(processorName string, outputName string) {} +func (n NoopArtifactRegistry) RemoveArtifact(processorName string, artifactName string) {} -func (n NoopOutputRegistry) Outputs(processorName string) []string { +func (n NoopArtifactRegistry) Artifacts(processorName string) []string { return nil } -type OutputProcessingContext struct { +type ArtifactProcessingContext struct { context.Context Specifications SpecificationGroup - Outputs []ProcessingOutput + Artifacts []ProcessingArtifact Logger Logger - outputRegistry OutputRegistry - processorName string + artifactRegistry ArtifactRegistry + processorName string } -func (c *OutputProcessingContext) AddToRegistry(outputName string) { - c.outputRegistry.AddOutput(c.processorName, outputName) +func (c *ArtifactProcessingContext) AddToRegistry(artifactName string) { + c.artifactRegistry.AddArtifact(c.processorName, artifactName) } -func (c *OutputProcessingContext) RemoveFromRegistry(outputName string) { - c.outputRegistry.RemoveOutput(c.processorName, outputName) +func (c *ArtifactProcessingContext) RemoveFromRegistry(artifactName string) { + c.artifactRegistry.RemoveArtifact(c.processorName, artifactName) } -func (c *OutputProcessingContext) RegistryOutputs() []string { - return c.outputRegistry.Outputs(c.processorName) +func (c *ArtifactProcessingContext) RegistryArtifacts() []string { + return c.artifactRegistry.Artifacts(c.processorName) } -// OutputProcessor are services responsible for processing outputs of SpecProcessors. -type OutputProcessor interface { - // Process performs the processing of outputs generated by SpecificationProcessor. - Process(ctx OutputProcessingContext) error +// ArtifactProcessor are services responsible for processing artifacts of SpecProcessors. +type ArtifactProcessor interface { + // Process performs the processing of artifacts generated by SpecificationProcessor. + Process(ctx ArtifactProcessingContext) error // Name returns the name of this processor. Name() string } -func GetContextOutput[T any](ctx ProcessingContext, name string) (v T) { - output := ctx.Output(name) - v, _ = output.Value.(T) +func GetContextArtifact[T any](ctx ProcessingContext, name string) (v T) { + artifact := ctx.Artifact(name) + v, _ = artifact.Value.(T) return v } diff --git a/processing_test.go b/processing_test.go index d37d1f0..42b788a 100644 --- a/processing_test.go +++ b/processing_test.go @@ -6,95 +6,95 @@ import ( "testing" ) -// MockOutputRegistry is a mock implementation of OutputRegistry -type MockOutputRegistry struct { +// MockArtifactRegistry is a mock implementation of ArtifactRegistry +type MockArtifactRegistry struct { mock.Mock } -func (m *MockOutputRegistry) Load() error { +func (m *MockArtifactRegistry) Load() error { args := m.Called() return args.Error(0) } -func (m *MockOutputRegistry) Save() error { +func (m *MockArtifactRegistry) Save() error { args := m.Called() return args.Error(0) } -func (m *MockOutputRegistry) AddOutput(processorName string, outputName string) { - m.Called(processorName, outputName) +func (m *MockArtifactRegistry) AddArtifact(processorName string, artifactName string) { + m.Called(processorName, artifactName) } -func (m *MockOutputRegistry) RemoveOutput(processorName string, outputName string) { - m.Called(processorName, outputName) +func (m *MockArtifactRegistry) RemoveArtifact(processorName string, artifactName string) { + m.Called(processorName, artifactName) } -func (m *MockOutputRegistry) Outputs(processorName string) []string { +func (m *MockArtifactRegistry) Artifacts(processorName string) []string { args := m.Called(processorName) return args.Get(0).([]string) } -func TestOutputProcessingContext__AddToRegistry(t *testing.T) { +func TestArtifactProcessingContext__AddToRegistry(t *testing.T) { // Arrange - mockRegistry := new(MockOutputRegistry) - ctx := &OutputProcessingContext{ - outputRegistry: mockRegistry, - processorName: "testProcessor", + mockRegistry := new(MockArtifactRegistry) + ctx := &ArtifactProcessingContext{ + artifactRegistry: mockRegistry, + processorName: "testProcessor", } - outputName := "outputFile.txt" + artifactName := "artifactFile.txt" - mockRegistry.On("AddOutput", "testProcessor", outputName).Return() + mockRegistry.On("AddArtifact", "testProcessor", artifactName).Return() // Act - ctx.AddToRegistry(outputName) + ctx.AddToRegistry(artifactName) // Assert mockRegistry.AssertExpectations(t) } -func TestOutputProcessingContext__RemoveFromRegistry(t *testing.T) { +func TestArtifactProcessingContext__RemoveFromRegistry(t *testing.T) { // Arrange - mockRegistry := new(MockOutputRegistry) - ctx := &OutputProcessingContext{ - outputRegistry: mockRegistry, - processorName: "testProcessor", + mockRegistry := new(MockArtifactRegistry) + ctx := &ArtifactProcessingContext{ + artifactRegistry: mockRegistry, + processorName: "testProcessor", } - outputName := "outputFile.txt" + artifactName := "artifactFile.txt" - mockRegistry.On("RemoveOutput", "testProcessor", outputName).Return() + mockRegistry.On("RemoveArtifact", "testProcessor", artifactName).Return() // Act - ctx.RemoveFromRegistry(outputName) + ctx.RemoveFromRegistry(artifactName) // Assert mockRegistry.AssertExpectations(t) } -func TestOutputProcessingContext__RegistryOutputs(t *testing.T) { +func TestArtifactProcessingContext__RegistryArtifacts(t *testing.T) { // Arrange - mockRegistry := new(MockOutputRegistry) - ctx := &OutputProcessingContext{ - outputRegistry: mockRegistry, - processorName: "testProcessor", + mockRegistry := new(MockArtifactRegistry) + ctx := &ArtifactProcessingContext{ + artifactRegistry: mockRegistry, + processorName: "testProcessor", } - expectedOutputs := []string{"file1.txt", "file2.txt"} + expectedArtifacts := []string{"file1.txt", "file2.txt"} - mockRegistry.On("Outputs", "testProcessor").Return(expectedOutputs) + mockRegistry.On("Artifacts", "testProcessor").Return(expectedArtifacts) // Act - outputs := ctx.RegistryOutputs() + artifacts := ctx.RegistryArtifacts() // Assert - assert.Equal(t, expectedOutputs, outputs) + assert.Equal(t, expectedArtifacts, artifacts) mockRegistry.AssertExpectations(t) } -func TestNoopOutputRegistry_Load(t *testing.T) { +func TestNoopArtifactRegistry_Load(t *testing.T) { // Arrange - registry := NoopOutputRegistry{} + registry := NoopArtifactRegistry{} // Act err := registry.Load() @@ -103,9 +103,9 @@ func TestNoopOutputRegistry_Load(t *testing.T) { assert.NoError(t, err, "Load should not return an error") } -func TestNoopOutputRegistry_Save(t *testing.T) { +func TestNoopArtifactRegistry_Save(t *testing.T) { // Arrange - registry := NoopOutputRegistry{} + registry := NoopArtifactRegistry{} // Act err := registry.Save() @@ -114,35 +114,35 @@ func TestNoopOutputRegistry_Save(t *testing.T) { assert.NoError(t, err, "Save should not return an error") } -func TestNoopOutputRegistry_AddOutput(t *testing.T) { +func TestNoopArtifactRegistry_AddArtifact(t *testing.T) { // Arrange - registry := NoopOutputRegistry{} + registry := NoopArtifactRegistry{} // Act - registry.AddOutput("processor1", "outputFile.txt") + registry.AddArtifact("processor1", "artifactFile.txt") // Assert // No state to assert since it's a no-op, just ensure it doesn't panic or error. } -func TestNoopOutputRegistry_RemoveOutput(t *testing.T) { +func TestNoopArtifactRegistry_RemoveArtifact(t *testing.T) { // Arrange - registry := NoopOutputRegistry{} + registry := NoopArtifactRegistry{} // Act - registry.RemoveOutput("processor1", "outputFile.txt") + registry.RemoveArtifact("processor1", "artifactFile.txt") // Assert // No state to assert since it's a no-op, just ensure it doesn't panic or error. } -func TestNoopOutputRegistry_Outputs(t *testing.T) { +func TestNoopArtifactRegistry_Artifacts(t *testing.T) { // Arrange - registry := NoopOutputRegistry{} + registry := NoopArtifactRegistry{} // Act - outputs := registry.Outputs("processor1") + artifacts := registry.Artifacts("processor1") // Assert - assert.Nil(t, outputs, "Outputs should return nil for NoopOutputRegistry") + assert.Nil(t, artifacts, "Artifacts should return nil for NoopArtifactRegistry") } diff --git a/specter.go b/specter.go index 590be64..67de7ec 100644 --- a/specter.go +++ b/specter.go @@ -13,7 +13,7 @@ type ExecutionMode string // LintMode will cause a Specter instance to run until the lint step only. const LintMode ExecutionMode = "lint" -// PreviewMode will cause a Specter instance to run until the processing step only, no output will be processed. +// PreviewMode will cause a Specter instance to run until the processing step only, no artifact will be processed. const PreviewMode ExecutionMode = "preview" // FullMode will cause a Specter instance to be run fully. @@ -21,13 +21,13 @@ const FullMode ExecutionMode = "full" // Specter is the service responsible to run a specter pipeline. type Specter struct { - SourceLoaders []SourceLoader - Loaders []SpecificationLoader - Processors []SpecificationProcessor - OutputProcessors []OutputProcessor - OutputRegistry OutputRegistry - Logger Logger - ExecutionMode ExecutionMode + SourceLoaders []SourceLoader + Loaders []SpecificationLoader + Processors []SpecificationProcessor + ArtifactProcessors []ArtifactProcessor + ArtifactRegistry ArtifactRegistry + Logger Logger + ExecutionMode ExecutionMode } type Stats struct { @@ -36,7 +36,7 @@ type Stats struct { NbSourceLocations int NbSources int NbSpecifications int - NbOutputs int + NbArtifacts int } func (s Stats) ExecutionTime() time.Duration { @@ -46,14 +46,14 @@ func (s Stats) ExecutionTime() time.Duration { type RunResult struct { Sources []Source Specification []Specification - Outputs []ProcessingOutput + Artifacts []ProcessingArtifact Stats Stats } // Run the pipeline from start to finish. func (s Specter) Run(ctx context.Context, sourceLocations []string) (RunResult, error) { var run RunResult - var outputs []ProcessingOutput + var artifacts []ProcessingArtifact defer func() { run.Stats.EndedAt = time.Now() @@ -63,7 +63,7 @@ func (s Specter) Run(ctx context.Context, sourceLocations []string) (RunResult, s.Logger.Info(fmt.Sprintf("Number of source locations: %d", run.Stats.NbSourceLocations)) s.Logger.Info(fmt.Sprintf("Number of sources: %d", run.Stats.NbSources)) s.Logger.Info(fmt.Sprintf("Number of specifications: %d", run.Stats.NbSpecifications)) - s.Logger.Info(fmt.Sprintf("Number of outputs: %d", run.Stats.NbOutputs)) + s.Logger.Info(fmt.Sprintf("Number of artifacts: %d", run.Stats.NbArtifacts)) }() run.Stats.StartedAt = time.Now() @@ -90,9 +90,9 @@ func (s Specter) Run(ctx context.Context, sourceLocations []string) (RunResult, } // Process Specifications - outputs, err = s.ProcessSpecifications(ctx, specifications) - run.Stats.NbOutputs = len(outputs) - run.Outputs = outputs + artifacts, err = s.ProcessSpecifications(ctx, specifications) + run.Stats.NbArtifacts = len(artifacts) + run.Artifacts = artifacts if err != nil { e := errors.WrapWithMessage(err, errors.InternalErrorCode, "failed processing specifications") s.Logger.Error(e.Error()) @@ -103,9 +103,9 @@ func (s Specter) Run(ctx context.Context, sourceLocations []string) (RunResult, return run, nil } - // Process Output - if err = s.ProcessOutputs(ctx, specifications, outputs); err != nil { - e := errors.WrapWithMessage(err, errors.InternalErrorCode, "failed processing outputs") + // Process Artifact + if err = s.ProcessArtifacts(ctx, specifications, artifacts); err != nil { + e := errors.WrapWithMessage(err, errors.InternalErrorCode, "failed processing artifacts") s.Logger.Error(e.Error()) return run, e } @@ -199,11 +199,11 @@ func (s Specter) LoadSpecifications(ctx context.Context, sources []Source) ([]Sp } // ProcessSpecifications sends the specifications to processors. -func (s Specter) ProcessSpecifications(ctx context.Context, specs []Specification) ([]ProcessingOutput, error) { +func (s Specter) ProcessSpecifications(ctx context.Context, specs []Specification) ([]ProcessingArtifact, error) { pctx := ProcessingContext{ Context: ctx, Specifications: specs, - Outputs: nil, + Artifacts: nil, Logger: s.Logger, } @@ -212,56 +212,58 @@ func (s Specter) ProcessSpecifications(ctx context.Context, specs []Specificatio if err := CheckContextDone(ctx); err != nil { return nil, err } - outputs, err := p.Process(pctx) + artifacts, err := p.Process(pctx) if err != nil { return nil, errors.WrapWithMessage(err, errors.InternalErrorCode, fmt.Sprintf("processor %q failed", p.Name())) } - pctx.Outputs = append(pctx.Outputs, outputs...) + pctx.Artifacts = append(pctx.Artifacts, artifacts...) } - s.Logger.Info(fmt.Sprintf("%d outputs generated.", len(pctx.Outputs))) - for _, o := range pctx.Outputs { + s.Logger.Info(fmt.Sprintf("%d artifacts generated.", len(pctx.Artifacts))) + for _, o := range pctx.Artifacts { s.Logger.Info(fmt.Sprintf("-> %s", o.Name)) } s.Logger.Success("Specifications processed successfully.") - return pctx.Outputs, nil + return pctx.Artifacts, nil } -// ProcessOutputs sends a list of ProcessingOutputs to the registered OutputProcessors. -func (s Specter) ProcessOutputs(ctx context.Context, specifications []Specification, outputs []ProcessingOutput) error { - if s.OutputRegistry == nil { - s.OutputRegistry = NoopOutputRegistry{} +// ProcessArtifacts sends a list of ProcessingArtifacts to the registered ArtifactProcessors. +func (s Specter) ProcessArtifacts(ctx context.Context, specifications []Specification, artifacts []ProcessingArtifact) error { + if s.ArtifactRegistry == nil { + s.ArtifactRegistry = NoopArtifactRegistry{} } - octx := OutputProcessingContext{ - Context: ctx, - Specifications: specifications, - Outputs: outputs, - Logger: s.Logger, - outputRegistry: s.OutputRegistry, - } - - s.Logger.Info("\nProcessing outputs ...") - if err := s.OutputRegistry.Load(); err != nil { - return fmt.Errorf("failed loading output registry: %w", err) + s.Logger.Info("\nProcessing artifacts ...") + if err := s.ArtifactRegistry.Load(); err != nil { + return fmt.Errorf("failed loading artifact registry: %w", err) } - for _, p := range s.OutputProcessors { + for _, p := range s.ArtifactProcessors { if err := CheckContextDone(ctx); err != nil { return err } + + octx := ArtifactProcessingContext{ + Context: ctx, + Specifications: specifications, + Artifacts: artifacts, + Logger: s.Logger, + artifactRegistry: s.ArtifactRegistry, + processorName: p.Name(), + } + err := p.Process(octx) if err != nil { - return errors.WrapWithMessage(err, errors.InternalErrorCode, fmt.Sprintf("output processor %q failed", p.Name())) + return errors.WrapWithMessage(err, errors.InternalErrorCode, fmt.Sprintf("artifact processor %q failed", p.Name())) } } - if err := s.OutputRegistry.Save(); err != nil { - return fmt.Errorf("failed saving output registry: %w", err) + if err := s.ArtifactRegistry.Save(); err != nil { + return fmt.Errorf("failed saving artifact registry: %w", err) } - s.Logger.Success("Outputs processed successfully.") + s.Logger.Success("Artifacts processed successfully.") return nil } @@ -308,10 +310,10 @@ func WithProcessors(processors ...SpecificationProcessor) Option { } } -// WithOutputProcessors configures the OutputProcessor of a Specter instance. -func WithOutputProcessors(processors ...OutputProcessor) Option { +// WithArtifactProcessors configures the ArtifactProcessor of a Specter instance. +func WithArtifactProcessors(processors ...ArtifactProcessor) Option { return func(s *Specter) { - s.OutputProcessors = append(s.OutputProcessors, processors...) + s.ArtifactProcessors = append(s.ArtifactProcessors, processors...) } } @@ -321,9 +323,9 @@ func WithExecutionMode(m ExecutionMode) Option { s.ExecutionMode = m } } -func WithOutputRegistry(r OutputRegistry) Option { +func WithArtifactRegistry(r ArtifactRegistry) Option { return func(s *Specter) { - s.OutputRegistry = r + s.ArtifactRegistry = r } } diff --git a/specter_test.go b/specter_test.go index 00122da..126f626 100644 --- a/specter_test.go +++ b/specter_test.go @@ -5,11 +5,11 @@ import ( "testing" ) -func TestWithOutputRegistry(t *testing.T) { +func TestWithArtifactRegistry(t *testing.T) { s := &Specter{} - r := &JSONOutputRegistry{} - WithOutputRegistry(r)(s) - assert.Equal(t, r, s.OutputRegistry) + r := &JSONArtifactRegistry{} + WithArtifactRegistry(r)(s) + assert.Equal(t, r, s.ArtifactRegistry) } func TestWithDefaultLogger(t *testing.T) { diff --git a/writefileartfproc.go b/writefileartfproc.go new file mode 100644 index 0000000..567195d --- /dev/null +++ b/writefileartfproc.go @@ -0,0 +1,97 @@ +package specter + +import ( + "fmt" + "github.com/morebec/go-errors/errors" + "os" + "sync" +) + +const WriteFileArtifactsProcessorErrorCode = "write_file_artifacts_processor_error" + +type WriteFileArtifactsProcessorConfig struct { +} + +// WriteFileArtifactProcessor is a processor responsible for writing ProcessingArtifact referring to files. +// To perform its work this processor looks at the processing context for any FileArtifact. +type WriteFileArtifactProcessor struct { + config WriteFileArtifactsProcessorConfig +} + +func NewWriteFilesProcessor(conf WriteFileArtifactsProcessorConfig) *WriteFileArtifactProcessor { + return &WriteFileArtifactProcessor{ + config: conf, + } +} + +func (f WriteFileArtifactProcessor) Name() string { + return "file_artifacts_processor" +} + +// FileArtifact is a data structure that can be used by a SpecificationProcessor to artifact files that can be written by tje WriteFileArtifactProcessor. +type FileArtifact struct { + Path string + Data []byte + Mode os.FileMode +} + +func (f WriteFileArtifactProcessor) Process(ctx ArtifactProcessingContext) error { + ctx.Logger.Info("Writing artifact files ...") + + var files []FileArtifact + for _, o := range ctx.Artifacts { + fo, ok := o.Value.(FileArtifact) + if !ok { + continue + } + files = append(files, fo) + } + + if err := f.cleanRegistry(ctx); err != nil { + ctx.Logger.Error("failed cleaning artifact registry") + return err + } + + errs := errors.NewGroup(WriteFileArtifactsProcessorErrorCode) + + // Write files concurrently to speed up process. + var wg sync.WaitGroup + for _, file := range files { + wg.Add(1) + go func(ctx ArtifactProcessingContext, file FileArtifact) { + defer wg.Done() + ctx.Logger.Info(fmt.Sprintf("Writing file %q ...", file.Path)) + err := os.WriteFile(file.Path, file.Data, os.ModePerm) + if err != nil { + ctx.Logger.Error(fmt.Sprintf("failed writing artifact file at %q", file.Path)) + errs = errs.Append(err) + } + ctx.AddToRegistry(file.Path) + }(ctx, file) + } + wg.Wait() + + ctx.Logger.Success("Artifact files written successfully.") + + return nil +} + +func (f WriteFileArtifactProcessor) cleanRegistry(ctx ArtifactProcessingContext) error { + var wg sync.WaitGroup + for _, o := range ctx.RegistryArtifacts() { + wg.Add(1) + go func(ctx ArtifactProcessingContext, o string) { + defer wg.Done() + if err := os.Remove(o); err != nil { + if errors.Is(err, os.ErrNotExist) { + return + } + panic(errors.Wrap(err, "failed cleaning artifact registry files")) + } + ctx.RemoveFromRegistry(o) + }(ctx, o) + } + wg.Wait() + + return nil +} diff --git a/writefileoutproc.go b/writefileoutproc.go deleted file mode 100644 index 2ec5adb..0000000 --- a/writefileoutproc.go +++ /dev/null @@ -1,97 +0,0 @@ -package specter - -import ( - "fmt" - "github.com/morebec/go-errors/errors" - "os" - "sync" -) - -const WriteFileOutputsProcessorErrorCode = "write_file_outputs_processor_error" - -type WriteFileOutputsProcessorConfig struct { -} - -// WriteFileOutputsProcessor is a processor responsible for writing ProcessingOutput referring to files. -// To perform its work this processor looks at the processing context for any FileOutput. -type WriteFileOutputsProcessor struct { - config WriteFileOutputsProcessorConfig -} - -func NewWriteFilesProcessor(conf WriteFileOutputsProcessorConfig) *WriteFileOutputsProcessor { - return &WriteFileOutputsProcessor{ - config: conf, - } -} - -func (f WriteFileOutputsProcessor) Name() string { - return "file_outputs_processor" -} - -// FileOutput is a data structure that can be used by a SpecificationProcessor to output files that can be written by tje WriteFileOutputsProcessor. -type FileOutput struct { - Path string - Data []byte - Mode os.FileMode -} - -func (f WriteFileOutputsProcessor) Process(ctx OutputProcessingContext) error { - ctx.Logger.Info("Writing output files ...") - - var files []FileOutput - for _, o := range ctx.Outputs { - fo, ok := o.Value.(FileOutput) - if !ok { - continue - } - files = append(files, fo) - } - - if err := f.cleanRegistry(ctx); err != nil { - ctx.Logger.Error("failed cleaning output registry") - return err - } - - errs := errors.NewGroup(WriteFileOutputsProcessorErrorCode) - - // Write files concurrently to speed up process. - var wg sync.WaitGroup - for _, file := range files { - wg.Add(1) - go func(ctx OutputProcessingContext, file FileOutput) { - defer wg.Done() - ctx.Logger.Info(fmt.Sprintf("Writing file %q ...", file.Path)) - err := os.WriteFile(file.Path, file.Data, os.ModePerm) - if err != nil { - ctx.Logger.Error(fmt.Sprintf("failed writing output file at %q", file.Path)) - errs = errs.Append(err) - } - ctx.AddToRegistry(file.Path) - }(ctx, file) - } - wg.Wait() - - ctx.Logger.Success("Output files written successfully.") - - return nil -} - -func (f WriteFileOutputsProcessor) cleanRegistry(ctx OutputProcessingContext) error { - var wg sync.WaitGroup - for _, o := range ctx.RegistryOutputs() { - wg.Add(1) - go func(ctx OutputProcessingContext, o string) { - defer wg.Done() - if err := os.Remove(o); err != nil { - if errors.Is(err, os.ErrNotExist) { - return - } - panic(errors.Wrap(err, "failed cleaning output registry files")) - } - ctx.RemoveFromRegistry(o) - }(ctx, o) - } - wg.Wait() - - return nil -}