Skip to content

Commit

Permalink
Rename spec* to unit (#21)
Browse files Browse the repository at this point in the history
  • Loading branch information
jwillp authored Sep 4, 2024
1 parent 875af13 commit 7cac376
Show file tree
Hide file tree
Showing 27 changed files with 1,266 additions and 1,252 deletions.
20 changes: 17 additions & 3 deletions pkg/specter/artifactproc.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2024 Morébec
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specter

import (
Expand Down Expand Up @@ -81,9 +95,9 @@ func (n ProcessorArtifactRegistry) FindAll() ([]ArtifactRegistryEntry, error) {

type ArtifactProcessingContext struct {
context.Context
Specifications SpecificationGroup
Artifacts []Artifact
Logger Logger
Units UnitGroup
Artifacts []Artifact
Logger Logger

ArtifactRegistry ProcessorArtifactRegistry
processorName string
Expand Down
2 changes: 1 addition & 1 deletion pkg/specter/artifactproc_fileartifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const DefaultWriteMode WriteMode = WriteOnceMode

var _ Artifact = (*FileArtifact)(nil)

// FileArtifact is a data structure that can be used by a SpecificationProcessor to generate file artifacts
// FileArtifact is a data structure that can be used by a UnitProcessor to generate file artifacts
// that can be written by the FileArtifactProcessor.
type FileArtifact struct {
Path string
Expand Down
36 changes: 18 additions & 18 deletions pkg/specter/assembly.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,50 +35,50 @@ type PipelineOption func(s *Pipeline)

// WithLogger configures the Logger of a Pipeline instance.
func WithLogger(l Logger) PipelineOption {
return func(s *Pipeline) {
s.Logger = l
return func(p *Pipeline) {
p.Logger = l
}
}

// WithSourceLoaders configures the SourceLoader of a Pipeline instance.
func WithSourceLoaders(loaders ...SourceLoader) PipelineOption {
return func(s *Pipeline) {
s.SourceLoaders = append(s.SourceLoaders, loaders...)
return func(p *Pipeline) {
p.SourceLoaders = append(p.SourceLoaders, loaders...)
}
}

// WithLoaders configures the SpecificationLoader of a Pipeline instance.
func WithLoaders(loaders ...SpecificationLoader) PipelineOption {
return func(s *Pipeline) {
s.Loaders = append(s.Loaders, loaders...)
// WithLoaders configures the UnitLoader of a Pipeline instance.
func WithLoaders(loaders ...UnitLoader) PipelineOption {
return func(p *Pipeline) {
p.Loaders = append(p.Loaders, loaders...)
}
}

// WithProcessors configures the SpecProcess of a Pipeline instance.
func WithProcessors(processors ...SpecificationProcessor) PipelineOption {
return func(s *Pipeline) {
s.Processors = append(s.Processors, processors...)
// WithProcessors configures the UnitProcess of a Pipeline instance.
func WithProcessors(processors ...UnitProcessor) PipelineOption {
return func(p *Pipeline) {
p.Processors = append(p.Processors, processors...)
}
}

// WithArtifactProcessors configures the ArtifactProcessor of a Pipeline instance.
func WithArtifactProcessors(processors ...ArtifactProcessor) PipelineOption {
return func(s *Pipeline) {
s.ArtifactProcessors = append(s.ArtifactProcessors, processors...)
return func(p *Pipeline) {
p.ArtifactProcessors = append(p.ArtifactProcessors, processors...)
}
}

// WithTimeProvider configures the TimeProvider of a Pipeline instance.
func WithTimeProvider(tp TimeProvider) PipelineOption {
return func(s *Pipeline) {
s.TimeProvider = tp
return func(p *Pipeline) {
p.TimeProvider = tp
}
}

// WithArtifactRegistry configures the ArtifactRegistry of a Pipeline instance.
func WithArtifactRegistry(r ArtifactRegistry) PipelineOption {
return func(s *Pipeline) {
s.ArtifactRegistry = r
return func(p *Pipeline) {
p.ArtifactRegistry = r
}
}

Expand Down
36 changes: 18 additions & 18 deletions pkg/specter/assembly_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,53 +23,53 @@ import (
)

func TestWithDefaultLogger(t *testing.T) {
s := NewPipeline(WithDefaultLogger())
assert.IsType(t, &DefaultLogger{}, s.Logger)
p := NewPipeline(WithDefaultLogger())
assert.IsType(t, &DefaultLogger{}, p.Logger)
}

func TestWithSourceLoaders(t *testing.T) {
loader := &FileSystemSourceLoader{}
s := NewPipeline(WithSourceLoaders(loader))
require.Contains(t, s.SourceLoaders, loader)
p := NewPipeline(WithSourceLoaders(loader))
require.Contains(t, p.SourceLoaders, loader)
}

func TestWithLoaders(t *testing.T) {
loader := &specterutils.HCLGenericSpecLoader{}
s := NewPipeline(WithLoaders(loader))
require.Contains(t, s.Loaders, loader)
loader := &specterutils.HCLGenericUnitLoader{}
p := NewPipeline(WithLoaders(loader))
require.Contains(t, p.Loaders, loader)
}

func TestWithProcessors(t *testing.T) {
processor := specterutils.LintingProcessor{}
s := NewPipeline(WithProcessors(processor))
require.Contains(t, s.Processors, processor)
p := NewPipeline(WithProcessors(processor))
require.Contains(t, p.Processors, processor)
}

func TestWithArtifactProcessors(t *testing.T) {
processor := FileArtifactProcessor{}
s := NewPipeline(WithArtifactProcessors(processor))
require.Contains(t, s.ArtifactProcessors, processor)
p := NewPipeline(WithArtifactProcessors(processor))
require.Contains(t, p.ArtifactProcessors, processor)
}

func TestWithTimeProvider(t *testing.T) {
tp := CurrentTimeProvider()
s := NewPipeline(WithTimeProvider(tp))
require.NotNil(t, s.TimeProvider)
p := NewPipeline(WithTimeProvider(tp))
require.NotNil(t, p.TimeProvider)
}

func TestWithArtifactRegistry(t *testing.T) {
registry := &InMemoryArtifactRegistry{}
s := NewPipeline(WithArtifactRegistry(registry))
require.Equal(t, s.ArtifactRegistry, registry)
p := NewPipeline(WithArtifactRegistry(registry))
require.Equal(t, p.ArtifactRegistry, registry)
}

func TestWithJSONArtifactRegistry(t *testing.T) {
fs := &mockFileSystem{}
filePath := DefaultJSONArtifactRegistryFileName

s := NewPipeline(WithJSONArtifactRegistry(filePath, fs))
require.IsType(t, &JSONArtifactRegistry{}, s.ArtifactRegistry)
registry := s.ArtifactRegistry.(*JSONArtifactRegistry)
p := NewPipeline(WithJSONArtifactRegistry(filePath, fs))
require.IsType(t, &JSONArtifactRegistry{}, p.ArtifactRegistry)
registry := p.ArtifactRegistry.(*JSONArtifactRegistry)

assert.Equal(t, registry.FileSystem, fs)
assert.Equal(t, registry.FilePath, filePath)
Expand Down
2 changes: 1 addition & 1 deletion pkg/specter/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ package specter

type ArtifactID string

// Artifact represents a result or output generated by a SpecificationProcessor.
// Artifact represents a result or output generated by a UnitProcessor.
// An artifact is a unit of data or information produced as part of the processing workflow.
// It can be a transient, in-memory object, or it might represent more permanent entities such as
// files on disk, records in a database, deployment units, or other forms of data artifacts.
Expand Down
72 changes: 36 additions & 36 deletions pkg/specter/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,15 @@ const RunThrough RunMode = "run-through"
const defaultRunMode = PreviewMode

const SourceLoadingFailedErrorCode = "specter.source_loading_failed"
const SpecificationLoadingFailedErrorCode = "specter.specification_loading_failed"
const SpecificationProcessingFailedErrorCode = "specter.specification_processing_failed"
const UnitLoadingFailedErrorCode = "specter.unit_loading_failed"
const UnitProcessingFailedErrorCode = "specter.unit_processing_failed"
const ArtifactProcessingFailedErrorCode = "specter.artifact_processing_failed"

// Pipeline is the service responsible to run a specter pipeline.
type Pipeline struct {
SourceLoaders []SourceLoader
Loaders []SpecificationLoader
Processors []SpecificationProcessor
Loaders []UnitLoader
Processors []UnitProcessor
ArtifactProcessors []ArtifactProcessor
ArtifactRegistry ArtifactRegistry
Logger Logger
Expand All @@ -53,7 +53,7 @@ type PipelineResult struct {

SourceLocations []string
Sources []Source
Specifications []Specification
Units []Unit
Artifacts []Artifact
RunMode RunMode
}
Expand Down Expand Up @@ -92,18 +92,18 @@ func (p Pipeline) Run(ctx context.Context, sourceLocations []string, runMode Run
return result, e
}

// Load Specifications
result.Specifications, err = p.loadSpecifications(ctx, result.Sources)
// Load Units
result.Units, err = p.loadUnits(ctx, result.Sources)
if err != nil {
e := errors.WrapWithMessage(err, SpecificationLoadingFailedErrorCode, "failed loading specifications")
e := errors.WrapWithMessage(err, UnitLoadingFailedErrorCode, "failed loading units")
p.Logger.Error(e.Error())
return result, e
}

// Process Specifications
result.Artifacts, err = p.processSpecifications(ctx, result.Specifications)
// Process Units
result.Artifacts, err = p.processUnits(ctx, result.Units)
if err != nil {
e := errors.WrapWithMessage(err, SpecificationProcessingFailedErrorCode, "failed processing specifications")
e := errors.WrapWithMessage(err, UnitProcessingFailedErrorCode, "failed processing units")
p.Logger.Error(e.Error())
return result, e
}
Expand All @@ -114,7 +114,7 @@ func (p Pipeline) Run(ctx context.Context, sourceLocations []string, runMode Run
}

// Process Artifact
if err = p.processArtifacts(ctx, result.Specifications, result.Artifacts); err != nil {
if err = p.processArtifacts(ctx, result.Units, result.Artifacts); err != nil {
e := errors.WrapWithMessage(err, ArtifactProcessingFailedErrorCode, "failed processing artifacts")
p.Logger.Error(e.Error())
return result, e
Expand All @@ -131,7 +131,7 @@ func (p Pipeline) logResult(run PipelineResult) {
p.Logger.Info(fmt.Sprintf("Run time: %s", run.ExecutionTime()))
p.Logger.Info(fmt.Sprintf("Number of source locations: %d", len(run.SourceLocations)))
p.Logger.Info(fmt.Sprintf("Number of sources: %d", len(run.Sources)))
p.Logger.Info(fmt.Sprintf("Number of specifications: %d", len(run.Specifications)))
p.Logger.Info(fmt.Sprintf("Number of units: %d", len(run.Units)))
p.Logger.Info(fmt.Sprintf("Number of artifacts: %d", len(run.Artifacts)))
}

Expand Down Expand Up @@ -171,12 +171,12 @@ func (p Pipeline) loadSources(ctx context.Context, sourceLocations []string) ([]
return sources, errors.GroupOrNil(errs)
}

// loadSpecifications performs the loading of Specifications.
func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]Specification, error) {
p.Logger.Info("\nLoading specifications ...")
// loadUnits performs the loading of Units.
func (p Pipeline) loadUnits(ctx context.Context, sources []Source) ([]Unit, error) {
p.Logger.Info("\nLoading units ...")

// Load specifications
var specifications []Specification
// Load units
var units []Unit
var sourcesNotLoaded []Source
errs := errors.NewGroup(errors.InternalErrorCode)

Expand All @@ -190,14 +190,14 @@ func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]S
continue
}

loadedSpecs, err := l.Load(src)
loadedUnits, err := l.Load(src)
if err != nil {
p.Logger.Error(err.Error())
errs = errs.Append(err)
continue
}

specifications = append(specifications, loadedSpecs...)
units = append(units, loadedUnits...)
wasLoaded = true
}

Expand All @@ -211,24 +211,24 @@ func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]S
p.Logger.Warning(fmt.Sprintf("%q could not be loaded.", src.Location))
}

p.Logger.Warning("%d specifications were not loaded.")
p.Logger.Warning("%d units were not loaded.")
}

p.Logger.Info(fmt.Sprintf("%d specifications loaded.", len(specifications)))
p.Logger.Info(fmt.Sprintf("%d units loaded.", len(units)))

return specifications, errors.GroupOrNil(errs)
return units, errors.GroupOrNil(errs)
}

// processSpecifications sends the specifications to processors.
func (p Pipeline) processSpecifications(ctx context.Context, specs []Specification) ([]Artifact, error) {
// processUnits sends the units to processors.
func (p Pipeline) processUnits(ctx context.Context, units []Unit) ([]Artifact, error) {
pctx := ProcessingContext{
Context: ctx,
Specifications: specs,
Artifacts: nil,
Logger: p.Logger,
Context: ctx,
Units: units,
Artifacts: nil,
Logger: p.Logger,
}

p.Logger.Info("\nProcessing specifications ...")
p.Logger.Info("\nProcessing units ...")
for _, processor := range p.Processors {
if err := ctx.Err(); err != nil {
return nil, err
Expand All @@ -245,12 +245,12 @@ func (p Pipeline) processSpecifications(ctx context.Context, specs []Specificati
p.Logger.Info(fmt.Sprintf("-> %s", o.ID()))
}

p.Logger.Success("Specifications processed successfully.")
p.Logger.Success("Units processed successfully.")
return pctx.Artifacts, nil
}

// processArtifacts sends a list of ProcessingArtifacts to the registered ArtifactProcessors.
func (p Pipeline) processArtifacts(ctx context.Context, specifications []Specification, artifacts []Artifact) error {
func (p Pipeline) processArtifacts(ctx context.Context, units []Unit, artifacts []Artifact) error {
if p.ArtifactRegistry == nil {
p.ArtifactRegistry = &InMemoryArtifactRegistry{}
}
Expand All @@ -273,10 +273,10 @@ func (p Pipeline) processArtifacts(ctx context.Context, specifications []Specifi

processorName := processor.Name()
artifactCtx := ArtifactProcessingContext{
Context: ctx,
Specifications: specifications,
Artifacts: artifacts,
Logger: p.Logger,
Context: ctx,
Units: units,
Artifacts: artifacts,
Logger: p.Logger,
ArtifactRegistry: ProcessorArtifactRegistry{
processorName: processorName,
registry: p.ArtifactRegistry,
Expand Down
Loading

0 comments on commit 7cac376

Please sign in to comment.