diff --git a/pkg/specter/artifactproc_test.go b/pkg/specter/artifactproc_test.go index 0eae069..381e58e 100644 --- a/pkg/specter/artifactproc_test.go +++ b/pkg/specter/artifactproc_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package specter import ( diff --git a/pkg/specter/pipelinedefault.go b/pkg/specter/pipelinedefault.go index 0863112..ff716f1 100644 --- a/pkg/specter/pipelinedefault.go +++ b/pkg/specter/pipelinedefault.go @@ -262,43 +262,56 @@ func (s unitLoadingStage) Run(ctx PipelineContext, sources []Source) ([]Unit, er return nil, err } - errs := errors.NewGroup(errors.InternalErrorCode) + units, err := s.run(ctx, sources) + if err != nil { + return units, s.Hooks.OnError(ctx, errors.WrapWithMessage(err, UnitLoadingFailedErrorCode, "failed to load units")) + } + + if err := s.Hooks.After(ctx); err != nil { + return nil, err + } + + return units, nil +} + +func (s unitLoadingStage) run(ctx PipelineContext, sources []Source) ([]Unit, error) { + var units []Unit for _, src := range sources { if err := ctx.Err(); err != nil { - return nil, s.handleError(ctx, err) + return nil, err } if err := s.Hooks.BeforeSource(ctx, src); err != nil { return nil, err } - for _, l := range s.Loaders { - if !l.SupportsSource(src) { - continue - } - - loadedUnits, err := l.Load(src) - if err != nil { - errs = errs.Append(err) - continue - } - ctx.Units = append(ctx.Units, loadedUnits...) + uns, err := s.runLoader(src) + if err != nil { + return nil, err } + units = append(units, uns...) if err := s.Hooks.AfterSource(ctx, src); err != nil { return nil, err } } - - if err := s.Hooks.After(ctx); err != nil { - return nil, err - } - - return ctx.Units, s.handleError(ctx, errors.GroupOrNil(errs)) + return units, nil } -func (s unitLoadingStage) handleError(ctx PipelineContext, err error) error { - return s.Hooks.OnError(ctx, err) +func (s unitLoadingStage) runLoader(src Source) ([]Unit, error) { + var units []Unit + for _, l := range s.Loaders { + if !l.SupportsSource(src) { + continue + } + + loadedUnits, err := l.Load(src) + if err != nil { + return nil, err + } + units = append(units, loadedUnits...) + } + return units, nil } type unitProcessingStage struct { diff --git a/pkg/specter/pipelinedefault_test.go b/pkg/specter/pipelinedefault_test.go index e7fc292..5f29814 100644 --- a/pkg/specter/pipelinedefault_test.go +++ b/pkg/specter/pipelinedefault_test.go @@ -443,6 +443,132 @@ func Test_sourceLoadingStage_Run(t *testing.T) { assert.False(t, recorder.afterSourceLocationCalled) assert.False(t, recorder.afterCalled) }) + + t.Run("should return the loaded sources", func(t *testing.T) { + locations := []string{"/path/to/file"} + expectedSources := []specter.Source{ + { + Location: "/path/to/file/0", + }, + { + Location: "/path/to/file/1", + }, + } + stage := specter.DefaultSourceLoadingStage{ + SourceLoaders: []specter.SourceLoader{ + specter.FunctionalSourceLoader{ + SupportsFunc: func(location string) bool { return true }, + LoadFunc: func(location string) ([]specter.Source, error) { + return []specter.Source{expectedSources[0]}, nil + }, + }, + specter.FunctionalSourceLoader{ + SupportsFunc: func(location string) bool { return true }, + LoadFunc: func(location string) ([]specter.Source, error) { + return []specter.Source{expectedSources[1]}, nil + }, + }, + }, + } + + sources, err := stage.Run(specter.PipelineContext{Context: context.Background()}, locations) + + require.NoError(t, err) + require.Equal(t, expectedSources, sources) + }) +} + +func Test_unitLoadingStage_Run(t *testing.T) { + t.Run("should call all hooks under normal processing", func(t *testing.T) { + recorder := unitLoadingStageHooksCallRecorder{} + + stage := specter.DefaultUnitLoadingStage{ + Loaders: []specter.UnitLoader{ + specter.FunctionalUnitLoader{ + LoadFunc: func(s specter.Source) ([]specter.Unit, error) { + return nil, nil + }, + SupportsSourceFunc: func(s specter.Source) bool { + return true + }}, + }, + Hooks: &recorder, + } + + units, err := stage.Run( + specter.PipelineContext{Context: context.Background()}, + []specter.Source{ + {Location: "/path/to/file"}, + }, + ) + require.NoError(t, err) + require.Nil(t, units) + + assert.True(t, recorder.beforeCalled) + assert.True(t, recorder.beforeSourceCalled) + assert.True(t, recorder.afterSourceCalled) + assert.True(t, recorder.afterCalled) + }) + + t.Run("should call hooks until error", func(t *testing.T) { + recorder := unitLoadingStageHooksCallRecorder{} + + stage := specter.DefaultUnitLoadingStage{ + Loaders: []specter.UnitLoader{ + specter.FunctionalUnitLoader{ + LoadFunc: func(s specter.Source) ([]specter.Unit, error) { + return nil, assert.AnError + }, + SupportsSourceFunc: func(s specter.Source) bool { + return true + }}, + }, + Hooks: &recorder, + } + units, err := stage.Run( + specter.PipelineContext{Context: context.Background()}, + []specter.Source{ + {Location: "/path/to/file"}, + }, + ) + require.Error(t, err) + require.Nil(t, units) + + assert.True(t, recorder.beforeCalled) + assert.True(t, recorder.beforeSourceCalled) + assert.True(t, recorder.onErrorCalled) + assert.False(t, recorder.afterSourceCalled) + assert.False(t, recorder.afterCalled) + }) + + t.Run("should return the loaded units", func(t *testing.T) { + + expectedUnits := []specter.Unit{ + testutils.NewUnitStub("", "", specter.Source{}), + testutils.NewUnitStub("", "", specter.Source{}), + } + stage := specter.DefaultUnitLoadingStage{ + Loaders: []specter.UnitLoader{ + specter.FunctionalUnitLoader{ + LoadFunc: func(s specter.Source) ([]specter.Unit, error) { + return expectedUnits, nil + }, + SupportsSourceFunc: func(s specter.Source) bool { + return true + }}, + }, + } + + units, err := stage.Run( + specter.PipelineContext{Context: context.Background()}, + []specter.Source{ + {Location: "/path/to/file"}, + }, + ) + + require.NoError(t, err) + require.Equal(t, expectedUnits, units) + }) } func Test_unitProcessingStage_Run(t *testing.T) { @@ -490,6 +616,33 @@ func Test_unitProcessingStage_Run(t *testing.T) { assert.False(t, recorder.afterProcessorCalled) assert.False(t, recorder.afterCalled) }) + + t.Run("should return artifacts of processors", func(t *testing.T) { + expectedArtifacts := []specter.Artifact{ + &specter.FileArtifact{ + Path: "/path/to/file/0", + }, + &specter.FileArtifact{ + Path: "/path/to/file/1", + }, + } + stage := specter.DefaultUnitProcessingStage{ + Processors: []specter.UnitProcessor{ + specter.NewUnitProcessorFunc("", func(specter.UnitProcessingContext) ([]specter.Artifact, error) { + return []specter.Artifact{expectedArtifacts[0]}, nil + }), + specter.NewUnitProcessorFunc("", func(specter.UnitProcessingContext) ([]specter.Artifact, error) { + return []specter.Artifact{expectedArtifacts[1]}, nil + }), + }, + } + artifacts, err := stage.Run(specter.PipelineContext{Context: context.Background()}, []specter.Unit{ + testutils.NewUnitStub("", "", specter.Source{}), + }) + require.NoError(t, err) + + require.Equal(t, expectedArtifacts, artifacts) + }) } func Test_artifactProcessingStage_Run(t *testing.T) { @@ -535,6 +688,27 @@ func Test_artifactProcessingStage_Run(t *testing.T) { assert.False(t, recorder.afterProcessorCalled) assert.False(t, recorder.afterCalled) }) + + t.Run("should process artifacts", func(t *testing.T) { + expectedArtifacts := []specter.Artifact{ + &specter.FileArtifact{ + Path: "/path/to/file/0", + }, + &specter.FileArtifact{ + Path: "/path/to/file/1", + }, + } + stage := specter.DefaultArtifactProcessingStage{ + Processors: []specter.ArtifactProcessor{ + specter.NewArtifactProcessorFunc("", func(ctx specter.ArtifactProcessingContext) error { + assert.Equal(t, expectedArtifacts, ctx.Artifacts) + return nil + }), + }, + } + err := stage.Run(specter.PipelineContext{Context: context.Background()}, expectedArtifacts) + require.NoError(t, err) + }) } type FailingSourceLoadingStage struct{} @@ -561,6 +735,39 @@ func (f FailingArtifactProcessingStage) Run(specter.PipelineContext, []specter.A return assert.AnError } +type unitLoadingStageHooksCallRecorder struct { + beforeCalled bool + afterCalled bool + beforeSourceCalled bool + afterSourceCalled bool + onErrorCalled bool +} + +func (u *unitLoadingStageHooksCallRecorder) Before(specter.PipelineContext) error { + u.beforeCalled = true + return nil +} + +func (u *unitLoadingStageHooksCallRecorder) After(specter.PipelineContext) error { + u.afterCalled = true + return nil +} + +func (u *unitLoadingStageHooksCallRecorder) BeforeSource(specter.PipelineContext, specter.Source) error { + u.beforeSourceCalled = true + return nil +} + +func (u *unitLoadingStageHooksCallRecorder) AfterSource(specter.PipelineContext, specter.Source) error { + u.afterSourceCalled = true + return nil +} + +func (u *unitLoadingStageHooksCallRecorder) OnError(_ specter.PipelineContext, err error) error { + u.onErrorCalled = true + return err +} + type sourceLoadingStageHooksCallRecorder struct { beforeCalled bool afterCalled bool diff --git a/pkg/specter/pipelinedefaultexport_test.go b/pkg/specter/pipelinedefaultexport_test.go index 215ce99..3cd4e44 100644 --- a/pkg/specter/pipelinedefaultexport_test.go +++ b/pkg/specter/pipelinedefaultexport_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package specter type DefaultSourceLoadingStage = sourceLoadingStage diff --git a/pkg/specter/unitloading.go b/pkg/specter/unitloading.go index e4ad801..4c53fe7 100644 --- a/pkg/specter/unitloading.go +++ b/pkg/specter/unitloading.go @@ -184,3 +184,16 @@ func MapUnitGroup[T any](g UnitGroup, p func(u Unit) T) []T { return mapped } + +type FunctionalUnitLoader struct { + LoadFunc func(s Source) ([]Unit, error) + SupportsSourceFunc func(s Source) bool +} + +func (u FunctionalUnitLoader) Load(s Source) ([]Unit, error) { + return u.LoadFunc(s) +} + +func (u FunctionalUnitLoader) SupportsSource(s Source) bool { + return u.SupportsSourceFunc(s) +} diff --git a/pkg/specter/unitloading_test.go b/pkg/specter/unitloading_test.go index 74bce95..8b63f1e 100644 --- a/pkg/specter/unitloading_test.go +++ b/pkg/specter/unitloading_test.go @@ -375,3 +375,30 @@ func TestUnitWithIDsMatcher(t *testing.T) { }) } } + +func TestFunctionalUnitLoader(t *testing.T) { + t.Run("functions should be called", func(t *testing.T) { + expectedSource := specter.Source{ + Location: "/path/to/file", + } + expectedUnits := []specter.Unit{ + specter.UnitOf(0, "", "", expectedSource), + } + + f := specter.FunctionalUnitLoader{ + LoadFunc: func(s specter.Source) ([]specter.Unit, error) { + assert.Equal(t, expectedSource, s) + return expectedUnits, nil + }, + SupportsSourceFunc: func(s specter.Source) bool { + assert.Equal(t, expectedSource, s) + return true + }, + } + assert.True(t, f.SupportsSource(expectedSource)) + + units, err := f.Load(expectedSource) + require.NoError(t, err) + assert.Equal(t, expectedUnits, units) + }) +} diff --git a/pkg/testutils/artifactproc.go b/pkg/testutils/artifactproc.go index 6feaac6..202d238 100644 --- a/pkg/testutils/artifactproc.go +++ b/pkg/testutils/artifactproc.go @@ -1,3 +1,17 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( diff --git a/pkg/testutils/errors.go b/pkg/testutils/errors.go index b7334c8..02aea6d 100644 --- a/pkg/testutils/errors.go +++ b/pkg/testutils/errors.go @@ -1,3 +1,17 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import ( diff --git a/pkg/testutils/filesystem.go b/pkg/testutils/filesystem.go index 2ee56fd..43ca903 100644 --- a/pkg/testutils/filesystem.go +++ b/pkg/testutils/filesystem.go @@ -1,3 +1,17 @@ +// Copyright 2024 Morébec +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package testutils import (