Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Secret detection capability #206

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

Secret detection capability #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
06baaad
update to latest syft and grype
matthyx Jan 19, 2024
279d8fa
add test with schema v1 image to validate fix
matthyx Jan 24, 2024
848a94b
use stereoscope fork to support maxImageSize
matthyx Jan 24, 2024
d322bd3
clean temp dir after SBOM generation
matthyx Jan 24, 2024
a4536c6
Initial commit of the secret detection package
slashben Jan 22, 2024
397ce6a
use fileresolver to access image layers
matthyx Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions adapters/v1/domain_to_syft_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ func Test_domainJSONToSyft(t *testing.T) {
artifactRelationships: 129,
files: 78,
source: source{
id: "e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501",
name: "alpine",
id: "fd6275a37d2472b9d3be70c3261087b8d65e441c21342ae7313096312bcda2b3",
name: "library/alpine",
},
},
}, {
Expand Down
124 changes: 45 additions & 79 deletions adapters/v1/syft.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,20 @@ import (

helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers"

"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/clio"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/cmd/syft/cli/eventloop"
"github.com/anchore/syft/cmd/syft/cli/options"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/eapache/go-resiliency/deadline"
"github.com/hashicorp/go-multierror"
"github.com/kubescape/go-logger"
"github.com/kubescape/go-logger/helpers"
"github.com/kubescape/kubevuln/core/domain"
"github.com/kubescape/kubevuln/core/ports"
"github.com/kubescape/kubevuln/internal/tools"
"github.com/kubescape/kubevuln/pkg/secretdetector"
"go.opentelemetry.io/otel"
)

Expand All @@ -34,7 +34,6 @@ type SyftAdapter struct {
}

var _ ports.SBOMCreator = (*SyftAdapter)(nil)
var ErrImageTooLarge = fmt.Errorf("image size exceeds maximum allowed size")

// NewSyftAdapter initializes the SyftAdapter struct
func NewSyftAdapter(scanTimeout time.Duration, maxImageSize int64) *SyftAdapter {
Expand Down Expand Up @@ -78,13 +77,14 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, name, imageID string, opti
InsecureSkipTLSVerify: options.InsecureSkipTLSVerify,
InsecureUseHTTP: options.InsecureUseHTTP,
Credentials: credentials,
MaxImageSize: s.maxImageSize,
}

syftOpts := defaultPackagesOptions()

// prepare temporary directory for image download
t := file.NewTempDirGenerator("stereoscope")
defer func(t *file.TempDirGenerator) {
t := stereoscopeFile.NewTempDirGenerator("stereoscope")
defer func(t *stereoscopeFile.TempDirGenerator) {
err := t.Cleanup()
if err != nil {
logger.L().Ctx(ctx).Warning("failed to cleanup temp dir", helpers.Error(err),
Expand All @@ -106,7 +106,7 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, name, imageID string, opti
src, err = detectSource(imageID, syftOpts, &registryOptions)
}
switch {
case errors.Is(err, ErrImageTooLarge):
case errors.Is(err, image.ErrImageTooLarge):
logger.L().Ctx(ctx).Warning("Image exceeds size limit",
helpers.Int("maxImageSize", int(s.maxImageSize)),
helpers.String("imageID", imageID))
Expand All @@ -125,11 +125,41 @@ func (s *SyftAdapter) CreateSBOM(ctx context.Context, name, imageID string, opti
var syftSBOM *sbom.SBOM
dl := deadline.New(s.scanTimeout)
err = dl.Run(func(stopper <-chan struct{}) error {
// make sure we clean the temp dir
defer func(src source.Source) {
if err := src.Close(); err != nil {
logger.L().Ctx(ctx).Warning("failed to close source", helpers.Error(err),
helpers.String("imageID", imageID))
}
}(src)
// generate SBOM
logger.L().Debug("generating SBOM",
helpers.String("imageID", imageID))
syftSBOM, err = generateSBOM(name, s.Version(), src, &syftOpts.Catalog)
return err
id := clio.Identification{
Name: name,
Version: s.Version(),
}
syftSBOM, err = syft.CreateSBOM(ctx, src, syftOpts.Catalog.ToSBOMConfig(id))
if err != nil {
return fmt.Errorf("failed to generate SBOM: %w", err)
}
// scan for secrets
resolver, err := src.FileResolver(source.SquashedScope)
if err != nil {
return fmt.Errorf("unable to get file resolver: %w", err)
}
fileDetectionConfig := &secretdetector.FileDetectionConfig{
SkipBinaryFiles: false,
SizeThreshold: 0,
}
Comment on lines +151 to +154
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd set SkipBinaryFiles to true in the first release. We need to make the scanning more efficient before we turn this on

result, err := secretdetector.SearchDirectoryForSecrets(resolver, fileDetectionConfig)
if err != nil {
return fmt.Errorf("failed to search directory for secrets: %w", err)
}
logger.L().Info("searched directory for secrets",
helpers.String("imageID", imageID),
helpers.Interface("result", result))
return nil
})
switch err {
case deadline.ErrTimedOut:
Expand Down Expand Up @@ -181,16 +211,7 @@ func defaultPackagesOptions() *packagesOptions {
}

func detectSource(userInput string, opts *packagesOptions, registryOptions *image.RegistryOptions) (source.Source, error) {
detection, err := source.Detect(
userInput,
source.DetectConfig{
DefaultImageSource: opts.DefaultImagePullSource,
},
)
if err != nil {
return nil, fmt.Errorf("could not determine source: %w", err)
}

var err error
var platform *image.Platform

if opts.Platform != "" {
Expand All @@ -200,13 +221,8 @@ func detectSource(userInput string, opts *packagesOptions, registryOptions *imag
}
}

hashers, err := Hashers(opts.Source.File.Digests...)
if err != nil {
return nil, fmt.Errorf("invalid hash: %w", err)
}

src, err := detection.NewSource(
source.DetectionSourceConfig{
src, err := source.NewFromStereoscopeImage(
source.StereoscopeImageConfig{
Alias: source.Alias{
Name: opts.Source.Name,
Version: opts.Source.Version,
Expand All @@ -216,64 +232,14 @@ func detectSource(userInput string, opts *packagesOptions, registryOptions *imag
Exclude: source.ExcludeConfig{
Paths: opts.Exclusions,
},
DigestAlgorithms: hashers,
BasePath: opts.BasePath,
Reference: userInput,
From: image.DetermineDefaultImagePullSource(userInput),
},
)

return src, err
}

func generateSBOM(toolName string, toolVersion string, src source.Source, opts *options.Catalog) (*sbom.SBOM, error) {
tasks, err := eventloop.Tasks(opts)
if err != nil {
return nil, err
}

s := sbom.SBOM{
Source: src.Describe(),
Descriptor: sbom.Descriptor{
Name: toolName,
Version: toolVersion,
Configuration: opts,
},
}

err = buildRelationships(&s, src, tasks)

return &s, err
}

func buildRelationships(s *sbom.SBOM, src source.Source, tasks []eventloop.Task) error {
var errs error

var relationships []<-chan artifact.Relationship
for _, task := range tasks {
c := make(chan artifact.Relationship)
relationships = append(relationships, c)
go func(task eventloop.Task) {
err := eventloop.RunTask(task, &s.Artifacts, src, c)
if err != nil {
errs = multierror.Append(errs, err)
}
}(task)
}

s.Relationships = append(s.Relationships, mergeRelationships(relationships...)...)

return errs
}

func mergeRelationships(cs ...<-chan artifact.Relationship) (relationships []artifact.Relationship) {
for _, c := range cs {
for n := range c {
relationships = append(relationships, n)
}
}

return relationships
}

// Version returns Syft's version which is used to tag SBOMs
func (s *SyftAdapter) Version() string {
return tools.PackageVersion("github.com/anchore/syft")
Expand Down
19 changes: 12 additions & 7 deletions adapters/v1/syft_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ func Test_syftAdapter_CreateSBOM(t *testing.T) {
imageID: "library/hello-world@sha256:aa0cc8055b82dc2509bed2e19b275c8f463506616377219d9642221ab53cf9fe",
format: string(fileContent("testdata/hello-world-sbom.format.json")),
},
{
name: "schema v1 image produces well-formed SBOM",
imageID: "quay.io/jitesoft/debian:stretch-slim",
format: string(fileContent("testdata/stretch-slim-sbom.format.json")),
},
{
name: "valid image produces well-formed SBOM",
imageID: "library/alpine@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501",
Expand All @@ -55,13 +60,13 @@ func Test_syftAdapter_CreateSBOM(t *testing.T) {
},
},
},
// {
// name: "big image produces incomplete SBOM",
// imageID: "library/alpine@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501",
// format: "null",
// maxImageSize: 1,
// wantIncomplete: true,
// },
{
name: "big image produces incomplete SBOM",
imageID: "library/alpine@sha256:e2e16842c9b54d985bf1ef9242a313f36b856181f188de21313820e177002501",
format: "null",
maxImageSize: 1,
wantIncomplete: true,
},
{
name: "system tests image",
imageID: "public-registry.systest-ns-bpf7:5000/nginx:test",
Expand Down
Loading
Loading