refactor(test): run Trivy once

Signed-off-by: Nikita Pivkin <nikita.pivkin@smartforce.io>

Author: nikpivkin

Makefile

@@ -8,7 +8,7 @@ test:
 
 .PHONY: integration-test
 test-integration:
-	go test -v -timeout 15m -tags=integration ./integration/...
+	go test -v -timeout 5m -tags=integration ./integration/...
 
 .PHONY: rego
 rego: fmt-rego test-rego

integration/check_examples_test.go

@@ -3,11 +3,12 @@
 package integration
 
 import (
-	"fmt"
 	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,6 +23,49 @@ import (
 
 func TestValidateCheckExamples(t *testing.T) {
 	cacheDir := setupCache(t)
+	targetDir := setupTarget(t)
+	outputFile := filepath.Join(t.TempDir(), "report.json")
+
+	args := []string{
+		"conf",
+		"--skip-check-update",
+		"--quiet",
+		"--format", "json",
+		"--output", outputFile,
+		"--cache-dir", cacheDir,
+		targetDir,
+	}
+	runTrivy(t, args)
+
+	report := readTrivyReport(t, outputFile)
+
+	verifyExamples(t, report, targetDir)
+}
+
+func setupCache(t *testing.T) string {
+	t.Helper()
+
+	cmd := exec.Command("make", "create-bundle")
+	cmd.Dir = ".."
+	require.NoError(t, cmd.Run())
+	defer os.Remove("../bundle.tar.gz")
+
+	cacheDir := t.TempDir()
+
+	policyDir := filepath.Join(cacheDir, "policy", "content")
+	require.NoError(t, os.MkdirAll(policyDir, os.ModePerm))
+
+	cmd = exec.Command("tar", "-zxf", "bundle.tar.gz", "-C", policyDir)
+	cmd.Dir = ".."
+	require.NoError(t, cmd.Run())
+
+	return cacheDir
+}
+
+func setupTarget(t *testing.T) string {
+	t.Helper()
+
+	targetDir := t.TempDir()
 
 	// TODO(nikpivkin): load examples from fs
 	rego.LoadAndRegister()
@@ -32,85 +76,82 @@ func TestValidateCheckExamples(t *testing.T) {
 			continue
 		}
 
-		t.Run(r.AVDID, func(t *testing.T) {
-			examples, path, err := examples.GetCheckExamples(r.Rule)
-			require.NoError(t, err)
+		examples, path, err := examples.GetCheckExamples(r.Rule)
+		require.NoError(t, err)
 
-			if path == "" {
-				return
-			}
+		if path == "" {
+			continue
+		}
 
-			for provider, providerExamples := range examples {
-				validateExamples(t, providerExamples.Bad.ToStrings(), provider, cacheDir, r.AVDID, true)
-				validateExamples(t, providerExamples.Good.ToStrings(), provider, cacheDir, r.AVDID, false)
-			}
-		})
+		for provider, providerExamples := range examples {
+			writeExamples(t, providerExamples.Bad.ToStrings(), provider, targetDir, r.AVDID, "bad")
+			writeExamples(t, providerExamples.Good.ToStrings(), provider, targetDir, r.AVDID, "good")
+		}
 	}
+
+	return targetDir
 }
 
-func validateExamples(t *testing.T, examples []string, provider, cacheDir, avdID string, expected bool) {
+func writeExamples(t *testing.T, examples []string, provider, cacheDir string, id string, typ string) {
 	for i, example := range examples {
-		fileName := fmt.Sprintf("test-%d%s", i, extensionByProvider(provider))
-		t.Run(fileName, func(t *testing.T) {
-			targetFile := filepath.Join(t.TempDir(), fileName)
-
-			require.NoError(t, os.WriteFile(targetFile, []byte(example), fs.ModePerm))
-
-			outputFile := filepath.Join(t.TempDir(), "report.json")
-
-			args := []string{
-				"conf",
-				"--skip-check-update",
-				"--quiet",
-				"--format", "json",
-				"--output", outputFile,
-				"--cache-dir", cacheDir,
-				targetFile,
-			}
-			runTrivy(t, args)
-
-			report := readTrivyReport(t, outputFile)
-
-			assert.Equal(t, expected, reportContainsMisconfig(report, fileName, avdID))
-		})
+		name := "test" + extensionByProvider(provider)
+		file := filepath.Join(cacheDir, id, provider, typ, strconv.Itoa(i), name)
+		require.NoError(t, os.MkdirAll(filepath.Dir(file), fs.ModePerm))
+		require.NoError(t, os.WriteFile(file, []byte(example), fs.ModePerm))
 	}
 }
 
-func setupCache(t *testing.T) string {
-	t.Helper()
+func verifyExamples(t *testing.T, report types.Report, targetDir string) {
+	got := getFailureIDs(report)
 
-	cmd := exec.Command("make", "create-bundle")
-	cmd.Dir = ".."
+	err := filepath.Walk(targetDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
 
-	require.NoError(t, cmd.Run())
-	defer os.Remove("bundle.tar.gz")
+		relPath, err := filepath.Rel(targetDir, path)
+		require.NoError(t, err)
 
-	cacheDir := t.TempDir()
+		parts := strings.Split(relPath, string(os.PathSeparator))
+		require.Len(t, parts, 5)
 
-	policyDir := filepath.Join(cacheDir, "policy", "content")
-	require.NoError(t, os.MkdirAll(policyDir, os.ModePerm))
+		id, _, exampleType := parts[0], parts[1], parts[2]
 
-	cmd = exec.Command("tar", "-zxf", "bundle.tar.gz", "-C", policyDir)
-	cmd.Dir = ".."
-	require.NoError(t, cmd.Run())
+		shouldBePresent := exampleType == "bad"
 
-	return cacheDir
+		t.Run(relPath, func(t *testing.T) {
+			if shouldBePresent {
+				ids, exists := got[relPath]
+				assert.True(t, exists)
+				assert.Contains(t, ids, id)
+			} else {
+				ids, exists := got[relPath]
+				if exists {
+					assert.NotContains(t, ids, id)
+				}
+			}
+		})
+		return nil
+	})
+
+	require.NoError(t, err)
 }
 
-func reportContainsMisconfig(report types.Report, path string, id string) bool {
-	for _, res := range report.Results {
-		if res.Target != path {
-			continue
-		}
+func getFailureIDs(report types.Report) map[string][]string {
+	ids := make(map[string][]string)
 
-		for _, misconf := range res.Misconfigurations {
-			if misconf.AVDID == id && misconf.Status == types.MisconfStatusFailure {
-				return true
+	for _, result := range report.Results {
+		for _, misconf := range result.Misconfigurations {
+			if misconf.Status == types.MisconfStatusFailure {
+				ids[result.Target] = append(ids[result.Target], misconf.AVDID)
 			}
 		}
 	}
 
-	return false
+	return ids
 }
 
 func extensionByProvider(provider string) string {