From 79c0558533515d82fbbf2b187261494c5f54fcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Thu, 26 Sep 2024 09:49:37 +0200 Subject: [PATCH 1/2] feat: make evaluation path live in payload world MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- pkg/commands/scan/options.go | 23 +++++++++++++++++------ pkg/json-engine/compiler.go | 6 ++++-- pkg/json-engine/model.go | 11 ++++++----- pkg/server/model/response.go | 2 +- test/api/go/main/main.go | 2 +- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/pkg/commands/scan/options.go b/pkg/commands/scan/options.go index 46993edc..027eb754 100644 --- a/pkg/commands/scan/options.go +++ b/pkg/commands/scan/options.go @@ -105,15 +105,26 @@ func (c *options) run(cmd *cobra.Command, _ []string) error { for _, response := range responses { for _, policy := range response.Policies { for _, rule := range policy.Rules { + status := "PASSED" if rule.Error != nil { - out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "ERROR:", rule.Error.Error()) + status = fmt.Sprintf("ERROR: %s", rule.Error.Error()) } else if len(rule.Violations) != 0 { - out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED") - out.println(rule.Violations.Error()) - } else { - // TODO: handle skip, warn - out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "PASSED") + status = "FAILED" } + out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s, ID=%s)", status, policy.Policy.Name, rule.Rule.Name, rule.Identifier)) + if len(rule.Violations) != 0 { + out.println(rule.Violations.Error(" ")) + } + + // if rule.Error != nil { + // out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "ERROR:", rule.Error.Error()) + // } else if len(rule.Violations) != 0 { + // out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED") + // out.println(rule.Violations.Error()) + // } else { + // // TODO: handle skip, warn + // out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "PASSED") + // } } } } diff --git a/pkg/json-engine/compiler.go b/pkg/json-engine/compiler.go index c0fc5766..0150b097 100644 --- a/pkg/json-engine/compiler.go +++ b/pkg/json-engine/compiler.go @@ -194,9 +194,11 @@ func (c *compiler) compileAssertion( errs, err := check(resource, bindings) if len(errs) != 0 { result.ErrorList = errs + message := fmt.Sprintf("(CHECK=%s)", path.String()) if in.Message != nil { - result.Message = in.Message.Format(resource, bindings, compilers.Jp.Options()...) + message = fmt.Sprintf("%s %s", in.Message.Format(resource, bindings, compilers.Jp.Options()...), message) } + result.Message = message } return result, err }, nil @@ -228,7 +230,7 @@ func (c *compiler) compileAssertionTree( return nil, err } return func(resource any, bindings binding.Bindings) (field.ErrorList, error) { - return check.Assert(path, resource, bindings) + return check.Assert(nil, resource, bindings) }, nil } diff --git a/pkg/json-engine/model.go b/pkg/json-engine/model.go index bcb30f7c..8dcf687a 100644 --- a/pkg/json-engine/model.go +++ b/pkg/json-engine/model.go @@ -1,6 +1,7 @@ package jsonengine import ( + "fmt" "strings" "time" @@ -39,13 +40,13 @@ type Result struct { Message string } -func (r Result) Error() string { +func (r Result) Error(prefix string) string { var lines []string if r.Message != "" { - lines = append(lines, "-> "+r.Message) + lines = append(lines, prefix+"-> "+r.Message) } for _, err := range r.ErrorList { - lines = append(lines, " -> "+err.Error()) + lines = append(lines, prefix+fmt.Sprintf(" -> %s (PATH=%s)", err.ErrorBody(), err.Field)) } return strings.Join(lines, "\n") } @@ -53,10 +54,10 @@ func (r Result) Error() string { //nolint:errname type Results []Result -func (r Results) Error() string { +func (r Results) Error(prefix string) string { var lines []string for _, err := range r { - lines = append(lines, err.Error()) + lines = append(lines, err.Error(prefix)) } return strings.Join(lines, "\n") } diff --git a/pkg/server/model/response.go b/pkg/server/model/response.go index 66c866e1..1df57e30 100644 --- a/pkg/server/model/response.go +++ b/pkg/server/model/response.go @@ -50,7 +50,7 @@ func makeMessage(rule jsonengine.RuleResponse) string { return rule.Error.Error() } if len(rule.Violations) != 0 { - return rule.Violations.Error() + return rule.Violations.Error("") } return "" } diff --git a/test/api/go/main/main.go b/test/api/go/main/main.go index fbe4cef5..d664a9c9 100644 --- a/test/api/go/main/main.go +++ b/test/api/go/main/main.go @@ -69,7 +69,7 @@ func main() { if rule.Error != nil { logger.Printf("error: %s/%s -> %s: %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Error) } else if len(rule.Violations) != 0 { - logger.Printf("fail: %s/%s -> %s\n%s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Violations.Error()) + logger.Printf("fail: %s/%s -> %s\n%s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Violations.Error("")) } else { logger.Printf("pass: %s/%s -> %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier) } From 39759bee0684b89c1e409aaf34cb904b08508190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Thu, 26 Sep 2024 10:13:45 +0200 Subject: [PATCH 2/2] tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- pkg/commands/scan/options.go | 6 ++++- test/commands/scan/bindings/out.txt | 2 +- test/commands/scan/cel/out.txt | 2 +- test/commands/scan/dockerfile/out.txt | 10 ++++---- test/commands/scan/escaped/out.txt | 2 +- test/commands/scan/foo-bar/out.txt | 2 +- test/commands/scan/payload-yaml/out.txt | 6 ++--- test/commands/scan/pod-all-latest/out.txt | 2 +- test/commands/scan/pod-no-latest/out.txt | 23 +++++++++++-------- test/commands/scan/scripted/out.txt | 2 +- test/commands/scan/tf-ec2/out.txt | 2 +- test/commands/scan/tf-ecs-cluster/01-out.txt | 2 +- test/commands/scan/tf-ecs-cluster/02-out.txt | 2 +- test/commands/scan/tf-ecs-service/01-out.txt | 2 +- test/commands/scan/tf-ecs-service/02-out.txt | 2 +- .../scan/tf-ecs-task-definition/out.txt | 2 +- test/commands/scan/tf-plan/out.txt | 6 ++--- test/commands/scan/tf-s3/out.txt | 5 ++-- test/commands/scan/wildcard/out.txt | 5 ++-- 19 files changed, 47 insertions(+), 38 deletions(-) diff --git a/pkg/commands/scan/options.go b/pkg/commands/scan/options.go index 027eb754..b35af4b5 100644 --- a/pkg/commands/scan/options.go +++ b/pkg/commands/scan/options.go @@ -111,7 +111,11 @@ func (c *options) run(cmd *cobra.Command, _ []string) error { } else if len(rule.Violations) != 0 { status = "FAILED" } - out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s, ID=%s)", status, policy.Policy.Name, rule.Rule.Name, rule.Identifier)) + if rule.Identifier != "" { + out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s, ID=%s)", status, policy.Policy.Name, rule.Rule.Name, rule.Identifier)) + } else { + out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s)", status, policy.Policy.Name, rule.Rule.Name)) + } if len(rule.Violations) != 0 { out.println(rule.Violations.Error(" ")) } diff --git a/test/commands/scan/bindings/out.txt b/test/commands/scan/bindings/out.txt index 2fab40f8..24464f4c 100644 --- a/test/commands/scan/bindings/out.txt +++ b/test/commands/scan/bindings/out.txt @@ -4,5 +4,5 @@ Loading bindings ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / foo-bar-4 / PASSED +- PASSED (POLICY=test, RULE=foo-bar-4) Done diff --git a/test/commands/scan/cel/out.txt b/test/commands/scan/cel/out.txt index 7592126b..b1d970d9 100644 --- a/test/commands/scan/cel/out.txt +++ b/test/commands/scan/cel/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / foo-bar-4 / PASSED +- PASSED (POLICY=test, RULE=foo-bar-4) Done diff --git a/test/commands/scan/dockerfile/out.txt b/test/commands/scan/dockerfile/out.txt index bbb5e855..2cfa3185 100644 --- a/test/commands/scan/dockerfile/out.txt +++ b/test/commands/scan/dockerfile/out.txt @@ -2,9 +2,9 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- check-dockerfile / deny-external-calls / FAILED --> HTTP calls are not allowed - -> spec.rules[0].assert.all[0].check.~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')): Invalid value: true: Expected value: false --> wget is not allowed - -> spec.rules[0].assert.all[3].check.~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')): Invalid value: true: Expected value: false +- FAILED (POLICY=check-dockerfile, RULE=deny-external-calls) + -> HTTP calls are not allowed (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: true: Expected value: false (PATH=~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://'))) + -> wget is not allowed (CHECK=spec.rules[0].assert.all[3]) + -> Invalid value: true: Expected value: false (PATH=~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget'))) Done diff --git a/test/commands/scan/escaped/out.txt b/test/commands/scan/escaped/out.txt index 7592126b..b1d970d9 100644 --- a/test/commands/scan/escaped/out.txt +++ b/test/commands/scan/escaped/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / foo-bar-4 / PASSED +- PASSED (POLICY=test, RULE=foo-bar-4) Done diff --git a/test/commands/scan/foo-bar/out.txt b/test/commands/scan/foo-bar/out.txt index 7592126b..b1d970d9 100644 --- a/test/commands/scan/foo-bar/out.txt +++ b/test/commands/scan/foo-bar/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / foo-bar-4 / PASSED +- PASSED (POLICY=test, RULE=foo-bar-4) Done diff --git a/test/commands/scan/payload-yaml/out.txt b/test/commands/scan/payload-yaml/out.txt index 446ccf7a..4948beae 100644 --- a/test/commands/scan/payload-yaml/out.txt +++ b/test/commands/scan/payload-yaml/out.txt @@ -2,7 +2,7 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED --> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} - -> spec.rules[0].assert.all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} +- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=aws_s3_bucket.example) + -> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} (PATH=values.tags) Done diff --git a/test/commands/scan/pod-all-latest/out.txt b/test/commands/scan/pod-all-latest/out.txt index 980a4d52..1cd5bc9c 100644 --- a/test/commands/scan/pod-all-latest/out.txt +++ b/test/commands/scan/pod-all-latest/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / pod-no-latest / webserver PASSED +- PASSED (POLICY=test, RULE=pod-no-latest, ID=webserver) Done diff --git a/test/commands/scan/pod-no-latest/out.txt b/test/commands/scan/pod-no-latest/out.txt index 073c2a76..b09a5c00 100644 --- a/test/commands/scan/pod-no-latest/out.txt +++ b/test/commands/scan/pod-no-latest/out.txt @@ -2,14 +2,17 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / pod-no-latest / webserver FAILED - -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[0].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[1].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[2].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')): Invalid value: true: Expected value: false - -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')): Invalid value: true: Expected value: false +- FAILED (POLICY=test, RULE=pod-no-latest, ID=webserver) + -> (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag))) + -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag))) + -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag))) + -> (CHECK=spec.rules[0].assert.all[1]) + -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[0].image.(ends_with(@, ':latest'))) + -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[1].image.(ends_with(@, ':latest'))) + -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[2].image.(ends_with(@, ':latest'))) + -> (CHECK=spec.rules[0].assert.all[2]) + -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest'))) + -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest'))) + -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest'))) Done diff --git a/test/commands/scan/scripted/out.txt b/test/commands/scan/scripted/out.txt index 7592126b..b1d970d9 100644 --- a/test/commands/scan/scripted/out.txt +++ b/test/commands/scan/scripted/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- test / foo-bar-4 / PASSED +- PASSED (POLICY=test, RULE=foo-bar-4) Done diff --git a/test/commands/scan/tf-ec2/out.txt b/test/commands/scan/tf-ec2/out.txt index e9f5d7c8..32ceaf11 100644 --- a/test/commands/scan/tf-ec2/out.txt +++ b/test/commands/scan/tf-ec2/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- required-ec2-tags / require-team-tag / PASSED +- PASSED (POLICY=required-ec2-tags, RULE=require-team-tag) Done diff --git a/test/commands/scan/tf-ecs-cluster/01-out.txt b/test/commands/scan/tf-ecs-cluster/01-out.txt index 952024fd..8507aa21 100644 --- a/test/commands/scan/tf-ecs-cluster/01-out.txt +++ b/test/commands/scan/tf-ecs-cluster/01-out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 3 resources against 1 policy ) ... -- required-container-insights / required-container-insights / PASSED +- PASSED (POLICY=required-container-insights, RULE=required-container-insights) Done diff --git a/test/commands/scan/tf-ecs-cluster/02-out.txt b/test/commands/scan/tf-ecs-cluster/02-out.txt index 899ad158..50fa07dc 100644 --- a/test/commands/scan/tf-ecs-cluster/02-out.txt +++ b/test/commands/scan/tf-ecs-cluster/02-out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 3 resources against 1 policy ) ... -- ecs-cluster-enable-logging / ecs-cluster-enable-logging / PASSED +- PASSED (POLICY=ecs-cluster-enable-logging, RULE=ecs-cluster-enable-logging) Done diff --git a/test/commands/scan/tf-ecs-service/01-out.txt b/test/commands/scan/tf-ecs-service/01-out.txt index 5dbe755e..d5238fe1 100644 --- a/test/commands/scan/tf-ecs-service/01-out.txt +++ b/test/commands/scan/tf-ecs-service/01-out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- required-latest-platform-fargate / required-latest-platform / PASSED +- PASSED (POLICY=required-latest-platform-fargate, RULE=required-latest-platform) Done diff --git a/test/commands/scan/tf-ecs-service/02-out.txt b/test/commands/scan/tf-ecs-service/02-out.txt index 1a83a981..398bd6e0 100644 --- a/test/commands/scan/tf-ecs-service/02-out.txt +++ b/test/commands/scan/tf-ecs-service/02-out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- ecs-public-ip / ecs-public-ip / PASSED +- PASSED (POLICY=ecs-public-ip, RULE=ecs-public-ip) Done diff --git a/test/commands/scan/tf-ecs-task-definition/out.txt b/test/commands/scan/tf-ecs-task-definition/out.txt index 0595d9c7..51d05e64 100644 --- a/test/commands/scan/tf-ecs-task-definition/out.txt +++ b/test/commands/scan/tf-ecs-task-definition/out.txt @@ -2,5 +2,5 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- fs-read-only / require-fs-read-only / PASSED +- PASSED (POLICY=fs-read-only, RULE=require-fs-read-only) Done diff --git a/test/commands/scan/tf-plan/out.txt b/test/commands/scan/tf-plan/out.txt index 446ccf7a..4948beae 100644 --- a/test/commands/scan/tf-plan/out.txt +++ b/test/commands/scan/tf-plan/out.txt @@ -2,7 +2,7 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED --> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} - -> spec.rules[0].assert.all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} +- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=aws_s3_bucket.example) + -> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} (PATH=values.tags) Done diff --git a/test/commands/scan/tf-s3/out.txt b/test/commands/scan/tf-s3/out.txt index 8b0dfe10..a00fb800 100644 --- a/test/commands/scan/tf-s3/out.txt +++ b/test/commands/scan/tf-s3/out.txt @@ -2,6 +2,7 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- s3 / check-tags / FAILED - -> spec.rules[0].assert.all[0].check.planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team')): Invalid value: false: Expected value: true +- FAILED (POLICY=s3, RULE=check-tags) + -> (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: false: Expected value: true (PATH=planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team'))) Done diff --git a/test/commands/scan/wildcard/out.txt b/test/commands/scan/wildcard/out.txt index 65fd2f40..4cf8bc62 100644 --- a/test/commands/scan/wildcard/out.txt +++ b/test/commands/scan/wildcard/out.txt @@ -2,6 +2,7 @@ Loading policies ... Loading payload ... Pre processing ... Running ( evaluating 1 resource against 1 policy ) ... -- required-s3-tags / require-team-tag / bucket1 FAILED - -> spec.rules[0].assert.all[0].check.tags.(wildcard('?*', Team)): Invalid value: true: Expected value: false +- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=bucket1) + -> (CHECK=spec.rules[0].assert.all[0]) + -> Invalid value: true: Expected value: false (PATH=tags.(wildcard('?*', Team))) Done