feat: make evaluation path live in payload world (#522)

* feat: make evaluation path live in payload world Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * tests Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> --------- Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
kyverno · Sep 26, 2024 · 0500f6b · 0500f6b
1 parent 120cf5c
commit 0500f6b
Show file tree

Hide file tree

Showing 23 changed files with 74 additions and 51 deletions.
diff --git a/pkg/commands/scan/options.go b/pkg/commands/scan/options.go
@@ -105,15 +105,30 @@ func (c *options) run(cmd *cobra.Command, _ []string) error {
 	for _, response := range responses {
 		for _, policy := range response.Policies {
 			for _, rule := range policy.Rules {
+				status := "PASSED"
 				if rule.Error != nil {
-					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "ERROR:", rule.Error.Error())
+					status = fmt.Sprintf("ERROR: %s", rule.Error.Error())
 				} else if len(rule.Violations) != 0 {
-					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED")
-					out.println(rule.Violations.Error())
+					status = "FAILED"
+				}
+				if rule.Identifier != "" {
+					out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s, ID=%s)", status, policy.Policy.Name, rule.Rule.Name, rule.Identifier))
 				} else {
-					// TODO: handle skip, warn
-					out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "PASSED")
+					out.println(fmt.Sprintf("- %s (POLICY=%s, RULE=%s)", status, policy.Policy.Name, rule.Rule.Name))
+				}
+				if len(rule.Violations) != 0 {
+					out.println(rule.Violations.Error(" "))
 				}
+
+				// if rule.Error != nil {
+				// 	out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "ERROR:", rule.Error.Error())
+				// } else if len(rule.Violations) != 0 {
+				// 	out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "FAILED")
+				// 	out.println(rule.Violations.Error())
+				// } else {
+				// 	// TODO: handle skip, warn
+				// 	out.println("-", policy.Policy.Name, "/", rule.Rule.Name, "/", rule.Identifier, "PASSED")
+				// }
 			}
 		}
 	}

diff --git a/pkg/json-engine/compiler.go b/pkg/json-engine/compiler.go
@@ -194,9 +194,11 @@ func (c *compiler) compileAssertion(
 		errs, err := check(resource, bindings)
 		if len(errs) != 0 {
 			result.ErrorList = errs
+			message := fmt.Sprintf("(CHECK=%s)", path.String())
 			if in.Message != nil {
-				result.Message = in.Message.Format(resource, bindings, compilers.Jp.Options()...)
+				message = fmt.Sprintf("%s %s", in.Message.Format(resource, bindings, compilers.Jp.Options()...), message)
 			}
+			result.Message = message
 		}
 		return result, err
 	}, nil
@@ -228,7 +230,7 @@ func (c *compiler) compileAssertionTree(
 		return nil, err
 	}
 	return func(resource any, bindings binding.Bindings) (field.ErrorList, error) {
-		return check.Assert(path, resource, bindings)
+		return check.Assert(nil, resource, bindings)
 	}, nil
 }
 

diff --git a/pkg/json-engine/model.go b/pkg/json-engine/model.go
@@ -1,6 +1,7 @@
 package jsonengine
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
@@ -39,24 +40,24 @@ type Result struct {
 	Message string
 }
 
-func (r Result) Error() string {
+func (r Result) Error(prefix string) string {
 	var lines []string
 	if r.Message != "" {
-		lines = append(lines, "-> "+r.Message)
+		lines = append(lines, prefix+"-> "+r.Message)
 	}
 	for _, err := range r.ErrorList {
-		lines = append(lines, " -> "+err.Error())
+		lines = append(lines, prefix+fmt.Sprintf(" -> %s (PATH=%s)", err.ErrorBody(), err.Field))
 	}
 	return strings.Join(lines, "\n")
 }
 
 //nolint:errname
 type Results []Result
 
-func (r Results) Error() string {
+func (r Results) Error(prefix string) string {
 	var lines []string
 	for _, err := range r {
-		lines = append(lines, err.Error())
+		lines = append(lines, err.Error(prefix))
 	}
 	return strings.Join(lines, "\n")
 }

diff --git a/pkg/server/model/response.go b/pkg/server/model/response.go
@@ -50,7 +50,7 @@ func makeMessage(rule jsonengine.RuleResponse) string {
 		return rule.Error.Error()
 	}
 	if len(rule.Violations) != 0 {
-		return rule.Violations.Error()
+		return rule.Violations.Error("")
 	}
 	return ""
 }
diff --git a/test/api/go/main/main.go b/test/api/go/main/main.go
@@ -69,7 +69,7 @@ func main() {
 			if rule.Error != nil {
 				logger.Printf("error: %s/%s -> %s: %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Error)
 			} else if len(rule.Violations) != 0 {
-				logger.Printf("fail: %s/%s -> %s\n%s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Violations.Error())
+				logger.Printf("fail: %s/%s -> %s\n%s", policy.Policy.Name, rule.Rule.Name, rule.Identifier, rule.Violations.Error(""))
 			} else {
 				logger.Printf("pass: %s/%s -> %s", policy.Policy.Name, rule.Rule.Name, rule.Identifier)
 			}

diff --git a/test/commands/scan/bindings/out.txt b/test/commands/scan/bindings/out.txt
@@ -4,5 +4,5 @@ Loading bindings ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / foo-bar-4 /  PASSED
+- PASSED (POLICY=test, RULE=foo-bar-4)
 Done
diff --git a/test/commands/scan/cel/out.txt b/test/commands/scan/cel/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / foo-bar-4 /  PASSED
+- PASSED (POLICY=test, RULE=foo-bar-4)
 Done
diff --git a/test/commands/scan/dockerfile/out.txt b/test/commands/scan/dockerfile/out.txt
@@ -2,9 +2,9 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- check-dockerfile / deny-external-calls /  FAILED
--> HTTP calls are not allowed
- -> spec.rules[0].assert.all[0].check.~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')): Invalid value: true: Expected value: false
--> wget is not allowed
- -> spec.rules[0].assert.all[3].check.~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')): Invalid value: true: Expected value: false
+- FAILED (POLICY=check-dockerfile, RULE=deny-external-calls)
+ -> HTTP calls are not allowed (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: true: Expected value: false (PATH=~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')))
+ -> wget is not allowed (CHECK=spec.rules[0].assert.all[3])
+  -> Invalid value: true: Expected value: false (PATH=~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')))
 Done
diff --git a/test/commands/scan/escaped/out.txt b/test/commands/scan/escaped/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / foo-bar-4 /  PASSED
+- PASSED (POLICY=test, RULE=foo-bar-4)
 Done
diff --git a/test/commands/scan/foo-bar/out.txt b/test/commands/scan/foo-bar/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / foo-bar-4 /  PASSED
+- PASSED (POLICY=test, RULE=foo-bar-4)
 Done
diff --git a/test/commands/scan/payload-yaml/out.txt b/test/commands/scan/payload-yaml/out.txt
@@ -2,7 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED
--> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
- -> spec.rules[0].assert.all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
+- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=aws_s3_bucket.example)
+ -> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} (PATH=values.tags)
 Done
diff --git a/test/commands/scan/pod-all-latest/out.txt b/test/commands/scan/pod-all-latest/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / pod-no-latest / webserver PASSED
+- PASSED (POLICY=test, RULE=pod-no-latest, ID=webserver)
 Done
diff --git a/test/commands/scan/pod-no-latest/out.txt b/test/commands/scan/pod-no-latest/out.txt
@@ -2,14 +2,17 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / pod-no-latest / webserver FAILED
- -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[0].check.spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[0].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[1].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[1].check.spec.~.containers->foo[2].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
- -> spec.rules[0].assert.all[2].check.~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')): Invalid value: true: Expected value: false
+- FAILED (POLICY=test, RULE=pod-no-latest, ID=webserver)
+ -> (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)))
+  -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)))
+  -> Invalid value: true: Expected value: false (PATH=spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)))
+ -> (CHECK=spec.rules[0].assert.all[1])
+  -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[0].image.(ends_with(@, ':latest')))
+  -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[1].image.(ends_with(@, ':latest')))
+  -> Invalid value: true: Expected value: false (PATH=spec.~.containers->foo[2].image.(ends_with(@, ':latest')))
+ -> (CHECK=spec.rules[0].assert.all[2])
+  -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')))
+  -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')))
+  -> Invalid value: true: Expected value: false (PATH=~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')))
 Done
diff --git a/test/commands/scan/scripted/out.txt b/test/commands/scan/scripted/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- test / foo-bar-4 /  PASSED
+- PASSED (POLICY=test, RULE=foo-bar-4)
 Done
diff --git a/test/commands/scan/tf-ec2/out.txt b/test/commands/scan/tf-ec2/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-ec2-tags / require-team-tag /  PASSED
+- PASSED (POLICY=required-ec2-tags, RULE=require-team-tag)
 Done
diff --git a/test/commands/scan/tf-ecs-cluster/01-out.txt b/test/commands/scan/tf-ecs-cluster/01-out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 3 resources against 1 policy ) ...
-- required-container-insights / required-container-insights /  PASSED
+- PASSED (POLICY=required-container-insights, RULE=required-container-insights)
 Done
diff --git a/test/commands/scan/tf-ecs-cluster/02-out.txt b/test/commands/scan/tf-ecs-cluster/02-out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 3 resources against 1 policy ) ...
-- ecs-cluster-enable-logging / ecs-cluster-enable-logging /  PASSED
+- PASSED (POLICY=ecs-cluster-enable-logging, RULE=ecs-cluster-enable-logging)
 Done
diff --git a/test/commands/scan/tf-ecs-service/01-out.txt b/test/commands/scan/tf-ecs-service/01-out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-latest-platform-fargate / required-latest-platform /  PASSED
+- PASSED (POLICY=required-latest-platform-fargate, RULE=required-latest-platform)
 Done
diff --git a/test/commands/scan/tf-ecs-service/02-out.txt b/test/commands/scan/tf-ecs-service/02-out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- ecs-public-ip / ecs-public-ip /  PASSED
+- PASSED (POLICY=ecs-public-ip, RULE=ecs-public-ip)
 Done
diff --git a/test/commands/scan/tf-ecs-task-definition/out.txt b/test/commands/scan/tf-ecs-task-definition/out.txt
@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- fs-read-only / require-fs-read-only /  PASSED
+- PASSED (POLICY=fs-read-only, RULE=require-fs-read-only)
 Done
diff --git a/test/commands/scan/tf-plan/out.txt b/test/commands/scan/tf-plan/out.txt
@@ -2,7 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED
--> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
- -> spec.rules[0].assert.all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
+- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=aws_s3_bucket.example)
+ -> Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"} (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"} (PATH=values.tags)
 Done
diff --git a/test/commands/scan/tf-s3/out.txt b/test/commands/scan/tf-s3/out.txt
@@ -2,6 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- s3 / check-tags /  FAILED
- -> spec.rules[0].assert.all[0].check.planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team')): Invalid value: false: Expected value: true
+- FAILED (POLICY=s3, RULE=check-tags)
+ -> (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: false: Expected value: true (PATH=planned_values.root_module.~.resources[0].values.(keys(tags_all)).(contains(@, 'Team')))
 Done
diff --git a/test/commands/scan/wildcard/out.txt b/test/commands/scan/wildcard/out.txt
@@ -2,6 +2,7 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / bucket1 FAILED
- -> spec.rules[0].assert.all[0].check.tags.(wildcard('?*', Team)): Invalid value: true: Expected value: false
+- FAILED (POLICY=required-s3-tags, RULE=require-team-tag, ID=bucket1)
+ -> (CHECK=spec.rules[0].assert.all[0])
+  -> Invalid value: true: Expected value: false (PATH=tags.(wildcard('?*', Team)))
 Done