[Enhancement] Easy checking of CLI output

go.mod

@@ -3,6 +3,7 @@ module github.com/kyverno/kuttl
 go 1.18
 
 require (
+	github.com/IGLOU-EU/go-wildcard v1.0.3
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/docker/docker v20.10.17+incompatible
 	github.com/dustin/go-humanize v1.0.0

go.sum

@@ -67,6 +67,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/IGLOU-EU/go-wildcard v1.0.3 h1:r8T46+8/9V1STciXJomTWRpPEv4nGJATDbJkdU0Nou0=
+github.com/IGLOU-EU/go-wildcard v1.0.3/go.mod h1:/qeV4QLmydCbwH0UMQJmXDryrFKJknWi/jjO8IiuQfY=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=

pkg/apis/testharness/v1beta1/commands.go

@@ -0,0 +1,49 @@
+package v1beta1
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/IGLOU-EU/go-wildcard"
+)
+
+func (c *CommandOutput) ValidateCommandOutput(stdoutOutput, stderrOutput strings.Builder) error {
+	var errs []string
+
+	if c.Stdout != nil {
+		if err := c.Stdout.validateOutput("stdout", stdoutOutput.String()); err != nil {
+			errs = append(errs, err.Error())
+		}
+	}
+	if c.Stderr != nil {
+		if err := c.Stderr.validateOutput("stderr", stderrOutput.String()); err != nil {
+			errs = append(errs, err.Error())
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("validation errors: %s", strings.Join(errs, "; "))
+	}
+	return nil
+}
+
+func (e *ExpectedOutput) validateOutput(outputType string, actualValue string) error {
+	expectedValue := e.ExpectedValue
+	switch e.MatchType {
+	case MatchContains:
+		if !strings.Contains(actualValue, expectedValue) {
+			return fmt.Errorf("expected %s to contain: %s, but it did not", outputType, expectedValue)
+		}
+
+	case MatchWildcard:
+		if !wildcard.Match(expectedValue, actualValue) {
+			return fmt.Errorf("%s did not match wildcard pattern: %s", outputType, expectedValue)
+		}
+
+	default: // MatchEquals
+		if actualValue != expectedValue {
+			return fmt.Errorf("expected exact %s: %s, got: %s", outputType, expectedValue, actualValue)
+		}
+	}
+
+	return nil
+}

pkg/apis/testharness/v1beta1/commands_test.go

@@ -0,0 +1,330 @@
+package v1beta1
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestValidateCommandOutput(t *testing.T) {
+	tests := []struct {
+		name         string
+		cmdOutput    CommandOutput
+		stdoutOutput strings.Builder
+		stderrOutput strings.Builder
+		wantErr      bool
+	}{
+		{
+			name: "stdout matches exactly",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Hello, World!",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stdout does not match exactly",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Hello",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stdout and stderr with contains match",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "World",
+				},
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "Err",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("An Error Occurred")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stdout wildcard match",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "Hello, *!",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stdout matches but stderr fails",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Hello, World!",
+				},
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Error",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Different Error")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stderr contains but stdout fails",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "Error",
+				},
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Hello",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Different Error")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stdout wildcard match with missing pattern",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "Hi, *!",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stderr wildcard match",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "*Error*",
+				},
+			},
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("This is an Error message!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stdout contains and stderr wildcard match",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "universe",
+				},
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "*Error*",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, universe!")
+				return b
+			}(),
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Some Error here!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stderr equals but stdout wildcard fails",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Some Error here!",
+				},
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "Greetings, *",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Some Error here!")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stdout contains with no stderr",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "planet",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, planet Earth!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stderr equals with no stdout",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchEquals,
+					ExpectedValue: "Error: File not found.",
+				},
+			},
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Error: File not found.")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "stdout contains but missing in actual output",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "Mars",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "stderr wildcard does not match",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchWildcard,
+					ExpectedValue: "Critical*Error*",
+				},
+			},
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("This is a simple Error message.")
+				return b
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "Empty Expected Output",
+			cmdOutput: CommandOutput{
+				Stderr: &ExpectedOutput{
+					MatchType:     MatchContains,
+					ExpectedValue: "",
+				},
+			},
+			stderrOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Empty Expected Output")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "default match type is not provided",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     "",
+					ExpectedValue: "Hello, World!",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "random match type is not provided",
+			cmdOutput: CommandOutput{
+				Stdout: &ExpectedOutput{
+					MatchType:     "abc",
+					ExpectedValue: "Hello, World!",
+				},
+			},
+			stdoutOutput: func() strings.Builder {
+				b := strings.Builder{}
+				b.WriteString("Hello, World!")
+				return b
+			}(),
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cmdOutput.ValidateCommandOutput(tt.stdoutOutput, tt.stderrOutput)
+			if tt.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}