Merge pull request #16 from gdt-dev/exec-not-contains

`assert.out.none` and `assert.err.none` to exec

README.md

@@ -491,25 +491,29 @@ the base `Spec` fields listed above):
   instead the operating system's `exec` family of calls is used.
 * `assert`: (optional) an object describing the conditions that will be
   asserted about the test action.
-* `assert.exit_code`: (optional) an integer with the expected exit code from the
+* `assert.exit-code`: (optional) an integer with the expected exit code from the
   executed command. The default successful exit code is 0 and therefore you do
   not need to specify this if you expect a successful exit code.
 * `assert.out`: (optional) a [`PipeExpect`][pipeexpect] object containing
   assertions about content in `stdout`.
 * `assert.out.is`: (optional) a string with the exact contents of `stdout` you expect
   to get.
-* `assert.out.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.out.all`: (optional) a string or list of strings that *all* must be
   present in `stdout`.
-* `assert.out.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.out.any`: (optional) a string or list of strings of which *at
   least one* must be present in `stdout`.
+* `assert.out.none`: (optional) a string or list of strings of which *none
+  should be present* in `stdout`.
 * `assert.err`: (optional) a [`PipeAssertions`][pipeexpect] object containing
   assertions about content in `stderr`.
 * `assert.err.is`: (optional) a string with the exact contents of `stderr` you expect
   to get.
-* `assert.err.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.err.all`: (optional) a string or list of strings that *all* must be
   present in `stderr`.
-* `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.err.any`: (optional) a string or list of strings of which *at
   least one* must be present in `stderr`.
+* `assert.err.none`: (optional) a string or list of strings of which *none
+  should be present* in `stderr`.
 * `on`: (optional) an object describing actions to take upon certain
   conditions.
 * `on.fail`: (optional) an object describing an action to take when any

errors/failure.go

@@ -19,6 +19,9 @@ var (
 	// ErrNotEqual is an ErrFailure when an expected thing doesn't equal an
 	// observed thing.
 	ErrNotEqual = fmt.Errorf("%w: not equal", ErrFailure)
+	// ErrIn is an ErrFailure when a thing unexpectedly appears in an
+	// container.
+	ErrIn = fmt.Errorf("%w: in", ErrFailure)
 	// ErrNotIn is an ErrFailure when an expected thing doesn't appear in an
 	// expected container.
 	ErrNotIn = fmt.Errorf("%w: not in", ErrFailure)
@@ -58,6 +61,14 @@ func NotEqual(exp, got interface{}) error {
 	return fmt.Errorf("%w: expected %v but got %v", ErrNotEqual, exp, got)
 }
 
+// In returns an ErrIn when a thing unexpectedly appears in a container.
+func In(element, container interface{}) error {
+	return fmt.Errorf(
+		"%w: expected %v not to contain %v",
+		ErrIn, container, element,
+	)
+}
+
 // NotIn returns an ErrNotIn when an expected thing doesn't appear in an
 // expected container.
 func NotIn(element, container interface{}) error {

plugin/exec/assertions.go

@@ -17,7 +17,7 @@ type Expect struct {
 	// ExitCode is the expected exit code for the executed command. The default
 	// (0) is the universal successful exit code, so you only need to set this
 	// if you expect a non-successful result from executing the command.
-	ExitCode int `yaml:"exit_code,omitempty"`
+	ExitCode int `yaml:"exit-code,omitempty"`
 	// Out has things that are expected in the stdout response
 	Out *PipeExpect `yaml:"out,omitempty"`
 	// Err has things that are expected in the stderr response
@@ -26,15 +26,15 @@ type Expect struct {
 
 // PipeExpect contains assertions about the contents of a pipe
 type PipeExpect struct {
-	// Is contains the exact match (minus whitespace) of the contents of the
-	// pipe
-	Is *string `yaml:"is,omitempty"`
-	// Contains is one or more strings that *all* must be present in the
+	// ContainsAll is one or more strings that *all* must be present in the
 	// contents of the pipe
-	Contains []string `yaml:"contains,omitempty"`
+	ContainsAll *gdttypes.FlexStrings `yaml:"contains,omitempty"`
+	// ContainsNone is one or more strings, *none of which* should be present in
+	// the contents of the pipe
+	ContainsNone *gdttypes.FlexStrings `yaml:"contains-none-of,omitempty"`
 	// ContainsOneOf is one or more strings of which *at least one* must be
 	// present in the contents of the pipe
-	ContainsOneOf []string `yaml:"contains_one_of,omitempty"`
+	ContainsAny *gdttypes.FlexStrings `yaml:"contains-one-of,omitempty"`
 }
 
 // pipeAssertions contains assertions about the contents of a pipe
@@ -83,35 +83,45 @@ func (a *pipeAssertions) OK() bool {
 
 	res := true
 	contents := strings.TrimSpace(a.pipe.String())
-	if a.Is != nil {
-		exp := *a.Is
-		got := contents
-		if exp != got {
-			a.Fail(errors.NotEqual(exp, got))
-			res = false
-		}
-	}
-	if len(a.Contains) > 0 {
-		for _, find := range a.Contains {
-			if !strings.Contains(contents, find) {
-				a.Fail(errors.NotIn(find, a.name))
+	if a.ContainsAll != nil {
+		// When there is just a single value, we use the NotEqual error,
+		// otherwise we use the NotIn error
+		vals := a.ContainsAll.Values()
+		if len(vals) == 1 {
+			if !strings.Contains(contents, vals[0]) {
+				a.Fail(errors.NotEqual(vals[0], contents))
 				res = false
 			}
+		} else {
+			for _, find := range vals {
+				if !strings.Contains(contents, find) {
+					a.Fail(errors.NotIn(find, a.name))
+					res = false
+				}
+			}
 		}
 	}
-	if len(a.ContainsOneOf) > 0 {
+	if a.ContainsAny != nil {
 		found := false
-		for _, find := range a.ContainsOneOf {
+		for _, find := range a.ContainsAny.Values() {
 			if idx := strings.Index(contents, find); idx > -1 {
 				found = true
 				break
 			}
 		}
 		if !found {
-			a.Fail(errors.NoneIn(a.ContainsOneOf, a.name))
+			a.Fail(errors.NoneIn(a.ContainsAny.Values(), a.name))
 			res = false
 		}
 	}
+	if a.ContainsNone != nil {
+		for _, find := range a.ContainsNone.Values() {
+			if strings.Contains(contents, find) {
+				a.Fail(errors.In(find, a.name))
+				res = false
+			}
+		}
+	}
 	return res
 }
 

plugin/exec/eval_test.go

@@ -138,6 +138,25 @@ func TestContainsOneOf(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestContainsNoneOf(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "ls-contains-none-of.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := context.TODO()
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
 func TestSleepTimeout(t *testing.T) {
 	require := require.New(t)
 

plugin/exec/parse.go

@@ -116,7 +116,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
-		case "exit_code":
+		case "exit_code", "exit-code":
 			if valNode.Kind != yaml.ScalarNode {
 				return errors.ExpectedScalarAt(valNode)
 			}
@@ -149,3 +149,51 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 	}
 	return nil
 }
+
+func (e *PipeExpect) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return errors.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return errors.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "all", "is", "contains", "contains-all", "contains_all":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsAll = &v
+		case "any", "contains-one-of", "contains-any", "contains_one_of", "contains_any":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsAny = &v
+		case "none", "none-of", "contains-none-of", "contains-none", "none_of", "contains_none_of", "contains_none":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsNone = &v
+		default:
+			return errors.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}

plugin/exec/testdata/ls-contains-none-of.yaml

@@ -0,0 +1,30 @@
+name: ls-contains-none-of
+description: a scenario that runs the `ls` command and checks the output does not contain a string
+tests:
+  - exec: ls -l
+    assert:
+      out:
+        none:
+         - notexisting.go
+  # Variants of contains-none-of
+  - exec: ls -l
+    assert:
+      out:
+        none-of: notexisting.go
+  - exec: ls -l
+    assert:
+      out:
+        contains_none_of:
+         - notexisting.go
+  - exec: ls -l
+    assert:
+      out:
+        none-of: notexisting.go
+  # To test the stderr assertions, we redirect stdout to stderr in a shell
+  # command...
+  - exec: "ls -l 1>&2"
+    shell: sh
+    assert:
+      err:
+        none:
+         - notexisting.go

plugin/exec/testdata/ls-contains-one-of.yaml

@@ -1,20 +1,37 @@
 name: ls-contains-one-of
 description: a scenario that runs the `ls` command and checks the output contains one of a set of strings
 tests:
+  - exec: ls -l
+    assert:
+      out:
+        any:
+         - thisdoesnotexist
+         - neitherdoesthisexist
+         - parse.go
+  # Variants of contains-any
   - exec: ls -l
     assert:
       out:
         contains_one_of:
          - thisdoesnotexist
          - neitherdoesthisexist
          - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains-any:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        any: parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"
     shell: sh
     assert:
       err:
-        contains_one_of:
+        any:
          - thisdoesnotexist
          - neitherdoesthisexist
          - parse.go

plugin/exec/testdata/ls-contains.yaml

@@ -6,6 +6,29 @@ tests:
       out:
         contains:
          - parse.go
+  # Variants of contains-all
+  - exec: ls -l
+    assert:
+      out:
+        is: parse.go
+  - exec: ls -l
+    assert:
+      out:
+        is:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains: parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains-all:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        all: parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"

plugin/exec/testdata/ls-with-exit-code.yaml

@@ -3,4 +3,4 @@ description: a scenario that runs the `ls` command expecting a non-0 exit code
 tests:
   - exec: ls /this/dir/does/not/exist
     assert:
-      exit_code: 2
+      exit-code: 2

plugin/exec/testdata/mac-ls-with-exit-code.yaml

@@ -3,4 +3,4 @@ description: a scenario that runs the `ls` command expecting a non-0 exit code o
 tests:
   - exec: ls /this/dir/does/not/exist
     assert:
-      exit_code: 1
+      exit-code: 1

plugin/exec/testdata/windows-sleep-timeout.yaml

@@ -11,4 +11,4 @@ tests:
     # the context's deadline cancels the pipe and results in a 1 result
     # code on Windows...
     assert:
-      exit_code: 1
+      exit-code: 1