support on.fail.exec actions

README.md

@@ -510,6 +510,17 @@ the base `Spec` fields listed above):
   present in `stderr`.
 * `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
   least one* must be present in `stderr`.
+* `on`: (optional) an object describing actions to take upon certain
+  conditions.
+* `on.fail`: (optional) an object describing an action to take when any
+  assertion fails for the test action.
+* `on.fail.exec`: a string with the exact command to execute upon test
+  assertion failure. You may execute more than one command but must include the
+  `on.fail.shell` field to indicate that the command should be run in a shell.
+* `on.fail.shell`: (optional) a string with the specific shell to use in executing the
+  command to run upon test assertion failure. If empty (the default), no shell
+  is used to execute the command and instead the operating system's `exec` family
+  of calls is used.
 
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26

plugin/exec/action.go

@@ -0,0 +1,97 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package exec
+
+import (
+	"bytes"
+	"context"
+	"os/exec"
+	"testing"
+
+	gdtcontext "github.com/gdt-dev/gdt/context"
+	"github.com/gdt-dev/gdt/debug"
+	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/google/shlex"
+)
+
+// Action describes a single execution of one or more commands via the
+// operating system's `exec` family of functions.
+type Action struct {
+	// Exec is the exact command to execute.
+	//
+	// You may execute more than one command but must include the `shell` field
+	// to indicate that the command should be run in a shell. It is best
+	// practice, however, to simply use multiple `exec` specs instead of
+	// executing multiple commands in a single shell call.
+	Exec string `yaml:"exec"`
+	// Shell is the specific shell to use in executing the command. If empty
+	// (the default), no shell is used to execute the command and instead the
+	// operating system's `exec` family of calls is used.
+	Shell string `yaml:"shell,omitempty"`
+}
+
+// Do performs a single command or shell execution returning the corresponding
+// exit code and any runtime error. The `outbuf` and `errbuf` buffers will be
+// filled with the contents of the command's stdout and stderr pipes
+// respectively.
+func (a *Action) Do(
+	ctx context.Context,
+	t *testing.T,
+	outbuf *bytes.Buffer,
+	errbuf *bytes.Buffer,
+	exitcode *int,
+) error {
+	var target string
+	var args []string
+	if a.Shell == "" {
+		// Parse time already validated exec string parses into valid shell
+		// args
+		args, _ = shlex.Split(a.Exec)
+		target = args[0]
+		args = args[1:]
+	} else {
+		target = a.Shell
+		args = []string{"-c", a.Exec}
+	}
+
+	debug.Println(ctx, t, "exec: %s %s", target, args)
+
+	var cmd *exec.Cmd
+	cmd = exec.CommandContext(ctx, target, args...)
+
+	outpipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	errpipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if gdtcontext.TimedOut(ctx, err) {
+		return gdterrors.ErrTimeoutExceeded
+	}
+	if err != nil {
+		return err
+	}
+	if outbuf != nil {
+		outbuf.ReadFrom(outpipe)
+	}
+	if errbuf != nil {
+		errbuf.ReadFrom(errpipe)
+	}
+
+	err = cmd.Wait()
+	if gdtcontext.TimedOut(ctx, err) {
+		return gdterrors.ErrTimeoutExceeded
+	}
+	if err != nil && exitcode != nil {
+		eerr, _ := err.(*exec.ExitError)
+		ec := eerr.ExitCode()
+		*exitcode = ec
+	}
+	return nil
+}

plugin/exec/eval.go

@@ -7,12 +7,8 @@ package exec
 import (
 	"bytes"
 	"context"
-	"os/exec"
 	"testing"
 
-	"github.com/google/shlex"
-
-	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/gdt-dev/gdt/result"
@@ -25,57 +21,39 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
-	var err error
-	var cmd *exec.Cmd
-	var target string
-	var args []string
-	if s.Shell == "" {
-		// Parse time already validated exec string parses into valid shell
-		// args
-		args, _ = shlex.Split(s.Exec)
-		target = args[0]
-		args = args[1:]
-	} else {
-		target = s.Shell
-		args = []string{"-c", s.Exec}
-	}
-
-	debug.Println(ctx, t, "exec: %s %s", target, args)
-	cmd = exec.CommandContext(ctx, target, args...)
-
-	outpipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
-	}
-	errpipe, err := cmd.StderrPipe()
-	if err != nil {
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
-	}
+	var ec int
 
-	err = cmd.Start()
-	if gdtcontext.TimedOut(ctx, err) {
-		return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
-	}
-	if err != nil {
+	if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
+		if err == gdterrors.ErrTimeoutExceeded {
+			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
+		}
 		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
 	}
-	outbuf.ReadFrom(outpipe)
-	errbuf.ReadFrom(errpipe)
-
-	err = cmd.Wait()
-	if gdtcontext.TimedOut(ctx, err) {
-		return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
-	}
-	ec := 0
-	if err != nil {
-		eerr, _ := err.(*exec.ExitError)
-		ec = eerr.ExitCode()
-	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK() {
 		for _, fail := range a.Failures() {
 			t.Error(fail)
 		}
+		if s.On != nil {
+			if s.On.Fail != nil {
+				outbuf.Reset()
+				errbuf.Reset()
+				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
+				if err != nil {
+					debug.Println(ctx, t, "error in on.fail.exec: %s", err)
+				}
+				if outbuf.Len() > 0 {
+					debug.Println(
+						ctx, t, "on.fail.exec: stdout: %s", outbuf.String(),
+					)
+				}
+				if errbuf.Len() > 0 {
+					debug.Println(
+						ctx, t, "on.fail.exec: stderr: %s", errbuf.String(),
+					)
+				}
+			}
+		}
 	}
 	return result.New(result.WithFailures(a.Failures()...))
 }

plugin/exec/eval_test.go

@@ -235,3 +235,50 @@ func TestTimeoutCascade(t *testing.T) {
 	require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
 	require.Contains(debugout, "using timeout of 20ms (expected: true)")
 }
+
+// Unfortunately there's not really any good way of testing things like this
+// except by manually causing an assertion to fail in the test case and
+// checking to see if the `on.fail` action was taken and debug output emitted
+// to the console.
+//
+// When I change the `testdata/on-fail-exec.yaml` file to have a failed
+// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the
+// correct behaviour:
+//
+// === RUN   TestOnFail
+// === RUN   TestOnFail/on-fail-exec
+//
+//	action.go:59: exec: echo [cat]
+//	eval.go:35: assertion failed: not equal: expected dat but got cat
+//	action.go:59: exec: echo [bad kitty]
+//	eval.go:46: on.fail.exec: stdout: bad kitty
+//
+// === NAME  TestOnFail
+//
+//	eval_test.go:256:
+//	    	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
+//	    	Error:      	Should be false
+//	    	Test:       	TestOnFail
+//
+// --- FAIL: TestOnFail (0.00s)
+//
+//	--- FAIL: TestOnFail/on-fail-exec (0.00s)
+func TestOnFail(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "on-fail-exec.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+	require.False(t.Failed())
+}

plugin/exec/on.go

@@ -0,0 +1,29 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package exec
+
+// On describes actions that can be taken upon certain conditions.
+type On struct {
+	// Fail contains one or more actions to take if any of a Spec's assertions
+	// fail.
+	//
+	// For example, if you wanted to grep a log file in the event that no
+	// connectivity on a particular IP:PORT combination could be made you might
+	// do this:
+	//
+	// ```yaml
+	// tests:
+	//  - exec: nc -z $HOST $PORT
+	//    on:
+	//      fail:
+	//        exec: grep ERROR /var/log/myapp.log
+	// ```
+	//
+	// The `grep ERROR /var/log/myapp.log` command will only be executed if
+	// there is no connectivity to $HOST:$PORT and the results of that grep
+	// will be directed to the test's output. You can use the `gdt.WithDebug()`
+	// function to configure additional `io.Writer`s to direct this output to.
+	Fail *Action `yaml:"fail,omitempty"`
+}

plugin/exec/parse.go

@@ -72,6 +72,15 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 				return err
 			}
 			s.Assert = e
+		case "on":
+			if valNode.Kind != yaml.MappingNode {
+				return errors.ExpectedMapAt(valNode)
+			}
+			var o *On
+			if err := valNode.Decode(&o); err != nil {
+				return err
+			}
+			s.On = o
 		default:
 			if lo.Contains(gdttypes.BaseSpecFields, key) {
 				continue

plugin/exec/parse_test.go

@@ -57,7 +57,9 @@ func TestSimpleCommand(t *testing.T) {
 				Index:    0,
 				Defaults: &gdttypes.Defaults{},
 			},
-			Exec: "ls",
+			Action: gdtexec.Action{
+				Exec: "ls",
+			},
 		},
 	}
 	assert.Equal(expTests, s.Tests)

plugin/exec/spec.go

@@ -12,19 +12,11 @@ import (
 // operating system's `exec` family of functions.
 type Spec struct {
 	gdttypes.Spec
-	// Exec is the exact command to execute.
-	//
-	// You may execute more than one command but must include the `shell` field
-	// to indicate that the command should be run in a shell. It is best
-	// practice, however, to simply use multiple `exec` specs instead of
-	// executing multiple commands in a single shell call.
-	Exec string `yaml:"exec"`
-	// Shell is the specific shell to use in executing the command. If empty
-	// (the default), no shell is used to execute the command and instead the
-	// operating system's `exec` family of calls is used.
-	Shell string `yaml:"shell,omitempty"`
+	Action
 	// Assert is an object containing the conditions that the Spec will assert.
 	Assert *Expect `yaml:"assert,omitempty"`
+	// On is an object containing actions to take upon certain conditions.
+	On *On `yaml:"on,omitempty"`
 }
 
 func (s *Spec) SetBase(b gdttypes.Spec) {

plugin/exec/testdata/on-fail-exec.yaml

@@ -0,0 +1,31 @@
+name: on-fail-exec
+description: a scenario that has an on.fail.exec clause
+tests:
+  - exec: echo "cat"
+    assert:
+      out:
+        is: cat
+    # Unfortunately there's not really any good way of testing things like this
+    # except by manually causing an assertion to fail in the test case and checking
+    # to see if the `on.fail` action was taken and debug output emitted to the
+    # console.
+    # 
+    # When I change `assert.out.is` above to "dat" instead of "cat", I get the
+    # correct behaviour:
+    # 
+    # === RUN   TestOnFail
+    # === RUN   TestOnFail/on-fail-exec
+    #     action.go:59: exec: echo [cat]
+    #     eval.go:35: assertion failed: not equal: expected dat but got cat
+    #     action.go:59: exec: echo [bad kitty]
+    #     eval.go:46: on.fail.exec: stdout: bad kitty
+    # === NAME  TestOnFail
+    #     eval_test.go:256:
+    #         	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
+    #         	Error:      	Should be false
+    #         	Test:       	TestOnFail
+    # --- FAIL: TestOnFail (0.00s)
+    #     --- FAIL: TestOnFail/on-fail-exec (0.00s)
+    on:
+      fail:
+        exec: echo "bad kitty"