support on.fail.exec actions

Users would like to be able to execute commands, collect log
information, grep for errors in output and other actions when a test
assertion fails.

For instance, if an application is deployed using Kubernetes and network
connectivity doesn't work for the application, the test author might
want to call kubectl logs in the event of a test failure.

Another example might be if you wanted to grep a log file in the event
that no connectivity on a particular IP:PORT combination could be made
you might do this:

```yaml
tests:
 - exec: nc -z $HOST $PORT
   on:
     fail:
       exec: grep ERROR /var/log/myapp.log
```

The grep ERROR /var/log/myapp.log command will only be executed if there
is no connectivity to $HOST:$PORT and the results of that grep will be
directed to the test's output. You can use the gdt.WithDebug() function
to configure additional io.Writers to direct this output to.

This patch adds support for the exec plugin's `on.fail` field, hopefully
in a way that is extensible for other plugins to use as an example (and
possible embed the `plugin/exec.Action` struct).

Addresses Issue #12

Signed-off-by: Jay Pipes <jaypipes@gmail.com>

README.md

@@ -510,6 +510,17 @@ the base `Spec` fields listed above):
   present in `stderr`.
 * `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
   least one* must be present in `stderr`.
+* `on`: (optional) an object describing actions to take upon certain
+  conditions.
+* `on.fail`: (optional) an object describing an action to take when any
+  assertion fails for the test action.
+* `on.fail.exec`: a string with the exact command to execute upon test
+  assertion failure. You may execute more than one command but must include the
+  `on.fail.shell` field to indicate that the command should be run in a shell.
+* `on.fail.shell`: (optional) a string with the specific shell to use in executing the
+  command to run upon test assertion failure. If empty (the default), no shell
+  is used to execute the command and instead the operating system's `exec` family
+  of calls is used.
 
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26

plugin/exec/action.go

@@ -4,6 +4,18 @@
 
 package exec
 
+import (
+	"bytes"
+	"context"
+	"os/exec"
+	"testing"
+
+	gdtcontext "github.com/gdt-dev/gdt/context"
+	"github.com/gdt-dev/gdt/debug"
+	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/google/shlex"
+)
+
 // Action describes a single execution of one or more commands via the
 // operating system's `exec` family of functions.
 type Action struct {
@@ -19,3 +31,67 @@ type Action struct {
 	// operating system's `exec` family of calls is used.
 	Shell string `yaml:"shell,omitempty"`
 }
+
+// Do performs a single command or shell execution returning the corresponding
+// exit code and any runtime error. The `outbuf` and `errbuf` buffers will be
+// filled with the contents of the command's stdout and stderr pipes
+// respectively.
+func (a *Action) Do(
+	ctx context.Context,
+	t *testing.T,
+	outbuf *bytes.Buffer,
+	errbuf *bytes.Buffer,
+	exitcode *int,
+) error {
+	var target string
+	var args []string
+	if a.Shell == "" {
+		// Parse time already validated exec string parses into valid shell
+		// args
+		args, _ = shlex.Split(a.Exec)
+		target = args[0]
+		args = args[1:]
+	} else {
+		target = a.Shell
+		args = []string{"-c", a.Exec}
+	}
+
+	debug.Println(ctx, t, "exec: %s %s", target, args)
+
+	var cmd *exec.Cmd
+	cmd = exec.CommandContext(ctx, target, args...)
+
+	outpipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	errpipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if gdtcontext.TimedOut(ctx, err) {
+		return gdterrors.ErrTimeoutExceeded
+	}
+	if err != nil {
+		return err
+	}
+	if outbuf != nil {
+		outbuf.ReadFrom(outpipe)
+	}
+	if errbuf != nil {
+		errbuf.ReadFrom(errpipe)
+	}
+
+	err = cmd.Wait()
+	if gdtcontext.TimedOut(ctx, err) {
+		return gdterrors.ErrTimeoutExceeded
+	}
+	if err != nil && exitcode != nil {
+		eerr, _ := err.(*exec.ExitError)
+		ec := eerr.ExitCode()
+		*exitcode = ec
+	}
+	return nil
+}

plugin/exec/eval.go

@@ -7,12 +7,8 @@ package exec
 import (
 	"bytes"
 	"context"
-	"os/exec"
 	"testing"
 
-	"github.com/google/shlex"
-
-	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/gdt-dev/gdt/result"
@@ -25,57 +21,39 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
-	var err error
-	var cmd *exec.Cmd
-	var target string
-	var args []string
-	if s.Shell == "" {
-		// Parse time already validated exec string parses into valid shell
-		// args
-		args, _ = shlex.Split(s.Exec)
-		target = args[0]
-		args = args[1:]
-	} else {
-		target = s.Shell
-		args = []string{"-c", s.Exec}
-	}
-
-	debug.Println(ctx, t, "exec: %s %s", target, args)
-	cmd = exec.CommandContext(ctx, target, args...)
-
-	outpipe, err := cmd.StdoutPipe()
-	if err != nil {
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
-	}
-	errpipe, err := cmd.StderrPipe()
-	if err != nil {
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
-	}
+	var ec int
 
-	err = cmd.Start()
-	if gdtcontext.TimedOut(ctx, err) {
-		return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
-	}
-	if err != nil {
+	if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
+		if err == gdterrors.ErrTimeoutExceeded {
+			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
+		}
 		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
 	}
-	outbuf.ReadFrom(outpipe)
-	errbuf.ReadFrom(errpipe)
-
-	err = cmd.Wait()
-	if gdtcontext.TimedOut(ctx, err) {
-		return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
-	}
-	ec := 0
-	if err != nil {
-		eerr, _ := err.(*exec.ExitError)
-		ec = eerr.ExitCode()
-	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK() {
 		for _, fail := range a.Failures() {
 			t.Error(fail)
 		}
+		if s.On != nil {
+			if s.On.Fail != nil {
+				outbuf.Reset()
+				errbuf.Reset()
+				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
+				if err != nil {
+					debug.Println(ctx, t, "error in on.fail.exec: %s", err)
+				}
+				if outbuf.Len() > 0 {
+					debug.Println(
+						ctx, t, "on.fail.exec: stdout: %s", outbuf.String(),
+					)
+				}
+				if errbuf.Len() > 0 {
+					debug.Println(
+						ctx, t, "on.fail.exec: stderr: %s", errbuf.String(),
+					)
+				}
+			}
+		}
 	}
 	return result.New(result.WithFailures(a.Failures()...))
 }

plugin/exec/eval_test.go

@@ -235,3 +235,50 @@ func TestTimeoutCascade(t *testing.T) {
 	require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
 	require.Contains(debugout, "using timeout of 20ms (expected: true)")
 }
+
+// Unfortunately there's not really any good way of testing things like this
+// except by manually causing an assertion to fail in the test case and
+// checking to see if the `on.fail` action was taken and debug output emitted
+// to the console.
+//
+// When I change the `testdata/on-fail-exec.yaml` file to have a failed
+// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the
+// correct behaviour:
+//
+// === RUN   TestOnFail
+// === RUN   TestOnFail/on-fail-exec
+//
+//	action.go:59: exec: echo [cat]
+//	eval.go:35: assertion failed: not equal: expected dat but got cat
+//	action.go:59: exec: echo [bad kitty]
+//	eval.go:46: on.fail.exec: stdout: bad kitty
+//
+// === NAME  TestOnFail
+//
+//	eval_test.go:256:
+//	    	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
+//	    	Error:      	Should be false
+//	    	Test:       	TestOnFail
+//
+// --- FAIL: TestOnFail (0.00s)
+//
+//	--- FAIL: TestOnFail/on-fail-exec (0.00s)
+func TestOnFail(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "on-fail-exec.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+	require.False(t.Failed())
+}

plugin/exec/on.go

@@ -0,0 +1,29 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package exec
+
+// On describes actions that can be taken upon certain conditions.
+type On struct {
+	// Fail contains one or more actions to take if any of a Spec's assertions
+	// fail.
+	//
+	// For example, if you wanted to grep a log file in the event that no
+	// connectivity on a particular IP:PORT combination could be made you might
+	// do this:
+	//
+	// ```yaml
+	// tests:
+	//  - exec: nc -z $HOST $PORT
+	//    on:
+	//      fail:
+	//        exec: grep ERROR /var/log/myapp.log
+	// ```
+	//
+	// The `grep ERROR /var/log/myapp.log` command will only be executed if
+	// there is no connectivity to $HOST:$PORT and the results of that grep
+	// will be directed to the test's output. You can use the `gdt.WithDebug()`
+	// function to configure additional `io.Writer`s to direct this output to.
+	Fail *Action `yaml:"fail,omitempty"`
+}

plugin/exec/parse.go

@@ -72,6 +72,15 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 				return err
 			}
 			s.Assert = e
+		case "on":
+			if valNode.Kind != yaml.MappingNode {
+				return errors.ExpectedMapAt(valNode)
+			}
+			var o *On
+			if err := valNode.Decode(&o); err != nil {
+				return err
+			}
+			s.On = o
 		default:
 			if lo.Contains(gdttypes.BaseSpecFields, key) {
 				continue

plugin/exec/spec.go

@@ -15,6 +15,8 @@ type Spec struct {
 	Action
 	// Assert is an object containing the conditions that the Spec will assert.
 	Assert *Expect `yaml:"assert,omitempty"`
+	// On is an object containing actions to take upon certain conditions.
+	On *On `yaml:"on,omitempty"`
 }
 
 func (s *Spec) SetBase(b gdttypes.Spec) {

plugin/exec/testdata/on-fail-exec.yaml

@@ -0,0 +1,31 @@
+name: on-fail-exec
+description: a scenario that has an on.fail.exec clause
+tests:
+  - exec: echo "cat"
+    assert:
+      out:
+        is: cat
+    # Unfortunately there's not really any good way of testing things like this
+    # except by manually causing an assertion to fail in the test case and checking
+    # to see if the `on.fail` action was taken and debug output emitted to the
+    # console.
+    # 
+    # When I change `assert.out.is` above to "dat" instead of "cat", I get the
+    # correct behaviour:
+    # 
+    # === RUN   TestOnFail
+    # === RUN   TestOnFail/on-fail-exec
+    #     action.go:59: exec: echo [cat]
+    #     eval.go:35: assertion failed: not equal: expected dat but got cat
+    #     action.go:59: exec: echo [bad kitty]
+    #     eval.go:46: on.fail.exec: stdout: bad kitty
+    # === NAME  TestOnFail
+    #     eval_test.go:256:
+    #         	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
+    #         	Error:      	Should be false
+    #         	Test:       	TestOnFail
+    # --- FAIL: TestOnFail (0.00s)
+    #     --- FAIL: TestOnFail/on-fail-exec (0.00s)
+    on:
+      fail:
+        exec: echo "bad kitty"