From 91a95e6c1d4a473937660db7e7bae1696e457bcf Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Tue, 8 Aug 2023 14:32:26 -0400 Subject: [PATCH 1/2] separate exec plugin Action from Spec Following commits will add the ability to reference an exec plugin Action from a nested Spec `on.fail` field, so we simply separate out the Action bit from the Spec in the exec plugin here. Signed-off-by: Jay Pipes --- plugin/exec/action.go | 21 +++++++++++++++++++++ plugin/exec/parse_test.go | 4 +++- plugin/exec/spec.go | 12 +----------- 3 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 plugin/exec/action.go diff --git a/plugin/exec/action.go b/plugin/exec/action.go new file mode 100644 index 0000000..c8c3b02 --- /dev/null +++ b/plugin/exec/action.go @@ -0,0 +1,21 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package exec + +// Action describes a single execution of one or more commands via the +// operating system's `exec` family of functions. +type Action struct { + // Exec is the exact command to execute. + // + // You may execute more than one command but must include the `shell` field + // to indicate that the command should be run in a shell. It is best + // practice, however, to simply use multiple `exec` specs instead of + // executing multiple commands in a single shell call. + Exec string `yaml:"exec"` + // Shell is the specific shell to use in executing the command. If empty + // (the default), no shell is used to execute the command and instead the + // operating system's `exec` family of calls is used. + Shell string `yaml:"shell,omitempty"` +} diff --git a/plugin/exec/parse_test.go b/plugin/exec/parse_test.go index 32bdfd1..bc0e697 100644 --- a/plugin/exec/parse_test.go +++ b/plugin/exec/parse_test.go @@ -57,7 +57,9 @@ func TestSimpleCommand(t *testing.T) { Index: 0, Defaults: &gdttypes.Defaults{}, }, - Exec: "ls", + Action: gdtexec.Action{ + Exec: "ls", + }, }, } assert.Equal(expTests, s.Tests) diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go index 90f87f8..2448552 100644 --- a/plugin/exec/spec.go +++ b/plugin/exec/spec.go @@ -12,17 +12,7 @@ import ( // operating system's `exec` family of functions. type Spec struct { gdttypes.Spec - // Exec is the exact command to execute. - // - // You may execute more than one command but must include the `shell` field - // to indicate that the command should be run in a shell. It is best - // practice, however, to simply use multiple `exec` specs instead of - // executing multiple commands in a single shell call. - Exec string `yaml:"exec"` - // Shell is the specific shell to use in executing the command. If empty - // (the default), no shell is used to execute the command and instead the - // operating system's `exec` family of calls is used. - Shell string `yaml:"shell,omitempty"` + Action // Assert is an object containing the conditions that the Spec will assert. Assert *Expect `yaml:"assert,omitempty"` } From d2552cfa5a1adb0d829a365d854081ef53a74ede Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Tue, 8 Aug 2023 19:01:50 -0400 Subject: [PATCH 2/2] support `on.fail.exec` actions Users would like to be able to execute commands, collect log information, grep for errors in output and other actions when a test assertion fails. For instance, if an application is deployed using Kubernetes and network connectivity doesn't work for the application, the test author might want to call kubectl logs in the event of a test failure. Another example might be if you wanted to grep a log file in the event that no connectivity on a particular IP:PORT combination could be made you might do this: ```yaml tests: - exec: nc -z $HOST $PORT on: fail: exec: grep ERROR /var/log/myapp.log ``` The grep ERROR /var/log/myapp.log command will only be executed if there is no connectivity to $HOST:$PORT and the results of that grep will be directed to the test's output. You can use the gdt.WithDebug() function to configure additional io.Writers to direct this output to. This patch adds support for the exec plugin's `on.fail` field, hopefully in a way that is extensible for other plugins to use as an example (and possible embed the `plugin/exec.Action` struct). Addresses Issue #12 Signed-off-by: Jay Pipes --- README.md | 11 ++++ plugin/exec/action.go | 76 ++++++++++++++++++++++++++ plugin/exec/eval.go | 72 +++++++++--------------- plugin/exec/eval_test.go | 47 ++++++++++++++++ plugin/exec/on.go | 29 ++++++++++ plugin/exec/parse.go | 9 +++ plugin/exec/spec.go | 2 + plugin/exec/testdata/on-fail-exec.yaml | 31 +++++++++++ 8 files changed, 230 insertions(+), 47 deletions(-) create mode 100644 plugin/exec/on.go create mode 100644 plugin/exec/testdata/on-fail-exec.yaml diff --git a/README.md b/README.md index c03ed75..a81992f 100644 --- a/README.md +++ b/README.md @@ -510,6 +510,17 @@ the base `Spec` fields listed above): present in `stderr`. * `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at least one* must be present in `stderr`. +* `on`: (optional) an object describing actions to take upon certain + conditions. +* `on.fail`: (optional) an object describing an action to take when any + assertion fails for the test action. +* `on.fail.exec`: a string with the exact command to execute upon test + assertion failure. You may execute more than one command but must include the + `on.fail.shell` field to indicate that the command should be run in a shell. +* `on.fail.shell`: (optional) a string with the specific shell to use in executing the + command to run upon test assertion failure. If empty (the default), no shell + is used to execute the command and instead the operating system's `exec` family + of calls is used. [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26 diff --git a/plugin/exec/action.go b/plugin/exec/action.go index c8c3b02..9884972 100644 --- a/plugin/exec/action.go +++ b/plugin/exec/action.go @@ -4,6 +4,18 @@ package exec +import ( + "bytes" + "context" + "os/exec" + "testing" + + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/gdt-dev/gdt/debug" + gdterrors "github.com/gdt-dev/gdt/errors" + "github.com/google/shlex" +) + // Action describes a single execution of one or more commands via the // operating system's `exec` family of functions. type Action struct { @@ -19,3 +31,67 @@ type Action struct { // operating system's `exec` family of calls is used. Shell string `yaml:"shell,omitempty"` } + +// Do performs a single command or shell execution returning the corresponding +// exit code and any runtime error. The `outbuf` and `errbuf` buffers will be +// filled with the contents of the command's stdout and stderr pipes +// respectively. +func (a *Action) Do( + ctx context.Context, + t *testing.T, + outbuf *bytes.Buffer, + errbuf *bytes.Buffer, + exitcode *int, +) error { + var target string + var args []string + if a.Shell == "" { + // Parse time already validated exec string parses into valid shell + // args + args, _ = shlex.Split(a.Exec) + target = args[0] + args = args[1:] + } else { + target = a.Shell + args = []string{"-c", a.Exec} + } + + debug.Println(ctx, t, "exec: %s %s", target, args) + + var cmd *exec.Cmd + cmd = exec.CommandContext(ctx, target, args...) + + outpipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + errpipe, err := cmd.StderrPipe() + if err != nil { + return err + } + + err = cmd.Start() + if gdtcontext.TimedOut(ctx, err) { + return gdterrors.ErrTimeoutExceeded + } + if err != nil { + return err + } + if outbuf != nil { + outbuf.ReadFrom(outpipe) + } + if errbuf != nil { + errbuf.ReadFrom(errpipe) + } + + err = cmd.Wait() + if gdtcontext.TimedOut(ctx, err) { + return gdterrors.ErrTimeoutExceeded + } + if err != nil && exitcode != nil { + eerr, _ := err.(*exec.ExitError) + ec := eerr.ExitCode() + *exitcode = ec + } + return nil +} diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go index 68fef54..243680f 100644 --- a/plugin/exec/eval.go +++ b/plugin/exec/eval.go @@ -7,12 +7,8 @@ package exec import ( "bytes" "context" - "os/exec" "testing" - "github.com/google/shlex" - - gdtcontext "github.com/gdt-dev/gdt/context" "github.com/gdt-dev/gdt/debug" gdterrors "github.com/gdt-dev/gdt/errors" "github.com/gdt-dev/gdt/result" @@ -25,57 +21,39 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result { outbuf := &bytes.Buffer{} errbuf := &bytes.Buffer{} - var err error - var cmd *exec.Cmd - var target string - var args []string - if s.Shell == "" { - // Parse time already validated exec string parses into valid shell - // args - args, _ = shlex.Split(s.Exec) - target = args[0] - args = args[1:] - } else { - target = s.Shell - args = []string{"-c", s.Exec} - } - - debug.Println(ctx, t, "exec: %s %s", target, args) - cmd = exec.CommandContext(ctx, target, args...) - - outpipe, err := cmd.StdoutPipe() - if err != nil { - return result.New(result.WithRuntimeError(ExecRuntimeError(err))) - } - errpipe, err := cmd.StderrPipe() - if err != nil { - return result.New(result.WithRuntimeError(ExecRuntimeError(err))) - } + var ec int - err = cmd.Start() - if gdtcontext.TimedOut(ctx, err) { - return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) - } - if err != nil { + if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil { + if err == gdterrors.ErrTimeoutExceeded { + return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) + } return result.New(result.WithRuntimeError(ExecRuntimeError(err))) } - outbuf.ReadFrom(outpipe) - errbuf.ReadFrom(errpipe) - - err = cmd.Wait() - if gdtcontext.TimedOut(ctx, err) { - return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)) - } - ec := 0 - if err != nil { - eerr, _ := err.(*exec.ExitError) - ec = eerr.ExitCode() - } a := newAssertions(s.Assert, ec, outbuf, errbuf) if !a.OK() { for _, fail := range a.Failures() { t.Error(fail) } + if s.On != nil { + if s.On.Fail != nil { + outbuf.Reset() + errbuf.Reset() + err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil) + if err != nil { + debug.Println(ctx, t, "error in on.fail.exec: %s", err) + } + if outbuf.Len() > 0 { + debug.Println( + ctx, t, "on.fail.exec: stdout: %s", outbuf.String(), + ) + } + if errbuf.Len() > 0 { + debug.Println( + ctx, t, "on.fail.exec: stderr: %s", errbuf.String(), + ) + } + } + } } return result.New(result.WithFailures(a.Failures()...)) } diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go index 3518af3..96ed19c 100644 --- a/plugin/exec/eval_test.go +++ b/plugin/exec/eval_test.go @@ -235,3 +235,50 @@ func TestTimeoutCascade(t *testing.T) { require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]") require.Contains(debugout, "using timeout of 20ms (expected: true)") } + +// Unfortunately there's not really any good way of testing things like this +// except by manually causing an assertion to fail in the test case and +// checking to see if the `on.fail` action was taken and debug output emitted +// to the console. +// +// When I change the `testdata/on-fail-exec.yaml` file to have a failed +// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the +// correct behaviour: +// +// === RUN TestOnFail +// === RUN TestOnFail/on-fail-exec +// +// action.go:59: exec: echo [cat] +// eval.go:35: assertion failed: not equal: expected dat but got cat +// action.go:59: exec: echo [bad kitty] +// eval.go:46: on.fail.exec: stdout: bad kitty +// +// === NAME TestOnFail +// +// eval_test.go:256: +// Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256 +// Error: Should be false +// Test: TestOnFail +// +// --- FAIL: TestOnFail (0.00s) +// +// --- FAIL: TestOnFail/on-fail-exec (0.00s) +func TestOnFail(t *testing.T) { + require := require.New(t) + + fp := filepath.Join("testdata", "on-fail-exec.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader( + f, + scenario.WithPath(fp), + ) + require.Nil(err) + require.NotNil(s) + + ctx := gdtcontext.New(gdtcontext.WithDebug()) + err = s.Run(ctx, t) + require.Nil(err) + require.False(t.Failed()) +} diff --git a/plugin/exec/on.go b/plugin/exec/on.go new file mode 100644 index 0000000..5f56e09 --- /dev/null +++ b/plugin/exec/on.go @@ -0,0 +1,29 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package exec + +// On describes actions that can be taken upon certain conditions. +type On struct { + // Fail contains one or more actions to take if any of a Spec's assertions + // fail. + // + // For example, if you wanted to grep a log file in the event that no + // connectivity on a particular IP:PORT combination could be made you might + // do this: + // + // ```yaml + // tests: + // - exec: nc -z $HOST $PORT + // on: + // fail: + // exec: grep ERROR /var/log/myapp.log + // ``` + // + // The `grep ERROR /var/log/myapp.log` command will only be executed if + // there is no connectivity to $HOST:$PORT and the results of that grep + // will be directed to the test's output. You can use the `gdt.WithDebug()` + // function to configure additional `io.Writer`s to direct this output to. + Fail *Action `yaml:"fail,omitempty"` +} diff --git a/plugin/exec/parse.go b/plugin/exec/parse.go index b7a458b..a08e063 100644 --- a/plugin/exec/parse.go +++ b/plugin/exec/parse.go @@ -72,6 +72,15 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error { return err } s.Assert = e + case "on": + if valNode.Kind != yaml.MappingNode { + return errors.ExpectedMapAt(valNode) + } + var o *On + if err := valNode.Decode(&o); err != nil { + return err + } + s.On = o default: if lo.Contains(gdttypes.BaseSpecFields, key) { continue diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go index 2448552..2589369 100644 --- a/plugin/exec/spec.go +++ b/plugin/exec/spec.go @@ -15,6 +15,8 @@ type Spec struct { Action // Assert is an object containing the conditions that the Spec will assert. Assert *Expect `yaml:"assert,omitempty"` + // On is an object containing actions to take upon certain conditions. + On *On `yaml:"on,omitempty"` } func (s *Spec) SetBase(b gdttypes.Spec) { diff --git a/plugin/exec/testdata/on-fail-exec.yaml b/plugin/exec/testdata/on-fail-exec.yaml new file mode 100644 index 0000000..5cfc00f --- /dev/null +++ b/plugin/exec/testdata/on-fail-exec.yaml @@ -0,0 +1,31 @@ +name: on-fail-exec +description: a scenario that has an on.fail.exec clause +tests: + - exec: echo "cat" + assert: + out: + is: cat + # Unfortunately there's not really any good way of testing things like this + # except by manually causing an assertion to fail in the test case and checking + # to see if the `on.fail` action was taken and debug output emitted to the + # console. + # + # When I change `assert.out.is` above to "dat" instead of "cat", I get the + # correct behaviour: + # + # === RUN TestOnFail + # === RUN TestOnFail/on-fail-exec + # action.go:59: exec: echo [cat] + # eval.go:35: assertion failed: not equal: expected dat but got cat + # action.go:59: exec: echo [bad kitty] + # eval.go:46: on.fail.exec: stdout: bad kitty + # === NAME TestOnFail + # eval_test.go:256: + # Error Trace: /home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256 + # Error: Should be false + # Test: TestOnFail + # --- FAIL: TestOnFail (0.00s) + # --- FAIL: TestOnFail/on-fail-exec (0.00s) + on: + fail: + exec: echo "bad kitty"