Merge pull request #50 from jaypipes/exec-err

correct exit code assertion in exec plugin

.github/workflows/fmtcheck.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
      - name: harden runner
-       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          egress-policy: block
          disable-sudo: true
@@ -26,8 +26,9 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
-       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:

.github/workflows/lint.yml

@@ -27,6 +27,7 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
      - name: harden runner
-       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          egress-policy: block
          disable-sudo: true
@@ -29,8 +29,9 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
-       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:

plugin/exec/assertions.go

@@ -166,9 +166,13 @@ func newAssertions(
 	outPipe *bytes.Buffer,
 	errPipe *bytes.Buffer,
 ) api.Assertions {
+	expExitCode := 0
+	if e != nil {
+		expExitCode = e.ExitCode
+	}
 	a := &assertions{
 		failures:    []error{},
-		expExitCode: exitCode,
+		expExitCode: expExitCode,
 		exitCode:    exitCode,
 	}
 	if e != nil {

plugin/exec/eval_test.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"flag"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -71,6 +72,52 @@ func TestExitCode(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestFailExecExitCodeNotSpecified(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "ls-fail-no-exit-code.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
+func TestExecFailExitCodeNotSpecified(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecExitCodeNotSpecified",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+	debugout := string(outerr)
+	ec := 2
+	// Yay, different exit codes for the same not found error...
+	if runtime.GOOS == "darwin" {
+		ec = 1
+	}
+	msg := fmt.Sprintf(
+		"assertion failed: not equal: expected 0 but got %d", ec,
+	)
+	require.Contains(debugout, msg)
+}
+
 func TestShellList(t *testing.T) {
 	require := require.New(t)
 

plugin/exec/testdata/ls-fail-no-exit-code.yaml

@@ -0,0 +1,4 @@
+name: ls-fail-no-exit-code
+description: a scenario that runs the `ls` command with a non-0 exit code and no assertion on exit code
+tests:
+  - exec: ls /this/dir/does/not/exist