Expose task attribute to control env command substitution pattern mat…

…ching

README.md

@@ -156,6 +156,17 @@ would be substituted with an empty string before the action was
 executed and we would not receive the expected output from the
 command.
 
+The matching of the substitution pattern `$[...]` can be problematic
+for statements like
+
+`$[bash -c "if [ \"${DEPLOY_ENV}\" == \"production\" ]; then echo production; else echo staging; fi"]`
+
+In this case, the opening `$[` will be matched in a non-greedy manner
+by the closing `]` of the bash `if` statement and the command will
+fail. `ork` exposes a task attribute called `env_subst_greedy`
+(default: false) which can be used to enforce the desired behaviour
+(in this case it must be set to true).
+
 ### Task dependencies
 
 `ork` also supports task dependencies (with cyclic dependency

env.go

@@ -15,11 +15,11 @@ type Env map[string]string
 // does not mutate `this` in any way
 // all env values will be parsed to detect substitution patterns $[...]
 // which will be executed as actions whose output will be interpolated in the env value
-func (this Env) Apply() error {
+func (this Env) Apply(greedyEnvSubst bool) error {
 	// apply key, value entries
 	for key, value := range this {
 		val := ""
-		for _, token := range parseEnvTokens(value) {
+		for _, token := range parseEnvTokens(value, greedyEnvSubst) {
 			v, err := token.expand()
 			if err != nil {
 				return fmt.Errorf("key %s: %s: %v", key, value, err)
@@ -44,8 +44,13 @@ type envToken struct {
 // split the statement into discrete tokens that will:
 // - either be executed and replaced with the execution output
 // - or will just be used as is
-func parseEnvTokens(statement string) []envToken {
-	re := regexp.MustCompile(`\$\[.*?\]`)
+func parseEnvTokens(statement string, greedyEnvSubst bool) []envToken {
+	var re *regexp.Regexp
+	if greedyEnvSubst {
+		re = regexp.MustCompile(`\$\[.*\]+`)
+	} else {
+		re = regexp.MustCompile(`\$\[.*?\]`)
+	}
 
 	tokens := []envToken{}
 	matches := re.FindAllStringIndex(statement, -1)

env_test.go

@@ -10,13 +10,14 @@ import (
 func Test_envValues(t *testing.T) {
 	kases := []struct {
 		statement string
+		greedy    bool
 		expected  []envToken
 	}{
-		{"", []envToken{{"", false}}},
-		{"12 12", []envToken{{"12 12", false}}},
-		{"$[echo foo]", []envToken{{"echo foo", true}}},
-		{"$[bash -c \"echo $(echo foo)\"]", []envToken{{"bash -c \"echo $(echo foo)\"", true}}},
-		{"1-$[foo]-2-$[echo foo ]-3-$[bar]-4", []envToken{
+		{"", false, []envToken{{"", false}}},
+		{"12 12", false, []envToken{{"12 12", false}}},
+		{"$[echo foo]", false, []envToken{{"echo foo", true}}},
+		{`$[bash -c "echo $(echo foo)"]`, false, []envToken{{"bash -c \"echo $(echo foo)\"", true}}},
+		{"1-$[foo]-2-$[echo foo ]-3-$[bar]-4", false, []envToken{
 			{"1-", false},
 			{"foo", true},
 			{"-2-", false},
@@ -25,9 +26,12 @@ func Test_envValues(t *testing.T) {
 			{"bar", true},
 			{"-4", false},
 		}},
+		{`$[bash -c "if [ foo = foo]; then echo foo; else echo bar; fi"]`, true, []envToken{
+			{"bash -c \"if [ foo = foo]; then echo foo; else echo bar; fi\"", true},
+		}},
 	}
 
 	for idx, kase := range kases {
-		assert.Equal(t, kase.expected, parseEnvTokens(kase.statement), fmt.Sprintf("case index=%d", idx))
+		assert.Equal(t, kase.expected, parseEnvTokens(kase.statement, kase.greedy), fmt.Sprintf("case index=%d", idx))
 	}
 }

orkfile_test.go

@@ -193,6 +193,36 @@ tasks:
 			task:    "foo",
 			outputs: []string{"a"},
 		},
+		// ===================================
+		{
+			test: "env can execute non-trivial bash statements",
+			yml: `
+global:
+  env:
+    - MY_VAR_6: production
+tasks:
+  - name: foo
+    env_subst_greedy: true
+    env:
+      - MY_VAR_7: $[bash -c "if [ \"${MY_VAR_6}\" == \"production\" ]; then echo production; else echo staging; fi"]
+    actions:
+      - echo $MY_VAR_7
+`,
+			task:    "foo",
+			outputs: []string{"production"},
+		},
+		// ===================================
+		{
+			test: "action can execute arbitrary commands",
+			yml: `
+tasks:
+  - name: foo
+    actions:
+      - python -c "import sys; sys.stdout.write('hello from python');"
+`,
+			task:    "foo",
+			outputs: []string{"hello from python"},
+		},
 	}
 
 	// set this to a kase.test value to run one test only

task.go

@@ -6,17 +6,18 @@ import (
 )
 
 type Task struct {
-	Name        string   `yaml:"name"`
-	Default     string   `yaml:"default"` // used in the global task
-	Description string   `yaml:"description"`
-	WorkingDir  string   `yaml:"working_dir"`
-	Env         []Env    `yaml:"env"`
-	ExpandEnv   *bool    `yaml:"expand_env"`
-	Actions     []string `yaml:"actions"`
-	DependsOn   []string `yaml:"depends_on"`
-	Tasks       []*Task  `yaml:"tasks"`
-	OnSuccess   []string `yaml:"on_success"`
-	OnFailure   []string `yaml:"on_failure"`
+	Name           string   `yaml:"name"`
+	Default        string   `yaml:"default"` // used in the global task
+	Description    string   `yaml:"description"`
+	WorkingDir     string   `yaml:"working_dir"`
+	Env            []Env    `yaml:"env"`
+	ExpandEnv      *bool    `yaml:"expand_env"`
+	GreedyEnvSubst *bool    `yaml:"env_subst_greedy"`
+	Actions        []string `yaml:"actions"`
+	DependsOn      []string `yaml:"depends_on"`
+	Tasks          []*Task  `yaml:"tasks"`
+	OnSuccess      []string `yaml:"on_success"`
+	OnFailure      []string `yaml:"on_failure"`
 }
 
 func (t *Task) Info() string {
@@ -85,7 +86,7 @@ func (t *Task) execute(inventory Inventory, logger Logger, cdt map[string]struct
 	// apply the environment
 	logger.Debugf("[%s] applying task environment", t.Name)
 	for _, e := range t.Env {
-		if err = e.Apply(); err != nil {
+		if err = e.Apply(t.IsEnvSubstGreedy()); err != nil {
 			err = fmt.Errorf("[%s] failed to apply environment: %v", t.Name, err)
 			return
 		}
@@ -104,6 +105,13 @@ func (t *Task) execute(inventory Inventory, logger Logger, cdt map[string]struct
 	return
 }
 
+func (t *Task) IsEnvSubstGreedy() bool {
+	if t.GreedyEnvSubst == nil {
+		return false
+	}
+	return *t.GreedyEnvSubst
+}
+
 func executeAction(action string, expandEnv *bool, chdir string, logger Logger) error {
 	ee := true
 	if expandEnv != nil {