From b919e4f38abaa4d0f1c97f1fb267a62abf1e90c0 Mon Sep 17 00:00:00 2001 From: David Chung Date: Wed, 8 Feb 2017 10:41:33 -0800 Subject: [PATCH] Template improvements (#390) Signed-off-by: David Chung --- examples/flavor/swarm/flavor.go | 4 +- examples/flavor/swarm/templates.go | 6 +- pkg/template/funcs.go | 92 ++++++++++++++++++------------ pkg/template/funcs_test.go | 17 ++++++ pkg/template/integration_test.go | 39 +++++++++++++ pkg/template/template.go | 57 ++++++++++++++++-- 6 files changed, 171 insertions(+), 44 deletions(-) diff --git a/examples/flavor/swarm/flavor.go b/examples/flavor/swarm/flavor.go index ec850cb26..a9977f528 100644 --- a/examples/flavor/swarm/flavor.go +++ b/examples/flavor/swarm/flavor.go @@ -160,7 +160,7 @@ func (s *baseFlavor) prepare(role string, flavorProperties *types.Any, instanceS log.Warningln("Worker prepare:", err) } - swarmID := "?" + swarmID = "?" if swarmStatus != nil { swarmID = swarmStatus.ID } @@ -342,7 +342,7 @@ func (c *templateContext) Funcs() []template.Function { }, }, { - Name: "SWARM_MANAGER_IP", + Name: "SWARM_MANAGER_ADDR", Description: []string{"IP of the Swarm manager / leader"}, Func: func() (string, error) { if c.nodeInfo == nil { diff --git a/examples/flavor/swarm/templates.go b/examples/flavor/swarm/templates.go index f6cbb59c7..34e013156 100644 --- a/examples/flavor/swarm/templates.go +++ b/examples/flavor/swarm/templates.go @@ -20,7 +20,7 @@ EOF kill -s HUP $(cat /var/run/docker.pid) sleep 5 -{{ if eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP }} +{{ if and ( eq INSTANCE_LOGICAL_ID SPEC.SwarmJoinIP ) (not SWARM_INITIALIZED) }} {{/* The first node of the special allocations will initialize the swarm. */}} docker swarm init --advertise-addr {{ INSTANCE_LOGICAL_ID }} @@ -34,7 +34,7 @@ sleep 5 {{ else }} {{/* The rest of the nodes will join as followers in the manager group. */}} - docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SPEC.SwarmJoinIP }}:2377 + docker swarm join --token {{ SWARM_JOIN_TOKENS.Manager }} {{ SWARM_MANAGER_ADDR }} {{ end }} ` @@ -59,7 +59,7 @@ kill -s HUP $(cat /var/run/docker.pid) sleep 5 -docker swarm join --token {{ SWARM_JOIN_TOKENS.Worker }} {{ SPEC.SwarmJoinIP }}:2377 +docker swarm join --token {{ SWARM_JOIN_TOKENS.Worker }} {{ SWARM_MANAGER_ADDR }} ` ) diff --git a/pkg/template/funcs.go b/pkg/template/funcs.go index c5d84b6d8..90e3d7cf4 100644 --- a/pkg/template/funcs.go +++ b/pkg/template/funcs.go @@ -1,6 +1,7 @@ package template import ( + "bytes" "encoding/json" "fmt" "reflect" @@ -10,6 +11,24 @@ import ( "github.com/jmespath/go-jmespath" ) +// DeepCopyObject makes a deep copy of the argument, using encoding/gob encode/decode. +func DeepCopyObject(from interface{}) (interface{}, error) { + var mod bytes.Buffer + enc := json.NewEncoder(&mod) + dec := json.NewDecoder(&mod) + err := enc.Encode(from) + if err != nil { + return nil, err + } + + copy := reflect.New(reflect.TypeOf(from)) + err = dec.Decode(copy.Interface()) + if err != nil { + return nil, err + } + return reflect.Indirect(copy).Interface(), nil +} + // QueryObject applies a JMESPath query specified by the expression, against the target object. func QueryObject(exp string, target interface{}) (interface{}, error) { query, err := jmespath.Compile(exp) @@ -119,8 +138,14 @@ func (t *Template) DefaultFuncs() []Function { Description: []string{ "Source / evaluate the template at the input location (as URL).", "This will make all of the global variables declared there visible in this template's context.", + "Similar to 'source' in bash, sourcing another template means applying it in the same context ", + "as the calling template. The context (e.g. variables) of the calling template as a result can be mutated.", }, - Func: func(p string) (string, error) { + Func: func(p string, opt ...interface{}) (string, error) { + var o interface{} + if len(opt) > 0 { + o = opt[0] + } loc := p if strings.Index(loc, "str://") == -1 { buff, err := getURL(t.url, p) @@ -133,19 +158,16 @@ func (t *Template) DefaultFuncs() []Function { if err != nil { return "", err } - // copy the binds in the parent scope into the child - for k, v := range t.binds { - sourced.binds[k] = v - } - // inherit the functions defined for this template - for k, v := range t.funcs { - sourced.AddFunc(k, v) - } // set this as the parent of the sourced template so its global can mutate the globals in this sourced.parent = t + sourced.forkFrom(t) + sourced.context = t.context + if o == nil { + o = sourced.context + } // TODO(chungers) -- let the sourced template define new functions that can be called in the parent. - return sourced.Render(nil) + return sourced.Render(o) }, }, { @@ -153,28 +175,37 @@ func (t *Template) DefaultFuncs() []Function { Description: []string{ "Render content found at URL as template and include here.", "The optional second parameter is the context to use when rendering the template.", + "Conceptually similar to exec in bash, where the template included is applied using a fork ", + "of current context in the calling template. Any mutations to the context via 'global' will not ", + "be visible in the calling template's context.", }, Func: func(p string, opt ...interface{}) (string, error) { var o interface{} if len(opt) > 0 { o = opt[0] } - loc, err := getURL(t.url, p) - if err != nil { - return "", err + loc := p + if strings.Index(loc, "str://") == -1 { + buff, err := getURL(t.url, p) + if err != nil { + return "", err + } + loc = buff } included, err := NewTemplate(loc, t.options) if err != nil { return "", err } - // copy the binds in the parent scope into the child - for k, v := range t.binds { - included.binds[k] = v + dotCopy, err := included.forkFrom(t) + if err != nil { + return "", err } - // inherit the functions defined for this template - for k, v := range t.funcs { - included.AddFunc(k, v) + included.context = dotCopy + + if o == nil { + o = included.context } + return included.Render(o) }, }, @@ -193,10 +224,10 @@ func (t *Template) DefaultFuncs() []Function { "Defines a variable with the first argument as name and last argument value as the default.", "It's also ok to pass a third optional parameter, in the middle, as the documentation string.", }, - Func: func(name string, args ...interface{}) (string, error) { + Func: func(name string, args ...interface{}) (Void, error) { if _, has := t.defaults[name]; has { // not sure if this is good, but should complain loudly - return "", fmt.Errorf("already defined: %v", name) + return voidValue, fmt.Errorf("already defined: %v", name) } var doc string var value interface{} @@ -210,7 +241,7 @@ func (t *Template) DefaultFuncs() []Function { value = args[1] } t.AddDef(name, value, doc) - return "", nil + return voidValue, nil }, }, { @@ -220,11 +251,9 @@ func (t *Template) DefaultFuncs() []Function { "This is similar to def (which sets the default value).", "Global variables are propagated to all templates that are rendered via the 'include' function.", }, - Func: func(name string, v interface{}) interface{} { - for here := t; here != nil; here = here.parent { - here.updateGlobal(name, v) - } - return "" + Func: func(n string, v interface{}) Void { + t.Global(n, v) + return voidValue }, }, { @@ -233,14 +262,7 @@ func (t *Template) DefaultFuncs() []Function { "References / gets the variable named after the first argument.", "The values must be set first by either def or global.", }, - Func: func(name string) interface{} { - if found, has := t.binds[name]; has { - return found - } else if v, has := t.defaults[name]; has { - return v.Value - } - return nil - }, + Func: t.Ref, }, { Name: "q", diff --git a/pkg/template/funcs_test.go b/pkg/template/funcs_test.go index 4f30e8c2b..28cbe2d6c 100644 --- a/pkg/template/funcs_test.go +++ b/pkg/template/funcs_test.go @@ -25,6 +25,23 @@ type testCloud struct { ResourceList []interface{} } +func TestDeepCopyObject(t *testing.T) { + resource := "disk" + input := testCloud{ + Parameters: []testParameter{{ParameterKey: "foo", ParameterValue: "bar"}}, + Resources: []testResource{{ResourceType: "test", ResourceTypePtr: &resource}}, + } + + copy, err := DeepCopyObject(input) + require.NoError(t, err) + require.Equal(t, input, copy) + inputStr, err := ToJSON(input) + require.NoError(t, err) + copyStr, err := ToJSON(copy) + require.NoError(t, err) + require.Equal(t, inputStr, copyStr) +} + func TestQueryObjectEncodeDecode(t *testing.T) { param1 := testParameter{ diff --git a/pkg/template/integration_test.go b/pkg/template/integration_test.go index a94149304..7aeef7fd4 100644 --- a/pkg/template/integration_test.go +++ b/pkg/template/integration_test.go @@ -179,3 +179,42 @@ func TestSourceAndGlobal(t *testing.T) { require.NoError(t, err) require.Equal(t, "foo=100", view) } + +func TestIncludeAndGlobal(t *testing.T) { + r := `{{ global \"foo\" 100 }}` // the child template tries to mutate the global + s := `{{ include "str://` + r + `" }}foo={{ref "foo"}}` + tt, err := NewTemplate("str://"+s, Options{}) + require.NoError(t, err) + tt.Global("foo", 200) // set the global of the calling / parent template + view, err := tt.Render(nil) + require.NoError(t, err) + require.Equal(t, "foo=200", view) // parent's not affected by child template +} + +func TestSourceAndGlobalWithContext(t *testing.T) { + ctx := map[string]interface{}{ + "a": 1, + "b": 2, + } + r := `{{ global \"foo\" 100 }}{{$void := set . \"a\" 100}}` // sourced mutates the context + s := `{{ source "str://` + r + `" }}a={{.a}}` + tt, err := NewTemplate("str://"+s, Options{}) + require.NoError(t, err) + view, err := tt.Render(ctx) + require.NoError(t, err) + require.Equal(t, "a=100", view) // the sourced template mutated the calling template's context. +} + +func TestIncludeAndGlobalWithContext(t *testing.T) { + ctx := map[string]interface{}{ + "a": 1, + "b": 2, + } + r := `{{ global \"foo\" 100 }}{{$void := set . \"a\" 100}}` // included tries to mutate the context + s := `{{ include "str://` + r + `" }}a={{.a}}` + tt, err := NewTemplate("str://"+s, Options{}) + require.NoError(t, err) + view, err := tt.Render(ctx) + require.NoError(t, err) + require.Equal(t, "a=1", view) // the included template cannot mutate the calling template's context. +} diff --git a/pkg/template/template.go b/pkg/template/template.go index 959abbd7b..ee5675f30 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -71,8 +71,9 @@ type Template struct { body []byte parsed *template.Template funcs map[string]interface{} - binds map[string]interface{} + globals map[string]interface{} defaults map[string]defaultValue + context interface{} registered []Function lock sync.Mutex @@ -80,6 +81,12 @@ type Template struct { parent *Template } +// Void is used in the template functions return value type to indicate a void. +// Golang template does not allow functions with no return types to be bound. +type Void string + +const voidValue Void = "" + // NewTemplate fetches the content at the url and returns a template. If the string begins // with str:// as scheme, then the rest of the string is interpreted as the body of the template. func NewTemplate(s string, opt Options) (*Template, error) { @@ -111,7 +118,7 @@ func NewTemplateFromBytes(buff []byte, contextURL string, opt Options) (*Templat url: contextURL, body: buff, funcs: map[string]interface{}{}, - binds: map[string]interface{}{}, + globals: map[string]interface{}{}, defaults: map[string]defaultValue{}, }, nil } @@ -145,10 +152,51 @@ func (t *Template) AddDef(name string, val interface{}, doc ...string) *Template return t } +// Ref returns the value keyed by name in the context of this template. See 'ref' template function. +func (t *Template) Ref(name string) interface{} { + if found, has := t.globals[name]; has { + return found + } else if v, has := t.defaults[name]; has { + return v.Value + } + return nil +} + +// Dot returns the '.' in this template. +func (t *Template) Dot() interface{} { + return t.context +} + +func (t *Template) forkFrom(parent *Template) (dotCopy interface{}, err error) { + t.lock.Lock() + defer t.lock.Unlock() + + // copy the globals in the parent scope into the child + for k, v := range parent.globals { + t.globals[k] = v + } + // inherit the functions defined for this template + for k, v := range parent.funcs { + t.AddFunc(k, v) + } + if parent.context != nil { + return DeepCopyObject(parent.context) + } + return nil, nil +} + +// Global sets the a key, value in the context of this template. It is visible to all the 'included' +// and 'sourced' templates by the calling template. +func (t *Template) Global(name string, value interface{}) { + for here := t; here != nil; here = here.parent { + here.updateGlobal(name, value) + } +} + func (t *Template) updateGlobal(name string, value interface{}) { t.lock.Lock() defer t.lock.Unlock() - t.binds[name] = value + t.globals[name] = value } // Validate parses the template and checks for validity. @@ -220,6 +268,7 @@ func (t *Template) Execute(output io.Writer, context interface{}) error { if err := t.build(toContext(context)); err != nil { return err } + t.context = context return t.parsed.Execute(output, context) } @@ -240,7 +289,7 @@ func (t *Template) Render(context interface{}) (string, error) { return "", err } var buff bytes.Buffer - err := t.parsed.Execute(&buff, context) + err := t.Execute(&buff, context) return buff.String(), err }