diff --git a/.github/workflows/gate-tests.yml b/.github/workflows/gate-tests.yml index 19bf432..f3f129a 100644 --- a/.github/workflows/gate-tests.yml +++ b/.github/workflows/gate-tests.yml @@ -38,7 +38,7 @@ jobs: - name: run non-Kind tests env: SKIP_KIND: 1 - run: go test -v ./... + run: make test test-all: strategy: matrix: @@ -69,4 +69,4 @@ jobs: with: cluster_name: kind - name: run all tests - run: go test -v ./... + run: make test-all diff --git a/Makefile b/Makefile index ce0acc9..5a63fd6 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,44 @@ VERSION ?= $(shell git describe --tags --always --dirty) +CLUSTERS = $(shell kind get clusters) .PHONY: test +kind-install: +ifeq (, $(shell command -v kind)) + go install sigs.k8s.io/kind@v0.23.0 +endif + # We clear the test cache because some of the tests require an out-of-band KinD # cluster to run against and we want to re-run tests against that KinD cluster # instead of from cached unit test results. -test: +clear-test-cache: + @echo -n "clearing Go test cache ... " @go clean -testcache - @go test -v ./... + @echo "ok." + +kind-clear-clusters: +ifneq (, $(shell command -v kind)) +ifneq (, $(CLUSTERS)) + @echo -n "clearing KinD clusters ... " + @for c in $(CLUSTERS); do kind delete cluster -q --name $$c; done + @echo "ok." +endif +endif + +kind-create-cluster: +ifneq (, $(shell command -v kind)) + @echo -n "creating 'kind' cluster ... " + @kind create cluster -q + @echo "ok." + @sleep 5 +endif + +test: clear-test-cache kind-clear-clusters kind-create-cluster test-kind-simple + +test-kind-simple: + @go test -v ./parse_test.go + @go test -v ./eval_test.go + +test-all: test kind-clear-clusters + @go test -v ./fixtures/kind/kind_test.go + @go test -v ./placement_test.go diff --git a/eval_test.go b/eval_test.go index dbad7ea..3e95c65 100644 --- a/eval_test.go +++ b/eval_test.go @@ -5,21 +5,19 @@ package kube_test import ( - "bufio" - "bytes" - "fmt" - "os" "path/filepath" "testing" "github.com/gdt-dev/gdt" gdtcontext "github.com/gdt-dev/gdt/context" - kindfix "github.com/gdt-dev/kube/fixtures/kind" "github.com/stretchr/testify/require" + + kindfix "github.com/gdt-dev/kube/fixtures/kind" + "github.com/gdt-dev/kube/testutil" ) func TestListPodsEmpty(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "list-pods-empty.yaml") @@ -36,7 +34,7 @@ func TestListPodsEmpty(t *testing.T) { } func TestGetPodNotFound(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "get-pod-not-found.yaml") @@ -53,7 +51,7 @@ func TestGetPodNotFound(t *testing.T) { } func TestCreateUnknownResource(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "create-unknown-resource.yaml") @@ -70,7 +68,7 @@ func TestCreateUnknownResource(t *testing.T) { } func TestDeleteResourceNotFound(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "delete-resource-not-found.yaml") @@ -87,7 +85,7 @@ func TestDeleteResourceNotFound(t *testing.T) { } func TestDeleteUnknownResource(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "delete-unknown-resource.yaml") @@ -104,7 +102,7 @@ func TestDeleteUnknownResource(t *testing.T) { } func TestPodCreateGetDelete(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "create-get-delete-pod.yaml") @@ -121,7 +119,7 @@ func TestPodCreateGetDelete(t *testing.T) { } func TestMatches(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "matches.yaml") @@ -138,7 +136,7 @@ func TestMatches(t *testing.T) { } func TestConditions(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "conditions.yaml") @@ -155,7 +153,7 @@ func TestConditions(t *testing.T) { } func TestJSON(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "json.yaml") @@ -172,7 +170,7 @@ func TestJSON(t *testing.T) { } func TestApply(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "apply-deployment.yaml") @@ -189,7 +187,7 @@ func TestApply(t *testing.T) { } func TestEnvvarSubstitution(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) t.Setenv("pod_name", "foo") @@ -208,7 +206,7 @@ func TestEnvvarSubstitution(t *testing.T) { } func TestWithLabels(t *testing.T) { - skipIfNoKind(t) + testutil.SkipIfNoKind(t) require := require.New(t) fp := filepath.Join("testdata", "list-pods-with-labels.yaml") @@ -223,41 +221,3 @@ func TestWithLabels(t *testing.T) { err = s.Run(ctx, t) require.Nil(err) } - -func TestPlacementSpread(t *testing.T) { - skipIfNoKind(t) - require := require.New(t) - - fp := filepath.Join("testdata", "placement-spread.yaml") - - s, err := gdt.From(fp) - require.Nil(err) - require.NotNil(s) - - kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml") - - var b bytes.Buffer - w := bufio.NewWriter(&b) - ctx := gdtcontext.New(gdtcontext.WithDebug(w)) - - ctx = gdtcontext.RegisterFixture( - ctx, "kind-three-workers-three-zones", - kindfix.New( - kindfix.WithClusterName("kind-three-workers-three-zones"), - kindfix.WithConfigPath(kindCfgPath), - ), - ) - - err = s.Run(ctx, t) - require.Nil(err) - - w.Flush() - fmt.Println(b.String()) -} - -func skipIfNoKind(t *testing.T) { - _, found := os.LookupEnv("SKIP_KIND") - if found { - t.Skipf("skipping KinD-requiring test") - } -} diff --git a/fixtures/kind/kind.go b/fixtures/kind/kind.go index 03cf1f6..1982eda 100644 --- a/fixtures/kind/kind.go +++ b/fixtures/kind/kind.go @@ -5,8 +5,11 @@ package kind import ( + "context" "strings" + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/gdt-dev/gdt/debug" gdttypes "github.com/gdt-dev/gdt/types" "github.com/samber/lo" "sigs.k8s.io/kind/pkg/cluster" @@ -20,9 +23,29 @@ import ( type KindFixture struct { // provider is the KinD cluster provider provider *cluster.Provider - // cfgStr contains the stringified KUBECONFIG that KinD returns in its - // Provider.KubeConfig() call - cfgStr string + // deleteOnStop indicates that the KinD cluster should be deleted when + // the fixture is stopped. Fixtures are stopped when test scenarios + // utilizing the fixture have executed all their test steps. + // + // By default, KinD clusters that were already running when the fixture was + // started are not deleted. This is to prevent the deletion of KinD + // clusters that were in use outside of a gdt-kube execution. To override + // this behaviour and always delete the KinD cluster on stop, use the + // WithDeleteOnStop() modifier. + deleteOnStop bool + // retainOnStop indicates that the KinD cluster should *not* be deleted + // when the fixture is stopped. Fixtures are stopped when test scenarios + // utilizing the fixture have executed all their test steps. + // + // By default, KinD clusters that were *not* already running when the fixture was + // started are deleted when the fixture stops. This is to clean up KinD + // clusters that were created and used by the gdt-kube execution. To override + // this behaviour and always retain the KinD cluster on stop, use the + // WithRetainOnStop() modifier. + retainOnStop bool + // runningBeforeStart is true when the KinD cluster was already running + // when the fixture was started. + runningBeforeStart bool // ClusterName is the name of the KinD cluster. If not specified, gdt will // use the default cluster name that KinD uses, which is just "kind" ClusterName string @@ -34,20 +57,35 @@ type KindFixture struct { ConfigPath string } -func (f *KindFixture) Start() { +func (f *KindFixture) Start(ctx context.Context) { + ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.start") + defer func() { + ctx = gdtcontext.PopTrace(ctx) + }() if f.ClusterName == "" { f.ClusterName = kindconst.DefaultClusterName } if f.isRunning() { + debug.Println(ctx, "cluster %s already running", f.ClusterName) + f.runningBeforeStart = true return } opts := []cluster.CreateOption{} if f.ConfigPath != "" { + debug.Println( + ctx, "using custom kind config %s for cluster %s", + f.ConfigPath, f.ClusterName, + ) opts = append(opts, cluster.CreateWithConfigFile(f.ConfigPath)) } if err := f.provider.Create(f.ClusterName, opts...); err != nil { panic(err) } + debug.Println(ctx, "cluster %s successfully created", f.ClusterName) + if !f.retainOnStop { + f.deleteOnStop = true + debug.Println(ctx, "cluster %s will be deleted on stop", f.ClusterName) + } } func (f *KindFixture) isRunning() bool { @@ -61,7 +99,26 @@ func (f *KindFixture) isRunning() bool { return lo.Contains(clusterNames, f.ClusterName) } -func (f *KindFixture) Stop() {} +func (f *KindFixture) Stop(ctx context.Context) { + ctx = gdtcontext.PushTrace(ctx, "fixtures.kind.stop") + defer func() { + ctx = gdtcontext.PopTrace(ctx) + }() + if !f.isRunning() { + debug.Println(ctx, "cluster %s not running", f.ClusterName) + return + } + if f.runningBeforeStart && !f.deleteOnStop { + debug.Println(ctx, "cluster %s was running before start and deleteOnStop=false so not deleting", f.ClusterName) + return + } + if f.deleteOnStop { + if err := f.provider.Delete(f.ClusterName, ""); err != nil { + panic(err) + } + debug.Println(ctx, "cluster %s successfully deleted", f.ClusterName) + } +} func (f *KindFixture) HasState(key string) bool { lkey := strings.ToLower(key) @@ -119,6 +176,36 @@ func WithConfigPath(path string) KindFixtureModifier { } } +// WithDeleteOnStop instructs gdt-kube to always delete the KinD cluster when +// the fixture stops. Fixtures are stopped when test scenarios utilizing the +// fixture have executed all their test steps. +// +// By default, KinD clusters that were already running when the fixture was +// started are not deleted. This is to prevent the deletion of KinD +// clusters that were in use outside of a gdt-kube execution. To override +// this behaviour and always delete the KinD cluster on stop, use the +// WithDeleteOnStop() modifier. +func WithDeleteOnStop() KindFixtureModifier { + return func(f *KindFixture) { + f.deleteOnStop = true + } +} + +// WithRetainOnStop instructs gdt-kube that the KinD cluster should *not* be +// deleted when the fixture is stopped. Fixtures are stopped when test +// scenarios utilizing the fixture have executed all their test steps. +// +// By default, KinD clusters that were *not* already running when the fixture +// was started are deleted when the fixture stops. This is to clean up KinD +// clusters that were created and used by the gdt-kube execution. To override +// this behaviour and always retain the KinD cluster on stop, use the +// WithRetainOnStop() modifier. +func WithRetainOnStop() KindFixtureModifier { + return func(f *KindFixture) { + f.retainOnStop = true + } +} + // New returns a fixture that exposes Kubernetes configuration/context // information about a KinD cluster. If no such KinD cluster exists, one will // be created. If the KinD cluster is created, it is destroyed at the end of diff --git a/fixtures/kind/kind_test.go b/fixtures/kind/kind_test.go index 556c0a0..3778bba 100644 --- a/fixtures/kind/kind_test.go +++ b/fixtures/kind/kind_test.go @@ -5,6 +5,9 @@ package kind_test import ( + "bufio" + "bytes" + "fmt" "os" "path/filepath" "testing" @@ -25,10 +28,19 @@ func TestDefaultSingleControlPlane(t *testing.T) { require.Nil(err) require.NotNil(s) - ctx := gdtcontext.New() - ctx = gdtcontext.RegisterFixture(ctx, "kind", kindfix.New()) + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) + ctx = gdtcontext.RegisterFixture( + ctx, "kind", + kindfix.New( + kindfix.WithDeleteOnStop(), + ), + ) err = s.Run(ctx, t) + w.Flush() + fmt.Println(b.String()) require.Nil(err) } @@ -44,7 +56,9 @@ func TestOneControlPlaneOneWorker(t *testing.T) { kindCfgPath := filepath.Join("testdata", "kind-config-one-cp-one-worker.yaml") - ctx := gdtcontext.New() + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) ctx = gdtcontext.RegisterFixture( ctx, "kind-one-cp-one-worker", kindfix.New( @@ -54,6 +68,8 @@ func TestOneControlPlaneOneWorker(t *testing.T) { ) err = s.Run(ctx, t) + w.Flush() + fmt.Println(b.String()) require.Nil(err) } diff --git a/go.mod b/go.mod index 104af8d..38d2b71 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/cenkalti/backoff/v4 v4.2.1 - github.com/gdt-dev/gdt v1.5.0 + github.com/gdt-dev/gdt v1.6.2 github.com/samber/lo v1.38.1 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ce6ea22..803368e 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/gdt-dev/gdt v1.5.0 h1:HnLbiU8zXKK73SCGgm0R68dMvfry/J+pACHDODty48Y= -github.com/gdt-dev/gdt v1.5.0/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU= +github.com/gdt-dev/gdt v1.6.2 h1:ZHugSKIpMdO8hLQ1pGMPs0xGrLwQIutliDFfTjCkdys= +github.com/gdt-dev/gdt v1.6.2/go.mod h1:qkAfKZpEIYy4ymXcDvcZpfxgVvRDQTpSqeU/ze/EobU= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/placement_test.go b/placement_test.go new file mode 100644 index 0000000..977d4e3 --- /dev/null +++ b/placement_test.go @@ -0,0 +1,51 @@ +// Use and distribution licensed under the Apache license version 2. +// +// See the COPYING file in the root project directory for full text. + +package kube_test + +import ( + "bufio" + "bytes" + "fmt" + "path/filepath" + "testing" + + "github.com/gdt-dev/gdt" + gdtcontext "github.com/gdt-dev/gdt/context" + "github.com/stretchr/testify/require" + + kindfix "github.com/gdt-dev/kube/fixtures/kind" + "github.com/gdt-dev/kube/testutil" +) + +func TestPlacementSpread(t *testing.T) { + testutil.SkipIfNoKind(t) + require := require.New(t) + + fp := filepath.Join("testdata", "placement-spread.yaml") + + s, err := gdt.From(fp) + require.Nil(err) + require.NotNil(s) + + kindCfgPath := filepath.Join("testdata", "kind-config-three-workers-three-zones.yaml") + + var b bytes.Buffer + w := bufio.NewWriter(&b) + ctx := gdtcontext.New(gdtcontext.WithDebug(w)) + + ctx = gdtcontext.RegisterFixture( + ctx, "kind-three-workers-three-zones", + kindfix.New( + kindfix.WithClusterName("kind-three-workers-three-zones"), + kindfix.WithConfigPath(kindCfgPath), + ), + ) + + err = s.Run(ctx, t) + require.Nil(err) + + w.Flush() + fmt.Println(b.String()) +} diff --git a/testutil/kind.go b/testutil/kind.go new file mode 100644 index 0000000..db488ff --- /dev/null +++ b/testutil/kind.go @@ -0,0 +1,13 @@ +package testutil + +import ( + "os" + "testing" +) + +func SkipIfNoKind(t *testing.T) { + _, found := os.LookupEnv("SKIP_KIND") + if found { + t.Skipf("skipping KinD-requiring test") + } +}