From 7700702d8845f6f6e19d534775cf995f00437e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Wed, 3 Apr 2024 13:04:25 +0300 Subject: [PATCH 1/3] General refactoring for chainsaw migraiton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- cmd/uptest/main.go | 13 +- internal/config/config.go | 32 +- internal/prepare.go | 2 +- internal/runner.go | 12 +- internal/templates/00-apply.yaml.tmpl | 73 +- .../templates/00-assert.yaml.tmpl.license | 3 - .../templates/01-assert.yaml.tmpl.license | 3 - internal/templates/01-update.yaml.tmpl | 71 +- .../templates/02-assert.yaml.tmpl.license | 3 - internal/templates/02-import.yaml.tmpl | 112 +- .../templates/03-assert.yaml.tmpl.license | 3 - internal/templates/03-delete.yaml.tmpl | 72 +- internal/templates/embed.go | 20 - internal/templates/renderer.go | 6 +- internal/templates/renderer_test.go | 1294 +++++++++++++---- internal/tester.go | 140 +- 16 files changed, 1408 insertions(+), 451 deletions(-) delete mode 100644 internal/templates/00-assert.yaml.tmpl.license delete mode 100644 internal/templates/01-assert.yaml.tmpl.license delete mode 100644 internal/templates/02-assert.yaml.tmpl.license delete mode 100644 internal/templates/03-assert.yaml.tmpl.license diff --git a/cmd/uptest/main.go b/cmd/uptest/main.go index e7b0b77..7f165c0 100644 --- a/cmd/uptest/main.go +++ b/cmd/uptest/main.go @@ -11,6 +11,7 @@ import ( "strings" "gopkg.in/alecthomas/kingpin.v2" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/crossplane/uptest/internal" "github.com/crossplane/uptest/internal/config" @@ -34,13 +35,17 @@ var ( teardownScript = e2e.Flag("teardown-script", "Script that will be executed after running tests.").Default("").String() defaultTimeout = e2e.Flag("default-timeout", "Default timeout in seconds for the test.\n"+ - "Timeout could be overridden per resource using \"uptest.upbound.io/timeout\" annotation.").Default("1200").Int() + "Timeout could be overridden per resource using \"uptest.upbound.io/timeout\" annotation.").Default("1200s").Duration() defaultConditions = e2e.Flag("default-conditions", "Comma separated list of default conditions to wait for a successful test.\n"+ "Conditions could be overridden per resource using \"uptest.upbound.io/conditions\" annotation.").Default("Ready").String() skipDelete = e2e.Flag("skip-delete", "Skip the delete step of the test.").Default("false").Bool() - testDir = e2e.Flag("test-directory", "Directory where kuttl test case will be generated and executed.").Envar("UPTEST_TEST_DIR").Default(filepath.Join(os.TempDir(), "uptest-e2e")).String() + testDir = e2e.Flag("test-directory", "Directory where chainsaw test case will be generated and executed.").Envar("UPTEST_TEST_DIR").Default(filepath.Join(os.TempDir(), "uptest-e2e")).String() onlyCleanUptestResources = e2e.Flag("only-clean-uptest-resources", "While deletion step, only clean resources that were created by uptest").Default("false").Bool() + + renderOnly = e2e.Flag("render-only", "Only render test files. Do not run the tests.").Default("false").Bool() + logCollectInterval = e2e.Flag("log-collect-interval", "Specifies the interval duration for collecting logs. "+ + "The duration should be provided in a format understood by the tool, such as seconds (s), minutes (m), or hours (h). For example, '30s' for 30 seconds, '5m' for 5 minutes, or '1h' for one hour.").Default("30s").Duration() ) func main() { @@ -88,10 +93,12 @@ func e2eTests() { SetupScriptPath: setupPath, TeardownScriptPath: teardownPath, DefaultConditions: strings.Split(*defaultConditions, ","), - DefaultTimeout: *defaultTimeout, + DefaultTimeout: v1.Duration{Duration: *defaultTimeout}, Directory: *testDir, SkipDelete: *skipDelete, OnlyCleanUptestResources: *onlyCleanUptestResources, + RenderOnly: *renderOnly, + LogCollectionInterval: *logCollectInterval, } kingpin.FatalIfError(internal.RunTest(o), "cannot run e2e tests successfully") diff --git a/internal/config/config.go b/internal/config/config.go index e417ebb..9e061fb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,7 +5,12 @@ // Package config contains configuration options for configuring uptest runtime. package config -import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +import ( + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) const ( // AnnotationKeyTimeout defines a test time for the annotated resource. @@ -47,12 +52,15 @@ type AutomatedTest struct { SetupScriptPath string TeardownScriptPath string - DefaultTimeout int + DefaultTimeout v1.Duration DefaultConditions []string SkipDelete bool OnlyCleanUptestResources bool + + RenderOnly bool + LogCollectionInterval time.Duration } // Manifest represents a resource loaded from an example resource manifest file. @@ -62,26 +70,30 @@ type Manifest struct { YAML string } -// TestCase represents a test-case to be run by kuttl. +// TestCase represents a test-case to be run by chainsaw. type TestCase struct { - Timeout int + Timeout time.Duration SetupScriptPath string TeardownScriptPath string SkipUpdate bool SkipImport bool OnlyCleanUptestResources bool + + TestDirectory string } // Resource represents a Kubernetes object to be tested and asserted // by uptest. type Resource struct { - Name string - Namespace string - KindGroup string - YAML string - - Timeout int + Name string + Namespace string + KindGroup string + YAML string + APIVersion string + Kind string + + Timeout v1.Duration Conditions []string PreAssertScriptPath string PostAssertScriptPath string diff --git a/internal/prepare.go b/internal/prepare.go index 90aab61..c09d5b1 100644 --- a/internal/prepare.go +++ b/internal/prepare.go @@ -4,7 +4,7 @@ // Package internal implements the uptest runtime for running // automated tests using resource example manifests -// using kuttl. +// using chainsaw. package internal import ( diff --git a/internal/runner.go b/internal/runner.go index 5fd661a..b081981 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -15,11 +15,13 @@ import ( // RunTest runs the specified automated test func RunTest(o *config.AutomatedTest) error { - defer func() { - if err := os.RemoveAll(o.Directory); err != nil { - fmt.Println(fmt.Sprint(err, "cannot clean the test directory")) - } - }() + if !o.RenderOnly { + defer func() { + if err := os.RemoveAll(o.Directory); err != nil { + fmt.Println(fmt.Sprint(err, "cannot clean the test directory")) + } + }() + } // Read examples and inject data source values to manifests manifests, err := newPreparer(o.ManifestPaths, withDataSource(o.DataSourcePath), withTestDirectory(o.Directory)).prepareManifests() diff --git a/internal/templates/00-apply.yaml.tmpl b/internal/templates/00-apply.yaml.tmpl index 2734913..11f5123 100644 --- a/internal/templates/00-apply.yaml.tmpl +++ b/internal/templates/00-apply.yaml.tmpl @@ -1,11 +1,66 @@ # This file belongs to the resource apply step. {{ if .TestCase.SetupScriptPath -}} -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: {{ .TestCase.SetupScriptPath }} -{{ end }} -{{- range $resource := .Resources -}} ---- -{{ $resource.YAML }} -{{- end }} \ No newline at end of file +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: {{ .TestCase.Timeout }} + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: {{ .TestCase.SetupScriptPath }} + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: {{ .TestCase.TestDirectory }} + - script: + content: | + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + ${KUBECTL} annotate {{ $resource.KindGroup }}/{{ $resource.Name }} upjet.upbound.io/test=true --overwrite + {{- end }} + {{- end }} + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if $resource.PreAssertScriptPath }} + - command: + entrypoint: {{ $resource.PreAssertScriptPath }} + {{- end }} + - assert: + resource: + apiVersion: {{ $resource.APIVersion }} + kind: {{ $resource.Kind }} + metadata: + name: {{ $resource.Name }} + {{- if $resource.Namespace }} + namespace: {{ $resource.Namespace }} + {{- end }} + status: + {{- range $condition := $resource.Conditions }} + ((conditions[?type == '{{ $condition }}'])[0]): + status: "True" + {{- end }} + {{- if $resource.PostAssertScriptPath }} + - command: + entrypoint: {{ $resource.PostAssertScriptPath }} + {{- end }} + {{- end }} +{{ end }} \ No newline at end of file diff --git a/internal/templates/00-assert.yaml.tmpl.license b/internal/templates/00-assert.yaml.tmpl.license deleted file mode 100644 index 2072b73..0000000 --- a/internal/templates/00-assert.yaml.tmpl.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2024 The Crossplane Authors - -SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/internal/templates/01-assert.yaml.tmpl.license b/internal/templates/01-assert.yaml.tmpl.license deleted file mode 100644 index 2072b73..0000000 --- a/internal/templates/01-assert.yaml.tmpl.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2024 The Crossplane Authors - -SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/internal/templates/01-update.yaml.tmpl b/internal/templates/01-update.yaml.tmpl index d9bbd9f..abc404e 100644 --- a/internal/templates/01-update.yaml.tmpl +++ b/internal/templates/01-update.yaml.tmpl @@ -1,17 +1,56 @@ # This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if not $resource.Namespace }} -{{- if $resource.Root }} -- command: ${KUBECTL} patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"spec":{"forProvider":{{ $resource.UpdateParameter }}}}' -{{- end }} -{{- end }} -{{- end }} +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: {{ .TestCase.Timeout }} + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + {{- if $resource.Root }} + - script: + content: | + ${KUBECTL} --subresource=status patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"status":{"conditions":[]}}' + ${KUBECTL} patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"spec":{"forProvider":{{ $resource.UpdateParameter }}}}' + {{- end }} + {{- end }} + {{- end }} + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + {{- if $resource.Root }} + try: + - assert: + resource: + apiVersion: {{ $resource.APIVersion }} + kind: {{ $resource.Kind }} + metadata: + name: {{ $resource.Name }} + status: + {{- range $condition := $resource.Conditions }} + ((conditions[?type == '{{ $condition }}'])[0]): + status: "True" + {{- end }} + - script: + content: ${KUBECTL} get {{ $resource.KindGroup }}/{{ $resource.Name }} -o=jsonpath='{.status.atProvider{{ $resource.UpdateAssertKey }}}' | grep -q "^{{ $resource.UpdateAssertValue }}$" + {{- end }} + {{- end }} + {{- end }} diff --git a/internal/templates/02-assert.yaml.tmpl.license b/internal/templates/02-assert.yaml.tmpl.license deleted file mode 100644 index 2072b73..0000000 --- a/internal/templates/02-assert.yaml.tmpl.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2024 The Crossplane Authors - -SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/internal/templates/02-import.yaml.tmpl b/internal/templates/02-import.yaml.tmpl index 5f73b8f..72c4b36 100644 --- a/internal/templates/02-import.yaml.tmpl +++ b/internal/templates/02-import.yaml.tmpl @@ -1,25 +1,89 @@ # This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if not $resource.Namespace }} -- script: /tmp/patch.sh {{ $resource.KindGroup }} {{ $resource.Name }} -{{- end }} -{{- end }} -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: {{ .TestCase.Timeout }} + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + ${KUBECTL} annotate {{ $resource.KindGroup }}/{{ $resource.Name }} crossplane.io/paused=true --overwrite + {{- end }} + {{- end }} + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + /tmp/patch.sh {{ $resource.KindGroup }} {{ $resource.Name }} + {{- end }} + {{- end }} + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + ${KUBECTL} annotate {{ $resource.KindGroup }}/{{ $resource.Name }} --all crossplane.io/paused=false --overwrite + {{- end }} + {{- end }} + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if not $resource.Namespace }} + - assert: + resource: + apiVersion: {{ $resource.APIVersion }} + kind: {{ $resource.Kind }} + metadata: + name: {{ $resource.Name }} + status: + {{- range $condition := $resource.Conditions }} + ((conditions[?type == '{{ $condition }}'])[0]): + status: "True" + {{- end }} + {{- end }} + {{- if not (or $resource.Namespace $resource.SkipImport) }} + - assert: + timeout: 1m + resource: + apiVersion: {{ $resource.APIVersion }} + kind: {{ $resource.Kind }} + metadata: + name: {{ $resource.Name }} + ("status.atProvider.id" == "metadata.annotations.uptest-old-id"): true + {{- end }} + {{- end }} diff --git a/internal/templates/03-assert.yaml.tmpl.license b/internal/templates/03-assert.yaml.tmpl.license deleted file mode 100644 index 2072b73..0000000 --- a/internal/templates/03-assert.yaml.tmpl.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2024 The Crossplane Authors - -SPDX-License-Identifier: CC0-1.0 \ No newline at end of file diff --git a/internal/templates/03-delete.yaml.tmpl b/internal/templates/03-delete.yaml.tmpl index a2e2d3a..dd08913 100644 --- a/internal/templates/03-delete.yaml.tmpl +++ b/internal/templates/03-delete.yaml.tmpl @@ -1,23 +1,51 @@ # This file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if $resource.PreDeleteScriptPath }} -- command: {{ $resource.PreDeleteScriptPath }} -{{- end }} -{{- if $resource.Namespace }} -- command: ${KUBECTL} delete {{ $resource.KindGroup }}/{{ $resource.Name }} --wait=false --namespace {{ $resource.Namespace }} --ignore-not-found -{{- else }} -- command: ${KUBECTL} delete {{ $resource.KindGroup }}/{{ $resource.Name }} --wait=false --ignore-not-found -{{- end }} -{{- if $resource.PostDeleteScriptPath }} -- command: {{ $resource.PostDeleteScriptPath }} -{{- end }} -{{- end }} +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: delete +spec: + timeouts: + exec: {{ .TestCase.Timeout }} + steps: + - name: Delete Resources + description: Delete resources. If needs ordered deletion, the pre-delete scripts were used. + try: + - script: + content: | + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end -}} + {{- if $resource.PreDeleteScriptPath }} + {{ $resource.PreDeleteScriptPath }} + {{- end }} + {{- if $resource.Namespace }} + ${KUBECTL} delete {{ $resource.KindGroup }}/{{ $resource.Name }} --wait=false --namespace {{ $resource.Namespace }} --ignore-not-found + {{- else }} + ${KUBECTL} delete {{ $resource.KindGroup }}/{{ $resource.Name }} --wait=false --ignore-not-found + {{- end }} + {{- if $resource.PostDeleteScriptPath }} + {{ $resource.PostDeleteScriptPath }} + {{- end }} + {{- end }} + - name: Assert Deletion + description: Assert deletion of resources. + try: + {{- range $resource := .Resources }} + {{- if eq $resource.KindGroup "secret." -}} + {{continue}} + {{- end }} + - wait: + apiVersion: {{ $resource.APIVersion }} + kind: {{ $resource.Kind }} + name: {{ $resource.Name }} + {{- if $resource.Namespace }} + namespace: {{ $resource.Namespace }} + {{- end }} + for: + deletion: {} + {{- end }} + {{- if .TestCase.TeardownScriptPath }} + - command: + entrypoint: {{ .TestCase.TeardownScriptPath }} + {{- end }} diff --git a/internal/templates/embed.go b/internal/templates/embed.go index 92ee066..cadeb6c 100644 --- a/internal/templates/embed.go +++ b/internal/templates/embed.go @@ -11,37 +11,17 @@ import _ "embed" //go:embed 00-apply.yaml.tmpl var inputFileTemplate string -// assertFileTemplate is the template for the assert file. -// -//go:embed 00-assert.yaml.tmpl -var assertFileTemplate string - // updateFileTemplate is the template for the update file. // //go:embed 01-update.yaml.tmpl var updateFileTemplate string -// assertUpdatedFileTemplate is the template for update assert file. -// -//go:embed 01-assert.yaml.tmpl -var assertUpdatedFileTemplate string - // deleteFileTemplate is the template for the import file. // //go:embed 02-import.yaml.tmpl var importFileTemplate string -// assertDeletedFileTemplate is the template for import assert file. -// -//go:embed 02-assert.yaml.tmpl -var assertImportedFileTemplate string - // deleteFileTemplate is the template for the delete file. // //go:embed 03-delete.yaml.tmpl var deleteFileTemplate string - -// assertDeletedFileTemplate is the template for delete assert file. -// -//go:embed 03-assert.yaml.tmpl -var assertDeletedFileTemplate string diff --git a/internal/templates/renderer.go b/internal/templates/renderer.go index 2350cd0..bd736f8 100644 --- a/internal/templates/renderer.go +++ b/internal/templates/renderer.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: CC0-1.0 -// Package templates contains utilities for rendering kuttl test cases using +// Package templates contains utilities for rendering chainsaw test cases using // the templates contained in the package. package templates @@ -17,13 +17,9 @@ import ( var fileTemplates = map[string]string{ "00-apply.yaml": inputFileTemplate, - "00-assert.yaml": assertFileTemplate, "01-update.yaml": updateFileTemplate, - "01-assert.yaml": assertUpdatedFileTemplate, "02-import.yaml": importFileTemplate, - "02-assert.yaml": assertImportedFileTemplate, "03-delete.yaml": deleteFileTemplate, - "03-assert.yaml": assertDeletedFileTemplate, } // Render renders the specified list of resources as a test case diff --git a/internal/templates/renderer_test.go b/internal/templates/renderer_test.go index c3fab10..9822d4b 100644 --- a/internal/templates/renderer_test.go +++ b/internal/templates/renderer_test.go @@ -5,7 +5,9 @@ package templates import ( + "fmt" "testing" + "time" "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/google/go-cmp/cmp" @@ -43,12 +45,6 @@ type: Opaque data: key: dmFsdWU= ` - - namespaceManifest = `apiVersion: v1 -kind: Namespace -metadata: - name: test-namespace -` ) func TestRender(t *testing.T) { @@ -67,11 +63,15 @@ func TestRender(t *testing.T) { "SuccessSingleResource": { args: args{ tc: &config.TestCase{ - Timeout: 10, + SetupScriptPath: "/tmp/setup.sh", + Timeout: 10 * time.Minute, + TestDirectory: "/tmp/test-input.yaml", }, resources: []config.Resource{ { Name: "example-bucket", + APIVersion: "bucket.s3.aws.upbound.io/v1alpha1", + Kind: "Bucket", KindGroup: "s3.aws.upbound.io", YAML: bucketManifest, Conditions: []string{"Test"}, @@ -80,71 +80,209 @@ func TestRender(t *testing.T) { }, want: want{ out: map[string]string{ - "00-apply.yaml": "# This file belongs to the resource apply step.\n---\n" + bucketManifest, - "00-assert.yaml": `# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s + "00-apply.yaml": `# This file belongs to the resource apply step. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: /tmp/setup.sh + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: /tmp/test-input.yaml + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" `, "01-update.yaml": `# This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -`, - "01-assert.yaml": `# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -`, - "02-assert.yaml": `# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- script: new_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.status.atProvider.id}')" && old_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.metadata.annotations.uptest-old-id}')" && [ "$new_id" = "$old_id" ] +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. `, "02-import.yaml": `# This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -- script: /tmp/patch.sh s3.aws.upbound.io example-bucket -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite -`, +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + function check_endpoints { + endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) + for endpoint in ${endpoints[@]}; do + port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') + if [[ -z "${port}" ]]; then + echo "$endpoint - No served ports" + return 1 + else + echo "$endpoint - Ports present" + fi + done + } + + function check_endpoints_main { + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if check_endpoints; then + return 0 + else + printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 + fi + ((attempt++)) + sleep 5 + done + return 1 + } + + function patch { + kindgroup=$1; + name=$2; + if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then + return 0; + else + return 1; + fi; + }; - "03-assert.yaml": `# This assert file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the delete assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the delete assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=delete --timeout 10s -- command: ${KUBECTL} wait managed --all --for=delete --timeout 10s + function patch_main { + kindgroup=$1 + name=$2 + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if patch "$kindgroup" "$name"; then + echo "Successfully patched $kindgroup/$name" + ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite + break + else + printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 + fi + ((attempt++)) + sleep 5 + done + if [[ $attempt -gt $max_attempts ]]; then + echo "Failed to patch $kindgroup/$name after $max_attempts attempts" + return 1 + fi + return 0 + } + + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + check_endpoints_main + patch_main s3.aws.upbound.io example-bucket + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + timeout: 1m + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + ("status.atProvider.id" == "metadata.annotations.uptest-old-id"): true `, "03-delete.yaml": `# This file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} delete s3.aws.upbound.io/example-bucket --wait=false --ignore-not-found +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: delete +spec: + timeouts: + assert: 10m0s + exec: 1m + steps: + - name: Delete Resources + description: Delete resources. If needs ordered deletion, the pre-delete scripts were used. + try: + - script: + content: | + ${KUBECTL} delete s3.aws.upbound.io/example-bucket --wait=false --ignore-not-found + - name: Assert Deletion + description: Assert deletion of resources. + try: + - wait: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + name: example-bucket + for: + deletion: {} `, }, }, @@ -152,13 +290,16 @@ commands: "SuccessMultipleResource": { args: args{ tc: &config.TestCase{ - Timeout: 10, + Timeout: 10 * time.Minute, SetupScriptPath: "/tmp/setup.sh", TeardownScriptPath: "/tmp/teardown.sh", + TestDirectory: "/tmp/test-input.yaml", }, resources: []config.Resource{ { YAML: bucketManifest, + APIVersion: "bucket.s3.aws.upbound.io/v1alpha1", + Kind: "Bucket", Name: "example-bucket", KindGroup: "s3.aws.upbound.io", PreAssertScriptPath: "/tmp/bucket/pre-assert.sh", @@ -167,6 +308,8 @@ commands: }, { YAML: claimManifest, + APIVersion: "cluster.gcp.platformref.upbound.io/v1alpha1", + Kind: "Cluster", Name: "test-cluster-claim", KindGroup: "cluster.gcp.platformref.upbound.io", Namespace: "upbound-system", @@ -180,93 +323,241 @@ commands: KindGroup: "secret.", Namespace: "upbound-system", }, - { - YAML: namespaceManifest, - Name: "test-namespace", - KindGroup: "namespace.", - }, }, }, want: want{ out: map[string]string{ "00-apply.yaml": `# This file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: /tmp/setup.sh -` + "---\n" + bucketManifest + "---\n" + claimManifest + "---\n" + secretManifest + "---\n" + namespaceManifest, - "00-assert.yaml": `# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: /tmp/bucket/pre-assert.sh -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Ready --timeout 10s --namespace upbound-system -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Synced --timeout 10s --namespace upbound-system -- command: /tmp/claim/post-assert.sh +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: /tmp/setup.sh + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: /tmp/test-input.yaml + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + - command: + entrypoint: /tmp/bucket/pre-assert.sh + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + resource: + apiVersion: cluster.gcp.platformref.upbound.io/v1alpha1 + kind: Cluster + metadata: + name: test-cluster-claim + namespace: upbound-system + status: + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + - command: + entrypoint: /tmp/claim/post-assert.sh `, "01-update.yaml": `# This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -`, - "01-assert.yaml": `# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -`, - "02-assert.yaml": `# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- script: new_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.status.atProvider.id}')" && old_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.metadata.annotations.uptest-old-id}')" && [ "$new_id" = "$old_id" ] +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. `, "02-import.yaml": `# This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -- script: /tmp/patch.sh s3.aws.upbound.io example-bucket -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite -`, - "03-assert.yaml": `# This assert file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the delete assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the delete assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=delete --timeout 10s -- script: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=delete --timeout 10s --namespace upbound-system -- command: ${KUBECTL} wait managed --all --for=delete --timeout 10s -- command: /tmp/teardown.sh +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + function check_endpoints { + endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) + for endpoint in ${endpoints[@]}; do + port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') + if [[ -z "${port}" ]]; then + echo "$endpoint - No served ports" + return 1 + else + echo "$endpoint - Ports present" + fi + done + } + + function check_endpoints_main { + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if check_endpoints; then + return 0 + else + printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 + fi + ((attempt++)) + sleep 5 + done + return 1 + } + + function patch { + kindgroup=$1; + name=$2; + if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then + return 0; + else + return 1; + fi; + }; + + function patch_main { + kindgroup=$1 + name=$2 + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if patch "$kindgroup" "$name"; then + echo "Successfully patched $kindgroup/$name" + ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite + break + else + printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 + fi + ((attempt++)) + sleep 5 + done + if [[ $attempt -gt $max_attempts ]]; then + echo "Failed to patch $kindgroup/$name after $max_attempts attempts" + return 1 + fi + return 0 + } + + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + check_endpoints_main + patch_main s3.aws.upbound.io example-bucket + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + timeout: 1m + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + ("status.atProvider.id" == "metadata.annotations.uptest-old-id"): true `, "03-delete.yaml": `# This file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} delete s3.aws.upbound.io/example-bucket --wait=false --ignore-not-found -- command: /tmp/bucket/post-delete.sh -- command: /tmp/claim/pre-delete.sh -- command: ${KUBECTL} delete cluster.gcp.platformref.upbound.io/test-cluster-claim --wait=false --namespace upbound-system --ignore-not-found +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: delete +spec: + timeouts: + assert: 10m0s + exec: 1m + steps: + - name: Delete Resources + description: Delete resources. If needs ordered deletion, the pre-delete scripts were used. + try: + - script: + content: | + ${KUBECTL} delete s3.aws.upbound.io/example-bucket --wait=false --ignore-not-found + /tmp/bucket/post-delete.sh + /tmp/claim/pre-delete.sh + ${KUBECTL} delete cluster.gcp.platformref.upbound.io/test-cluster-claim --wait=false --namespace upbound-system --ignore-not-found + - name: Assert Deletion + description: Assert deletion of resources. + try: + - wait: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + name: example-bucket + for: + deletion: {} + - wait: + apiVersion: cluster.gcp.platformref.upbound.io/v1alpha1 + kind: Cluster + name: test-cluster-claim + namespace: upbound-system + for: + deletion: {} + - command: + entrypoint: /tmp/teardown.sh `, }, }, @@ -301,11 +592,15 @@ func TestRenderWithSkipDelete(t *testing.T) { "SuccessSingleResource": { args: args{ tc: &config.TestCase{ - Timeout: 10, + SetupScriptPath: "/tmp/setup.sh", + Timeout: 10 * time.Minute, + TestDirectory: "/tmp/test-input.yaml", }, resources: []config.Resource{ { Name: "example-bucket", + APIVersion: "bucket.s3.aws.upbound.io/v1alpha1", + Kind: "Bucket", KindGroup: "s3.aws.upbound.io", YAML: bucketManifest, Conditions: []string{"Test"}, @@ -314,54 +609,183 @@ func TestRenderWithSkipDelete(t *testing.T) { }, want: want{ out: map[string]string{ - "00-apply.yaml": "# This file belongs to the resource apply step.\n---\n" + bucketManifest, - "00-assert.yaml": `# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s + "00-apply.yaml": `# This file belongs to the resource apply step. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: /tmp/setup.sh + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: /tmp/test-input.yaml + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" `, "01-update.yaml": `# This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -`, - "01-assert.yaml": `# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -`, - "02-assert.yaml": `# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- script: new_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.status.atProvider.id}')" && old_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.metadata.annotations.uptest-old-id}')" && [ "$new_id" = "$old_id" ] +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. `, "02-import.yaml": `# This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -- script: /tmp/patch.sh s3.aws.upbound.io example-bucket -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + function check_endpoints { + endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) + for endpoint in ${endpoints[@]}; do + port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') + if [[ -z "${port}" ]]; then + echo "$endpoint - No served ports" + return 1 + else + echo "$endpoint - Ports present" + fi + done + } + + function check_endpoints_main { + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if check_endpoints; then + return 0 + else + printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 + fi + ((attempt++)) + sleep 5 + done + return 1 + } + + function patch { + kindgroup=$1; + name=$2; + if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then + return 0; + else + return 1; + fi; + }; + + function patch_main { + kindgroup=$1 + name=$2 + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if patch "$kindgroup" "$name"; then + echo "Successfully patched $kindgroup/$name" + ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite + break + else + printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 + fi + ((attempt++)) + sleep 5 + done + if [[ $attempt -gt $max_attempts ]]; then + echo "Failed to patch $kindgroup/$name after $max_attempts attempts" + return 1 + fi + return 0 + } + + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + check_endpoints_main + patch_main s3.aws.upbound.io example-bucket + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + timeout: 1m + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + ("status.atProvider.id" == "metadata.annotations.uptest-old-id"): true `, }, }, @@ -369,13 +793,16 @@ commands: "SkipImport": { args: args{ tc: &config.TestCase{ - Timeout: 10, + Timeout: 10 * time.Minute, SetupScriptPath: "/tmp/setup.sh", TeardownScriptPath: "/tmp/teardown.sh", + TestDirectory: "/tmp/test-input.yaml", }, resources: []config.Resource{ { YAML: bucketManifest, + APIVersion: "bucket.s3.aws.upbound.io/v1alpha1", + Kind: "Bucket", Name: "example-bucket", KindGroup: "s3.aws.upbound.io", PreAssertScriptPath: "/tmp/bucket/pre-assert.sh", @@ -386,6 +813,8 @@ commands: { YAML: claimManifest, Name: "test-cluster-claim", + APIVersion: "cluster.gcp.platformref.upbound.io/v1alpha1", + Kind: "Cluster", KindGroup: "cluster.gcp.platformref.upbound.io", Namespace: "upbound-system", PostAssertScriptPath: "/tmp/claim/post-assert.sh", @@ -398,71 +827,195 @@ commands: KindGroup: "secret.", Namespace: "upbound-system", }, - { - YAML: namespaceManifest, - Name: "test-namespace", - KindGroup: "namespace.", - }, }, }, want: want{ out: map[string]string{ "00-apply.yaml": `# This file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: /tmp/setup.sh -` + "---\n" + bucketManifest + "---\n" + claimManifest + "---\n" + secretManifest + "---\n" + namespaceManifest, - "00-assert.yaml": `# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: /tmp/bucket/pre-assert.sh -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Ready --timeout 10s --namespace upbound-system -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Synced --timeout 10s --namespace upbound-system -- command: /tmp/claim/post-assert.sh +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: /tmp/setup.sh + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: /tmp/test-input.yaml + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + - command: + entrypoint: /tmp/bucket/pre-assert.sh + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + resource: + apiVersion: cluster.gcp.platformref.upbound.io/v1alpha1 + kind: Cluster + metadata: + name: test-cluster-claim + namespace: upbound-system + status: + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + - command: + entrypoint: /tmp/claim/post-assert.sh `, "01-update.yaml": `# This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -`, - "01-assert.yaml": `# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -`, - "02-assert.yaml": `# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. `, "02-import.yaml": `# This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -- script: /tmp/patch.sh s3.aws.upbound.io example-bucket -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + function check_endpoints { + endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) + for endpoint in ${endpoints[@]}; do + port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') + if [[ -z "${port}" ]]; then + echo "$endpoint - No served ports" + return 1 + else + echo "$endpoint - Ports present" + fi + done + } + + function check_endpoints_main { + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if check_endpoints; then + return 0 + else + printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 + fi + ((attempt++)) + sleep 5 + done + return 1 + } + + function patch { + kindgroup=$1; + name=$2; + if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then + return 0; + else + return 1; + fi; + }; + + function patch_main { + kindgroup=$1 + name=$2 + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if patch "$kindgroup" "$name"; then + echo "Successfully patched $kindgroup/$name" + ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite + break + else + printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 + fi + ((attempt++)) + sleep 5 + done + if [[ $attempt -gt $max_attempts ]]; then + echo "Failed to patch $kindgroup/$name after $max_attempts attempts" + return 1 + fi + return 0 + } + + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + check_endpoints_main + patch_main s3.aws.upbound.io example-bucket + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" `, }, }, @@ -470,13 +1023,16 @@ commands: "SuccessMultipleResource": { args: args{ tc: &config.TestCase{ - Timeout: 10, + Timeout: 10 * time.Minute, SetupScriptPath: "/tmp/setup.sh", TeardownScriptPath: "/tmp/teardown.sh", + TestDirectory: "/tmp/test-input.yaml", }, resources: []config.Resource{ { YAML: bucketManifest, + APIVersion: "bucket.s3.aws.upbound.io/v1alpha1", + Kind: "Bucket", Name: "example-bucket", KindGroup: "s3.aws.upbound.io", PreAssertScriptPath: "/tmp/bucket/pre-assert.sh", @@ -485,6 +1041,8 @@ commands: }, { YAML: claimManifest, + APIVersion: "cluster.gcp.platformref.upbound.io/v1alpha1", + Kind: "Cluster", Name: "test-cluster-claim", KindGroup: "cluster.gcp.platformref.upbound.io", Namespace: "upbound-system", @@ -498,72 +1056,203 @@ commands: KindGroup: "secret.", Namespace: "upbound-system", }, - { - YAML: namespaceManifest, - Name: "test-namespace", - KindGroup: "namespace.", - }, }, }, want: want{ out: map[string]string{ "00-apply.yaml": `# This file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: /tmp/setup.sh -` + "---\n" + bucketManifest + "---\n" + claimManifest + "---\n" + secretManifest + "---\n" + namespaceManifest, - "00-assert.yaml": `# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -- command: /tmp/bucket/pre-assert.sh -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Ready --timeout 10s --namespace upbound-system -- command: ${KUBECTL} wait cluster.gcp.platformref.upbound.io/test-cluster-claim --for=condition=Synced --timeout 10s --namespace upbound-system -- command: /tmp/claim/post-assert.sh +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: apply +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Run Setup Script + description: Setup the test environment by running the setup script. + try: + - command: + entrypoint: /tmp/setup.sh + - name: Apply Resources + description: Apply resources to the cluster. + try: + - apply: + file: /tmp/test-input.yaml + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite + - name: Assert Status Conditions + description: | + Assert applied resources. Firstly run the pre-assert script if exists. + Then check the status conditions. Finally run the post-assert script if + exists. + try: + - command: + entrypoint: /tmp/bucket/pre-assert.sh + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + resource: + apiVersion: cluster.gcp.platformref.upbound.io/v1alpha1 + kind: Cluster + metadata: + name: test-cluster-claim + namespace: upbound-system + status: + ((conditions[?type == 'Ready'])[0]): + status: "True" + ((conditions[?type == 'Synced'])[0]): + status: "True" + - command: + entrypoint: /tmp/claim/post-assert.sh `, "01-update.yaml": `# This file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -`, - "01-assert.yaml": `# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -`, - "02-assert.yaml": `# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 10 -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -- command: ${KUBECTL} wait s3.aws.upbound.io/example-bucket --for=condition=Test --timeout 10s -- script: new_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.status.atProvider.id}')" && old_id="$(${KUBECTL} get s3.aws.upbound.io/example-bucket -o=jsonpath='{.metadata.annotations.uptest-old-id}')" && [ "$new_id" = "$old_id" ] +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: update +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 1m + steps: + - name: Update Root Resource + description: | + Update the root resource by using the specified update-parameter in annotation. + Before updating the resources, the status conditions are cleaned. + try: + - name: Assert Updated Resource + description: | + Assert update operation. Firstly check the status conditions. Then assert + the updated field in status.atProvider. `, "02-import.yaml": `# This file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=true --overwrite -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 -- command: sleep 10 -- command: ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s -- script: ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh -- script: curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh -- script: /tmp/check_endpoints.sh -- script: /tmp/patch.sh s3.aws.upbound.io example-bucket -- command: ${KUBECTL} annotate managed --all crossplane.io/paused=false --overwrite +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: import +spec: + timeouts: + apply: 1m + assert: 10m0s + exec: 3m + steps: + - name: Remove State + description: | + Removes the resource statuses from MRs and controllers. For controllers + the scale down&up was applied. For MRs status conditions are patched. + Also, for the assertion step, the ID before import was stored in the + uptest-old-id annotation. + try: + - script: + content: | + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 + - sleep: + duration: 10s + - script: + content: | + function check_endpoints { + endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) + for endpoint in ${endpoints[@]}; do + port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') + if [[ -z "${port}" ]]; then + echo "$endpoint - No served ports" + return 1 + else + echo "$endpoint - Ports present" + fi + done + } + + function check_endpoints_main { + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if check_endpoints; then + return 0 + else + printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 + fi + ((attempt++)) + sleep 5 + done + return 1 + } + + function patch { + kindgroup=$1; + name=$2; + if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then + return 0; + else + return 1; + fi; + }; + + function patch_main { + kindgroup=$1 + name=$2 + attempt=1 + max_attempts=10 + while [[ $attempt -le $max_attempts ]]; do + if patch "$kindgroup" "$name"; then + echo "Successfully patched $kindgroup/$name" + ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite + break + else + printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 + fi + ((attempt++)) + sleep 5 + done + if [[ $attempt -gt $max_attempts ]]; then + echo "Failed to patch $kindgroup/$name after $max_attempts attempts" + return 1 + fi + return 0 + } + + ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s + ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 + check_endpoints_main + patch_main s3.aws.upbound.io example-bucket + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite + - name: Assert Status Conditions and IDs + description: | + Assert imported resources. Firstly check the status conditions. Then + compare the stored ID and the new populated ID. For successful test, + the ID must be the same. + try: + - assert: + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + status: + ((conditions[?type == 'Test'])[0]): + status: "True" + - assert: + timeout: 1m + resource: + apiVersion: bucket.s3.aws.upbound.io/v1alpha1 + kind: Bucket + metadata: + name: example-bucket + ("status.atProvider.id" == "metadata.annotations.uptest-old-id"): true `, }, }, @@ -572,6 +1261,7 @@ commands: for name, tc := range tests { t.Run(name, func(t *testing.T) { got, err := Render(tc.args.tc, tc.args.resources, true) + fmt.Println(got) if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { t.Errorf("Render(...): -want error, +got error:\n%s", diff) } diff --git a/internal/tester.go b/internal/tester.go index 68619de..d65afde 100644 --- a/internal/tester.go +++ b/internal/tester.go @@ -14,6 +14,10 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/crossplane/crossplane-runtime/pkg/errors" @@ -21,6 +25,13 @@ import ( "github.com/crossplane/uptest/internal/templates" ) +var testFiles = []string{ + "00-apply.yaml", + "01-update.yaml", + "02-import.yaml", + "03-delete.yaml", +} + func newTester(ms []config.Manifest, opts *config.AutomatedTest) *tester { return &tester{ options: opts, @@ -34,35 +45,93 @@ type tester struct { } func (t *tester) executeTests() error { - if err := t.writeKuttlFiles(); err != nil { - return errors.Wrap(err, "cannot write kuttl test files") + if err := writeTestFile(t.manifests, t.options.Directory); err != nil { + return err + } + + resources, err := t.writeChainsawFiles() + if err != nil { + return errors.Wrap(err, "cannot write chainsaw test files") + } + + fmt.Printf("Written test files: %s\n", t.options.Directory) + + if t.options.RenderOnly { + return nil } - fmt.Println("Running kuttl tests at " + t.options.Directory) - cmd := exec.Command("bash", "-c", fmt.Sprintf(`"${KUTTL}" test --start-kind=false --skip-cluster-delete %s --timeout %d 2>&1`, t.options.Directory, t.options.DefaultTimeout)) // #nosec G204 - stdout, _ := cmd.StdoutPipe() - if err := cmd.Start(); err != nil { - return errors.Wrap(err, "cannot start kuttl") + + fmt.Println("Running chainsaw tests at " + t.options.Directory) + for _, tf := range testFiles { + if !checkFileExists(filepath.Join(t.options.Directory, caseDirectory, tf)) { + continue + } + cmd := exec.Command("bash", "-c", fmt.Sprintf(`"${CHAINSAW}" test --test-dir %s --test-file %s --skip-delete --parallel 1 2>&1`, filepath.Join(t.options.Directory, caseDirectory), tf)) // #nosec G204 + stdout, _ := cmd.StdoutPipe() + if err := cmd.Start(); err != nil { + return errors.Wrap(err, "cannot start chainsaw") + } + + var mutex sync.Mutex + // Start ticker for kubectl command every 30 seconds + ticker := time.NewTicker(t.options.LogCollectionInterval) + done := make(chan bool) + go logCollector(done, ticker, &mutex, resources)() + + sc := bufio.NewScanner(stdout) + sc.Split(bufio.ScanLines) + for sc.Scan() { + mutex.Lock() + fmt.Println(sc.Text()) + mutex.Unlock() + } + if err := cmd.Wait(); err != nil { + return errors.Wrap(err, "cannot wait for chainsaw") + } + + ticker.Stop() + done <- true } - sc := bufio.NewScanner(stdout) - sc.Split(bufio.ScanLines) - for sc.Scan() { - fmt.Println(sc.Text()) + return nil +} + +func logCollector(done chan bool, ticker *time.Ticker, mutex sync.Locker, resources []config.Resource) func() { + return func() { + for { + select { + case <-done: + return + case <-ticker.C: + mutex.Lock() + fmt.Printf("crossplane trace logs %s\n", time.Now()) + for _, r := range resources { + kubectlCmd := exec.Command("bash", "-c", fmt.Sprintf(`"${CROSSPLANE_CLI}" beta trace %s %s -o wide`, r.KindGroup, r.Name)) //nolint:gosec // Disabling gosec to allow dynamic shell command execution + output, err := kubectlCmd.CombinedOutput() + if err != nil { + fmt.Println("Error executing crossplane:", err) + } else { + fmt.Println(string(output)) + } + } + mutex.Unlock() + } + } } - return errors.Wrap(cmd.Wait(), "kuttl failed") } func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { //nolint:gocyclo // TODO: can we break this? tc := &config.TestCase{ - Timeout: t.options.DefaultTimeout, + Timeout: t.options.DefaultTimeout.Duration, SetupScriptPath: t.options.SetupScriptPath, TeardownScriptPath: t.options.TeardownScriptPath, OnlyCleanUptestResources: t.options.OnlyCleanUptestResources, + TestDirectory: "test-input.yaml", } examples := make([]config.Resource, 0, len(t.manifests)) for _, m := range t.manifests { obj := m.Object groupVersionKind := obj.GroupVersionKind() + apiVersion, kind := groupVersionKind.ToAPIVersionAndKind() kg := strings.ToLower(groupVersionKind.Kind + "." + groupVersionKind.Group) example := config.Resource{ @@ -72,17 +141,20 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { YAML: m.YAML, Timeout: t.options.DefaultTimeout, Conditions: t.options.DefaultConditions, + APIVersion: apiVersion, + Kind: kind, } var err error annotations := obj.GetAnnotations() if v, ok := annotations[config.AnnotationKeyTimeout]; ok { - example.Timeout, err = strconv.Atoi(v) + d, err := strconv.Atoi(v) if err != nil { return nil, nil, errors.Wrap(err, "timeout value is not valid") } - if example.Timeout > tc.Timeout { - tc.Timeout = example.Timeout + example.Timeout = v1.Duration{Duration: time.Duration(d)} + if example.Timeout.Duration > tc.Timeout { + tc.Timeout = example.Timeout.Duration } } @@ -140,7 +212,7 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { if disableImport == "true" { tc.SkipImport = true } - if updateParameter == "" { + if updateParameter == "" || obj.GetNamespace() != "" { tc.SkipUpdate = true } example.Root = true @@ -153,24 +225,43 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { return tc, examples, nil } -func (t *tester) writeKuttlFiles() error { +func (t *tester) writeChainsawFiles() ([]config.Resource, error) { tc, examples, err := t.prepareConfig() if err != nil { - return errors.Wrap(err, "cannot build examples config") + return nil, errors.Wrap(err, "cannot build examples config") } files, err := templates.Render(tc, examples, t.options.SkipDelete) if err != nil { - return errors.Wrap(err, "cannot render kuttl templates") + return nil, errors.Wrap(err, "cannot render chainsaw templates") } for k, v := range files { if err := os.WriteFile(filepath.Join(filepath.Join(t.options.Directory, caseDirectory), k), []byte(v), fs.ModePerm); err != nil { - return errors.Wrapf(err, "cannot write file %q", k) + return nil, errors.Wrapf(err, "cannot write file %q", k) } } - return nil + return examples, nil +} + +func writeTestFile(manifests []config.Manifest, directory string) error { + file, err := os.Create(filepath.Join(directory, caseDirectory, "test-input.yaml")) //nolint:gosec // Safe disable of gosec: path components are controlled, mitigating directory traversal risks. + if err != nil { + return err + } + defer file.Close() //nolint:errcheck // Ignoring error on file close as any failures do not impact the functionality and are logged at a higher level. + + writer := bufio.NewWriter(file) + for _, manifest := range manifests { + if _, err := writer.WriteString("---\n"); err != nil { + return err + } + if _, err = writer.WriteString(manifest.YAML + "\n"); err != nil { + return err + } + } + return writer.Flush() } func convertToJSONPath(data map[string]interface{}, currentPath string) (string, string) { @@ -185,3 +276,8 @@ func convertToJSONPath(data map[string]interface{}, currentPath string) (string, } return currentPath, "" } + +func checkFileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !errors.Is(err, os.ErrNotExist) +} From aae5026d251d9c1d0f8e5d518d3e98d01b8b1233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Thu, 23 May 2024 13:32:13 +0300 Subject: [PATCH 2/3] Fix unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- internal/templates/renderer_test.go | 353 +++------------------------- 1 file changed, 27 insertions(+), 326 deletions(-) diff --git a/internal/templates/renderer_test.go b/internal/templates/renderer_test.go index 9822d4b..313513b 100644 --- a/internal/templates/renderer_test.go +++ b/internal/templates/renderer_test.go @@ -5,7 +5,6 @@ package templates import ( - "fmt" "testing" "time" @@ -161,78 +160,19 @@ spec: try: - script: content: | - ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket crossplane.io/paused=true --overwrite ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 - sleep: duration: 10s - script: content: | - function check_endpoints { - endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) - for endpoint in ${endpoints[@]}; do - port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') - if [[ -z "${port}" ]]; then - echo "$endpoint - No served ports" - return 1 - else - echo "$endpoint - Ports present" - fi - done - } - - function check_endpoints_main { - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if check_endpoints; then - return 0 - else - printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 - fi - ((attempt++)) - sleep 5 - done - return 1 - } - - function patch { - kindgroup=$1; - name=$2; - if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then - return 0; - else - return 1; - fi; - }; - - function patch_main { - kindgroup=$1 - name=$2 - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if patch "$kindgroup" "$name"; then - echo "Successfully patched $kindgroup/$name" - ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite - break - else - printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 - fi - ((attempt++)) - sleep 5 - done - if [[ $attempt -gt $max_attempts ]]; then - echo "Failed to patch $kindgroup/$name after $max_attempts attempts" - return 1 - fi - return 0 - } - ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 - check_endpoints_main - patch_main s3.aws.upbound.io example-bucket + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + /tmp/patch.sh s3.aws.upbound.io example-bucket ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite - name: Assert Status Conditions and IDs description: | @@ -265,8 +205,7 @@ metadata: name: delete spec: timeouts: - assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Delete Resources description: Delete resources. If needs ordered deletion, the pre-delete scripts were used. @@ -424,78 +363,19 @@ spec: try: - script: content: | - ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket crossplane.io/paused=true --overwrite ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 - sleep: duration: 10s - script: content: | - function check_endpoints { - endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) - for endpoint in ${endpoints[@]}; do - port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') - if [[ -z "${port}" ]]; then - echo "$endpoint - No served ports" - return 1 - else - echo "$endpoint - Ports present" - fi - done - } - - function check_endpoints_main { - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if check_endpoints; then - return 0 - else - printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 - fi - ((attempt++)) - sleep 5 - done - return 1 - } - - function patch { - kindgroup=$1; - name=$2; - if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then - return 0; - else - return 1; - fi; - }; - - function patch_main { - kindgroup=$1 - name=$2 - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if patch "$kindgroup" "$name"; then - echo "Successfully patched $kindgroup/$name" - ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite - break - else - printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 - fi - ((attempt++)) - sleep 5 - done - if [[ $attempt -gt $max_attempts ]]; then - echo "Failed to patch $kindgroup/$name after $max_attempts attempts" - return 1 - fi - return 0 - } - ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 - check_endpoints_main - patch_main s3.aws.upbound.io example-bucket + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + /tmp/patch.sh s3.aws.upbound.io example-bucket ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite - name: Assert Status Conditions and IDs description: | @@ -528,8 +408,7 @@ metadata: name: delete spec: timeouts: - assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Delete Resources description: Delete resources. If needs ordered deletion, the pre-delete scripts were used. @@ -690,78 +569,19 @@ spec: try: - script: content: | - ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket crossplane.io/paused=true --overwrite ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 - sleep: duration: 10s - script: content: | - function check_endpoints { - endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) - for endpoint in ${endpoints[@]}; do - port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') - if [[ -z "${port}" ]]; then - echo "$endpoint - No served ports" - return 1 - else - echo "$endpoint - Ports present" - fi - done - } - - function check_endpoints_main { - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if check_endpoints; then - return 0 - else - printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 - fi - ((attempt++)) - sleep 5 - done - return 1 - } - - function patch { - kindgroup=$1; - name=$2; - if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then - return 0; - else - return 1; - fi; - }; - - function patch_main { - kindgroup=$1 - name=$2 - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if patch "$kindgroup" "$name"; then - echo "Successfully patched $kindgroup/$name" - ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite - break - else - printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 - fi - ((attempt++)) - sleep 5 - done - if [[ $attempt -gt $max_attempts ]]; then - echo "Failed to patch $kindgroup/$name after $max_attempts attempts" - return 1 - fi - return 0 - } - ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 - check_endpoints_main - patch_main s3.aws.upbound.io example-bucket + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + /tmp/patch.sh s3.aws.upbound.io example-bucket ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite - name: Assert Status Conditions and IDs description: | @@ -928,78 +748,19 @@ spec: try: - script: content: | - ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket crossplane.io/paused=true --overwrite ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 - sleep: duration: 10s - script: content: | - function check_endpoints { - endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) - for endpoint in ${endpoints[@]}; do - port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') - if [[ -z "${port}" ]]; then - echo "$endpoint - No served ports" - return 1 - else - echo "$endpoint - Ports present" - fi - done - } - - function check_endpoints_main { - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if check_endpoints; then - return 0 - else - printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 - fi - ((attempt++)) - sleep 5 - done - return 1 - } - - function patch { - kindgroup=$1; - name=$2; - if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then - return 0; - else - return 1; - fi; - }; - - function patch_main { - kindgroup=$1 - name=$2 - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if patch "$kindgroup" "$name"; then - echo "Successfully patched $kindgroup/$name" - ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite - break - else - printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 - fi - ((attempt++)) - sleep 5 - done - if [[ $attempt -gt $max_attempts ]]; then - echo "Failed to patch $kindgroup/$name after $max_attempts attempts" - return 1 - fi - return 0 - } - ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 - check_endpoints_main - patch_main s3.aws.upbound.io example-bucket + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + /tmp/patch.sh s3.aws.upbound.io example-bucket ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite - name: Assert Status Conditions and IDs description: | @@ -1157,78 +918,19 @@ spec: try: - script: content: | - ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=true --overwrite + ${KUBECTL} annotate s3.aws.upbound.io/example-bucket crossplane.io/paused=true --overwrite ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=0 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=0 - sleep: duration: 10s - script: content: | - function check_endpoints { - endpoints=( $("${KUBECTL}" -n "${CROSSPLANE_NAMESPACE}" get endpoints --no-headers | grep '^provider-' | awk '{print $1}') ) - for endpoint in ${endpoints[@]}; do - port=$(${KUBECTL} -n "${CROSSPLANE_NAMESPACE}" get endpoints "$endpoint" -o jsonpath='{.subsets[*].ports[0].port}') - if [[ -z "${port}" ]]; then - echo "$endpoint - No served ports" - return 1 - else - echo "$endpoint - Ports present" - fi - done - } - - function check_endpoints_main { - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if check_endpoints; then - return 0 - else - printf "Retrying... (%d/%d)\n" "$attempt" "$max_attempts" >&2 - fi - ((attempt++)) - sleep 5 - done - return 1 - } - - function patch { - kindgroup=$1; - name=$2; - if ${KUBECTL} --subresource=status patch "$kindgroup/$name" --type=merge -p '{"status":{"conditions":[]}}' ; then - return 0; - else - return 1; - fi; - }; - - function patch_main { - kindgroup=$1 - name=$2 - attempt=1 - max_attempts=10 - while [[ $attempt -le $max_attempts ]]; do - if patch "$kindgroup" "$name"; then - echo "Successfully patched $kindgroup/$name" - ${KUBECTL} annotate "$kindgroup/$name" uptest-old-id=$(${KUBECTL} get "$kindgroup/$name" -o=jsonpath='{.status.atProvider.id}') --overwrite - break - else - printf "Retrying... (%d/%d) for %s/%s\n" "$attempt" "$max_attempts" "$kindgroup" "$name" >&2 - fi - ((attempt++)) - sleep 5 - done - if [[ $attempt -gt $max_attempts ]]; then - echo "Failed to patch $kindgroup/$name after $max_attempts attempts" - return 1 - fi - return 0 - } - ${KUBECTL} scale deployment crossplane -n ${CROSSPLANE_NAMESPACE} --replicas=1 --timeout 10s ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} get deploy --no-headers -o custom-columns=":metadata.name" | grep "provider-" | xargs ${KUBECTL} -n ${CROSSPLANE_NAMESPACE} scale deploy --replicas=1 - check_endpoints_main - patch_main s3.aws.upbound.io example-bucket + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/check_endpoints.sh -o /tmp/check_endpoints.sh && chmod +x /tmp/check_endpoints.sh + curl -sL https://raw.githubusercontent.com/crossplane/uptest/main/hack/patch.sh -o /tmp/patch.sh && chmod +x /tmp/patch.sh + /tmp/check_endpoints.sh + /tmp/patch.sh s3.aws.upbound.io example-bucket ${KUBECTL} annotate s3.aws.upbound.io/example-bucket --all crossplane.io/paused=false --overwrite - name: Assert Status Conditions and IDs description: | @@ -1261,7 +963,6 @@ spec: for name, tc := range tests { t.Run(name, func(t *testing.T) { got, err := Render(tc.args.tc, tc.args.resources, true) - fmt.Println(got) if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" { t.Errorf("Render(...): -want error, +got error:\n%s", diff) } From 72c37705cf7bb0012f03f913e852daf9814b9aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Mon, 12 Aug 2024 01:07:12 +0300 Subject: [PATCH 3/3] - Use log package instead of fmt - Use configurable timeout for all timeout types - Use CommandContext for applying the global timeout - For preventing memory leak extract the test function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- cmd/uptest/main.go | 3 +- internal/config/config.go | 5 +- internal/prepare.go | 3 +- internal/runner.go | 4 +- internal/templates/00-apply.yaml.tmpl | 8 +- internal/templates/00-assert.yaml.tmpl | 30 ------ internal/templates/01-assert.yaml.tmpl | 19 ---- internal/templates/01-update.yaml.tmpl | 4 +- internal/templates/02-assert.yaml.tmpl | 22 ---- internal/templates/02-import.yaml.tmpl | 4 +- internal/templates/03-assert.yaml.tmpl | 26 ----- internal/templates/renderer_test.go | 80 +++++++-------- internal/tester.go | 134 ++++++++++++++----------- 13 files changed, 133 insertions(+), 209 deletions(-) delete mode 100644 internal/templates/00-assert.yaml.tmpl delete mode 100644 internal/templates/01-assert.yaml.tmpl delete mode 100644 internal/templates/02-assert.yaml.tmpl delete mode 100644 internal/templates/03-assert.yaml.tmpl diff --git a/cmd/uptest/main.go b/cmd/uptest/main.go index 7f165c0..508592d 100644 --- a/cmd/uptest/main.go +++ b/cmd/uptest/main.go @@ -11,7 +11,6 @@ import ( "strings" "gopkg.in/alecthomas/kingpin.v2" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/crossplane/uptest/internal" "github.com/crossplane/uptest/internal/config" @@ -93,7 +92,7 @@ func e2eTests() { SetupScriptPath: setupPath, TeardownScriptPath: teardownPath, DefaultConditions: strings.Split(*defaultConditions, ","), - DefaultTimeout: v1.Duration{Duration: *defaultTimeout}, + DefaultTimeout: *defaultTimeout, Directory: *testDir, SkipDelete: *skipDelete, OnlyCleanUptestResources: *onlyCleanUptestResources, diff --git a/internal/config/config.go b/internal/config/config.go index 9e061fb..15e5424 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,7 +8,6 @@ package config import ( "time" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -52,7 +51,7 @@ type AutomatedTest struct { SetupScriptPath string TeardownScriptPath string - DefaultTimeout v1.Duration + DefaultTimeout time.Duration DefaultConditions []string SkipDelete bool @@ -93,7 +92,7 @@ type Resource struct { APIVersion string Kind string - Timeout v1.Duration + Timeout time.Duration Conditions []string PreAssertScriptPath string PostAssertScriptPath string diff --git a/internal/prepare.go b/internal/prepare.go index c09d5b1..f9f2e53 100644 --- a/internal/prepare.go +++ b/internal/prepare.go @@ -11,6 +11,7 @@ import ( "bytes" "fmt" "io" + "log" "math/rand" "os" "path/filepath" @@ -93,7 +94,7 @@ func (p *preparer) prepareManifests() ([]config.Manifest, error) { } if u != nil { if v, ok := u.GetAnnotations()["upjet.upbound.io/manual-intervention"]; ok { - fmt.Printf("Skipping %s with name %s since it requires the following manual intervention: %s\n", u.GroupVersionKind().String(), u.GetName(), v) + log.Printf("Skipping %s with name %s since it requires the following manual intervention: %s\n", u.GroupVersionKind().String(), u.GetName(), v) continue } y, err := yaml.Marshal(u) diff --git a/internal/runner.go b/internal/runner.go index b081981..2538d78 100644 --- a/internal/runner.go +++ b/internal/runner.go @@ -5,7 +5,7 @@ package internal import ( - "fmt" + "log" "os" "github.com/crossplane/crossplane-runtime/pkg/errors" @@ -18,7 +18,7 @@ func RunTest(o *config.AutomatedTest) error { if !o.RenderOnly { defer func() { if err := os.RemoveAll(o.Directory); err != nil { - fmt.Println(fmt.Sprint(err, "cannot clean the test directory")) + log.Printf("Cannot clean the test directory: %s\n", err.Error()) } }() } diff --git a/internal/templates/00-apply.yaml.tmpl b/internal/templates/00-apply.yaml.tmpl index 11f5123..e5eb05b 100644 --- a/internal/templates/00-apply.yaml.tmpl +++ b/internal/templates/00-apply.yaml.tmpl @@ -6,9 +6,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: {{ .TestCase.Timeout }} assert: {{ .TestCase.Timeout }} - exec: 1m + exec: {{ .TestCase.Timeout }} steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -32,8 +32,8 @@ spec: {{- end }} - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: {{- range $resource := .Resources }} diff --git a/internal/templates/00-assert.yaml.tmpl b/internal/templates/00-assert.yaml.tmpl deleted file mode 100644 index 12c287b..0000000 --- a/internal/templates/00-assert.yaml.tmpl +++ /dev/null @@ -1,30 +0,0 @@ -# This assert file belongs to the resource apply step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: {{ .TestCase.Timeout }} -commands: -- command: ${KUBECTL} annotate managed --all upjet.upbound.io/test=true --overwrite -- script: if [ -n "${CROSSPLANE_CLI}" ]; then ${KUBECTL} get composite --no-headers -o name | while read -r comp; do [ -n "$comp" ] && ${CROSSPLANE_CLI} beta trace "$comp"; done; fi -- script: echo "Dump MR manifests for the apply assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the apply assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if $resource.PreAssertScriptPath }} -- command: {{ $resource.PreAssertScriptPath }} -{{- end }} -{{- range $condition := $resource.Conditions }} -{{- if $resource.Namespace }} -- command: ${KUBECTL} wait {{ $resource.KindGroup }}/{{ $resource.Name }} --for=condition={{ $condition }} --timeout 10s --namespace {{ $resource.Namespace }} -{{- else }} -- command: ${KUBECTL} wait {{ $resource.KindGroup }}/{{ $resource.Name }} --for=condition={{ $condition }} --timeout 10s -{{- end }} -{{- end }} -{{- if $resource.PostAssertScriptPath }} -- command: {{ $resource.PostAssertScriptPath }} -{{- end }} -{{- end }} diff --git a/internal/templates/01-assert.yaml.tmpl b/internal/templates/01-assert.yaml.tmpl deleted file mode 100644 index 307ed9e..0000000 --- a/internal/templates/01-assert.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -# This assert file belongs to the resource update step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: {{ .TestCase.Timeout }} -commands: -- script: echo "Dump MR manifests for the update assertion step:"; ${KUBECTL} get managed -o yaml -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if not $resource.Namespace }} -{{- if $resource.Root }} -- script: ${KUBECTL} get {{ $resource.KindGroup }}/{{ $resource.Name }} -o=jsonpath='{.status.atProvider{{ $resource.UpdateAssertKey }}}' | grep -q "^{{ $resource.UpdateAssertValue }}$" -{{- end }} -{{- end }} -{{- end }} diff --git a/internal/templates/01-update.yaml.tmpl b/internal/templates/01-update.yaml.tmpl index abc404e..5a4fc37 100644 --- a/internal/templates/01-update.yaml.tmpl +++ b/internal/templates/01-update.yaml.tmpl @@ -5,9 +5,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: {{ .TestCase.Timeout }} assert: {{ .TestCase.Timeout }} - exec: 1m + exec: {{ .TestCase.Timeout }} steps: - name: Update Root Resource description: | diff --git a/internal/templates/02-assert.yaml.tmpl b/internal/templates/02-assert.yaml.tmpl deleted file mode 100644 index ac3aed5..0000000 --- a/internal/templates/02-assert.yaml.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -# This assert file belongs to the resource import step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: {{ .TestCase.Timeout }} -commands: -- script: echo "Dump MR manifests for the import assertion step:"; ${KUBECTL} get managed -o yaml -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- range $condition := $resource.Conditions }} -{{- if not $resource.Namespace }} -- command: ${KUBECTL} wait {{ $resource.KindGroup }}/{{ $resource.Name }} --for=condition={{ $condition }} --timeout 10s -{{- end }} -{{- end }} -{{- if not (or $resource.Namespace $resource.SkipImport) }} -- script: new_id="$(${KUBECTL} get {{ $resource.KindGroup }}/{{ $resource.Name }} -o=jsonpath='{.status.atProvider.id}')" && old_id="$(${KUBECTL} get {{ $resource.KindGroup }}/{{ $resource.Name }} -o=jsonpath='{.metadata.annotations.uptest-old-id}')" && [ "$new_id" = "$old_id" ] -{{- end }} -{{- end }} diff --git a/internal/templates/02-import.yaml.tmpl b/internal/templates/02-import.yaml.tmpl index 72c4b36..85eaf8f 100644 --- a/internal/templates/02-import.yaml.tmpl +++ b/internal/templates/02-import.yaml.tmpl @@ -5,9 +5,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: {{ .TestCase.Timeout }} assert: {{ .TestCase.Timeout }} - exec: 3m + exec: {{ .TestCase.Timeout }} steps: - name: Remove State description: | diff --git a/internal/templates/03-assert.yaml.tmpl b/internal/templates/03-assert.yaml.tmpl deleted file mode 100644 index e0041ca..0000000 --- a/internal/templates/03-assert.yaml.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -# This assert file belongs to the resource delete step. -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: {{ .TestCase.Timeout }} -commands: -- script: echo "Dump MR manifests for the delete assertion step:"; ${KUBECTL} get managed -o yaml -- script: echo "Dump Claim manifests for the delete assertion step:" || ${KUBECTL} get claim --all-namespaces -o yaml -{{- range $resource := .Resources }} -{{- if eq $resource.KindGroup "secret." -}} - {{continue}} -{{- end -}} -{{- if eq $resource.KindGroup "namespace." -}} - {{continue}} -{{- end -}} -{{- if $resource.Namespace }} -- script: ${KUBECTL} wait {{ $resource.KindGroup }}/{{ $resource.Name }} --for=delete --timeout 10s --namespace {{ $resource.Namespace }} -{{- else }} -- command: ${KUBECTL} wait {{ $resource.KindGroup }}/{{ $resource.Name }} --for=delete --timeout 10s -{{- end }} -{{- end }} -{{- if not .TestCase.OnlyCleanUptestResources }} -- command: ${KUBECTL} wait managed --all --for=delete --timeout 10s -{{- end }} -{{- if .TestCase.TeardownScriptPath }} -- command: {{ .TestCase.TeardownScriptPath }} -{{- end }} diff --git a/internal/templates/renderer_test.go b/internal/templates/renderer_test.go index 313513b..ad9cf85 100644 --- a/internal/templates/renderer_test.go +++ b/internal/templates/renderer_test.go @@ -86,9 +86,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -105,8 +105,8 @@ spec: ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: - assert: @@ -126,9 +126,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Update Root Resource description: | @@ -147,9 +147,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 3m + exec: 10m0s steps: - name: Remove State description: | @@ -273,9 +273,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -292,8 +292,8 @@ spec: ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: - command: @@ -329,9 +329,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Update Root Resource description: | @@ -350,9 +350,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 3m + exec: 10m0s steps: - name: Remove State description: | @@ -495,9 +495,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -514,8 +514,8 @@ spec: ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: - assert: @@ -535,9 +535,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Update Root Resource description: | @@ -556,9 +556,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 3m + exec: 10m0s steps: - name: Remove State description: | @@ -658,9 +658,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -677,8 +677,8 @@ spec: ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: - command: @@ -714,9 +714,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Update Root Resource description: | @@ -735,9 +735,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 3m + exec: 10m0s steps: - name: Remove State description: | @@ -828,9 +828,9 @@ metadata: name: apply spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Run Setup Script description: Setup the test environment by running the setup script. @@ -847,8 +847,8 @@ spec: ${KUBECTL} annotate s3.aws.upbound.io/example-bucket upjet.upbound.io/test=true --overwrite - name: Assert Status Conditions description: | - Assert applied resources. Firstly run the pre-assert script if exists. - Then check the status conditions. Finally run the post-assert script if + Assert applied resources. First, run the pre-assert script if exists. + Then, check the status conditions. Finally run the post-assert script if it exists. try: - command: @@ -884,9 +884,9 @@ metadata: name: update spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 1m + exec: 10m0s steps: - name: Update Root Resource description: | @@ -905,9 +905,9 @@ metadata: name: import spec: timeouts: - apply: 1m + apply: 10m0s assert: 10m0s - exec: 3m + exec: 10m0s steps: - name: Remove State description: | diff --git a/internal/tester.go b/internal/tester.go index d65afde..88d9bd4 100644 --- a/internal/tester.go +++ b/internal/tester.go @@ -6,9 +6,11 @@ package internal import ( "bufio" + "context" "encoding/json" "fmt" "io/fs" + "log" "os" "os/exec" "path/filepath" @@ -17,8 +19,6 @@ import ( "sync" "time" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/uptest/internal/config" @@ -46,81 +46,101 @@ type tester struct { func (t *tester) executeTests() error { if err := writeTestFile(t.manifests, t.options.Directory); err != nil { - return err + return errors.Wrap(err, "cannot write test manifest files") } - resources, err := t.writeChainsawFiles() + resources, timeout, err := t.writeChainsawFiles() if err != nil { return errors.Wrap(err, "cannot write chainsaw test files") } - fmt.Printf("Written test files: %s\n", t.options.Directory) + log.Printf("Written test files: %s\n", t.options.Directory) if t.options.RenderOnly { return nil } - fmt.Println("Running chainsaw tests at " + t.options.Directory) + log.Println("Running chainsaw tests at " + t.options.Directory) + startTime := time.Now() for _, tf := range testFiles { if !checkFileExists(filepath.Join(t.options.Directory, caseDirectory, tf)) { + log.Println("Skipping test " + tf) continue } - cmd := exec.Command("bash", "-c", fmt.Sprintf(`"${CHAINSAW}" test --test-dir %s --test-file %s --skip-delete --parallel 1 2>&1`, filepath.Join(t.options.Directory, caseDirectory), tf)) // #nosec G204 - stdout, _ := cmd.StdoutPipe() - if err := cmd.Start(); err != nil { - return errors.Wrap(err, "cannot start chainsaw") + if err := executeSingleTestFile(t, tf, timeout-time.Since(startTime), resources); err != nil { + return errors.Wrap(err, "cannot execute test "+tf) } + } + return nil +} - var mutex sync.Mutex - // Start ticker for kubectl command every 30 seconds - ticker := time.NewTicker(t.options.LogCollectionInterval) - done := make(chan bool) - go logCollector(done, ticker, &mutex, resources)() +func executeSingleTestFile(t *tester, tf string, timeout time.Duration, resources []config.Resource) error { + chainsawCommand := fmt.Sprintf(`"${CHAINSAW}" test --test-dir %s --test-file %s --skip-delete --parallel 1 2>&1`, + filepath.Clean(filepath.Join(t.options.Directory, caseDirectory)), + filepath.Clean(tf)) - sc := bufio.NewScanner(stdout) - sc.Split(bufio.ScanLines) - for sc.Scan() { - mutex.Lock() - fmt.Println(sc.Text()) - mutex.Unlock() - } - if err := cmd.Wait(); err != nil { - return errors.Wrap(err, "cannot wait for chainsaw") - } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", "-c", chainsawCommand) // #nosec G204 + stdout, _ := cmd.StdoutPipe() + if err := cmd.Start(); err != nil { + return errors.Wrapf(err, "cannot start chainsaw: %s", chainsawCommand) + } + + // Start ticker for kubectl command every 30 seconds + ticker := time.NewTicker(t.options.LogCollectionInterval) + done := make(chan bool) + defer func() { ticker.Stop() - done <- true + close(done) + }() + + var mutex sync.Mutex + go logCollector(done, ticker, &mutex, resources) + + sc := bufio.NewScanner(stdout) + for sc.Scan() { + mutex.Lock() + log.Println(sc.Text()) + mutex.Unlock() + } + if sc.Err() != nil { + return errors.Wrap(sc.Err(), "cannot scan output") + } + if err := cmd.Wait(); err != nil { + return errors.Wrapf(err, "cannot wait for chainsaw: %s", chainsawCommand) } + return nil } -func logCollector(done chan bool, ticker *time.Ticker, mutex sync.Locker, resources []config.Resource) func() { - return func() { - for { - select { - case <-done: - return - case <-ticker.C: - mutex.Lock() - fmt.Printf("crossplane trace logs %s\n", time.Now()) - for _, r := range resources { - kubectlCmd := exec.Command("bash", "-c", fmt.Sprintf(`"${CROSSPLANE_CLI}" beta trace %s %s -o wide`, r.KindGroup, r.Name)) //nolint:gosec // Disabling gosec to allow dynamic shell command execution - output, err := kubectlCmd.CombinedOutput() - if err != nil { - fmt.Println("Error executing crossplane:", err) - } else { - fmt.Println(string(output)) - } +func logCollector(done chan bool, ticker *time.Ticker, mutex sync.Locker, resources []config.Resource) { + for { + select { + case <-done: + return + case <-ticker.C: + mutex.Lock() + log.Printf("crossplane trace logs %s\n", time.Now()) + for _, r := range resources { + traceCmd := exec.Command("bash", "-c", fmt.Sprintf(`"${CROSSPLANE_CLI}" beta trace %s %s -o wide`, r.KindGroup, r.Name)) //nolint:gosec // Disabling gosec to allow dynamic shell command execution + output, err := traceCmd.CombinedOutput() + if err != nil { + log.Println("Error executing crossplane:", err) + } else { + log.Println(string(output)) } - mutex.Unlock() } + mutex.Unlock() } } + } func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { //nolint:gocyclo // TODO: can we break this? tc := &config.TestCase{ - Timeout: t.options.DefaultTimeout.Duration, + Timeout: t.options.DefaultTimeout, SetupScriptPath: t.options.SetupScriptPath, TeardownScriptPath: t.options.TeardownScriptPath, OnlyCleanUptestResources: t.options.OnlyCleanUptestResources, @@ -152,9 +172,9 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { if err != nil { return nil, nil, errors.Wrap(err, "timeout value is not valid") } - example.Timeout = v1.Duration{Duration: time.Duration(d)} - if example.Timeout.Duration > tc.Timeout { - tc.Timeout = example.Timeout.Duration + example.Timeout = time.Duration(d) * time.Second + if example.Timeout > tc.Timeout { + tc.Timeout = example.Timeout } } @@ -210,9 +230,11 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { if exampleID, ok := annotations[config.AnnotationKeyExampleID]; ok { if exampleID == strings.ToLower(fmt.Sprintf("%s/%s/%s", strings.Split(groupVersionKind.Group, ".")[0], groupVersionKind.Version, groupVersionKind.Kind)) { if disableImport == "true" { + log.Println("Skipping import step because the root resource has disable import annotation") tc.SkipImport = true } if updateParameter == "" || obj.GetNamespace() != "" { + log.Println("Skipping update step because the root resource does not have the update parameter") tc.SkipUpdate = true } example.Root = true @@ -225,28 +247,28 @@ func (t *tester) prepareConfig() (*config.TestCase, []config.Resource, error) { return tc, examples, nil } -func (t *tester) writeChainsawFiles() ([]config.Resource, error) { +func (t *tester) writeChainsawFiles() ([]config.Resource, time.Duration, error) { tc, examples, err := t.prepareConfig() if err != nil { - return nil, errors.Wrap(err, "cannot build examples config") + return nil, 0, errors.Wrap(err, "cannot build examples config") } files, err := templates.Render(tc, examples, t.options.SkipDelete) if err != nil { - return nil, errors.Wrap(err, "cannot render chainsaw templates") + return nil, 0, errors.Wrap(err, "cannot render chainsaw templates") } for k, v := range files { if err := os.WriteFile(filepath.Join(filepath.Join(t.options.Directory, caseDirectory), k), []byte(v), fs.ModePerm); err != nil { - return nil, errors.Wrapf(err, "cannot write file %q", k) + return nil, 0, errors.Wrapf(err, "cannot write file %q", k) } } - return examples, nil + return examples, tc.Timeout, nil } func writeTestFile(manifests []config.Manifest, directory string) error { - file, err := os.Create(filepath.Join(directory, caseDirectory, "test-input.yaml")) //nolint:gosec // Safe disable of gosec: path components are controlled, mitigating directory traversal risks. + file, err := os.Create(filepath.Clean(filepath.Join(directory, caseDirectory, "test-input.yaml"))) if err != nil { return err } @@ -255,10 +277,10 @@ func writeTestFile(manifests []config.Manifest, directory string) error { writer := bufio.NewWriter(file) for _, manifest := range manifests { if _, err := writer.WriteString("---\n"); err != nil { - return err + return errors.Wrap(err, "cannot write the manifest delimiter") } if _, err = writer.WriteString(manifest.YAML + "\n"); err != nil { - return err + return errors.Wrap(err, "cannot write the manifest content") } } return writer.Flush()