Merge pull request #15 from sergenyalcin/chainsaw-migration

Migrate the underlying test framework from `kuttl` to `chainsaw`

cmd/uptest/main.go

@@ -34,13 +34,17 @@ var (
 	teardownScript = e2e.Flag("teardown-script", "Script that will be executed after running tests.").Default("").String()
 
 	defaultTimeout = e2e.Flag("default-timeout", "Default timeout in seconds for the test.\n"+
-		"Timeout could be overridden per resource using \"uptest.upbound.io/timeout\" annotation.").Default("1200").Int()
+		"Timeout could be overridden per resource using \"uptest.upbound.io/timeout\" annotation.").Default("1200s").Duration()
 	defaultConditions = e2e.Flag("default-conditions", "Comma separated list of default conditions to wait for a successful test.\n"+
 		"Conditions could be overridden per resource using \"uptest.upbound.io/conditions\" annotation.").Default("Ready").String()
 
 	skipDelete               = e2e.Flag("skip-delete", "Skip the delete step of the test.").Default("false").Bool()
-	testDir                  = e2e.Flag("test-directory", "Directory where kuttl test case will be generated and executed.").Envar("UPTEST_TEST_DIR").Default(filepath.Join(os.TempDir(), "uptest-e2e")).String()
+	testDir                  = e2e.Flag("test-directory", "Directory where chainsaw test case will be generated and executed.").Envar("UPTEST_TEST_DIR").Default(filepath.Join(os.TempDir(), "uptest-e2e")).String()
 	onlyCleanUptestResources = e2e.Flag("only-clean-uptest-resources", "While deletion step, only clean resources that were created by uptest").Default("false").Bool()
+
+	renderOnly         = e2e.Flag("render-only", "Only render test files. Do not run the tests.").Default("false").Bool()
+	logCollectInterval = e2e.Flag("log-collect-interval", "Specifies the interval duration for collecting logs. "+
+		"The duration should be provided in a format understood by the tool, such as seconds (s), minutes (m), or hours (h). For example, '30s' for 30 seconds, '5m' for 5 minutes, or '1h' for one hour.").Default("30s").Duration()
 )
 
 func main() {
@@ -92,6 +96,8 @@ func e2eTests() {
 		Directory:                *testDir,
 		SkipDelete:               *skipDelete,
 		OnlyCleanUptestResources: *onlyCleanUptestResources,
+		RenderOnly:               *renderOnly,
+		LogCollectionInterval:    *logCollectInterval,
 	}
 
 	kingpin.FatalIfError(internal.RunTest(o), "cannot run e2e tests successfully")

internal/config/config.go

@@ -5,7 +5,11 @@
 // Package config contains configuration options for configuring uptest runtime.
 package config
 
-import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+import (
+	"time"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
 
 const (
 	// AnnotationKeyTimeout defines a test time for the annotated resource.
@@ -47,12 +51,15 @@ type AutomatedTest struct {
 	SetupScriptPath    string
 	TeardownScriptPath string
 
-	DefaultTimeout    int
+	DefaultTimeout    time.Duration
 	DefaultConditions []string
 
 	SkipDelete bool
 
 	OnlyCleanUptestResources bool
+
+	RenderOnly            bool
+	LogCollectionInterval time.Duration
 }
 
 // Manifest represents a resource loaded from an example resource manifest file.
@@ -62,26 +69,30 @@ type Manifest struct {
 	YAML     string
 }
 
-// TestCase represents a test-case to be run by kuttl.
+// TestCase represents a test-case to be run by chainsaw.
 type TestCase struct {
-	Timeout            int
+	Timeout            time.Duration
 	SetupScriptPath    string
 	TeardownScriptPath string
 	SkipUpdate         bool
 	SkipImport         bool
 
 	OnlyCleanUptestResources bool
+
+	TestDirectory string
 }
 
 // Resource represents a Kubernetes object to be tested and asserted
 // by uptest.
 type Resource struct {
-	Name      string
-	Namespace string
-	KindGroup string
-	YAML      string
-
-	Timeout              int
+	Name       string
+	Namespace  string
+	KindGroup  string
+	YAML       string
+	APIVersion string
+	Kind       string
+
+	Timeout              time.Duration
 	Conditions           []string
 	PreAssertScriptPath  string
 	PostAssertScriptPath string

internal/prepare.go

@@ -4,13 +4,14 @@
 
 // Package internal implements the uptest runtime for running
 // automated tests using resource example manifests
-// using kuttl.
+// using chainsaw.
 package internal
 
 import (
 	"bytes"
 	"fmt"
 	"io"
+	"log"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -93,7 +94,7 @@ func (p *preparer) prepareManifests() ([]config.Manifest, error) {
 			}
 			if u != nil {
 				if v, ok := u.GetAnnotations()["upjet.upbound.io/manual-intervention"]; ok {
-					fmt.Printf("Skipping %s with name %s since it requires the following manual intervention: %s\n", u.GroupVersionKind().String(), u.GetName(), v)
+					log.Printf("Skipping %s with name %s since it requires the following manual intervention: %s\n", u.GroupVersionKind().String(), u.GetName(), v)
 					continue
 				}
 				y, err := yaml.Marshal(u)

internal/runner.go

@@ -5,7 +5,7 @@
 package internal
 
 import (
-	"fmt"
+	"log"
 	"os"
 
 	"github.com/crossplane/crossplane-runtime/pkg/errors"
@@ -15,11 +15,13 @@ import (
 
 // RunTest runs the specified automated test
 func RunTest(o *config.AutomatedTest) error {
-	defer func() {
-		if err := os.RemoveAll(o.Directory); err != nil {
-			fmt.Println(fmt.Sprint(err, "cannot clean the test directory"))
-		}
-	}()
+	if !o.RenderOnly {
+		defer func() {
+			if err := os.RemoveAll(o.Directory); err != nil {
+				log.Printf("Cannot clean the test directory: %s\n", err.Error())
+			}
+		}()
+	}
 
 	// Read examples and inject data source values to manifests
 	manifests, err := newPreparer(o.ManifestPaths, withDataSource(o.DataSourcePath), withTestDirectory(o.Directory)).prepareManifests()

internal/templates/00-apply.yaml.tmpl

@@ -1,11 +1,66 @@
 # This file belongs to the resource apply step.
 {{ if .TestCase.SetupScriptPath -}}
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-- command: {{ .TestCase.SetupScriptPath }}
-{{ end }}
-{{- range $resource := .Resources -}}
----
-{{ $resource.YAML }}
-{{- end }}
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: apply
+spec:
+  timeouts:
+    apply: {{ .TestCase.Timeout }}
+    assert: {{ .TestCase.Timeout }}
+    exec: {{ .TestCase.Timeout }}
+  steps:
+  - name: Run Setup Script
+    description: Setup the test environment by running the setup script.
+    try:
+    - command:
+        entrypoint: {{ .TestCase.SetupScriptPath }}
+  - name: Apply Resources
+    description: Apply resources to the cluster.
+    try:
+    - apply:
+        file: {{ .TestCase.TestDirectory }}
+    - script:
+        content: |
+    {{- range $resource := .Resources }}
+    {{- if eq $resource.KindGroup "secret." -}}
+      {{continue}}
+    {{- end -}}
+    {{- if not $resource.Namespace }}
+          ${KUBECTL} annotate {{ $resource.KindGroup }}/{{ $resource.Name }} upjet.upbound.io/test=true --overwrite
+    {{- end }}
+    {{- end }}
+  - name: Assert Status Conditions
+    description: |
+      Assert applied resources. First, run the pre-assert script if exists.
+      Then, check the status conditions. Finally run the post-assert script if it
+      exists.
+    try:
+    {{- range $resource := .Resources }}
+    {{- if eq $resource.KindGroup "secret." -}}
+      {{continue}}
+    {{- end -}}
+    {{- if $resource.PreAssertScriptPath }}
+    - command:
+        entrypoint: {{ $resource.PreAssertScriptPath }}
+    {{- end }}
+    - assert:
+        resource:
+          apiVersion: {{ $resource.APIVersion }}
+          kind: {{ $resource.Kind }}
+          metadata:
+            name: {{ $resource.Name }}
+            {{- if $resource.Namespace }}
+            namespace: {{ $resource.Namespace }}
+            {{- end }}
+          status:
+            {{- range $condition := $resource.Conditions }}
+            ((conditions[?type == '{{ $condition }}'])[0]):
+              status: "True"
+            {{- end }}
+    {{- if $resource.PostAssertScriptPath }}
+    - command:
+        entrypoint: {{ $resource.PostAssertScriptPath }}
+    {{- end }}
+    {{- end }}
+{{ end }}

internal/templates/01-update.yaml.tmpl

@@ -1,17 +1,56 @@
 # This file belongs to the resource update step.
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-{{- range $resource := .Resources }}
-{{- if eq $resource.KindGroup "secret." -}}
-  {{continue}}
-{{- end -}}
-{{- if eq $resource.KindGroup "namespace." -}}
-  {{continue}}
-{{- end -}}
-{{- if not $resource.Namespace }}
-{{- if $resource.Root }}
-- command: ${KUBECTL} patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"spec":{"forProvider":{{ $resource.UpdateParameter }}}}'
-{{- end }}
-{{- end }}
-{{- end }}
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: update
+spec:
+  timeouts:
+    apply: {{ .TestCase.Timeout }}
+    assert: {{ .TestCase.Timeout }}
+    exec: {{ .TestCase.Timeout }}
+  steps:
+  - name: Update Root Resource
+    description: |
+      Update the root resource by using the specified update-parameter in annotation.
+      Before updating the resources, the status conditions are cleaned.
+    try:
+    {{- range $resource := .Resources }}
+    {{- if eq $resource.KindGroup "secret." -}}
+      {{continue}}
+    {{- end -}}
+    {{- if not $resource.Namespace }}
+    {{- if $resource.Root }}
+    - script:
+        content: |
+          ${KUBECTL} --subresource=status patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"status":{"conditions":[]}}'
+          ${KUBECTL} patch {{ $resource.KindGroup }}/{{ $resource.Name }} --type=merge -p '{"spec":{"forProvider":{{ $resource.UpdateParameter }}}}'
+    {{- end }}
+    {{- end }}
+    {{- end }}
+  - name: Assert Updated Resource
+    description: |
+      Assert update operation. Firstly check the status conditions. Then assert
+      the updated field in status.atProvider.
+    {{- range $resource := .Resources }}
+    {{- if eq $resource.KindGroup "secret." -}}
+      {{continue}}
+    {{- end -}}
+    {{- if not $resource.Namespace }}
+    {{- if $resource.Root }}
+    try:
+    - assert:
+        resource:
+          apiVersion: {{ $resource.APIVersion }}
+          kind: {{ $resource.Kind }}
+          metadata:
+            name: {{ $resource.Name }}
+          status:
+            {{- range $condition := $resource.Conditions }}
+            ((conditions[?type == '{{ $condition }}'])[0]):
+              status: "True"
+            {{- end }}
+    - script:
+        content: ${KUBECTL} get {{ $resource.KindGroup }}/{{ $resource.Name }} -o=jsonpath='{.status.atProvider{{ $resource.UpdateAssertKey }}}' | grep -q "^{{ $resource.UpdateAssertValue }}$"
+    {{- end }}
+    {{- end }}
+    {{- end }}