adding e2e test for multi app run k8s (#1336)

Signed-off-by: Mukundan Sundararajan <65565396+mukundansundar@users.noreply.github.com>

Makefile

@@ -155,6 +155,13 @@ test: test-deps
 test-e2e-k8s: test-deps
 	gotestsum --jsonfile $(TEST_OUTPUT_FILE) --format standard-verbose -- -timeout 20m -count=1 -tags=e2e ./tests/e2e/kubernetes/...
 
+################################################################################
+# E2E Tests for K8s Template exec											       #
+################################################################################
+.PHONY: test-e2e-k8s-template
+test-e2e-k8s-template: test-deps
+	gotestsum --jsonfile $(TEST_OUTPUT_FILE) --format standard-verbose -- -timeout 20m -count=1 -tags=templatek8s ./tests/e2e/kubernetes/...
+
 ################################################################################
 # Build, E2E Tests for Kubernetes											   #
 ################################################################################

pkg/kubernetes/logs.go

@@ -185,7 +185,6 @@ func streamContainerLogsToDisk(ctx context.Context, appID string, appLogWriter,
 func getPods(ctx context.Context, appID string, podClient v1.PodInterface) (*corev1.PodList, error) {
 	listCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	labelSelector := fmt.Sprintf("%s=%s", daprAppIDKey, appID)
-	fmt.Println("Select", labelSelector)
 	podList, err := podClient.List(listCtx, metav1.ListOptions{
 		LabelSelector: labelSelector,
 	})

pkg/kubernetes/run.go

@@ -106,6 +106,7 @@ func Run(runFilePath string, config runfileconfig.RunFileConfig) (bool, error) {
 	daprsyscall.SetupShutdownNotify(sigCh)
 
 	runStates := []runState{}
+	print.InfoStatusEvent(os.Stdout, "This is a preview feature and subject to change in future releases.")
 
 	for _, app := range config.Apps {
 		print.StatusEvent(os.Stdout, print.LogInfo, "Validating config and starting app %q", app.RunConfig.AppID)
@@ -360,7 +361,11 @@ func writeYamlFile(app runfileconfig.App, svc serviceConfig, dep deploymentConfi
 }
 
 func deployYamlToK8s(yamlToDeployPath string) error {
-	_, err := utils.RunCmdAndWait("kubectl", "apply", "-f", yamlToDeployPath)
+	_, err := os.Stat(yamlToDeployPath)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("error given file %q does not exist", yamlToDeployPath)
+	}
+	_, err = utils.RunCmdAndWait("kubectl", "apply", "-f", yamlToDeployPath)
 	if err != nil {
 		return fmt.Errorf("error deploying the yaml %s to Kubernetes: %w", yamlToDeployPath, err)
 	}
@@ -369,9 +374,13 @@ func deployYamlToK8s(yamlToDeployPath string) error {
 
 func deleteYamlK8s(yamlToDeletePath string) error {
 	print.InfoStatusEvent(os.Stdout, "Deleting %q from Kubernetes", yamlToDeletePath)
-	_, err := utils.RunCmdAndWait("kubectl", "delete", "-f", yamlToDeletePath)
+	_, err := os.Stat(yamlToDeletePath)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("error given file %q does not exist", yamlToDeletePath)
+	}
+	_, err = utils.RunCmdAndWait("kubectl", "delete", "-f", yamlToDeletePath)
 	if err != nil {
-		return fmt.Errorf("error deploying the yaml %s to Kubernetes: %w", yamlToDeletePath, err)
+		return fmt.Errorf("error deleting the yaml %s from Kubernetes: %w", yamlToDeletePath, err)
 	}
 	return nil
 }
@@ -401,7 +410,10 @@ func gracefullyShutdownK8sDeployment(runStates []runState, client k8s.Interface,
 		}
 
 		// shutdown logs.
-		r.logCancel()
+		if r.logCancel != nil { // checking nil, in scenarios where deployments are not run correctly.
+			r.logCancel()
+		}
+
 		errs = append(errs, r.app.CloseAppLogFile(), r.app.CloseDaprdLogFile())
 	}
 	return errors.Join(errs...)
@@ -413,14 +425,14 @@ func monitorK8sPods(ctx context.Context, client k8s.Interface, namespace string,
 	wg := sync.WaitGroup{}
 
 	for _, r := range runStates {
+		wg.Add(1)
 		go func(appID string, wg *sync.WaitGroup) {
-			err := waitPodDeleted(ctx, client, namespace, r.app.AppID)
+			err := waitPodDeleted(ctx, client, namespace, appID)
 			if err != nil && strings.Contains(err.Error(), podWatchErrTemplate) {
 				print.WarningStatusEvent(os.Stderr, "Error monitoring Kubernetes pod(s) for app %q.", appID)
 			}
 			wg.Done()
 		}(r.app.AppID, &wg)
-		wg.Add(1)
 	}
 	wg.Wait()
 	// Send signal to gracefully close log writers and shut down process.

pkg/kubernetes/stop.go

@@ -17,10 +17,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	corev1 "k8s.io/api/core/v1"
 
+	"github.com/dapr/cli/pkg/print"
 	"github.com/dapr/cli/pkg/runfileconfig"
 )
 
@@ -32,20 +34,34 @@ func Stop(runFilePath string, config runfileconfig.RunFileConfig) error {
 		return fmt.Errorf("error getting k8s client for monitoring pod deletion: %w", cErr)
 	}
 
+	var err error
 	namespace := corev1.NamespaceDefault
 	for _, app := range config.Apps {
+		appError := false
 		deployDir := app.GetDeployDir()
 		serviceFilePath := filepath.Join(deployDir, serviceFileName)
 		deploymentFilePath := filepath.Join(deployDir, deploymentFileName)
 		if app.CreateService {
-			errs = append(errs, deleteYamlK8s(serviceFilePath))
+			err = deleteYamlK8s(serviceFilePath)
+			if err != nil {
+				appError = true
+			}
+			errs = append(errs, err)
 		}
-		errs = append(errs, deleteYamlK8s(deploymentFilePath))
-		ctx, cancel := context.WithTimeout(context.Background(), podCreationDeletionTimeout)
+		err = deleteYamlK8s(deploymentFilePath)
+		if err != nil {
+			appError = true
+		}
+		errs = append(errs, err)
+		if !appError {
+			ctx, cancel := context.WithTimeout(context.Background(), podCreationDeletionTimeout)
 
-		// Ignoring errors here as it will anyway be printed in the other dapr cli process.
-		waitPodDeleted(ctx, client, namespace, app.AppID)
-		cancel()
+			// Ignoring errors here as it will anyway be printed in the other dapr cli process.
+			waitPodDeleted(ctx, client, namespace, app.AppID)
+			cancel()
+		} else {
+			print.WarningStatusEvent(os.Stderr, "Error stopping deployment for app %q in file %q", app.AppID, runFilePath)
+		}
 	}
 	return errors.Join(errs...)
 }

tests/e2e/common/common.go

@@ -144,7 +144,7 @@ func UpgradeTest(details VersionDetails, opts TestOptions) func(t *testing.T) {
 	}
 }
 
-func EnsureUninstall(all bool) (string, error) {
+func EnsureUninstall(all bool, devEnabled bool) (string, error) {
 	daprPath := GetDaprPath()
 
 	var _command [10]string
@@ -153,6 +153,9 @@ func EnsureUninstall(all bool) (string, error) {
 	if all {
 		command = append(command, "--all")
 	}
+	if devEnabled {
+		command = append(command, "--dev")
+	}
 
 	command = append(command,
 		"-n", DaprTestNamespace,
@@ -179,6 +182,14 @@ func DeleteCRD(crds []string) func(*testing.T) {
 
 // Get Test Cases.
 
+func GetInstallOnlyTest(details VersionDetails, opts TestOptions) TestCase {
+	return TestCase{"install " + details.RuntimeVersion, installTest(details, opts)}
+}
+
+func GetUninstallOnlyTest(details VersionDetails, opts TestOptions) TestCase {
+	return TestCase{"uninstall " + details.RuntimeVersion, uninstallTest(opts.UninstallAll, opts.DevEnabled)} // waits for pod deletion.
+}
+
 func GetTestsOnInstall(details VersionDetails, opts TestOptions) []TestCase {
 	return []TestCase{
 		{"install " + details.RuntimeVersion, installTest(details, opts)},
@@ -799,7 +810,7 @@ func installTest(details VersionDetails, opts TestOptions) func(t *testing.T) {
 
 func uninstallTest(all bool, devEnabled bool) func(t *testing.T) {
 	return func(t *testing.T) {
-		output, err := EnsureUninstall(all)
+		output, err := EnsureUninstall(all, devEnabled)
 		t.Log(output)
 		require.NoError(t, err, "uninstall failed")
 		// wait for pods to be deleted completely.
@@ -833,7 +844,7 @@ func uninstallTest(all bool, devEnabled bool) func(t *testing.T) {
 
 func kubernetesTestOnUninstall() func(t *testing.T) {
 	return func(t *testing.T) {
-		_, err := EnsureUninstall(true)
+		_, err := EnsureUninstall(true, true)
 		require.NoError(t, err, "uninstall failed")
 		daprPath := GetDaprPath()
 		output, err := spawn.Command(daprPath, "uninstall", "-k")

tests/e2e/kubernetes/clean_env.go

@@ -0,0 +1,57 @@
+//go:build e2e || templatek8s
+// +build e2e templatek8s
+
+/*
+Copyright 2021 The Dapr Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/dapr/cli/tests/e2e/common"
+)
+
+var (
+	currentRuntimeVersion     string
+	currentDashboardVersion   string
+	currentVersionDetails     common.VersionDetails
+	clusterRoles1_9_X         = []string{"dapr-operator-admin", "dashboard-reader"}
+	clusterRoleBindings1_9_X  = []string{"dapr-operator", "dapr-role-tokenreview-binding", "dashboard-reader-global"}
+	clusterRoles1_10_X        = []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}
+	clusterRoleBindings1_10_X = []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}
+)
+
+// ensureCleanEnv function needs to be called in every Test function.
+// sets necessary variable values and uninstalls any previously installed `dapr`.
+func ensureCleanEnv(t *testing.T, useDaprLatestVersion bool) {
+	currentRuntimeVersion, currentDashboardVersion = common.GetVersionsFromEnv(t, useDaprLatestVersion)
+
+	currentVersionDetails = common.VersionDetails{
+		RuntimeVersion:       currentRuntimeVersion,
+		DashboardVersion:     currentDashboardVersion,
+		CustomResourceDefs:   []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"},
+		ImageVariant:         "",
+		UseDaprLatestVersion: useDaprLatestVersion,
+	}
+	if strings.HasPrefix(currentRuntimeVersion, "1.9.") {
+		currentVersionDetails.ClusterRoles = clusterRoles1_9_X
+		currentVersionDetails.ClusterRoleBindings = clusterRoleBindings1_9_X
+	} else {
+		currentVersionDetails.ClusterRoles = clusterRoles1_10_X
+		currentVersionDetails.ClusterRoleBindings = clusterRoleBindings1_10_X
+	}
+	// Ensure a clean environment
+	common.EnsureUninstall(true, true) // does not wait for pod deletion
+}

tests/e2e/kubernetes/kubernetes_test.go

@@ -17,45 +17,11 @@ limitations under the License.
 package kubernetes_test
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/dapr/cli/tests/e2e/common"
 )
 
-var (
-	currentRuntimeVersion     string
-	currentDashboardVersion   string
-	currentVersionDetails     common.VersionDetails
-	clusterRoles1_9_X         = []string{"dapr-operator-admin", "dashboard-reader"}
-	clusterRoleBindings1_9_X  = []string{"dapr-operator", "dapr-role-tokenreview-binding", "dashboard-reader-global"}
-	clusterRoles1_10_X        = []string{"dapr-dashboard", "dapr-injector", "dapr-operator-admin", "dapr-placement", "dapr-sentry"}
-	clusterRoleBindings1_10_X = []string{"dapr-operator-admin", "dapr-dashboard", "dapr-injector", "dapr-placement", "dapr-sentry"}
-)
-
-// ensureCleanEnv function needs to be called in every Test function.
-// sets necessary variable values and uninstalls any previously installed `dapr`.
-func ensureCleanEnv(t *testing.T, useDaprLatestVersion bool) {
-	currentRuntimeVersion, currentDashboardVersion = common.GetVersionsFromEnv(t, useDaprLatestVersion)
-
-	currentVersionDetails = common.VersionDetails{
-		RuntimeVersion:       currentRuntimeVersion,
-		DashboardVersion:     currentDashboardVersion,
-		CustomResourceDefs:   []string{"components.dapr.io", "configurations.dapr.io", "subscriptions.dapr.io", "resiliencies.dapr.io", "httpendpoints.dapr.io"},
-		ImageVariant:         "",
-		UseDaprLatestVersion: useDaprLatestVersion,
-	}
-	if strings.HasPrefix(currentRuntimeVersion, "1.9.") {
-		currentVersionDetails.ClusterRoles = clusterRoles1_9_X
-		currentVersionDetails.ClusterRoleBindings = clusterRoleBindings1_9_X
-	} else {
-		currentVersionDetails.ClusterRoles = clusterRoles1_10_X
-		currentVersionDetails.ClusterRoleBindings = clusterRoleBindings1_10_X
-	}
-	// Ensure a clean environment
-	common.EnsureUninstall(true) // does not wait for pod deletion
-}
-
 func TestKubernetesNonHAModeMTLSDisabled(t *testing.T) {
 	// ensure clean env for test
 	ensureCleanEnv(t, false)
@@ -393,7 +359,7 @@ func TestKubernetesUninstall(t *testing.T) {
 }
 
 func TestRenewCertWithIncorrectFlags(t *testing.T) {
-	common.EnsureUninstall(true)
+	common.EnsureUninstall(true, true)
 
 	tests := []common.TestCase{}
 	installOpts := common.TestOptions{