From b4b06ecbeb5f4fd11ca9b9f3299c8dd8f495d0ab Mon Sep 17 00:00:00 2001
From: Ryotaro Banno <ryotaro.banno@gmail.com>
Date: Tue, 27 Aug 2024 02:55:12 +0000
Subject: [PATCH] wip: add multik8s Ginkgo test replacing shell script with

---
 test/e2e/Makefile                      |  16 ++--
 test/e2e/multik8s/suite_test.go        | 101 +++++++++++++++++++++++++
 test/e2e/test-multiple-k8s-clusters.sh |  53 -------------
 3 files changed, 111 insertions(+), 59 deletions(-)
 create mode 100644 test/e2e/multik8s/suite_test.go
 delete mode 100755 test/e2e/test-multiple-k8s-clusters.sh

diff --git a/test/e2e/Makefile b/test/e2e/Makefile
index 71ea2baf..1fb90c8a 100644
--- a/test/e2e/Makefile
+++ b/test/e2e/Makefile
@@ -45,12 +45,7 @@ test-multiple-k8s-clusters:
 	$(MAKE) launch-cluster MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_PRIMARY)
 	$(MAKE) launch-cluster MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_SECONDARY)
 	$(MINIKUBE) profile $(MINIKUBE_PROFILE_PRIMARY)
-	env \
-		MINIKUBE=$(MINIKUBE) \
-		MINIKUBE_HOME=$(MINIKUBE_HOME) \
-		MINIKUBE_PROFILE_PRIMARY=$(MINIKUBE_PROFILE_PRIMARY) \
-		MINIKUBE_PROFILE_SECONDARY=$(MINIKUBE_PROFILE_SECONDARY) \
-		./test-multiple-k8s-clusters.sh
+	$(MAKE) do-test-multik8s
 
 .PHONY: clean
 clean:
@@ -166,3 +161,12 @@ do_test: $(GINKGO)
 	E2ETEST=1 \
 	KUBECTL=$(KUBECTL) \
 	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) singlek8s
+
+.PHONY: do-test-multik8s
+do-test-multik8s: $(GINKGO)
+	env \
+	PATH=${PATH} \
+	E2ETEST=1 \
+	KUBECTL0="$(MINIKUBE) -p $(MINIKUBE_PROFILE_PRIMARY) kubectl -- " \
+	KUBECTL1="$(MINIKUBE) -p $(MINIKUBE_PROFILE_SECONDARY) kubectl -- " \
+	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) multik8s
diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go
new file mode 100644
index 00000000..d3c3ff9e
--- /dev/null
+++ b/test/e2e/multik8s/suite_test.go
@@ -0,0 +1,101 @@
+package multik8s
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMtest(t *testing.T) {
+	if os.Getenv("E2ETEST") == "" {
+		t.Skip("Run under e2e/")
+	}
+
+	RegisterFailHandler(Fail)
+
+	SetDefaultEventuallyPollingInterval(time.Second)
+	SetDefaultEventuallyTimeout(3 * time.Minute)
+
+	RunSpecs(t, "rbd backup system test with multiple k8s clusters")
+}
+
+var _ = Describe("Mantle", func() {
+	Context("wait controller to be ready", waitControllerToBeReady)
+	Context("replication", replicationTestSuite)
+})
+
+var (
+	kubectlPrefix0 = os.Getenv("KUBECTL0")
+	kubectlPrefix1 = os.Getenv("KUBECTL1")
+)
+
+func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) {
+	var stdout, stderr bytes.Buffer
+	command := exec.Command(cmd, args...)
+	command.Stdout = &stdout
+	command.Stderr = &stderr
+
+	if len(input) != 0 {
+		command.Stdin = bytes.NewReader(input)
+	}
+
+	err := command.Run()
+	return stdout.Bytes(), stderr.Bytes(), err
+}
+
+// input can be nil
+func kubectl(clusterNo int, input []byte, args ...string) ([]byte, []byte, error) {
+	kubectlPrefix := ""
+	switch clusterNo {
+	case 0:
+		kubectlPrefix = kubectlPrefix0
+	case 1:
+		kubectlPrefix = kubectlPrefix1
+	default:
+		panic(fmt.Sprintf("invalid clusterNo: %d", clusterNo))
+	}
+	if len(kubectlPrefix) == 0 {
+		panic("Either KUBECTL0 or KUBECTL1 environment variable is not set")
+	}
+	fields := strings.Fields(kubectlPrefix)
+	fields = append(fields, args...)
+	return execAtLocal(fields[0], input, fields[1:]...)
+}
+
+func checkDeploymentReady(clusterNo int, namespace, name string) error {
+	_, stderr, err := kubectl(
+		clusterNo, nil,
+		"-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m",
+	)
+	if err != nil {
+		return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err)
+	}
+	return nil
+}
+
+func waitControllerToBeReady() {
+	It("wait for mantle-controller to be ready", func() {
+		Eventually(func() error {
+			return checkDeploymentReady(0, "rook-ceph", "mantle-controller")
+		}).Should(Succeed())
+
+		Eventually(func() error {
+			return checkDeploymentReady(0, "rook-ceph", "mantle-controller")
+		}).Should(Succeed())
+	})
+}
+
+func replicationTestSuite() {
+	Describe("make sure SyncToRemote becomes true after a MantleBackup is created", func() {
+		// FIXME
+		return
+	})
+}
diff --git a/test/e2e/test-multiple-k8s-clusters.sh b/test/e2e/test-multiple-k8s-clusters.sh
deleted file mode 100755
index a1288307..00000000
--- a/test/e2e/test-multiple-k8s-clusters.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/bash -xeu
-
-set -o pipefail
-
-# Exits with an "unbound variable" error if one of the following environment
-# variables is undefined, thanks to "-u" option to bash.
-echo "${MINIKUBE}"
-echo "${MINIKUBE_HOME}"
-echo "${MINIKUBE_PROFILE_PRIMARY}"
-echo "${MINIKUBE_PROFILE_SECONDARY}"
-
-cat <<EOS | ${MINIKUBE} -p ${MINIKUBE_PROFILE_PRIMARY} kubectl -- apply -f -
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: nginx
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: nginx
-  template:
-    metadata:
-      labels:
-        app: nginx
-    spec:
-      containers:
-      - name: nginx
-        image: nginx:1.27.0
-        ports:
-        - containerPort: 80
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  type: NodePort
-  ports:
-  - port: 8080
-    targetPort: 80
-    protocol: TCP
-    name: http
-  selector:
-    app: nginx
-EOS
-${MINIKUBE} -p ${MINIKUBE_PROFILE_PRIMARY} kubectl -- wait --for=jsonpath='{.status.readyReplicas}'=1 deploy/nginx
-
-${MINIKUBE} service list -p ${MINIKUBE_PROFILE_PRIMARY}
-URL=$(${MINIKUBE} service list -p ${MINIKUBE_PROFILE_PRIMARY} -o json | jq -r '.[].URLs | select(. | length > 0)[]' | head -1)
-
-# Exits with an errornous exit code if curl fails, thanks to "-e" option to bash.
-${MINIKUBE} -p ${MINIKUBE_PROFILE_SECONDARY} kubectl -- exec -it -n rook-ceph deploy/rook-ceph-tools -- curl -vvv ${URL} > /dev/null