diff --git a/.gitignore b/.gitignore index a3b201b5..4d42b5f3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ Dockerfile.cross *~ test/e2e/testdata/persistentvolumes.yaml +test/e2e/testdata/values-mantle-primary.yaml include/ diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 71ea2baf..2b907bbe 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -36,21 +36,49 @@ setup: .PHONY: test test: - $(MAKE) launch-cluster MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_PRIMARY) - $(MINIKUBE) profile $(MINIKUBE_PROFILE_PRIMARY) + $(MAKE) launch-minikube MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_PRIMARY) + $(MAKE) install-rook-ceph-operator + $(MAKE) install-rook-ceph-cluster1 + $(MAKE) install-rook-ceph-cluster2 + $(MAKE) install-mantle-cluster-wide + $(MAKE) install-mantle \ + NAMESPACE=$(CEPH_CLUSTER1_NAMESPACE) \ + HELM_RELEASE=mantle \ + VALUES_YAML=testdata/values-mantle1.yaml + $(MAKE) install-mantle \ + NAMESPACE=$(CEPH_CLUSTER2_NAMESPACE) \ + HELM_RELEASE=mantle2 \ + VALUES_YAML=testdata/values-mantle2.yaml $(MAKE) do_test .PHONY: test-multiple-k8s-clusters test-multiple-k8s-clusters: - $(MAKE) launch-cluster MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_PRIMARY) - $(MAKE) launch-cluster MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_SECONDARY) +# set up a k8s cluster for secondary mantle + $(MAKE) launch-minikube MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_SECONDARY) + $(MAKE) install-rook-ceph-operator + $(MAKE) install-rook-ceph-cluster1 + $(MAKE) install-mantle-cluster-wide + $(MAKE) install-mantle \ + NAMESPACE=$(CEPH_CLUSTER1_NAMESPACE) \ + HELM_RELEASE=mantle \ + VALUES_YAML=testdata/values-mantle-secondary.yaml + $(KUBECTL) apply -f testdata/secondary-mantle-service.yaml +# set up a k8s cluster for primary mantle + $(MAKE) launch-minikube MINIKUBE_PROFILE=$(MINIKUBE_PROFILE_PRIMARY) + $(MAKE) install-rook-ceph-operator + $(MAKE) install-rook-ceph-cluster1 + $(MAKE) install-mantle-cluster-wide + sed \ + -e "s%{ENDPOINT}%$$($(MINIKUBE) service list -p $(MINIKUBE_PROFILE_SECONDARY) -o json | jq -r '.[].URLs | select(. | length > 0)[]' | head -1 | sed -r 's/^http:\/\///')%" \ + testdata/values-mantle-primary-template.yaml \ + > testdata/values-mantle-primary.yaml + $(MAKE) install-mantle \ + NAMESPACE=$(CEPH_CLUSTER1_NAMESPACE) \ + HELM_RELEASE=mantle \ + VALUES_YAML=testdata/values-mantle-primary.yaml +# start testing $(MINIKUBE) profile $(MINIKUBE_PROFILE_PRIMARY) - env \ - MINIKUBE=$(MINIKUBE) \ - MINIKUBE_HOME=$(MINIKUBE_HOME) \ - MINIKUBE_PROFILE_PRIMARY=$(MINIKUBE_PROFILE_PRIMARY) \ - MINIKUBE_PROFILE_SECONDARY=$(MINIKUBE_PROFILE_SECONDARY) \ - ./test-multiple-k8s-clusters.sh + $(MAKE) do-test-multik8s .PHONY: clean clean: @@ -75,9 +103,9 @@ $(HELM): | $(BINDIR) $(CURL) https://get.helm.sh/helm-v$(HELM_VERSION)-linux-amd64.tar.gz \ | tar xvz -C $(BINDIR) --strip-components 1 linux-amd64/helm -.PHONY: launch-cluster -launch-cluster: MINIKUBE_PROFILE= -launch-cluster: +.PHONY: launch-minikube +launch-minikube: MINIKUBE_PROFILE= +launch-minikube: # TODO: Is there any better way to verify whether k8s cluster is available or not? if $(MINIKUBE) profile $(MINIKUBE_PROFILE) |& grep "not found" > /dev/null; then \ $(MINIKUBE) start \ @@ -92,8 +120,14 @@ launch-cluster: fi $(MINIKUBE) profile $(MINIKUBE_PROFILE) $(MAKE) image-build - $(MAKE) launch-rook-ceph - $(MAKE) setup-components + $(MAKE) create-loop-dev + sed \ + -e "s%{LOOP_DEV}%$(LOOP_DEV)%" \ + -e "s%{LOOP_DEV2}%$(LOOP_DEV2)%" \ + -e "s%{NODE_NAME}%$(NODE_NAME)%" \ + testdata/persistentvolumes-template.yaml \ + > testdata/persistentvolumes.yaml + $(KUBECTL) apply -f testdata/persistentvolumes.yaml .PHONY: create-loop-dev create-loop-dev: @@ -122,27 +156,26 @@ wait-deploy-ready: exit 1; \ fi -.PHONY: launch-rook-ceph -launch-rook-ceph: create-loop-dev +.PHONY: install-rook-ceph-operator +install-rook-ceph-operator: $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ --create-namespace --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values.yaml --wait \ rook-ceph rook-ceph - sed \ - -e "s%{LOOP_DEV}%$(LOOP_DEV)%" \ - -e "s%{LOOP_DEV2}%$(LOOP_DEV2)%" \ - -e "s%{NODE_NAME}%$(NODE_NAME)%" \ - testdata/persistentvolumes-template.yaml \ - > testdata/persistentvolumes.yaml - $(KUBECTL) apply -f testdata/persistentvolumes.yaml + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator + +.PHONY: install-rook-ceph-cluster1 +install-rook-ceph-cluster1: $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml \ --wait rook-ceph-cluster rook-ceph-cluster + $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0 + +.PHONY: install-rook-ceph-cluster2 +install-rook-ceph-cluster2: $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ --create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \ --set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \ --wait rook-ceph-cluster2 rook-ceph-cluster - $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-operator - $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER1_NAMESPACE) DEPLOY=rook-ceph-osd-0 $(MAKE) wait-deploy-ready NS=$(CEPH_CLUSTER2_NAMESPACE) DEPLOY=rook-ceph-osd-0 .PHONY: image-build @@ -151,13 +184,17 @@ image-build: $(MAKE) -C ../.. docker-build $(MINIKUBE) ssh -- docker images -.PHONY: setup-components -setup-components: +.PHONY: install-mantle-cluster-wide +install-mantle-cluster-wide: $(HELM) upgrade --install mantle-cluster-wide ../../charts/mantle-cluster-wide/ --wait - $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER1_NAMESPACE) mantle ../../charts/mantle/ --wait -f testdata/values-mantle.yaml - $(HELM) upgrade --install --namespace=$(CEPH_CLUSTER2_NAMESPACE) mantle2 ../../charts/mantle/ --wait - $(KUBECTL) rollout restart -n $(CEPH_CLUSTER1_NAMESPACE) deploy/mantle-controller - $(KUBECTL) rollout restart -n $(CEPH_CLUSTER2_NAMESPACE) deploy/mantle2-controller + +.PHONY: install-mantle +install-mantle: NAMESPACE= +install-mantle: HELM_RELEASE= +install-mantle: VALUES_YAML= +install-mantle: + $(HELM) upgrade --install --namespace=$(NAMESPACE) $(HELM_RELEASE) ../../charts/mantle/ --wait -f $(VALUES_YAML) + $(KUBECTL) rollout restart -n $(NAMESPACE) deploy/$(HELM_RELEASE)-controller .PHONY: do_test do_test: $(GINKGO) @@ -166,3 +203,12 @@ do_test: $(GINKGO) E2ETEST=1 \ KUBECTL=$(KUBECTL) \ $(GINKGO) --fail-fast -v $(GINKGO_FLAGS) singlek8s + +.PHONY: do-test-multik8s +do-test-multik8s: $(GINKGO) + env \ + PATH=${PATH} \ + E2ETEST=1 \ + KUBECTL_PRIMARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_PRIMARY) kubectl -- " \ + KUBECTL_SECONDARY="$(MINIKUBE) -p $(MINIKUBE_PROFILE_SECONDARY) kubectl -- " \ + $(GINKGO) --fail-fast -v $(GINKGO_FLAGS) multik8s diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go new file mode 100644 index 00000000..0c6536d8 --- /dev/null +++ b/test/e2e/multik8s/suite_test.go @@ -0,0 +1,91 @@ +package multik8s + +import ( + _ "embed" + "errors" + "os" + "testing" + "time" + + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + + mantlev1 "github.com/cybozu-go/mantle/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMtest(t *testing.T) { + if os.Getenv("E2ETEST") == "" { + t.Skip("Run under e2e/") + } + + RegisterFailHandler(Fail) + + SetDefaultEventuallyPollingInterval(time.Second) + SetDefaultEventuallyTimeout(3 * time.Minute) + + RunSpecs(t, "rbd backup system test with multiple k8s clusters") +} + +var _ = Describe("Mantle", func() { + Context("wait controller to be ready", waitControllerToBeReady) + Context("replication test", replicationTestSuite) +}) + +func waitControllerToBeReady() { + It("wait for mantle-controller to be ready", func() { + Eventually(func() error { + return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") + }).Should(Succeed()) + + Eventually(func() error { + return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") + }).Should(Succeed()) + }) +} + +func replicationTestSuite() { + Describe("replication test", func() { + It("should eventually set SyncedToRemote of a MantleBackup to True after it is created", func() { + namespace := util.GetUniqueName("ns-") + pvcName := util.GetUniqueName("pvc-") + backupName := util.GetUniqueName("mb-") + scName := util.GetUniqueName("sc-") + poolName := util.GetUniqueName("pool-") + + By("setting up the environment") + Eventually(func() error { + return createNamespace(primaryK8sCluster, namespace) + }).Should(Succeed()) + Eventually(func() error { + return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace, poolName, scName) + }).Should(Succeed()) + Eventually(func() error { + return applyPVCTemplate(primaryK8sCluster, namespace, pvcName, scName) + }).Should(Succeed()) + + By("creating a MantleBackup object") + Eventually(func() error { + return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName) + }).Should(Succeed()) + + By("checking MantleBackup's SyncedToRemote status") + Eventually(func() error { + mb, err := getMB(primaryK8sCluster, namespace, backupName) + if err != nil { + return err + } + cond := meta.FindStatusCondition(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) + if cond == nil { + return errors.New("couldn't find condition SyncedToRemote") + } + if cond.Status != metav1.ConditionTrue { + return errors.New("status of SyncedToRemote condition is not True") + } + return nil + }).Should(Succeed()) + }) + }) +} diff --git a/test/e2e/multik8s/testdata/mantlebackup-template.yaml b/test/e2e/multik8s/testdata/mantlebackup-template.yaml new file mode 100644 index 00000000..4e25d7e2 --- /dev/null +++ b/test/e2e/multik8s/testdata/mantlebackup-template.yaml @@ -0,0 +1,13 @@ +apiVersion: mantle.cybozu.io/v1 +kind: MantleBackup +metadata: + labels: + app.kubernetes.io/name: mantlebackup + app.kubernetes.io/instance: %s + app.kubernetes.io/part-of: mantle + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: mantle + name: %s + namespace: %s +spec: + pvc: %s diff --git a/test/e2e/multik8s/testdata/pvc-template.yaml b/test/e2e/multik8s/testdata/pvc-template.yaml new file mode 100644 index 00000000..0bcdf27e --- /dev/null +++ b/test/e2e/multik8s/testdata/pvc-template.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: %s +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: %s diff --git a/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml b/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml new file mode 100644 index 00000000..fb3a2210 --- /dev/null +++ b/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml @@ -0,0 +1,30 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: %s + namespace: %s +spec: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: %s +provisioner: rook-ceph.rbd.csi.ceph.com +parameters: + clusterID: %s + pool: %s + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: %s + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: %s + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: %s + csi.storage.k8s.io/fstype: ext4 +allowVolumeExpansion: true +reclaimPolicy: Delete diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go new file mode 100644 index 00000000..b3afee5f --- /dev/null +++ b/test/e2e/multik8s/util.go @@ -0,0 +1,130 @@ +package multik8s + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + + mantlev1 "github.com/cybozu-go/mantle/api/v1" +) + +const ( + cephClusterNamespace = "rook-ceph" + primaryK8sCluster = 1 + secondaryK8sCluster = 2 +) + +var ( + //go:embed testdata/pvc-template.yaml + testPVCTemplate string + //go:embed testdata/rbd-pool-sc-template.yaml + testRBDPoolSCTemplate string + //go:embed testdata/mantlebackup-template.yaml + testMantleBackupTemplate string + + kubectlPrefixPrimary = os.Getenv("KUBECTL_PRIMARY") + kubectlPrefixSecondary = os.Getenv("KUBECTL_SECONDARY") +) + +func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { + var stdout, stderr bytes.Buffer + command := exec.Command(cmd, args...) + command.Stdout = &stdout + command.Stderr = &stderr + + if len(input) != 0 { + command.Stdin = bytes.NewReader(input) + } + + err := command.Run() + return stdout.Bytes(), stderr.Bytes(), err +} + +// input can be nil +func kubectl(clusterNo int, input []byte, args ...string) ([]byte, []byte, error) { + kubectlPrefix := "" + switch clusterNo { + case primaryK8sCluster: + kubectlPrefix = kubectlPrefixPrimary + case secondaryK8sCluster: + kubectlPrefix = kubectlPrefixSecondary + default: + panic(fmt.Sprintf("invalid clusterNo: %d", clusterNo)) + } + if len(kubectlPrefix) == 0 { + panic("Either KUBECTL_PRIMARY or KUBECTL_SECONDARY environment variable is not set") + } + fields := strings.Fields(kubectlPrefix) + fields = append(fields, args...) + return execAtLocal(fields[0], input, fields[1:]...) +} + +func checkDeploymentReady(clusterNo int, namespace, name string) error { + _, stderr, err := kubectl( + clusterNo, nil, + "-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m", + ) + if err != nil { + return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func applyMantleBackupTemplate(clusterNo int, namespace, pvcName, backupName string) error { + manifest := fmt.Sprintf(testMantleBackupTemplate, backupName, backupName, namespace, pvcName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply mantlebackup failed. err: %w", err) + } + return nil +} + +func applyPVCTemplate(clusterNo int, namespace, name, storageClassName string) error { + manifest := fmt.Sprintf(testPVCTemplate, name, storageClassName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply pvc failed. err: %w", err) + } + return nil +} + +func createNamespace(clusterNo int, name string) error { + _, _, err := kubectl(clusterNo, nil, "create", "ns", name) + if err != nil { + return fmt.Errorf("kubectl create ns failed. err: %w", err) + } + return nil +} + +func applyRBDPoolAndSCTemplate(clusterNo int, namespace, poolName, storageClassName string) error { + manifest := fmt.Sprintf( + testRBDPoolSCTemplate, poolName, namespace, + storageClassName, namespace, poolName, namespace, namespace, namespace) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return err + } + return nil +} + +func getObject[T any](clusterNo int, kind, namespace, name string) (*T, error) { + stdout, _, err := kubectl(clusterNo, nil, "get", kind, "-n", namespace, name, "-o", "json") + if err != nil { + return nil, err + } + + var obj T + if err := json.Unmarshal(stdout, &obj); err != nil { + return nil, err + } + + return &obj, nil +} + +func getMB(clusterNo int, namespace, name string) (*mantlev1.MantleBackup, error) { + return getObject[mantlev1.MantleBackup](clusterNo, "mantlebackup", namespace, name) +} diff --git a/test/e2e/test-multiple-k8s-clusters.sh b/test/e2e/test-multiple-k8s-clusters.sh deleted file mode 100755 index a1288307..00000000 --- a/test/e2e/test-multiple-k8s-clusters.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/bash -xeu - -set -o pipefail - -# Exits with an "unbound variable" error if one of the following environment -# variables is undefined, thanks to "-u" option to bash. -echo "${MINIKUBE}" -echo "${MINIKUBE_HOME}" -echo "${MINIKUBE_PROFILE_PRIMARY}" -echo "${MINIKUBE_PROFILE_SECONDARY}" - -cat < 0)[]' | head -1) - -# Exits with an errornous exit code if curl fails, thanks to "-e" option to bash. -${MINIKUBE} -p ${MINIKUBE_PROFILE_SECONDARY} kubectl -- exec -it -n rook-ceph deploy/rook-ceph-tools -- curl -vvv ${URL} > /dev/null diff --git a/test/e2e/testdata/secondary-mantle-service.yaml b/test/e2e/testdata/secondary-mantle-service.yaml new file mode 100644 index 00000000..4f1d2839 --- /dev/null +++ b/test/e2e/testdata/secondary-mantle-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: mantle + namespace: rook-ceph +spec: + type: NodePort + ports: + - port: 58080 + targetPort: 58080 + protocol: TCP + name: grpc + selector: + app.kubernetes.io/instance: mantle + app.kubernetes.io/name: mantle diff --git a/test/e2e/testdata/values-mantle-primary-template.yaml b/test/e2e/testdata/values-mantle-primary-template.yaml new file mode 100644 index 00000000..b0871cdb --- /dev/null +++ b/test/e2e/testdata/values-mantle-primary-template.yaml @@ -0,0 +1,3 @@ +controller: + role: primary + mantleServiceEndpoint: {ENDPOINT} diff --git a/test/e2e/testdata/values-mantle-secondary.yaml b/test/e2e/testdata/values-mantle-secondary.yaml new file mode 100644 index 00000000..cdeb8a23 --- /dev/null +++ b/test/e2e/testdata/values-mantle-secondary.yaml @@ -0,0 +1,5 @@ +controller: + role: secondary + mantleServiceEndpoint: ":58080" + ports: + - containerPort: 58080 diff --git a/test/e2e/testdata/values-mantle.yaml b/test/e2e/testdata/values-mantle1.yaml similarity index 100% rename from test/e2e/testdata/values-mantle.yaml rename to test/e2e/testdata/values-mantle1.yaml diff --git a/test/e2e/testdata/values-mantle2.yaml b/test/e2e/testdata/values-mantle2.yaml new file mode 100644 index 00000000..e69de29b