From f1bf0f42b36f3a117558cfd6d1ef3810fab29106 Mon Sep 17 00:00:00 2001 From: Yuji Ito Date: Wed, 26 Jun 2024 09:29:35 +0000 Subject: [PATCH] tests for using multiple Rook/Ceph cluster environment Signed-off-by: Yuji Ito --- test/e2e/Makefile | 3 +- test/e2e/backup_test.go | 125 ++++------ test/e2e/multi_rook_ceph_test.go | 252 ++++++++++++++++++++ test/e2e/restore_test.go | 43 ++-- test/e2e/suite_test.go | 14 ++ test/e2e/testdata/rbd-pool-sc-template.yaml | 2 +- test/e2e/util.go | 136 ++++++++++- 7 files changed, 470 insertions(+), 105 deletions(-) create mode 100644 test/e2e/multi_rook_ceph_test.go diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 62b2c327..e9ce97bd 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -122,7 +122,8 @@ launch-rook-ceph: create-loop-dev > testdata/persistentvolumes.yaml $(KUBECTL) apply -f testdata/persistentvolumes.yaml $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ - --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml --wait rook-ceph-cluster rook-ceph-cluster + --namespace $(CEPH_CLUSTER1_NAMESPACE) -f testdata/values-cluster.yaml \ + --wait rook-ceph-cluster rook-ceph-cluster $(HELM) upgrade --install --version $(ROOK_CHART_VERSION) --repo https://charts.rook.io/release \ --create-namespace --namespace $(CEPH_CLUSTER2_NAMESPACE) -f testdata/values-cluster.yaml \ --set cephClusterSpec.dataDirHostPath=/var/lib/rook2 \ diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index a29098f5..215e1ce0 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,7 +1,6 @@ package e2e import ( - "encoding/json" "fmt" "strings" @@ -19,11 +18,9 @@ const ( ) type backupTest struct { - poolName string - storageClassName1 string - storageClassName2 string - tenantNamespace1 string - tenantNamespace2 string + poolName string + storageClassName string + tenantNamespace string pvcName1 string pvcName2 string @@ -34,11 +31,9 @@ type backupTest struct { func backupTestSuite() { test := &backupTest{ - poolName: util.GetUniqueName("pool-"), - storageClassName1: util.GetUniqueName("sc-"), - storageClassName2: util.GetUniqueName("sc-"), - tenantNamespace1: util.GetUniqueName("ns-"), - tenantNamespace2: util.GetUniqueName("ns-"), + poolName: util.GetUniqueName("pool-"), + storageClassName: util.GetUniqueName("sc-"), + tenantNamespace: util.GetUniqueName("ns-"), pvcName1: "rbd-pvc1", pvcName2: "rbd-pvc2", @@ -53,74 +48,47 @@ func backupTestSuite() { } func (test *backupTest) setupEnv() { - It("creating common resources", func() { - for _, ns := range []string{test.tenantNamespace1, test.tenantNamespace2} { - err := createNamespace(ns) - Expect(err).NotTo(HaveOccurred()) - } + It("setting up the test environment", func() { + fmt.Fprintf(GinkgoWriter, "%+v\n", *test) + }) - err := applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName1) + It("creating common resources", func() { + err := createNamespace(test.tenantNamespace) Expect(err).NotTo(HaveOccurred()) - err = applyRBDPoolAndSCTemplate(cephCluster2Namespace, test.poolName, test.storageClassName2) + err = applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName) Expect(err).NotTo(HaveOccurred()) for _, name := range []string{test.pvcName1, test.pvcName2} { By(fmt.Sprintf("Creating PVC, PV and RBD image (%s)", name)) - err = applyPVCTemplate(test.tenantNamespace1, name, test.storageClassName1) - Expect(err).NotTo(HaveOccurred()) - - pvName, err := getPVFromPVC(test.tenantNamespace1, name) - Expect(err).NotTo(HaveOccurred()) - - imageName, err := getImageNameFromPVName(pvName) + err = applyPVCTemplate(test.tenantNamespace, name, test.storageClassName) Expect(err).NotTo(HaveOccurred()) - - // Create a new RBD image in cephNamespace2 with the same name as imageName. - Eventually(func() error { - return createRBDImage(cephCluster2Namespace, test.poolName, imageName, 100) - }).Should(Succeed()) } }) - - It("waiting for mantle-controller to get ready", func() { - Eventually(func() error { - return checkDeploymentReady(cephCluster1Namespace, "mantle-controller") - }).Should(Succeed()) - }) } func (test *backupTest) teardownEnv() { - It("deleting MantleBackups", func() { - for _, mantleBackup := range []string{test.mantleBackupName1, test.mantleBackupName2, test.mantleBackupName3} { - _, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "mantlebackup", mantleBackup) - } + It("delete resources in the namespace: "+test.tenantNamespace, func() { + err := deleteNamespacedResource(test.tenantNamespace, "mantlebackup") + Expect(err).NotTo(HaveOccurred()) + err = deleteNamespacedResource(test.tenantNamespace, "pvc") + Expect(err).NotTo(HaveOccurred()) }) - It("deleting PVCs", func() { - for _, pvc := range []string{test.pvcName1, test.pvcName2} { - _, _, _ = kubectl("delete", "-n", test.tenantNamespace1, "pvc", pvc) - } - - By("Deleting RBD images in " + cephCluster2Namespace) - stdout, _, err := kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--", - "rbd", "ls", test.poolName, "--format=json") - if err == nil { - imageNames := []string{} - if err := json.Unmarshal(stdout, &imageNames); err == nil { - for _, imageName := range imageNames { - _, _, _ = kubectl("exec", "-n", cephCluster2Namespace, "deploy/rook-ceph-tools", "--", - "rbd", "rm", test.poolName+"/"+imageName) - } - } - } + It("delete namespace: "+test.tenantNamespace, func() { + _, stderr, err := kubectl("delete", "namespace", test.tenantNamespace) + Expect(err).NotTo(HaveOccurred(), string(stderr)) }) - It("deleting common resources", func() { - _, _, _ = kubectl("delete", "sc", test.storageClassName1, "--wait=false") - _, _, _ = kubectl("delete", "sc", test.storageClassName2, "--wait=false") - _, _, _ = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false") - _, _, _ = kubectl("delete", "-n", cephCluster2Namespace, "cephblockpool", test.poolName, "--wait=false") + It("clean up the SCs and RBD pools", func() { + _, stderr, err := kubectl("delete", "sc", test.storageClassName) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + + err = removeAllRBDImageAndSnap(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false") + Expect(err).NotTo(HaveOccurred()) }) } @@ -129,10 +97,10 @@ func (test *backupTest) testCase1() { createMantleBackupAndGetImage := func(mantleBackupName string) string { By("Creating MantleBackup") - err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName1, mantleBackupName) + err := applyMantleBackupTemplate(test.tenantNamespace, test.pvcName1, mantleBackupName) Expect(err).NotTo(HaveOccurred()) - pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName1) + pvName, err := getPVFromPVC(test.tenantNamespace, test.pvcName1) Expect(err).NotTo(HaveOccurred()) By("Waiting for RBD snapshot to be created") @@ -146,17 +114,6 @@ func (test *backupTest) testCase1() { return checkSnapshotExist(cephCluster1Namespace, test.poolName, imageName, mantleBackupName) }).Should(Succeed()) - By("Checking that the mantle-controller deployed for a certain Rook/Ceph cluster (i.e., " + - cephCluster1Namespace + ") doesn't create a snapshot for a MantleBackup for a different Rook/Ceph cluster (i.e., " + - cephCluster2Namespace + ")") - Consistently(func() error { - err := checkSnapshotExist(cephCluster2Namespace, test.poolName, imageName, mantleBackupName) - if err == nil { - return fmt.Errorf("a wrong snapshot exists. snapshotName: %s", mantleBackupName) - } - return nil - }).Should(Succeed()) - return imageName } @@ -170,10 +127,10 @@ func (test *backupTest) testCase1() { It("should create MantleBackups resources for different PVCs", func() { By("Creating a third MantleBackup for the other PVC") - err := applyMantleBackupTemplate(test.tenantNamespace1, test.pvcName2, test.mantleBackupName3) + err := applyMantleBackupTemplate(test.tenantNamespace, test.pvcName2, test.mantleBackupName3) Expect(err).NotTo(HaveOccurred()) - pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName2) + pvName, err := getPVFromPVC(test.tenantNamespace, test.pvcName2) Expect(err).NotTo(HaveOccurred()) By("Waiting for RBD snapshot to be created") @@ -189,12 +146,12 @@ func (test *backupTest) testCase1() { It("should not delete MantleBackup resource when delete backup target PVC", func() { By("Deleting backup target PVC") - _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "pvc", test.pvcName2) + _, _, err := kubectl("-n", test.tenantNamespace, "delete", "pvc", test.pvcName2) Expect(err).NotTo(HaveOccurred()) By("Checking backup target PVC deletion") Eventually(func() error { - stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "pvc", test.pvcName2) + stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "pvc", test.pvcName2) if err != nil { if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { return nil @@ -205,7 +162,7 @@ func (test *backupTest) testCase1() { }).Should(Succeed()) By("Checking that the status.conditions of the MantleBackup resource remain \"Bound\"") - stdout, _, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3, "-o", "json") + stdout, _, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName3, "-o", "json") Expect(err).NotTo(HaveOccurred()) var backup mantlev1.MantleBackup err = yaml.Unmarshal(stdout, &backup) @@ -216,7 +173,7 @@ func (test *backupTest) testCase1() { It("should delete MantleBackup resource", func() { By("Delete MantleBackup") - _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName1, "--wait=false") + _, _, err := kubectl("-n", test.tenantNamespace, "delete", "mantlebackup", test.mantleBackupName1, "--wait=false") Expect(err).NotTo(HaveOccurred()) By("Waiting for RBD snapshot to be deleted") @@ -231,7 +188,7 @@ func (test *backupTest) testCase1() { By("Checking MantleBackup resource deletion") Eventually(func() error { - stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName1) + stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName1) if err != nil { if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { return nil @@ -244,12 +201,12 @@ func (test *backupTest) testCase1() { It("should delete MantleBackup resource when backup target PVC is missing", func() { By("Deleting MantleBackup resource") - _, _, err := kubectl("-n", test.tenantNamespace1, "delete", "mantlebackup", test.mantleBackupName3) + _, _, err := kubectl("-n", test.tenantNamespace, "delete", "mantlebackup", test.mantleBackupName3) Expect(err).NotTo(HaveOccurred()) By("Checking MantleBackup resource deletion") Eventually(func() error { - stdout, stderr, err := kubectl("-n", test.tenantNamespace1, "get", "mantlebackup", test.mantleBackupName3) + stdout, stderr, err := kubectl("-n", test.tenantNamespace, "get", "mantlebackup", test.mantleBackupName3) if err != nil { if strings.Contains(string(stderr), kubectlIsNotFoundMessage) { return nil diff --git a/test/e2e/multi_rook_ceph_test.go b/test/e2e/multi_rook_ceph_test.go new file mode 100644 index 00000000..d21c0658 --- /dev/null +++ b/test/e2e/multi_rook_ceph_test.go @@ -0,0 +1,252 @@ +package e2e + +import ( + "fmt" + + "github.com/cybozu-go/mantle/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type multiRookCephTest struct { + poolName string // use the same pool name for both clusters + storageClassName1 string + storageClassName2 string + tenantNamespace1 string + tenantNamespace2 string + + pvcName string // use the same PVC name for both tenants + mantleBackupName string // use the same MantleBackup name for both tenants + mantleRestoreName string // use the same MantleRestore name for both tenants +} + +func multiRookCephTestSuite() { + test := &multiRookCephTest{ + poolName: util.GetUniqueName("pool-"), + storageClassName1: util.GetUniqueName("sc-"), + storageClassName2: util.GetUniqueName("sc-"), + tenantNamespace1: util.GetUniqueName("ns-"), + tenantNamespace2: util.GetUniqueName("ns-"), + + pvcName: util.GetUniqueName("pvc-"), + mantleBackupName: util.GetUniqueName("backup-"), + mantleRestoreName: util.GetUniqueName("restore-"), + } + + Describe("setup environment", test.setupEnv) + Describe("test main", test.testMain) + Describe("tearDown environment", test.teardownEnv) +} + +func (test *multiRookCephTest) setupEnv() { + It("setting up the test environment", func() { + fmt.Fprintf(GinkgoWriter, "%+v\n", *test) + }) + + It("creating common resources", func() { + for _, ns := range []string{test.tenantNamespace1, test.tenantNamespace2} { + err := createNamespace(ns) + Expect(err).NotTo(HaveOccurred()) + } + + err := applyRBDPoolAndSCTemplate(cephCluster1Namespace, test.poolName, test.storageClassName1) + Expect(err).NotTo(HaveOccurred()) + + err = applyRBDPoolAndSCTemplate(cephCluster2Namespace, test.poolName, test.storageClassName2) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (test *multiRookCephTest) teardownEnv() { + for _, ns := range []string{test.tenantNamespace1, test.tenantNamespace2} { + ns := ns + It("delete resources in the namespace: "+ns, func() { + err := deleteNamespacedResource(ns, "mantlerestore") + Expect(err).NotTo(HaveOccurred()) + err = deleteNamespacedResource(ns, "mantlebackup") + Expect(err).NotTo(HaveOccurred()) + err = deleteNamespacedResource(ns, "pvc") + Expect(err).NotTo(HaveOccurred()) + }) + + It("delete the namespace: "+ns, func() { + _, stderr, err := kubectl("delete", "namespace", ns) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + }) + } + + It("clean up the SCs and RBD pools", func() { + for _, sc := range []string{test.storageClassName1, test.storageClassName2} { + _, stderr, err := kubectl("delete", "sc", sc) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + } + + for _, ns := range []string{cephCluster1Namespace, cephCluster2Namespace} { + err := removeAllRBDImageAndSnap(ns, test.poolName) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = kubectl("delete", "-n", ns, "cephblockpool", test.poolName, "--wait=false") + Expect(err).NotTo(HaveOccurred()) + } + }) +} + +func (test *multiRookCephTest) testMain() { + testData1 := []byte("this is a test data for the tenant namespace 1") + testData2 := []byte("test data 2 should be in the tenant namespace 2") + var srcImageName1, srcImageName2, srcImageNameDummy string + var expectImageAndSnaps1, expectImageAndSnaps2 []string + var dummySnapInfo, dummyCloneInfo *rbdInfo + + It("create PVC, backup, and restore in the tenant namespace 1", func() { + test.createPVCBackupRestoreWithData(test.tenantNamespace1, test.storageClassName1, testData1) + + pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName) + Expect(err).NotTo(HaveOccurred()) + srcImageName1, err = getImageNameFromPVName(pvName) + Expect(err).NotTo(HaveOccurred()) + expectImageAndSnaps1 = []string{ + srcImageName1, + fmt.Sprintf("%s@%s", srcImageName1, test.mantleBackupName), + fmt.Sprintf("mantle-%s-%s", test.tenantNamespace1, test.mantleRestoreName), + } + }) + + It("create the dummy RBD image & snap in the ceph cluster 2 with the same name as ceph cluster 1", func() { + By("get source image name from the PV/PVC") + pvName, err := getPVFromPVC(test.tenantNamespace1, test.pvcName) + Expect(err).NotTo(HaveOccurred()) + srcImageName, err := getImageNameFromPVName(pvName) + Expect(err).NotTo(HaveOccurred()) + srcImageNameDummy = srcImageName + + By("create RBD image & snap in the ceph cluster 2") + err = createRBDImage(cephCluster2Namespace, test.poolName, srcImageNameDummy, 100) + Expect(err).NotTo(HaveOccurred()) + err = createRBDSnap(cephCluster2Namespace, test.poolName, srcImageNameDummy, test.mantleBackupName) + Expect(err).NotTo(HaveOccurred()) + snapName := fmt.Sprintf("%s@%s", srcImageNameDummy, test.mantleBackupName) + cloneName := fmt.Sprintf("mantle-%s-%s", test.tenantNamespace1, test.mantleRestoreName) + err = createRBDCloneImage(cephCluster2Namespace, test.poolName, snapName, cloneName) + Expect(err).NotTo(HaveOccurred()) + + By("get the dummy infos") + dummySnapInfo, err = getRBDInfo(cephCluster2Namespace, test.poolName, snapName) + Expect(err).NotTo(HaveOccurred()) + dummyCloneInfo, err = getRBDInfo(cephCluster2Namespace, test.poolName, cloneName) + Expect(err).NotTo(HaveOccurred()) + }) + + It("check the rbd images in the both clusters", func() { + imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps1).To(ConsistOf(expectImageAndSnaps1)) + + imageAndSnaps2, err := getImageAndSnapNames(cephCluster2Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps2).To(ConsistOf(expectImageAndSnaps1)) + }) + + It("create PVC, backup, and restore in the tenant namespace 2", func() { + test.createPVCBackupRestoreWithData(test.tenantNamespace2, test.storageClassName2, testData2) + + pvName, err := getPVFromPVC(test.tenantNamespace2, test.pvcName) + Expect(err).NotTo(HaveOccurred()) + srcImageName2, err = getImageNameFromPVName(pvName) + Expect(err).NotTo(HaveOccurred()) + expectImageAndSnaps2 = []string{ + srcImageName2, + fmt.Sprintf("%s@%s", srcImageName2, test.mantleBackupName), + fmt.Sprintf("mantle-%s-%s", test.tenantNamespace2, test.mantleRestoreName), + } + }) + + It("check the rbd images in the both clusters", func() { + imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps1).To(ConsistOf(expectImageAndSnaps1)) + backupData1, err := readTestData(test.tenantNamespace1, test.mantleRestoreName) + Expect(err).NotTo(HaveOccurred()) + Expect(backupData1).To(Equal(testData1)) + + imageAndSnaps2, err := getImageAndSnapNames(cephCluster2Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps2).To(ConsistOf(append(expectImageAndSnaps1, expectImageAndSnaps2...))) + backupData2, err := readTestData(test.tenantNamespace2, test.mantleRestoreName) + Expect(err).NotTo(HaveOccurred()) + Expect(backupData2).To(Equal(testData2)) + }) + + It("delete the resources in the tenant namespace 1", func() { + test.deleteBackupRestore(test.tenantNamespace1) + }) + + It("check the rbd images in the both clusters", func() { + imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + // after delete backup and restore, only the source image should be left + Expect(imageAndSnaps1).To(ConsistOf(srcImageName1)) + + imageAndSnaps2, err := getImageAndSnapNames(cephCluster2Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps2).To(ConsistOf(append(expectImageAndSnaps1, expectImageAndSnaps2...))) + }) + + It("delete the resources in the tenant namespace 2", func() { + test.deleteBackupRestore(test.tenantNamespace2) + }) + + It("check the rbd images in the both clusters", func() { + imageAndSnaps1, err := getImageAndSnapNames(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps1).To(ConsistOf(srcImageName1)) + + imageAndSnaps2, err := getImageAndSnapNames(cephCluster2Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + Expect(imageAndSnaps2).To(ConsistOf(append(expectImageAndSnaps1, srcImageName2))) + }) + + It("check there is no change in the dummy rbd image", func() { + snapName := fmt.Sprintf("%s@%s", srcImageNameDummy, test.mantleBackupName) + cloneName := fmt.Sprintf("mantle-%s-%s", test.tenantNamespace1, test.mantleRestoreName) + + info, err := getRBDInfo(cephCluster2Namespace, test.poolName, snapName) + Expect(err).NotTo(HaveOccurred()) + Expect(info.CreateTimestamp).To(Equal(dummySnapInfo.CreateTimestamp)) + Expect(info.ModifyTimestamp).To(Equal(dummySnapInfo.ModifyTimestamp)) + + info, err = getRBDInfo(cephCluster2Namespace, test.poolName, cloneName) + Expect(err).NotTo(HaveOccurred()) + Expect(info.CreateTimestamp).To(Equal(dummyCloneInfo.CreateTimestamp)) + Expect(info.ModifyTimestamp).To(Equal(dummyCloneInfo.ModifyTimestamp)) + }) + + It("clean up the dummy data", func() { + err := removeAllRBDImageAndSnap(cephCluster2Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + }) +} + +func (test *multiRookCephTest) createPVCBackupRestoreWithData(ns, sc string, data []byte) { + err := applyPVCTemplate(ns, test.pvcName, sc) + Expect(err).NotTo(HaveOccurred()) + err = writeTestData(ns, test.pvcName, data) + Expect(err).NotTo(HaveOccurred()) + + err = applyMantleBackupTemplate(ns, test.pvcName, test.mantleBackupName) + Expect(err).NotTo(HaveOccurred()) + + err = applyMantleRestoreTemplate(ns, test.mantleRestoreName, test.mantleBackupName) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() bool { + return isMantleRestoreReady(ns, test.mantleRestoreName) + }).Should(BeTrue()) +} + +func (test *multiRookCephTest) deleteBackupRestore(ns string) { + _, stderr, err := kubectl("delete", "mantlerestore", "-n", ns, test.mantleRestoreName) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + _, stderr, err = kubectl("delete", "mantlebackup", "-n", ns, test.mantleBackupName) + Expect(err).NotTo(HaveOccurred(), string(stderr)) +} diff --git a/test/e2e/restore_test.go b/test/e2e/restore_test.go index 6910ab47..d6ccb46d 100644 --- a/test/e2e/restore_test.go +++ b/test/e2e/restore_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "log/slog" - "strings" "time" mantlev1 "github.com/cybozu-go/mantle/api/v1" @@ -53,6 +52,10 @@ func restoreTestSuite() { } func (test *restoreTest) setupEnv() { + It("describing the test environment", func() { + fmt.Fprintf(GinkgoWriter, "%+v\n", *test) + }) + It("creating common resources", func() { // namespace err := createNamespace(test.tenantNamespace) @@ -72,6 +75,22 @@ func (test *restoreTest) setupEnv() { func (test *restoreTest) tearDownEnv() { test.cleanup() + + It("delete namespace: "+test.tenantNamespace, func() { + _, stderr, err := kubectl("delete", "namespace", test.tenantNamespace) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + }) + + It("clean up the SCs and RBD pools", func() { + _, stderr, err := kubectl("delete", "sc", test.storageClassName) + Expect(err).NotTo(HaveOccurred(), string(stderr)) + + err = removeAllRBDImageAndSnap(cephCluster1Namespace, test.poolName) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = kubectl("delete", "-n", cephCluster1Namespace, "cephblockpool", test.poolName, "--wait=false") + Expect(err).NotTo(HaveOccurred()) + }) } func (test *restoreTest) testRestore() { @@ -574,20 +593,10 @@ func (test *restoreTest) testRemoveImage() { } func (test *restoreTest) cleanup() { - test.deleteNamespacedResource(test.tenantNamespace, "mantlerestore") - test.deleteNamespacedResource(test.tenantNamespace, "mantlebackup") - test.deleteNamespacedResource(test.tenantNamespace, "pvc") -} - -func (test *restoreTest) deleteNamespacedResource(namespace, kind string) { - stdout, stderr, err := kubectl("get", kind, "-n", namespace, "-o", "jsonpath={.items[*].metadata.name}") - Expect(err).NotTo(HaveOccurred(), string(stderr)) - names := strings.Split(string(stdout), " ") - for _, name := range names { - if name == "" { - continue - } - _, stderr, err = kubectl("delete", kind, "-n", namespace, name) - Expect(err).NotTo(HaveOccurred(), string(stderr)) - } + err := deleteNamespacedResource(test.tenantNamespace, "mantlerestore") + Expect(err).NotTo(HaveOccurred()) + err = deleteNamespacedResource(test.tenantNamespace, "mantlebackup") + Expect(err).NotTo(HaveOccurred()) + err = deleteNamespacedResource(test.tenantNamespace, "pvc") + Expect(err).NotTo(HaveOccurred()) } diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 52a8db43..c6369440 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -29,6 +29,20 @@ func TestMtest(t *testing.T) { } var _ = Describe("Mantle", func() { + Context("wait environment", waitEnvironment) Context("backup", backupTestSuite) Context("restore", restoreTestSuite) + Context("multi Rook/Ceph env", multiRookCephTestSuite) }) + +func waitEnvironment() { + It("wait for mantle-controller to be ready", func() { + Eventually(func() error { + return checkDeploymentReady(cephCluster1Namespace, "mantle-controller") + }).Should(Succeed()) + + Eventually(func() error { + return checkDeploymentReady(cephCluster2Namespace, "mantle2-controller") + }).Should(Succeed()) + }) +} diff --git a/test/e2e/testdata/rbd-pool-sc-template.yaml b/test/e2e/testdata/rbd-pool-sc-template.yaml index 42f16d83..fb3a2210 100644 --- a/test/e2e/testdata/rbd-pool-sc-template.yaml +++ b/test/e2e/testdata/rbd-pool-sc-template.yaml @@ -15,7 +15,7 @@ metadata: name: %s provisioner: rook-ceph.rbd.csi.ceph.com parameters: - clusterID: rook-ceph + clusterID: %s pool: %s imageFormat: "2" imageFeatures: layering diff --git a/test/e2e/util.go b/test/e2e/util.go index f497a07f..73d5c770 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/exec" + "strings" "time" mantlev1 "github.com/cybozu-go/mantle/api/v1" @@ -108,8 +109,9 @@ func applyPVCTemplate(namespace, name, storageClassName string) error { } func applyRBDPoolAndSCTemplate(namespace, poolName, storageClassName string) error { - manifest := fmt.Sprintf(testRBDPoolSCTemplate, poolName, namespace, - storageClassName, poolName, namespace, namespace, namespace) + manifest := fmt.Sprintf( + testRBDPoolSCTemplate, poolName, namespace, + storageClassName, namespace, poolName, namespace, namespace, namespace) _, _, err := kubectlWithInput([]byte(manifest), "apply", "-n", namespace, "-f", "-") if err != nil { return err @@ -165,6 +167,100 @@ func createRBDImage(namespace, poolName, imageName string, siz uint) error { return nil } +func createRBDCloneImage(namespace, poolName, snapName, cloneName string) error { + _, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "clone", + "--rbd-default-clone-format", "2", + "--image-feature", "deep-flatten", + poolName+"/"+snapName, poolName+"/"+cloneName) + if err != nil { + return fmt.Errorf("rbd clone failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func removeRBDImage(namespace, poolName, imageName string) error { + _, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "rm", poolName+"/"+imageName) + if err != nil { + return fmt.Errorf("rbd rm failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func createRBDSnap(namespace, poolName, imageName, snapName string) error { + _, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "snap", "create", poolName+"/"+imageName+"@"+snapName) + if err != nil { + return fmt.Errorf("rbd snap create failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func removeRBDSnap(namespace, poolName, imageName, snapName string) error { + _, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "snap", "rm", poolName+"/"+imageName+"@"+snapName) + if err != nil { + return fmt.Errorf("rbd snap rm failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func removeAllRBDImageAndSnap(namespace, pool string) error { + targets, err := getImageAndSnapNames(namespace, pool) + if err != nil { + return err + } + + images := []string{} + // remove RBD snapshots first + for _, target := range targets { + if strings.Contains(target, "@") { + imageAndSnap := strings.Split(target, "@") + err = removeRBDSnap(namespace, pool, imageAndSnap[0], imageAndSnap[1]) + if err != nil { + return err + } + } else { + images = append(images, target) + } + } + + // remove RBD images + for _, image := range images { + err = removeRBDImage(namespace, pool, image) + if err != nil { + return err + } + } + return nil +} + +func deleteNamespacedResource(namespace, kind string) error { + stdout, stderr, err := kubectl("get", kind, "-n", namespace, "-o", "jsonpath={.items[*].metadata.name}") + if err != nil { + return fmt.Errorf("kubectl get failed. stderr: %s, err: %w", string(stderr), err) + } + + names := strings.Split(string(stdout), " ") + for _, name := range names { + if name == "" { + continue + } + _, stderr, err = kubectl("delete", kind, "-n", namespace, name) + if err != nil { + return fmt.Errorf("kubectl delete failed. ns: %s, kind: %s, name: %s, stderr: %s, err: %w", + namespace, kind, name, string(stderr), err) + } + } + + return nil +} + func getImageNameFromPVName(pvName string) (string, error) { stdout, stderr, err := kubectl("get", "pv", pvName, "-o", "json") if err != nil { @@ -299,3 +395,39 @@ func readTestData(namespace, pvc string) ([]byte, error) { return stdout, nil } + +func getImageAndSnapNames(namespace, pool string) ([]string, error) { + stdout, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "ls", "-p", pool, "--format", "json") + if err != nil { + return nil, fmt.Errorf("rbd ls failed. stderr: %s, err: %w", string(stderr), err) + } + + var images []string + err = json.Unmarshal(stdout, &images) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal output of rbd ls: %w", err) + } + + var snaps []string + for _, image := range images { + stdout, stderr, err := kubectl( + "-n", namespace, "exec", "deploy/rook-ceph-tools", "--", + "rbd", "snap", "ls", pool+"/"+image, "--format=json") + if err != nil { + return nil, fmt.Errorf("rbd snap ls failed. stderr: %s, err: %w", string(stderr), err) + } + var snapshots []controller.Snapshot + err = json.Unmarshal(stdout, &snapshots) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal output of rbd snap ls: %w", err) + } + + for _, s := range snapshots { + snaps = append(snaps, fmt.Sprintf("%s@%s", image, s.Name)) + } + } + + return append(images, snaps...), nil +}