diff --git a/test/e2e/multik8s/suite_test.go b/test/e2e/multik8s/suite_test.go index d3c3ff9e..d8bb552e 100644 --- a/test/e2e/multik8s/suite_test.go +++ b/test/e2e/multik8s/suite_test.go @@ -1,17 +1,25 @@ package multik8s import ( - "bytes" _ "embed" - "fmt" + "errors" "os" - "os/exec" - "strings" "testing" "time" + "github.com/cybozu-go/mantle/test/util" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + + mantlev1 "github.com/cybozu-go/mantle/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + cephClusterNamespace = "rook-ceph" + primaryK8sCluster = 0 + secondaryK8sCluster = 1 ) func TestMtest(t *testing.T) { @@ -32,70 +40,58 @@ var _ = Describe("Mantle", func() { Context("replication", replicationTestSuite) }) -var ( - kubectlPrefix0 = os.Getenv("KUBECTL0") - kubectlPrefix1 = os.Getenv("KUBECTL1") -) - -func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { - var stdout, stderr bytes.Buffer - command := exec.Command(cmd, args...) - command.Stdout = &stdout - command.Stderr = &stderr - - if len(input) != 0 { - command.Stdin = bytes.NewReader(input) - } - - err := command.Run() - return stdout.Bytes(), stderr.Bytes(), err -} - -// input can be nil -func kubectl(clusterNo int, input []byte, args ...string) ([]byte, []byte, error) { - kubectlPrefix := "" - switch clusterNo { - case 0: - kubectlPrefix = kubectlPrefix0 - case 1: - kubectlPrefix = kubectlPrefix1 - default: - panic(fmt.Sprintf("invalid clusterNo: %d", clusterNo)) - } - if len(kubectlPrefix) == 0 { - panic("Either KUBECTL0 or KUBECTL1 environment variable is not set") - } - fields := strings.Fields(kubectlPrefix) - fields = append(fields, args...) - return execAtLocal(fields[0], input, fields[1:]...) -} - -func checkDeploymentReady(clusterNo int, namespace, name string) error { - _, stderr, err := kubectl( - clusterNo, nil, - "-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m", - ) - if err != nil { - return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err) - } - return nil -} - func waitControllerToBeReady() { It("wait for mantle-controller to be ready", func() { Eventually(func() error { - return checkDeploymentReady(0, "rook-ceph", "mantle-controller") + return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") }).Should(Succeed()) Eventually(func() error { - return checkDeploymentReady(0, "rook-ceph", "mantle-controller") + return checkDeploymentReady(primaryK8sCluster, "rook-ceph", "mantle-controller") }).Should(Succeed()) }) } func replicationTestSuite() { - Describe("make sure SyncToRemote becomes true after a MantleBackup is created", func() { - // FIXME - return + Describe("reconciliation test", func() { + It("should eventually set SyncedToRemote of a MantleBackup to True after it is created", func() { + namespace := util.GetUniqueName("ns-") + pvcName := util.GetUniqueName("pvc-") + backupName := util.GetUniqueName("mb-") + scName := util.GetUniqueName("sc-") + poolName := util.GetUniqueName("pool-") + + By("setting up the environment") + Eventually(func() error { + return createNamespace(primaryK8sCluster, namespace) + }).Should(Succeed()) + Eventually(func() error { + return applyRBDPoolAndSCTemplate(primaryK8sCluster, cephClusterNamespace, poolName, scName) + }).Should(Succeed()) + Eventually(func() error { + return applyPVCTemplate(primaryK8sCluster, namespace, pvcName, scName) + }).Should(Succeed()) + + By("creating a MantleBackup object") + Eventually(func() error { + return applyMantleBackupTemplate(primaryK8sCluster, namespace, pvcName, backupName) + }).Should(Succeed()) + + By("checking MantleBackup's SyncedToRemote status") + Eventually(func() error { + mb, err := getMB(primaryK8sCluster, namespace, backupName) + if err != nil { + return err + } + cond := meta.FindStatusCondition(mb.Status.Conditions, mantlev1.BackupConditionSyncedToRemote) + if cond == nil { + return errors.New("couldn't find condition SyncedToRemote") + } + if cond.Status != metav1.ConditionTrue { + return errors.New("status of SyncedToRemote condition is not True") + } + return nil + }).Should(Succeed()) + }) }) } diff --git a/test/e2e/multik8s/testdata/mantlebackup-template.yaml b/test/e2e/multik8s/testdata/mantlebackup-template.yaml new file mode 100644 index 00000000..4e25d7e2 --- /dev/null +++ b/test/e2e/multik8s/testdata/mantlebackup-template.yaml @@ -0,0 +1,13 @@ +apiVersion: mantle.cybozu.io/v1 +kind: MantleBackup +metadata: + labels: + app.kubernetes.io/name: mantlebackup + app.kubernetes.io/instance: %s + app.kubernetes.io/part-of: mantle + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: mantle + name: %s + namespace: %s +spec: + pvc: %s diff --git a/test/e2e/multik8s/testdata/pvc-template.yaml b/test/e2e/multik8s/testdata/pvc-template.yaml new file mode 100644 index 00000000..0bcdf27e --- /dev/null +++ b/test/e2e/multik8s/testdata/pvc-template.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: %s +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: %s diff --git a/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml b/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml new file mode 100644 index 00000000..fb3a2210 --- /dev/null +++ b/test/e2e/multik8s/testdata/rbd-pool-sc-template.yaml @@ -0,0 +1,30 @@ +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: %s + namespace: %s +spec: + failureDomain: osd + replicated: + size: 1 + requireSafeReplicaSize: false +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: %s +provisioner: rook-ceph.rbd.csi.ceph.com +parameters: + clusterID: %s + pool: %s + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: %s + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: %s + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: %s + csi.storage.k8s.io/fstype: ext4 +allowVolumeExpansion: true +reclaimPolicy: Delete diff --git a/test/e2e/multik8s/util.go b/test/e2e/multik8s/util.go new file mode 100644 index 00000000..6921cc9b --- /dev/null +++ b/test/e2e/multik8s/util.go @@ -0,0 +1,124 @@ +package multik8s + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + + mantlev1 "github.com/cybozu-go/mantle/api/v1" +) + +var ( + //go:embed testdata/pvc-template.yaml + testPVCTemplate string + //go:embed testdata/rbd-pool-sc-template.yaml + testRBDPoolSCTemplate string + //go:embed testdata/mantlebackup-template.yaml + testMantleBackupTemplate string + + kubectlPrefix0 = os.Getenv("KUBECTL0") // primary k8s cluster + kubectlPrefix1 = os.Getenv("KUBECTL1") // secondary k8s cluster +) + +func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { + var stdout, stderr bytes.Buffer + command := exec.Command(cmd, args...) + command.Stdout = &stdout + command.Stderr = &stderr + + if len(input) != 0 { + command.Stdin = bytes.NewReader(input) + } + + err := command.Run() + return stdout.Bytes(), stderr.Bytes(), err +} + +// input can be nil +func kubectl(clusterNo int, input []byte, args ...string) ([]byte, []byte, error) { + kubectlPrefix := "" + switch clusterNo { + case 0: + kubectlPrefix = kubectlPrefix0 + case 1: + kubectlPrefix = kubectlPrefix1 + default: + panic(fmt.Sprintf("invalid clusterNo: %d", clusterNo)) + } + if len(kubectlPrefix) == 0 { + panic("Either KUBECTL0 or KUBECTL1 environment variable is not set") + } + fields := strings.Fields(kubectlPrefix) + fields = append(fields, args...) + return execAtLocal(fields[0], input, fields[1:]...) +} + +func checkDeploymentReady(clusterNo int, namespace, name string) error { + _, stderr, err := kubectl( + clusterNo, nil, + "-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m", + ) + if err != nil { + return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err) + } + return nil +} + +func applyMantleBackupTemplate(clusterNo int, namespace, pvcName, backupName string) error { + manifest := fmt.Sprintf(testMantleBackupTemplate, backupName, backupName, namespace, pvcName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply mantlebackup failed. err: %w", err) + } + return nil +} + +func applyPVCTemplate(clusterNo int, namespace, name, storageClassName string) error { + manifest := fmt.Sprintf(testPVCTemplate, name, storageClassName) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return fmt.Errorf("kubectl apply pvc failed. err: %w", err) + } + return nil +} + +func createNamespace(clusterNo int, name string) error { + _, _, err := kubectl(clusterNo, nil, "create", "ns", name) + if err != nil { + return fmt.Errorf("kubectl create ns failed. err: %w", err) + } + return nil +} + +func applyRBDPoolAndSCTemplate(clusterNo int, namespace, poolName, storageClassName string) error { + manifest := fmt.Sprintf( + testRBDPoolSCTemplate, poolName, namespace, + storageClassName, namespace, poolName, namespace, namespace, namespace) + _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") + if err != nil { + return err + } + return nil +} + +func getObject[T any](clusterNo int, kind, namespace, name string) (*T, error) { + stdout, _, err := kubectl(clusterNo, nil, "get", kind, "-n", namespace, name, "-o", "json") + if err != nil { + return nil, err + } + + var obj T + if err := json.Unmarshal(stdout, &obj); err != nil { + return nil, err + } + + return &obj, nil +} + +func getMB(clusterNo int, namespace, name string) (*mantlev1.MantleBackup, error) { + return getObject[mantlev1.MantleBackup](clusterNo, "mantlebackup", namespace, name) +}