-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
0612d31
commit 37de048
Showing
5 changed files
with
233 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| apiVersion: mantle.cybozu.io/v1 | ||
| kind: MantleBackup | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: mantlebackup | ||
| app.kubernetes.io/instance: %s | ||
| app.kubernetes.io/part-of: mantle | ||
| app.kubernetes.io/managed-by: kustomize | ||
| app.kubernetes.io/created-by: mantle | ||
| name: %s | ||
| namespace: %s | ||
| spec: | ||
| pvc: %s |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| --- | ||
| apiVersion: v1 | ||
| kind: PersistentVolumeClaim | ||
| metadata: | ||
| name: %s | ||
| spec: | ||
| accessModes: | ||
| - ReadWriteOnce | ||
| resources: | ||
| requests: | ||
| storage: 1Gi | ||
| storageClassName: %s |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| apiVersion: ceph.rook.io/v1 | ||
| kind: CephBlockPool | ||
| metadata: | ||
| name: %s | ||
| namespace: %s | ||
| spec: | ||
| failureDomain: osd | ||
| replicated: | ||
| size: 1 | ||
| requireSafeReplicaSize: false | ||
| --- | ||
| apiVersion: storage.k8s.io/v1 | ||
| kind: StorageClass | ||
| metadata: | ||
| name: %s | ||
| provisioner: rook-ceph.rbd.csi.ceph.com | ||
| parameters: | ||
| clusterID: %s | ||
| pool: %s | ||
| imageFormat: "2" | ||
| imageFeatures: layering | ||
| csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner | ||
| csi.storage.k8s.io/provisioner-secret-namespace: %s | ||
| csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner | ||
| csi.storage.k8s.io/controller-expand-secret-namespace: %s | ||
| csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node | ||
| csi.storage.k8s.io/node-stage-secret-namespace: %s | ||
| csi.storage.k8s.io/fstype: ext4 | ||
| allowVolumeExpansion: true | ||
| reclaimPolicy: Delete |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| package multik8s | ||
|
|
||
| import ( | ||
| "bytes" | ||
| _ "embed" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
|
|
||
| mantlev1 "github.com/cybozu-go/mantle/api/v1" | ||
| ) | ||
|
|
||
| var ( | ||
| //go:embed testdata/pvc-template.yaml | ||
| testPVCTemplate string | ||
| //go:embed testdata/rbd-pool-sc-template.yaml | ||
| testRBDPoolSCTemplate string | ||
| //go:embed testdata/mantlebackup-template.yaml | ||
| testMantleBackupTemplate string | ||
|
|
||
| kubectlPrefix0 = os.Getenv("KUBECTL0") // primary k8s cluster | ||
| kubectlPrefix1 = os.Getenv("KUBECTL1") // secondary k8s cluster | ||
| ) | ||
|
|
||
| func execAtLocal(cmd string, input []byte, args ...string) ([]byte, []byte, error) { | ||
| var stdout, stderr bytes.Buffer | ||
| command := exec.Command(cmd, args...) | ||
| command.Stdout = &stdout | ||
| command.Stderr = &stderr | ||
|
|
||
| if len(input) != 0 { | ||
| command.Stdin = bytes.NewReader(input) | ||
| } | ||
|
|
||
| err := command.Run() | ||
| return stdout.Bytes(), stderr.Bytes(), err | ||
| } | ||
|
|
||
| // input can be nil | ||
| func kubectl(clusterNo int, input []byte, args ...string) ([]byte, []byte, error) { | ||
| kubectlPrefix := "" | ||
| switch clusterNo { | ||
| case 0: | ||
| kubectlPrefix = kubectlPrefix0 | ||
| case 1: | ||
| kubectlPrefix = kubectlPrefix1 | ||
| default: | ||
| panic(fmt.Sprintf("invalid clusterNo: %d", clusterNo)) | ||
| } | ||
| if len(kubectlPrefix) == 0 { | ||
| panic("Either KUBECTL0 or KUBECTL1 environment variable is not set") | ||
| } | ||
| fields := strings.Fields(kubectlPrefix) | ||
| fields = append(fields, args...) | ||
| return execAtLocal(fields[0], input, fields[1:]...) | ||
| } | ||
|
|
||
| func checkDeploymentReady(clusterNo int, namespace, name string) error { | ||
| _, stderr, err := kubectl( | ||
| clusterNo, nil, | ||
| "-n", namespace, "wait", "--for=condition=Available", "deploy", name, "--timeout=1m", | ||
| ) | ||
| if err != nil { | ||
| return fmt.Errorf("kubectl wait deploy failed. stderr: %s, err: %w", string(stderr), err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func applyMantleBackupTemplate(clusterNo int, namespace, pvcName, backupName string) error { | ||
| manifest := fmt.Sprintf(testMantleBackupTemplate, backupName, backupName, namespace, pvcName) | ||
| _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-f", "-") | ||
| if err != nil { | ||
| return fmt.Errorf("kubectl apply mantlebackup failed. err: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func applyPVCTemplate(clusterNo int, namespace, name, storageClassName string) error { | ||
| manifest := fmt.Sprintf(testPVCTemplate, name, storageClassName) | ||
| _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") | ||
| if err != nil { | ||
| return fmt.Errorf("kubectl apply pvc failed. err: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func createNamespace(clusterNo int, name string) error { | ||
| _, _, err := kubectl(clusterNo, nil, "create", "ns", name) | ||
| if err != nil { | ||
| return fmt.Errorf("kubectl create ns failed. err: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func applyRBDPoolAndSCTemplate(clusterNo int, namespace, poolName, storageClassName string) error { | ||
| manifest := fmt.Sprintf( | ||
| testRBDPoolSCTemplate, poolName, namespace, | ||
| storageClassName, namespace, poolName, namespace, namespace, namespace) | ||
| _, _, err := kubectl(clusterNo, []byte(manifest), "apply", "-n", namespace, "-f", "-") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func getObject[T any](clusterNo int, kind, namespace, name string) (*T, error) { | ||
| stdout, _, err := kubectl(clusterNo, nil, "get", kind, "-n", namespace, name, "-o", "json") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| var obj T | ||
| if err := json.Unmarshal(stdout, &obj); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &obj, nil | ||
| } | ||
|
|
||
| func getMB(clusterNo int, namespace, name string) (*mantlev1.MantleBackup, error) { | ||
| return getObject[mantlev1.MantleBackup](clusterNo, "mantlebackup", namespace, name) | ||
| } |