From 040b5ddf9dc01bc6f7bde7dd0e23a05ea92c69b1 Mon Sep 17 00:00:00 2001 From: kouki Date: Wed, 18 Oct 2023 02:51:37 +0000 Subject: [PATCH 01/13] Enable TLS for minio used in testing Signed-off-by: kouki --- e2e/backup_test.go | 7 ++-- e2e/minio.yaml | 68 +++++++++++++++++++++++++++--------- e2e/testdata/backup.yaml | 28 +++++++-------- e2e/testdata/makebucket.yaml | 34 ++++++++++++++++++ 4 files changed, 103 insertions(+), 34 deletions(-) create mode 100644 e2e/testdata/makebucket.yaml diff --git a/e2e/backup_test.go b/e2e/backup_test.go index d9e6cdffd..650b949ba 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -19,6 +19,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +//go:embed testdata/makebucket.yaml +var makeBucketYAML string + //go:embed testdata/backup.yaml var backupYAML string @@ -33,9 +36,7 @@ var _ = Context("backup", func() { var restorePoint time.Time It("should create a bucket", func() { - kubectlSafe(nil, "run", "--command", "make-bucket", "--image=moco-backup:dev", "--", - "s3cmd", "--host=minio.default.svc:9000", "--host-bucket=minio.default.svc:9000", "--no-ssl", - "--access_key=minioadmin", "--secret_key=minioadmin", "mb", "s3://moco") + kubectlSafe([]byte(makeBucketYAML), "apply", "-f", "-") }) It("should construct a source cluster", func() { diff --git a/e2e/minio.yaml b/e2e/minio.yaml index 64bc6ee70..0f99ad3d3 100644 --- a/e2e/minio.yaml +++ b/e2e/minio.yaml @@ -1,3 +1,25 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: default + name: default-selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: default + name: minio-cert +spec: + commonName: minio cert + issuerRef: + kind: Issuer + name: default-selfsigned-issuer + secretName: minio-cert + dnsNames: + - minio.default.svc +--- apiVersion: v1 kind: Service metadata: @@ -5,10 +27,10 @@ metadata: name: minio spec: ports: - - name: minio - port: 9000 - targetPort: minio - protocol: TCP + - name: minio + port: 9000 + targetPort: minio + protocol: TCP selector: name: minio --- @@ -21,18 +43,30 @@ metadata: name: minio spec: containers: - - name: minio - image: minio/minio - args: - - server - - /data - ports: - name: minio - containerPort: 9000 - protocol: TCP - volumeMounts: - - name: data - mountPath: /data + image: minio/minio + args: + - server + - /data + ports: + - name: minio + containerPort: 9000 + protocol: TCP + volumeMounts: + - name: data + mountPath: /data + - name: secret-volume + mountPath: /root/.minio/certs volumes: - - name: data - emptyDir: {} + - name: data + emptyDir: {} + - name: secret-volume + secret: + secretName: minio-cert + items: + - key: ca.crt + path: public.crt + - key: tls.key + path: private.key + - key: ca.crt + path: CAs/public.crt diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index dfebab319..1bf0bfebf 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -27,13 +27,13 @@ spec: jobConfig: serviceAccountName: backup-owner env: - - name: AWS_ACCESS_KEY_ID - value: minioadmin - - name: AWS_SECRET_ACCESS_KEY - value: minioadmin + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin bucketConfig: bucketName: moco - endpointURL: http://minio.default.svc:9000 + endpointURL: https://minio.default.svc:9000 usePathStyle: true workVolume: emptyDir: {} @@ -50,13 +50,13 @@ spec: podTemplate: spec: containers: - - name: mysqld - image: ghcr.io/cybozu-go/moco/mysql:{{ . }} + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ . }} volumeClaimTemplates: - - metadata: - name: mysql-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/makebucket.yaml b/e2e/testdata/makebucket.yaml new file mode 100644 index 000000000..43ec390e6 --- /dev/null +++ b/e2e/testdata/makebucket.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: make-bucket + namespace: default +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - command: + - s3cmd + - --host=minio.default.svc:9000 + - --host-bucket=minio.default.svc:9000 + - --access_key=minioadmin + - --secret_key=minioadmin + - --ssl + - --no-check-certificate + - --ca-certs=/minio-cert/ca.crt + - mb + - s3://moco + image: moco-backup:dev + imagePullPolicy: IfNotPresent + name: make-bucket + volumeMounts: + - name: minio-cert + mountPath: /minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + items: + - key: ca.crt + path: ca.crt From 05345990e4e1f2238ad2c6408747a54c4909604f Mon Sep 17 00:00:00 2001 From: kouki Date: Wed, 18 Oct 2023 02:52:45 +0000 Subject: [PATCH 02/13] Add ca-cert-file-path arg for moco-backup Signed-off-by: kouki --- cmd/moco-backup/cmd/root.go | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/cmd/moco-backup/cmd/root.go b/cmd/moco-backup/cmd/root.go index a71419d1b..2abf72113 100644 --- a/cmd/moco-backup/cmd/root.go +++ b/cmd/moco-backup/cmd/root.go @@ -2,8 +2,11 @@ package cmd import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" + "net/http" "net/url" "os" @@ -15,12 +18,13 @@ import ( ) var commonArgs struct { - workDir string - threads int - region string - endpointURL string - usePathStyle bool - backendType string + workDir string + threads int + region string + endpointURL string + usePathStyle bool + backendType string + caCertFilePath string } func makeBucket(bucketName string) (bucket.Bucket, error) { @@ -45,6 +49,26 @@ func makeS3Bucket(bucketName string) (bucket.Bucket, error) { if commonArgs.usePathStyle { opts = append(opts, bucket.WithPathStyle()) } + if len(commonArgs.caCertFilePath) > 0 { + caCertFile, err := os.ReadFile(commonArgs.caCertFilePath) + if err != nil { + return nil, err + } + caCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + if ok := caCertPool.AppendCertsFromPEM(caCertFile); !ok { + return nil, fmt.Errorf("failed to add ca cert") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{ + RootCAs: caCertPool, + } + opts = append(opts, bucket.WithHTTPClient(&http.Client{ + Transport: transport, + })) + } return bucket.NewS3Bucket(bucketName, opts...) } @@ -95,4 +119,5 @@ func init() { pf.StringVar(&commonArgs.endpointURL, "endpoint", "", "Object storage API endpoint URL") pf.BoolVar(&commonArgs.usePathStyle, "use-path-style", false, "Use path-style S3 API") pf.StringVar(&commonArgs.backendType, "backend-type", "s3", "The identifier for the object storage to be used.") + pf.StringVar(&commonArgs.caCertFilePath, "ca-cert-file-path", "", "The file path using ca-certs") } From 1dc4695c9a05b6d01a2fe4dd2ff0c27e2a089749 Mon Sep 17 00:00:00 2001 From: kouki Date: Wed, 18 Oct 2023 03:00:54 +0000 Subject: [PATCH 03/13] Add caCertFilePath field for BackupPolicy Signed-off-by: kouki --- api/v1beta1/job_types.go | 3 +++ api/v1beta1/zz_generated.conversion.go | 2 ++ api/v1beta2/job_types.go | 3 +++ charts/moco/templates/generated/crds/moco_crds.yaml | 8 ++++++++ config/crd/bases/moco.cybozu.com_backuppolicies.yaml | 4 ++++ config/crd/bases/moco.cybozu.com_mysqlclusters.yaml | 4 ++++ ...resourcedefinition_backuppolicies.moco.cybozu.com.yaml | 4 ++++ ...mresourcedefinition_mysqlclusters.moco.cybozu.com.yaml | 4 ++++ controllers/mysqlcluster_controller.go | 3 +++ docs/crd_backuppolicy_v1beta1.md | 1 + docs/crd_backuppolicy_v1beta2.md | 1 + docs/crd_mysqlcluster_v1beta1.md | 1 + docs/crd_mysqlcluster_v1beta2.md | 1 + 13 files changed, 39 insertions(+) diff --git a/api/v1beta1/job_types.go b/api/v1beta1/job_types.go index 7bd4c73b2..13cf53786 100644 --- a/api/v1beta1/job_types.go +++ b/api/v1beta1/job_types.go @@ -218,4 +218,7 @@ type BucketConfig struct { // +kubebuilder:default=s3 // +optional BackendType string `json:"backendType,omitempty"` + + // +optional + CaCertFilePath string `json:"caCertFilePath,omitempty"` } diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 09ba74f0c..ec950f165 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -433,6 +433,7 @@ func autoConvert__BucketConfig_To_v1beta2_BucketConfig(in *BucketConfig, out *v1 out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType + out.CaCertFilePath = in.CaCertFilePath return nil } @@ -447,6 +448,7 @@ func autoConvert_v1beta2_BucketConfig_To__BucketConfig(in *v1beta2.BucketConfig, out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType + out.CaCertFilePath = in.CaCertFilePath return nil } diff --git a/api/v1beta2/job_types.go b/api/v1beta2/job_types.go index 60f13ed93..07a696fc0 100644 --- a/api/v1beta2/job_types.go +++ b/api/v1beta2/job_types.go @@ -201,6 +201,9 @@ type BucketConfig struct { // +kubebuilder:default=s3 // +optional BackendType string `json:"backendType,omitempty"` + + // +optional + CaCertFilePath string `json:"caCertFilePath,omitempty"` } // AffinityApplyConfiguration is the type defined to implement the DeepCopy method. diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index 2c74256c9..31f51db59 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -422,6 +422,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2452,6 +2454,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -7722,6 +7726,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -13606,6 +13612,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml index e46c5380f..5d8d390a7 100644 --- a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml +++ b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml @@ -462,6 +462,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2649,6 +2651,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index b034e42a1..78a8be823 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -3999,6 +3999,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -10430,6 +10432,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml index da23fbdc5..5b52c71a3 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml @@ -461,6 +461,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2648,6 +2650,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index c93e56071..55f9160be 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -4009,6 +4009,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -10440,6 +10442,8 @@ spec: description: The name of the bucket minLength: 1 type: string + caCertFilePath: + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 60b9a7cc3..05ba54862 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -997,6 +997,9 @@ func bucketArgs(bc mocov1beta2.BucketConfig) []string { if bc.BackendType != "" { args = append(args, "--backend-type="+bc.BackendType) } + if bc.CaCertFilePath != "" { + args = append(args, "--ca-cert-file-path="+bc.CaCertFilePath) + } return append(args, bc.BucketName) } diff --git a/docs/crd_backuppolicy_v1beta1.md b/docs/crd_backuppolicy_v1beta1.md index 4f1aa3fb6..6621cb678 100644 --- a/docs/crd_backuppolicy_v1beta1.md +++ b/docs/crd_backuppolicy_v1beta1.md @@ -60,6 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCertFilePath | | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_backuppolicy_v1beta2.md b/docs/crd_backuppolicy_v1beta2.md index 4f1aa3fb6..6621cb678 100644 --- a/docs/crd_backuppolicy_v1beta2.md +++ b/docs/crd_backuppolicy_v1beta2.md @@ -60,6 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCertFilePath | | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta1.md b/docs/crd_mysqlcluster_v1beta1.md index f82d11b5f..8e99cc6ba 100644 --- a/docs/crd_mysqlcluster_v1beta1.md +++ b/docs/crd_mysqlcluster_v1beta1.md @@ -181,6 +181,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCertFilePath | | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index 1f85b7c3d..49c1b3a7b 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -195,6 +195,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCertFilePath | | string | false | [Back to Custom Resources](#custom-resources) From d862e5d521c3adf2e31baea28a00c3f759c37428 Mon Sep 17 00:00:00 2001 From: kouki Date: Thu, 19 Oct 2023 05:01:12 +0000 Subject: [PATCH 04/13] fix test Signed-off-by: kouki --- e2e/backup_test.go | 7 +++++++ e2e/testdata/backup.yaml | 8 ++++++++ e2e/testdata/restore.yaml | 36 ++++++++++++++++++++++-------------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 650b949ba..8fe77f52c 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -41,6 +41,13 @@ var _ = Context("backup", func() { It("should construct a source cluster", func() { kubectlSafe(fillTemplate(backupYAML), "apply", "-f", "-") + secjson := kubectlSafe(nil, "get", "secret", "-o", "json", "minio-cert") + sec := &corev1.Secret{} + json.Unmarshal(secjson, sec) + sec.SetNamespace("backup") + secjson, err := json.Marshal(sec) + Expect(err).NotTo(HaveOccurred()) + kubectlSafe(secjson, "apply", "-f", "-") Eventually(func() error { cluster, err := getCluster("backup", "source") if err != nil { diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index 1bf0bfebf..f868abed1 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -31,10 +31,18 @@ spec: value: minioadmin - name: AWS_SECRET_ACCESS_KEY value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert bucketConfig: bucketName: moco endpointURL: https://minio.default.svc:9000 usePathStyle: true + caCertFilePath: /minio-cert/ca.crt workVolume: emptyDir: {} --- diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index 019f5c05c..912e64b1f 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -13,26 +13,34 @@ spec: jobConfig: serviceAccountName: backup-owner env: - - name: AWS_ACCESS_KEY_ID - value: minioadmin - - name: AWS_SECRET_ACCESS_KEY - value: minioadmin + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert bucketConfig: bucketName: moco - endpointURL: http://minio.default.svc:9000 + endpointURL: https://minio.default.svc:9000 usePathStyle: true + caCertFilePath: /minio-cert/ca.crt workVolume: emptyDir: {} podTemplate: spec: containers: - - name: mysqld - image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} volumeClaimTemplates: - - metadata: - name: mysql-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi From 16ce11dcee9ecba315535757b7e3025d22ff8d66 Mon Sep 17 00:00:00 2001 From: kouki Date: Fri, 20 Oct 2023 06:03:35 +0000 Subject: [PATCH 05/13] Add docs Signed-off-by: kouki --- CHANGELOG.md | 3 +++ api/v1beta1/job_types.go | 3 ++- api/v1beta1/zz_generated.conversion.go | 4 ++-- api/v1beta2/job_types.go | 3 ++- charts/moco/templates/generated/crds/moco_crds.yaml | 12 ++++++++---- cmd/moco-backup/cmd/root.go | 2 +- config/crd/bases/moco.cybozu.com_backuppolicies.yaml | 6 ++++-- config/crd/bases/moco.cybozu.com_mysqlclusters.yaml | 8 ++++++-- ...rcedefinition_backuppolicies.moco.cybozu.com.yaml | 6 ++++-- ...urcedefinition_mysqlclusters.moco.cybozu.com.yaml | 8 ++++++-- controllers/mysqlcluster_controller.go | 4 ++-- docs/crd_backuppolicy_v1beta1.md | 2 +- docs/crd_backuppolicy_v1beta2.md | 2 +- docs/crd_mysqlcluster_v1beta1.md | 2 +- docs/crd_mysqlcluster_v1beta2.md | 2 +- docs/moco-backup.md | 1 + e2e/testdata/backup.yaml | 2 +- e2e/testdata/restore.yaml | 2 +- 18 files changed, 47 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de192768..4a1f56433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added +- Add tls settings for BackupPolicy [#580](https://github.com/cybozu-go/moco/pull/580) + ## [0.17.0] - 2023-09-11 ### Breaking Changes diff --git a/api/v1beta1/job_types.go b/api/v1beta1/job_types.go index 13cf53786..d80166822 100644 --- a/api/v1beta1/job_types.go +++ b/api/v1beta1/job_types.go @@ -219,6 +219,7 @@ type BucketConfig struct { // +optional BackendType string `json:"backendType,omitempty"` + // Path to SSL CA certificate file instead of system default. // +optional - CaCertFilePath string `json:"caCertFilePath,omitempty"` + CaCerts string `json:"caCerts,omitempty"` } diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index ec950f165..d871c6321 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -433,7 +433,7 @@ func autoConvert__BucketConfig_To_v1beta2_BucketConfig(in *BucketConfig, out *v1 out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType - out.CaCertFilePath = in.CaCertFilePath + out.CaCerts = in.CaCerts return nil } @@ -448,7 +448,7 @@ func autoConvert_v1beta2_BucketConfig_To__BucketConfig(in *v1beta2.BucketConfig, out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType - out.CaCertFilePath = in.CaCertFilePath + out.CaCerts = in.CaCerts return nil } diff --git a/api/v1beta2/job_types.go b/api/v1beta2/job_types.go index 07a696fc0..1c4ce3b9c 100644 --- a/api/v1beta2/job_types.go +++ b/api/v1beta2/job_types.go @@ -202,8 +202,9 @@ type BucketConfig struct { // +optional BackendType string `json:"backendType,omitempty"` + // Path to SSL CA certificate file instead of system default. // +optional - CaCertFilePath string `json:"caCertFilePath,omitempty"` + CaCerts string `json:"caCerts,omitempty"` } // AffinityApplyConfiguration is the type defined to implement the DeepCopy method. diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index 31f51db59..5f1d8d551 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -422,7 +422,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. @@ -2454,7 +2455,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. @@ -7726,7 +7728,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. @@ -13612,7 +13615,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. diff --git a/cmd/moco-backup/cmd/root.go b/cmd/moco-backup/cmd/root.go index 2abf72113..22830fa3a 100644 --- a/cmd/moco-backup/cmd/root.go +++ b/cmd/moco-backup/cmd/root.go @@ -119,5 +119,5 @@ func init() { pf.StringVar(&commonArgs.endpointURL, "endpoint", "", "Object storage API endpoint URL") pf.BoolVar(&commonArgs.usePathStyle, "use-path-style", false, "Use path-style S3 API") pf.StringVar(&commonArgs.backendType, "backend-type", "s3", "The identifier for the object storage to be used.") - pf.StringVar(&commonArgs.caCertFilePath, "ca-cert-file-path", "", "The file path using ca-certs") + pf.StringVar(&commonArgs.caCertFilePath, "ca-certs", "", "Path to SSL CA certificate file instead of system default") } diff --git a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml index 5d8d390a7..5d9d5011a 100644 --- a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml +++ b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml @@ -462,7 +462,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. @@ -2651,7 +2652,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index 78a8be823..ce60432e1 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -3999,7 +3999,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead + of system ' type: string endpointURL: description: The API endpoint URL. @@ -10432,7 +10434,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead + of system ' type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml index 5b52c71a3..e100b67ea 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml @@ -461,7 +461,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. @@ -2650,7 +2651,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead of system ' type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index 55f9160be..c81722dd0 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -4009,7 +4009,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead + of system ' type: string endpointURL: description: The API endpoint URL. @@ -10442,7 +10444,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCertFilePath: + caCerts: + description: 'Path to SSL CA certificate file instead + of system ' type: string endpointURL: description: The API endpoint URL. diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 05ba54862..562017a6a 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -997,8 +997,8 @@ func bucketArgs(bc mocov1beta2.BucketConfig) []string { if bc.BackendType != "" { args = append(args, "--backend-type="+bc.BackendType) } - if bc.CaCertFilePath != "" { - args = append(args, "--ca-cert-file-path="+bc.CaCertFilePath) + if bc.CaCerts != "" { + args = append(args, "--ca-certs="+bc.CaCerts) } return append(args, bc.BucketName) diff --git a/docs/crd_backuppolicy_v1beta1.md b/docs/crd_backuppolicy_v1beta1.md index 6621cb678..6d69c232a 100644 --- a/docs/crd_backuppolicy_v1beta1.md +++ b/docs/crd_backuppolicy_v1beta1.md @@ -60,7 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCertFilePath | | string | false | +| caCerts | Path to SSL CA certificate file instead of system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_backuppolicy_v1beta2.md b/docs/crd_backuppolicy_v1beta2.md index 6621cb678..6d69c232a 100644 --- a/docs/crd_backuppolicy_v1beta2.md +++ b/docs/crd_backuppolicy_v1beta2.md @@ -60,7 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCertFilePath | | string | false | +| caCerts | Path to SSL CA certificate file instead of system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta1.md b/docs/crd_mysqlcluster_v1beta1.md index 8e99cc6ba..3a82194c8 100644 --- a/docs/crd_mysqlcluster_v1beta1.md +++ b/docs/crd_mysqlcluster_v1beta1.md @@ -181,7 +181,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCertFilePath | | string | false | +| caCerts | Path to SSL CA certificate file instead of system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index 49c1b3a7b..dd261520b 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -195,7 +195,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCertFilePath | | string | false | +| caCerts | Path to SSL CA certificate file instead of system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/moco-backup.md b/docs/moco-backup.md index 6e5315110..9f307d08f 100644 --- a/docs/moco-backup.md +++ b/docs/moco-backup.md @@ -19,6 +19,7 @@ Global Flags: --threads int The number of threads to be used (default 4) --use-path-style Use path-style S3 API --work-dir string The writable working directory (default "/work") + --ca-certs string Path to SSL CA certificate file instead of system default. ``` ## Subcommands diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index f868abed1..f67556b8d 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -42,7 +42,7 @@ spec: bucketName: moco endpointURL: https://minio.default.svc:9000 usePathStyle: true - caCertFilePath: /minio-cert/ca.crt + caCerts: /minio-cert/ca.crt workVolume: emptyDir: {} --- diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index 912e64b1f..c4b745882 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -28,7 +28,7 @@ spec: bucketName: moco endpointURL: https://minio.default.svc:9000 usePathStyle: true - caCertFilePath: /minio-cert/ca.crt + caCerts: /minio-cert/ca.crt workVolume: emptyDir: {} podTemplate: From b6e5545baf7eeaa4868f66eb6bf605f461303a8a Mon Sep 17 00:00:00 2001 From: kouki kamada Date: Tue, 24 Oct 2023 09:22:14 +0900 Subject: [PATCH 06/13] Update cmd/moco-backup/cmd/root.go Co-authored-by: morimoto-cybozu --- cmd/moco-backup/cmd/root.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/moco-backup/cmd/root.go b/cmd/moco-backup/cmd/root.go index 22830fa3a..7150291b4 100644 --- a/cmd/moco-backup/cmd/root.go +++ b/cmd/moco-backup/cmd/root.go @@ -62,9 +62,10 @@ func makeS3Bucket(bucketName string) (bucket.Bucket, error) { return nil, fmt.Errorf("failed to add ca cert") } transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{ - RootCAs: caCertPool, + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} } + transport.TLSClientConfig.RootCAs = caCertPool opts = append(opts, bucket.WithHTTPClient(&http.Client{ Transport: transport, })) From 3a59f14281c3cf19d9db8db1e7c90fc8a05ad0f3 Mon Sep 17 00:00:00 2001 From: kouki Date: Tue, 24 Oct 2023 00:27:38 +0000 Subject: [PATCH 07/13] Rename field, arg and their description Signed-off-by: kouki --- api/v1beta1/job_types.go | 4 ++-- api/v1beta1/zz_generated.conversion.go | 4 ++-- api/v1beta2/job_types.go | 4 ++-- .../moco/templates/generated/crds/moco_crds.yaml | 16 ++++++++-------- cmd/moco-backup/cmd/root.go | 2 +- .../bases/moco.cybozu.com_backuppolicies.yaml | 10 ++++++---- .../crd/bases/moco.cybozu.com_mysqlclusters.yaml | 12 ++++++------ ...efinition_backuppolicies.moco.cybozu.com.yaml | 10 ++++++---- ...definition_mysqlclusters.moco.cybozu.com.yaml | 12 ++++++------ controllers/mysqlcluster_controller.go | 4 ++-- docs/crd_backuppolicy_v1beta1.md | 2 +- docs/crd_backuppolicy_v1beta2.md | 2 +- docs/crd_mysqlcluster_v1beta1.md | 2 +- docs/crd_mysqlcluster_v1beta2.md | 2 +- e2e/testdata/backup.yaml | 2 +- e2e/testdata/restore.yaml | 2 +- 16 files changed, 47 insertions(+), 43 deletions(-) diff --git a/api/v1beta1/job_types.go b/api/v1beta1/job_types.go index d80166822..847f13b6a 100644 --- a/api/v1beta1/job_types.go +++ b/api/v1beta1/job_types.go @@ -219,7 +219,7 @@ type BucketConfig struct { // +optional BackendType string `json:"backendType,omitempty"` - // Path to SSL CA certificate file instead of system default. + // Path to SSL CA certificate file used in addition to system default. // +optional - CaCerts string `json:"caCerts,omitempty"` + CaCert string `json:"caCert,omitempty"` } diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index d871c6321..1849f4dac 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -433,7 +433,7 @@ func autoConvert__BucketConfig_To_v1beta2_BucketConfig(in *BucketConfig, out *v1 out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType - out.CaCerts = in.CaCerts + out.CaCert = in.CaCert return nil } @@ -448,7 +448,7 @@ func autoConvert_v1beta2_BucketConfig_To__BucketConfig(in *v1beta2.BucketConfig, out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType - out.CaCerts = in.CaCerts + out.CaCert = in.CaCert return nil } diff --git a/api/v1beta2/job_types.go b/api/v1beta2/job_types.go index 1c4ce3b9c..8e59547c6 100644 --- a/api/v1beta2/job_types.go +++ b/api/v1beta2/job_types.go @@ -202,9 +202,9 @@ type BucketConfig struct { // +optional BackendType string `json:"backendType,omitempty"` - // Path to SSL CA certificate file instead of system default. + // Path to SSL CA certificate file used in addition to system default. // +optional - CaCerts string `json:"caCerts,omitempty"` + CaCert string `json:"caCert,omitempty"` } // AffinityApplyConfiguration is the type defined to implement the DeepCopy method. diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index 5f1d8d551..c1c31c1dd 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -422,8 +422,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition t type: string endpointURL: description: The API endpoint URL. @@ -2455,8 +2455,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition t type: string endpointURL: description: The API endpoint URL. @@ -7728,8 +7728,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition t type: string endpointURL: description: The API endpoint URL. @@ -13615,8 +13615,8 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition t type: string endpointURL: description: The API endpoint URL. diff --git a/cmd/moco-backup/cmd/root.go b/cmd/moco-backup/cmd/root.go index 7150291b4..a42e287a2 100644 --- a/cmd/moco-backup/cmd/root.go +++ b/cmd/moco-backup/cmd/root.go @@ -120,5 +120,5 @@ func init() { pf.StringVar(&commonArgs.endpointURL, "endpoint", "", "Object storage API endpoint URL") pf.BoolVar(&commonArgs.usePathStyle, "use-path-style", false, "Use path-style S3 API") pf.StringVar(&commonArgs.backendType, "backend-type", "s3", "The identifier for the object storage to be used.") - pf.StringVar(&commonArgs.caCertFilePath, "ca-certs", "", "Path to SSL CA certificate file instead of system default") + pf.StringVar(&commonArgs.caCertFilePath, "ca-cert", "", "Path to SSL CA certificate file used in addition to system default") } diff --git a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml index 5d9d5011a..fda95959c 100644 --- a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml +++ b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml @@ -462,8 +462,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. @@ -2652,8 +2653,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index ce60432e1..21659d923 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -3999,9 +3999,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead - of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. @@ -10434,9 +10434,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead - of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml index e100b67ea..e905e470f 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml @@ -461,8 +461,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. @@ -2651,8 +2652,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index c81722dd0..7d698b41c 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -4009,9 +4009,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead - of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. @@ -10444,9 +10444,9 @@ spec: description: The name of the bucket minLength: 1 type: string - caCerts: - description: 'Path to SSL CA certificate file instead - of system ' + caCert: + description: Path to SSL CA certificate file used in addition + t type: string endpointURL: description: The API endpoint URL. diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 562017a6a..ce9d015b5 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -997,8 +997,8 @@ func bucketArgs(bc mocov1beta2.BucketConfig) []string { if bc.BackendType != "" { args = append(args, "--backend-type="+bc.BackendType) } - if bc.CaCerts != "" { - args = append(args, "--ca-certs="+bc.CaCerts) + if bc.CaCert != "" { + args = append(args, "--ca-cert="+bc.CaCert) } return append(args, bc.BucketName) diff --git a/docs/crd_backuppolicy_v1beta1.md b/docs/crd_backuppolicy_v1beta1.md index 6d69c232a..84f519a26 100644 --- a/docs/crd_backuppolicy_v1beta1.md +++ b/docs/crd_backuppolicy_v1beta1.md @@ -60,7 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCerts | Path to SSL CA certificate file instead of system default. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_backuppolicy_v1beta2.md b/docs/crd_backuppolicy_v1beta2.md index 6d69c232a..84f519a26 100644 --- a/docs/crd_backuppolicy_v1beta2.md +++ b/docs/crd_backuppolicy_v1beta2.md @@ -60,7 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCerts | Path to SSL CA certificate file instead of system default. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta1.md b/docs/crd_mysqlcluster_v1beta1.md index 3a82194c8..ccd95fb4d 100644 --- a/docs/crd_mysqlcluster_v1beta1.md +++ b/docs/crd_mysqlcluster_v1beta1.md @@ -181,7 +181,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCerts | Path to SSL CA certificate file instead of system default. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index dd261520b..5b88b1a28 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -195,7 +195,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | -| caCerts | Path to SSL CA certificate file instead of system default. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index f67556b8d..05167ae22 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -42,7 +42,7 @@ spec: bucketName: moco endpointURL: https://minio.default.svc:9000 usePathStyle: true - caCerts: /minio-cert/ca.crt + caCert: /minio-cert/ca.crt workVolume: emptyDir: {} --- diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index c4b745882..bf290733b 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -28,7 +28,7 @@ spec: bucketName: moco endpointURL: https://minio.default.svc:9000 usePathStyle: true - caCerts: /minio-cert/ca.crt + caCert: /minio-cert/ca.crt workVolume: emptyDir: {} podTemplate: From 0c74d0c2c450dec99a19c0ecc31f0077437a5c80 Mon Sep 17 00:00:00 2001 From: kouki Date: Tue, 24 Oct 2023 00:34:34 +0000 Subject: [PATCH 08/13] Check completion of make-bucket job Signed-off-by: kouki --- e2e/backup_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 8fe77f52c..3e970ce4e 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -37,6 +37,19 @@ var _ = Context("backup", func() { It("should create a bucket", func() { kubectlSafe([]byte(makeBucketYAML), "apply", "-f", "-") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "jobs", "make-bucket", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + for _, cond := range job.Status.Conditions { + if cond.Type != batchv1.JobComplete { + continue + } + g.Expect(cond.Status).To(Equal(corev1.ConditionTrue)) + } + }).Should(Succeed()) }) It("should construct a source cluster", func() { From 475a308354527f4a8cf3a3fa9abd05bad6baf644 Mon Sep 17 00:00:00 2001 From: kouki Date: Tue, 24 Oct 2023 05:04:15 +0000 Subject: [PATCH 09/13] fix configuration Signed-off-by: kouki --- e2e/minio.yaml | 4 +--- e2e/testdata/makebucket.yaml | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/e2e/minio.yaml b/e2e/minio.yaml index 0f99ad3d3..1b00b0efa 100644 --- a/e2e/minio.yaml +++ b/e2e/minio.yaml @@ -64,9 +64,7 @@ spec: secret: secretName: minio-cert items: - - key: ca.crt + - key: tls.crt path: public.crt - key: tls.key path: private.key - - key: ca.crt - path: CAs/public.crt diff --git a/e2e/testdata/makebucket.yaml b/e2e/testdata/makebucket.yaml index 43ec390e6..520321d62 100644 --- a/e2e/testdata/makebucket.yaml +++ b/e2e/testdata/makebucket.yaml @@ -15,7 +15,6 @@ spec: - --access_key=minioadmin - --secret_key=minioadmin - --ssl - - --no-check-certificate - --ca-certs=/minio-cert/ca.crt - mb - s3://moco From 4677acd35acc551d890db0ec33acbcf96b364686 Mon Sep 17 00:00:00 2001 From: kouki Date: Wed, 25 Oct 2023 01:46:36 +0000 Subject: [PATCH 10/13] Use Gomega matcher in Eventually Signed-off-by: kouki --- e2e/backup_test.go | 107 ++++++++++++----------------------- e2e/run_test.go | 22 +++++++ e2e/testdata/makebucket.yaml | 13 +---- 3 files changed, 58 insertions(+), 84 deletions(-) diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 3e970ce4e..23c07657a 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -4,8 +4,6 @@ import ( "bytes" _ "embed" "encoding/json" - "errors" - "fmt" "strconv" "strings" "text/template" @@ -43,12 +41,18 @@ var _ = Context("backup", func() { job := &batchv1.Job{} err = json.Unmarshal(out, job) g.Expect(err).NotTo(HaveOccurred()) +<<<<<<< Updated upstream for _, cond := range job.Status.Conditions { if cond.Type != batchv1.JobComplete { continue } g.Expect(cond.Status).To(Equal(corev1.ConditionTrue)) } +======= + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "make-bucket has not been finished") +>>>>>>> Stashed changes }).Should(Succeed()) }) @@ -61,21 +65,12 @@ var _ = Context("backup", func() { secjson, err := json.Marshal(sec) Expect(err).NotTo(HaveOccurred()) kubectlSafe(secjson, "apply", "-f", "-") - Eventually(func() error { + Eventually(func(g Gomega) { cluster, err := getCluster("backup", "source") - if err != nil { - return err - } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == metav1.ConditionTrue { - return nil - } - return fmt.Errorf("cluster is not healthy: %s", cond.Status) - } - return errors.New("no health condition") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue)) }).Should(Succeed()) kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--", @@ -88,24 +83,15 @@ var _ = Context("backup", func() { It("should take a full dump", func() { kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-1") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-1", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) job := &batchv1.Job{} - if err := json.Unmarshal(out, job); err != nil { - return err - } - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - } - return errors.New("backup-1 has not been finished") + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-1 has not been finished") }).Should(Succeed()) }) @@ -122,24 +108,15 @@ var _ = Context("backup", func() { time.Sleep(100 * time.Millisecond) kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-2") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-2", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) job := &batchv1.Job{} - if err := json.Unmarshal(out, job); err != nil { - return err - } - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - } - return errors.New("backup-2 has not been finished") + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-2 has not been finished") }).Should(Succeed()) cluster, err := getCluster("backup", "source") @@ -163,20 +140,12 @@ var _ = Context("backup", func() { Expect(err).NotTo(HaveOccurred()) kubectlSafe(buf.Bytes(), "apply", "-f", "-") - Eventually(func() error { + Eventually(func(g Gomega) { cluster, err := getCluster("backup", "target") - if err != nil { - return err - } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == metav1.ConditionTrue { - return nil - } - } - return errors.New("target is not healthy") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue), "target is not healthy") }).Should(Succeed()) out := kubectlSafe(nil, "moco", "-n", "backup", "mysql", "target", "--", @@ -189,19 +158,13 @@ var _ = Context("backup", func() { It("should delete clusters", func() { kubectlSafe(nil, "delete", "-n", "backup", "mysqlclusters", "--all") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "get", "-n", "backup", "pod", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) pods := &corev1.PodList{} - if err := json.Unmarshal(out, pods); err != nil { - return err - } - if len(pods.Items) > 0 { - return errors.New("wait until all Pods are deleted") - } - return nil + err = json.Unmarshal(out, pods) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(pods.Items)).To(BeNumerically(">", 0), "wait until all Pods are deleted") }).Should(Succeed()) }) }) diff --git a/e2e/run_test.go b/e2e/run_test.go index 0d64ed202..b7a625ce5 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -13,8 +13,10 @@ import ( . "github.com/onsi/gomega" dto "github.com/prometheus/client_model/go" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func kubectl(input []byte, args ...string) ([]byte, error) { @@ -57,6 +59,26 @@ func getCluster(ns, name string) (*mocov1beta2.MySQLCluster, error) { return cluster, nil } +func getClusterCondition(cluster *mocov1beta2.MySQLCluster, condType string) (metav1.Condition, error) { + for _, cond := range cluster.Status.Conditions { + if cond.Type != condType { + continue + } + return cond, nil + } + return metav1.Condition{}, fmt.Errorf("no %s condition", condType) +} + +func getJobCondition(job *batchv1.Job, condType batchv1.JobConditionType) (batchv1.JobCondition, error) { + for _, cond := range job.Status.Conditions { + if cond.Type != condType { + continue + } + return cond, nil + } + return batchv1.JobCondition{}, fmt.Errorf("no %s condition", condType) +} + func fillTemplate(tmpl string) []byte { return fillTemplateWithVersion(tmpl, mysqlVersion) } diff --git a/e2e/testdata/makebucket.yaml b/e2e/testdata/makebucket.yaml index 520321d62..18175e425 100644 --- a/e2e/testdata/makebucket.yaml +++ b/e2e/testdata/makebucket.yaml @@ -14,20 +14,9 @@ spec: - --host-bucket=minio.default.svc:9000 - --access_key=minioadmin - --secret_key=minioadmin - - --ssl - - --ca-certs=/minio-cert/ca.crt + - --no-ssl - mb - s3://moco image: moco-backup:dev imagePullPolicy: IfNotPresent name: make-bucket - volumeMounts: - - name: minio-cert - mountPath: /minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert - items: - - key: ca.crt - path: ca.crt From 3dbbdc0c6688d670ad2e296d59bdf87fcf77de97 Mon Sep 17 00:00:00 2001 From: kouki Date: Wed, 25 Oct 2023 02:01:47 +0000 Subject: [PATCH 11/13] Add test using minio with TLS enabled Signed-off-by: kouki --- e2e/Makefile | 1 + e2e/backup_test.go | 9 -- e2e/backup_tls_test.go | 161 +++++++++++++++++++++++++++++++ e2e/minio-tls.yaml | 70 ++++++++++++++ e2e/minio.yaml | 32 ------ e2e/testdata/backup.yaml | 10 +- e2e/testdata/backup_tls.yaml | 70 ++++++++++++++ e2e/testdata/makebucket_tls.yaml | 33 +++++++ e2e/testdata/restore.yaml | 10 +- e2e/testdata/restore_tls.yaml | 46 +++++++++ 10 files changed, 383 insertions(+), 59 deletions(-) create mode 100644 e2e/backup_tls_test.go create mode 100644 e2e/minio-tls.yaml create mode 100644 e2e/testdata/backup_tls.yaml create mode 100644 e2e/testdata/makebucket_tls.yaml create mode 100644 e2e/testdata/restore_tls.yaml diff --git a/e2e/Makefile b/e2e/Makefile index 963e926ed..d36089083 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -48,6 +48,7 @@ endif $(KUSTOMIZE) build . | $(KUBECTL) apply -f - $(KUBECTL) -n moco-system wait --for=condition=available --timeout=180s --all deployments $(KUBECTL) apply -f minio.yaml + $(KUBECTL) apply -f minio-tls.yaml $(KUBECTL) apply -f fake-gcs-server.yaml $(KUBECTL) wait --timeout=60s --for=condition=Ready --all pods diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 23c07657a..58e95eab0 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -41,18 +41,9 @@ var _ = Context("backup", func() { job := &batchv1.Job{} err = json.Unmarshal(out, job) g.Expect(err).NotTo(HaveOccurred()) -<<<<<<< Updated upstream - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - g.Expect(cond.Status).To(Equal(corev1.ConditionTrue)) - } -======= condComplete, err := getJobCondition(job, batchv1.JobComplete) g.Expect(err).NotTo(HaveOccurred()) g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "make-bucket has not been finished") ->>>>>>> Stashed changes }).Should(Succeed()) }) diff --git a/e2e/backup_tls_test.go b/e2e/backup_tls_test.go new file mode 100644 index 000000000..15e23f71c --- /dev/null +++ b/e2e/backup_tls_test.go @@ -0,0 +1,161 @@ +package e2e + +import ( + "bytes" + _ "embed" + "encoding/json" + "strconv" + "strings" + "text/template" + "time" + + mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//go:embed testdata/makebucket_tls.yaml +var makeBucketTLSYAML string + +//go:embed testdata/backup_tls.yaml +var backupTLSYAML string + +//go:embed testdata/restore_tls.yaml +var restoreTLSYAML string + +var _ = Context("backup-tls", func() { + if doUpgrade { + return + } + + var restorePoint time.Time + + It("should create a bucket", func() { + kubectlSafe([]byte(makeBucketTLSYAML), "apply", "-f", "-") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "jobs", "make-bucket-tls", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "make-bucket-tls has not been finished") + }).Should(Succeed()) + }) + + It("should construct a source cluster", func() { + kubectlSafe(fillTemplate(backupTLSYAML), "apply", "-f", "-") + secjson := kubectlSafe(nil, "get", "secret", "-o", "json", "minio-cert") + sec := &corev1.Secret{} + json.Unmarshal(secjson, sec) + sec.SetNamespace("backup-tls") + secjson, err := json.Marshal(sec) + Expect(err).NotTo(HaveOccurred()) + kubectlSafe(secjson, "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "source") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue)) + }).Should(Succeed()) + + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-e", "CREATE DATABASE test") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "-e", "CREATE TABLE t (id INT NOT NULL AUTO_INCREMENT, data VARCHAR(32) NOT NULL, PRIMARY KEY (id), KEY key1 (data), KEY key2 (data, id)) ENGINE=InnoDB") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('aaa')") + }) + + It("should take a full dump", func() { + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-1") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-1", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-tls-1 has not been finished") + }).Should(Succeed()) + }) + + It("should take an incremental backup", func() { + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('bbb')") + time.Sleep(1100 * time.Millisecond) + restorePoint = time.Now().UTC() + time.Sleep(1100 * time.Millisecond) + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-admin", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "FLUSH LOCAL BINARY LOGS") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('ccc')") + time.Sleep(100 * time.Millisecond) + + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-2") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-2", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-tls-2 has not been finished") + }).Should(Succeed()) + + cluster, err := getCluster("backup-tls", "source") + Expect(err).NotTo(HaveOccurred()) + Expect(cluster.Status.Backup.BinlogSize).NotTo(Equal(int64(0))) + }) + + It("should destroy the source then restore the backup data", func() { + kubectlSafe(nil, "-n", "backup-tls", "delete", "mysqlclusters", "source") + + tmpl, err := template.New("").Parse(restoreTLSYAML) + Expect(err).NotTo(HaveOccurred()) + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, struct { + MySQLVersion string + RestorePoint string + }{ + mysqlVersion, + restorePoint.Format(time.RFC3339), + }) + Expect(err).NotTo(HaveOccurred()) + + kubectlSafe(buf.Bytes(), "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "target") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue), "target is not healthy") + }).Should(Succeed()) + + out := kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "target", "--", + "-N", "-D", "test", "-e", "SELECT COUNT(*) FROM t") + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(2)) + }) + + It("should delete clusters", func() { + kubectlSafe(nil, "delete", "-n", "backup-tls", "mysqlclusters", "--all") + + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "-n", "backup-tls", "pod", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + pods := &corev1.PodList{} + err = json.Unmarshal(out, pods) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(pods.Items)).To(BeNumerically(">", 0), "wait until all Pods are deleted") + }).Should(Succeed()) + }) +}) diff --git a/e2e/minio-tls.yaml b/e2e/minio-tls.yaml new file mode 100644 index 000000000..770cf9bae --- /dev/null +++ b/e2e/minio-tls.yaml @@ -0,0 +1,70 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: default + name: default-selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: default + name: minio-cert +spec: + commonName: minio cert + issuerRef: + kind: Issuer + name: default-selfsigned-issuer + secretName: minio-cert + dnsNames: + - minio-tls.default.svc +--- +apiVersion: v1 +kind: Service +metadata: + namespace: default + name: minio-tls +spec: + ports: + - name: minio + port: 9000 + targetPort: minio + protocol: TCP + selector: + name: minio-tls +--- +apiVersion: v1 +kind: Pod +metadata: + namespace: default + name: minio-tls + labels: + name: minio-tls +spec: + containers: + - name: minio + image: minio/minio + args: + - server + - /data + ports: + - name: minio + containerPort: 9000 + protocol: TCP + volumeMounts: + - name: data + mountPath: /data + - name: secret-volume + mountPath: /root/.minio/certs + volumes: + - name: data + emptyDir: {} + - name: secret-volume + secret: + secretName: minio-cert + items: + - key: tls.crt + path: public.crt + - key: tls.key + path: private.key diff --git a/e2e/minio.yaml b/e2e/minio.yaml index 1b00b0efa..3c7bbb712 100644 --- a/e2e/minio.yaml +++ b/e2e/minio.yaml @@ -1,25 +1,3 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - namespace: default - name: default-selfsigned-issuer -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - namespace: default - name: minio-cert -spec: - commonName: minio cert - issuerRef: - kind: Issuer - name: default-selfsigned-issuer - secretName: minio-cert - dnsNames: - - minio.default.svc ---- apiVersion: v1 kind: Service metadata: @@ -55,16 +33,6 @@ spec: volumeMounts: - name: data mountPath: /data - - name: secret-volume - mountPath: /root/.minio/certs volumes: - name: data emptyDir: {} - - name: secret-volume - secret: - secretName: minio-cert - items: - - key: tls.crt - path: public.crt - - key: tls.key - path: private.key diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index 05167ae22..0ddde2c30 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -31,18 +31,10 @@ spec: value: minioadmin - name: AWS_SECRET_ACCESS_KEY value: minioadmin - volumeMounts: - - mountPath: /minio-cert - name: minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert bucketConfig: bucketName: moco - endpointURL: https://minio.default.svc:9000 + endpointURL: http://minio.default.svc:9000 usePathStyle: true - caCert: /minio-cert/ca.crt workVolume: emptyDir: {} --- diff --git a/e2e/testdata/backup_tls.yaml b/e2e/testdata/backup_tls.yaml new file mode 100644 index 000000000..525b12ffc --- /dev/null +++ b/e2e/testdata/backup_tls.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup-tls +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: backup-tls + name: mycnf +data: + innodb_log_file_size: "10M" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: backup-tls + name: backup-owner +--- +apiVersion: moco.cybozu.com/v1beta1 +kind: BackupPolicy +metadata: + namespace: backup-tls + name: daily +spec: + schedule: "@daily" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio-tls.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} +--- +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: source +spec: + mysqlConfigMapName: mycnf + replicas: 3 + backupPolicyName: daily + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ . }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/makebucket_tls.yaml b/e2e/testdata/makebucket_tls.yaml new file mode 100644 index 000000000..1ff37d6ea --- /dev/null +++ b/e2e/testdata/makebucket_tls.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: make-bucket-tls + namespace: default +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - command: + - s3cmd + - --host=minio-tls.default.svc:9000 + - --host-bucket=minio-tls.default.svc:9000 + - --access_key=minioadmin + - --secret_key=minioadmin + - --ssl + - --ca-certs=/minio-cert/ca.crt + - mb + - s3://moco + image: moco-backup:dev + imagePullPolicy: IfNotPresent + name: make-bucket-tls + volumeMounts: + - name: minio-cert + mountPath: /minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + items: + - key: ca.crt + path: ca.crt diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index bf290733b..7042a851a 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -17,18 +17,10 @@ spec: value: minioadmin - name: AWS_SECRET_ACCESS_KEY value: minioadmin - volumeMounts: - - mountPath: /minio-cert - name: minio-cert - volumes: - - name: minio-cert - secret: - secretName: minio-cert bucketConfig: bucketName: moco - endpointURL: https://minio.default.svc:9000 + endpointURL: http://minio.default.svc:9000 usePathStyle: true - caCert: /minio-cert/ca.crt workVolume: emptyDir: {} podTemplate: diff --git a/e2e/testdata/restore_tls.yaml b/e2e/testdata/restore_tls.yaml new file mode 100644 index 000000000..f15fb21d1 --- /dev/null +++ b/e2e/testdata/restore_tls.yaml @@ -0,0 +1,46 @@ +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: target +spec: + mysqlConfigMapName: mycnf + replicas: 1 + restore: + sourceName: source + sourceNamespace: backup-tls + restorePoint: "{{ .RestorePoint }}" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio-tls.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi From 767289a5a8a5bd2f14aa275837525defc6fb0ee2 Mon Sep 17 00:00:00 2001 From: kouki Date: Thu, 26 Oct 2023 06:00:52 +0000 Subject: [PATCH 12/13] Update docs about moco-backup global flags Signed-off-by: kouki --- docs/moco-backup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/moco-backup.md b/docs/moco-backup.md index 9f307d08f..b56dafdfb 100644 --- a/docs/moco-backup.md +++ b/docs/moco-backup.md @@ -19,7 +19,7 @@ Global Flags: --threads int The number of threads to be used (default 4) --use-path-style Use path-style S3 API --work-dir string The writable working directory (default "/work") - --ca-certs string Path to SSL CA certificate file instead of system default. + --ca-cert string Path to SSL CA certificate file used in addition to system default ``` ## Subcommands From 96c72662a6c7074c7d0232b564c9a878e202237f Mon Sep 17 00:00:00 2001 From: kouki Date: Fri, 27 Oct 2023 00:13:54 +0000 Subject: [PATCH 13/13] Fix to remove unnecessary secret Signed-off-by: kouki --- e2e/backup_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/e2e/backup_test.go b/e2e/backup_test.go index 58e95eab0..aa0772e50 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -49,13 +49,6 @@ var _ = Context("backup", func() { It("should construct a source cluster", func() { kubectlSafe(fillTemplate(backupYAML), "apply", "-f", "-") - secjson := kubectlSafe(nil, "get", "secret", "-o", "json", "minio-cert") - sec := &corev1.Secret{} - json.Unmarshal(secjson, sec) - sec.SetNamespace("backup") - secjson, err := json.Marshal(sec) - Expect(err).NotTo(HaveOccurred()) - kubectlSafe(secjson, "apply", "-f", "-") Eventually(func(g Gomega) { cluster, err := getCluster("backup", "source") g.Expect(err).NotTo(HaveOccurred())