From 62a7e0f018d9274f9e48ffac3d245db5bd1ed09d Mon Sep 17 00:00:00 2001 From: acekingke Date: Sun, 8 Oct 2023 17:12:38 +0800 Subject: [PATCH] *: add remote source from xtrabackup --- Dockerfile.sidecar | 2 +- api/v1alpha1/mysqlcluster_types.go | 7 ++- api/v1alpha1/zz_generated.deepcopy.go | 5 ++ api/v1beta1/mysqlcluster_conversion.go | 7 ++- api/v1beta1/zz_generated.conversion.go | 1 + .../mysql.radondb.com_mysqlclusters.yaml | 47 +++++++++++++++ .../mysql.radondb.com_mysqlclusters.yaml | 47 +++++++++++++++ .../samples/mysql_v1beta1_mysqlcluster.yaml | 10 +++- docs/en-us/backup_and_restoration_nfs.md | 6 +- docs/en-us/backup_and_restoration_s3.md | 2 +- docs/en-us/backup_cron.md | 2 +- docs/en-us/how_to_use_nfsbcp.md | 2 +- docs/en-us/rebuild.md | 2 +- docs/remotesource.md | 37 ++++++++++++ mysqlcluster/container/init_sidecar.go | 13 +++- mysqlcluster/container/mysql.go | 7 +++ mysqlcluster/mysqlcluster.go | 14 +++++ sidecar/config.go | 59 +++++++++++++++++-- sidecar/init.go | 4 ++ utils/constants.go | 3 + 20 files changed, 260 insertions(+), 17 deletions(-) create mode 100644 docs/remotesource.md diff --git a/Dockerfile.sidecar b/Dockerfile.sidecar index 581af4da..679bec9d 100644 --- a/Dockerfile.sidecar +++ b/Dockerfile.sidecar @@ -52,7 +52,7 @@ RUN set -ex; \ ARG XTRABACKUP_PKG=percona-xtrabackup-24 RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \ + apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc openssh-server sshpass;\ wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \ dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \ apt-get update; \ diff --git a/api/v1alpha1/mysqlcluster_types.go b/api/v1alpha1/mysqlcluster_types.go index 60fd2834..4d7be6c6 100644 --- a/api/v1alpha1/mysqlcluster_types.go +++ b/api/v1alpha1/mysqlcluster_types.go @@ -113,6 +113,10 @@ type MysqlClusterSpec struct { // Containing CA (ca.crt) and server cert (tls.crt), server private key (tls.key) for SSL // +optional TlsSecretName string `json:"tlsSecretName,omitempty"` + + // Bootstraping from remote data source + // +optional + SourceConfig *corev1.SecretProjection `json:"sourceConfig,omitempty"` } // ReadOnly define the ReadOnly pods @@ -345,7 +349,8 @@ const ( // ClusterUpdateState indicates whether the cluster is being updated. ClusterUpdateState ClusterState = "Updating" // ClusterReadyState indicates whether all containers in the pod are ready. - ClusterReadyState ClusterState = "Ready" + ClusterReadyState ClusterState = "Ready" + ClusterRemoteCompleted ClusterState = "RemoteCompleted" // ClusterCloseState indicates whether the cluster is closed. ClusterCloseState ClusterState = "Closed" // ClusterScaleInState indicates whether the cluster replicas is decreasing. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cdccbe7c..69da81f6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -299,6 +299,11 @@ func (in *MysqlClusterSpec) DeepCopyInto(out *MysqlClusterSpec) { *out = new(int) **out = **in } + if in.SourceConfig != nil { + in, out := &in.SourceConfig, &out.SourceConfig + *out = new(v1.SecretProjection) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MysqlClusterSpec. diff --git a/api/v1beta1/mysqlcluster_conversion.go b/api/v1beta1/mysqlcluster_conversion.go index 875b3633..e5c57372 100644 --- a/api/v1beta1/mysqlcluster_conversion.go +++ b/api/v1beta1/mysqlcluster_conversion.go @@ -95,7 +95,9 @@ func Convert_v1alpha1_MysqlClusterSpec_To_v1beta1_MysqlClusterSpec(in *v1alpha1. out.DataSource.S3Backup.SecretName = in.BackupSecretName out.DataSource.RestorePoint = in.RestorePoint } - + if in.SourceConfig != nil { + out.DataSource.Remote.SourceConfig = in.SourceConfig + } if len(in.NFSServerAddress) != 0 { ipStr := strings.Split(in.NFSServerAddress, ":") out.DataSource.RestorePoint = in.RestorePoint @@ -160,6 +162,9 @@ func Convert_v1beta1_MysqlClusterSpec_To_v1alpha1_MysqlClusterSpec(in *MysqlClus out.PodPolicy.PriorityClassName = in.PriorityClassName // in.DataSource in.Standby out.XenonOpts.EnableAutoRebuild = in.EnableAutoRebuild + if in.DataSource.Remote.SourceConfig != nil { + out.SourceConfig = in.DataSource.Remote.SourceConfig + } if len(in.DataSource.S3Backup.Name) != 0 { out.RestoreFrom = in.DataSource.S3Backup.Name out.BackupSecretName = in.DataSource.S3Backup.SecretName diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 8cee1cb9..506979ed 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -454,6 +454,7 @@ func autoConvert_v1alpha1_MysqlClusterSpec_To_v1beta1_MysqlClusterSpec(in *v1alp // WARNING: in.BothS3NFS requires manual conversion: does not exist in peer-type // WARNING: in.BackupScheduleJobsHistoryLimit requires manual conversion: does not exist in peer-type // WARNING: in.TlsSecretName requires manual conversion: does not exist in peer-type + // WARNING: in.SourceConfig requires manual conversion: does not exist in peer-type return nil } diff --git a/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml b/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml index 29bb282a..4da8fd8e 100644 --- a/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml +++ b/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml @@ -2278,6 +2278,53 @@ spec: description: RestorePoint is the target date and time to restore data. The format is "2006-01-02 15:04:05" type: string + sourceConfig: + description: Bootstraping from remote data source + properties: + items: + description: If unspecified, each key-value pair in the Data field + of the referenced Secret will be projected into the volume as + a file whose name is the key and content is the value. If specified, + the listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is specified + which is not present in the Secret, the volume setup will error + unless it is marked optional. Paths must be relative and may + not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 and + 0777 or a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the key + to. May not be an absolute path. May not contain the path + element '..'. May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + type: object tlsSecretName: description: Containing CA (ca.crt) and server cert (tls.crt), server private key (tls.key) for SSL diff --git a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml index 10ceedf7..40d731dd 100644 --- a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml +++ b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml @@ -2260,6 +2260,53 @@ spec: description: RestorePoint is the target date and time to restore data. The format is "2006-01-02 15:04:05" type: string + sourceConfig: + description: Bootstraping from remote data source + properties: + items: + description: If unspecified, each key-value pair in the Data field + of the referenced Secret will be projected into the volume as + a file whose name is the key and content is the value. If specified, + the listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is specified + which is not present in the Secret, the volume setup will error + unless it is marked optional. Paths must be relative and may + not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 and + 0777 or a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the key + to. May not be an absolute path. May not contain the path + element '..'. May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + type: object tlsSecretName: description: Containing CA (ca.crt) and server cert (tls.crt), server private key (tls.key) for SSL diff --git a/config/samples/mysql_v1beta1_mysqlcluster.yaml b/config/samples/mysql_v1beta1_mysqlcluster.yaml index 16c09ba7..7ea48068 100644 --- a/config/samples/mysql_v1beta1_mysqlcluster.yaml +++ b/config/samples/mysql_v1beta1_mysqlcluster.yaml @@ -14,7 +14,15 @@ spec: S3backup: name: "" secretName: "" - remote: {} + remote: + # sourceConfig: + # name: remotesecret + # items: + # - key: passwd + # path: passwd + # - key: host + # path: host + image: percona/percona-server:8.0.25 imagePullPolicy: Always logOpts: diff --git a/docs/en-us/backup_and_restoration_nfs.md b/docs/en-us/backup_and_restoration_nfs.md index 76c1dfff..e8b681ca 100644 --- a/docs/en-us/backup_and_restoration_nfs.md +++ b/docs/en-us/backup_and_restoration_nfs.md @@ -47,13 +47,13 @@ You can use `ClusterIp` to perform NFS backup. The cluster IP address in the exa ### 1. Configure the NFS server address ```yaml -# config/samples/mysql_v1alpha1_backup.yaml +# config/samples/mysql_v1beta1_backup.yaml nfsServerAddress: "10.96.253.82" ``` ### 2. Create a backup ```shell -kubectl apply -f config/samples/mysql_v1alpha1_backup.yaml +kubectl apply -f config/samples/mysql_v1beta1_backup.yaml ``` > Note: The backup CRD and MySQL cluster CRD must be in the same namespace. @@ -67,7 +67,7 @@ index.html initbackup sample_2022419101946 ## Restore the cluster from the NFS backup -Configure the `nfsServerAddress` attribute to the NFS server address in the `mysql_v1alpha1_cluster.yaml` file. +Configure the `nfsServerAddress` attribute to the NFS server address in the `mysql_v1alpha1_cluster.yaml` file or `mysql_v1beta1_cluster.yaml` file. ```yaml ... diff --git a/docs/en-us/backup_and_restoration_s3.md b/docs/en-us/backup_and_restoration_s3.md index c2676104..45718ef3 100644 --- a/docs/en-us/backup_and_restoration_s3.md +++ b/docs/en-us/backup_and_restoration_s3.md @@ -46,7 +46,7 @@ kubectl create -f config/samples/backup_secret.yaml ``` ### Step 2: Configure the backup Secret for the Operator cluster -Configure the `backupSecretName` property in `mysql_v1alpha1_mysqlcluster.yaml`, for example, `sample-backup-secre`. +Configure the `backupSecretName` property in `mysql_v1beta1_mysqlcluster.yaml`, for example, `sample-backup-secre`. ```yaml spec: diff --git a/docs/en-us/backup_cron.md b/docs/en-us/backup_cron.md index 716b4163..0fc2b7ac 100644 --- a/docs/en-us/backup_cron.md +++ b/docs/en-us/backup_cron.md @@ -1,4 +1,4 @@ -Contents +Contents(Deprecated) ============= - [Overview](#overview) - [Configuration of scheduled backups](#configuration-of-scheduled-backups) diff --git a/docs/en-us/how_to_use_nfsbcp.md b/docs/en-us/how_to_use_nfsbcp.md index 3be558c3..e08bd825 100644 --- a/docs/en-us/how_to_use_nfsbcp.md +++ b/docs/en-us/how_to_use_nfsbcp.md @@ -1,5 +1,5 @@ English | [简体中文](../en-cn/how_to_use_nfsbcp.md) -# Quickstart for nfsbcp +# Quickstart for nfsbcp （Deprecated） ## Contents * [Introduction](#Introduction) diff --git a/docs/en-us/rebuild.md b/docs/en-us/rebuild.md index 80da8d9a..8a066686 100644 --- a/docs/en-us/rebuild.md +++ b/docs/en-us/rebuild.md @@ -1,4 +1,4 @@ -English | [简体中文](../zh-cn/rebuild.md) +English | [简体中文](../zh-cn/rebuild.md) (Deprecated) # Reason for rebuilding Pod RadonDB cluster implements MySQL semisynchronous replication. Semisynchronous replication may cause the replica nodes to update more data than the source node. So, the Pod needs to be rebuilt when it is detected as invalid by Xenon. diff --git a/docs/remotesource.md b/docs/remotesource.md new file mode 100644 index 00000000..5242338d --- /dev/null +++ b/docs/remotesource.md @@ -0,0 +1,37 @@ +English + +# Quickstart remote migration from remote source + +## 1. perpare the remote source node + +We suppose you have got a existence mysql database, such as percona server, or other mysql release. +and mysql version is 8.0.25 or mysql 5.7, this document use 8.0.25 for example. we suppose the ip of the node is `172.16.0.29` , and the root's password is `rootpass`. Make sure the sshd is ok, and it can login by `ssh` + +## 2. create secret in k8s +Now, what's need you to do is that create secret file in k8s, and it must contain the keys ,`host` and `passwd`. for example, we can create a secret named `remotesecret` as follow: +``` + kubectl create secret generic remotesecret --from-literal=host=172.16.0.29 --from-literal=passwd=rootpass +``` +## 3. fill the fields in mysqlcluster.yaml +fill the dataSouce's `remote` field, you should fill the name which is the secret's name, in this example, name is `remotesecret`, and fill `items` as same as below: + +```yaml +spec: + ... + dataSource: + ... + remote: + sourceConfig: + name: remotesecret + items: + - key: passwd + path: passwd + - key: host + path: host + +``` +## 4. apply the yaml file, run it in k8s + +```sh +kubectl apply -f mysql_v1beta1_mysqlcluster.yaml +``` \ No newline at end of file diff --git a/mysqlcluster/container/init_sidecar.go b/mysqlcluster/container/init_sidecar.go index 418051ad..4723776c 100644 --- a/mysqlcluster/container/init_sidecar.go +++ b/mysqlcluster/container/init_sidecar.go @@ -147,6 +147,12 @@ func (c *initSidecar) getEnvVars() []corev1.EnvVar { Value: "1", }) } + if c.Spec.SourceConfig != nil { + envs = append(envs, corev1.EnvVar{ + Name: "REMOTESRC", + Value: "1", + }) + } return envs } @@ -252,6 +258,11 @@ func (c *initSidecar) getVolumeMounts() []corev1.VolumeMount { }, ) } - + if c.Spec.SourceConfig != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: utils.RemoteSourceVolume, + MountPath: utils.RemoteSourcePath, + }) + } return volumeMounts } diff --git a/mysqlcluster/container/mysql.go b/mysqlcluster/container/mysql.go index aa0f7fd3..67b0e6f3 100644 --- a/mysqlcluster/container/mysql.go +++ b/mysqlcluster/container/mysql.go @@ -190,5 +190,12 @@ func (c *mysql) getVolumeMounts() []corev1.VolumeMount { }, ) } + // Just for test + // if c.Spec.SourceConfig != nil { + // volumeMounts = append(volumeMounts, corev1.VolumeMount{ + // Name: utils.RemoteSourceVolume, + // MountPath: utils.RemoteSourcePath, + // }) + // } return volumeMounts } diff --git a/mysqlcluster/mysqlcluster.go b/mysqlcluster/mysqlcluster.go index 16d3d5b8..5d233ff8 100644 --- a/mysqlcluster/mysqlcluster.go +++ b/mysqlcluster/mysqlcluster.go @@ -286,6 +286,20 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume { EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }) + if c.Spec.SourceConfig != nil { + volumes = append(volumes, corev1.Volume{ + Name: utils.RemoteSourceVolume, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: c.Spec.SourceConfig, + }, + }, + }, + }, + }) + } return volumes } diff --git a/sidecar/config.go b/sidecar/config.go index 44246a5f..a097298d 100644 --- a/sidecar/config.go +++ b/sidecar/config.go @@ -130,7 +130,7 @@ type Config struct { XCloudS3AccessKey string XCloudS3SecretKey string XCloudS3Bucket string - + XRemoteDateSource string // Need Upgrade NeedUpgrade bool } @@ -212,10 +212,10 @@ func NewInitConfig() *Config { XCloudS3AccessKey: getEnvValue("S3_ACCESSKEY"), XCloudS3SecretKey: getEnvValue("S3_SECRETKEY"), XCloudS3Bucket: getEnvValue("S3_BUCKET"), - - ClusterName: getEnvValue("CLUSTER_NAME"), - CloneFlag: false, - GtidPurged: "", + XRemoteDateSource: getEnvValue("REMOTESRC"), + ClusterName: getEnvValue("CLUSTER_NAME"), + CloneFlag: false, + GtidPurged: "", // need upgrade NeedUpgrade: needUpgrade, } @@ -842,3 +842,52 @@ func (cfg *Config) ExecuteNFSRestore() error { cfg.BuildScript(gtid, restorePoint) return nil } + +func (cfg *Config) ExecuteRemoteSource() error { + // Check /var/lib/mysql exists or not. + log.Info("now get data from remote source") + if _, err := os.Stat(utils.DataVolumeMountPath); os.IsNotExist(err) { + err = os.MkdirAll(utils.DataVolumeMountPath, 0755) + if err != nil { + return fmt.Errorf("create /var/lib/mysql fail : %s", err) + } + // Change the owner of /var/lib/mysql + cmd := exec.Command("chown", "mysql.mysql", utils.DataVolumeMountPath) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to chown -R mysql.mysql : %s", err) + } + } + // Remove the data directory + cmd := exec.Command("rm", "-rf", utils.DataVolumeMountPath+"/*") + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to rm -rf %s : %s", utils.DataVolumeMountPath, err) + } + //sshpass -p rootpass ssh root@172.16.0.29 xtrabackup --user=root --password='' --backup --stream=xbstream --target-dir=./ > backup.xbstream 2>backup.log + CmdStr := "sshpass -p rootpass ssh -o 'StrictHostKeyChecking no' root@`cat /etc/rsrc/host` xtrabackup --user=root --password=`cat /etc/rsrc/passwd` --backup --stream=xbstream --target-dir=/ > backup.xbstream 2>backup.log;" + CmdStr += fmt.Sprintf("cd %s ; xbstream -x < /backup.xbstream", utils.DataVolumeMountPath) + cmd = exec.Command("/bin/bash", "-c", "--", CmdStr) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to rm -rf %s : %s", utils.DataVolumeMountPath, err) + } + // Prepare the append-only file + cmd = exec.Command("xtrabackup", "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--apply-log-only", "--target-dir="+utils.DataVolumeMountPath) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to xtrabackup prepare append-only: %s", err) + } + // Prepare the data directory + cmd = exec.Command("xtrabackup", "--defaults-file="+utils.MysqlConfVolumeMountPath+"/my.cnf", "--use-memory=3072M", "--prepare", "--target-dir="+utils.DataVolumeMountPath) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to xtrabackup prepare: %s", err) + } + cmd = exec.Command("chown", "-R", "mysql.mysql", utils.DataVolumeMountPath) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to chown -R mysql.mysql : %s", err) + } + return nil +} diff --git a/sidecar/init.go b/sidecar/init.go index 582fbdc4..9a37c1e9 100644 --- a/sidecar/init.go +++ b/sidecar/init.go @@ -316,6 +316,10 @@ func runInitCommand(cfg *Config, hasInitialized bool) error { } // Check has initialized again. hasInitialized, _ = checkIfPathExists(path.Join(dataPath, "mysql")) + } else if len(cfg.XRemoteDateSource) != 0 { + if err_r := cfg.ExecuteRemoteSource(); err_r != nil { + return fmt.Errorf("failed to remote source from %s: %s", cfg.XRestoreFrom, err_r) + } } } // Build init.sql after restore diff --git a/utils/constants.go b/utils/constants.go index 2875b78f..9007ed0d 100644 --- a/utils/constants.go +++ b/utils/constants.go @@ -135,6 +135,9 @@ const ( // TlsMountPath is the volume mount path for tls TlsMountPath = "/etc/mysql-ssl" + // RemoteSource + RemoteSourceVolume = "rsrc" + RemoteSourcePath = "/etc/rsrc" //extra env for readonly ROIbPool = "IB_POOL" ROIbInst = "IB_INST"