diff --git a/charts/mysql-operator/templates/cluster_rbac.yaml b/charts/mysql-operator/templates/cluster_rbac.yaml index 4f068d79..156c9b40 100644 --- a/charts/mysql-operator/templates/cluster_rbac.yaml +++ b/charts/mysql-operator/templates/cluster_rbac.yaml @@ -9,6 +9,16 @@ metadata: release: {{ .Release.Name | quote }} heritage: {{ .Release.Service | quote }} rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch - apiGroups: - apps resources: diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 32eeb3c0..9d828d02 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -24,6 +24,7 @@ import ( // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/apimachinery/pkg/runtime" @@ -44,6 +45,7 @@ import ( "github.com/radondb/radondb-mysql-kubernetes/controllers" "github.com/radondb/radondb-mysql-kubernetes/controllers/backup" "github.com/radondb/radondb-mysql-kubernetes/internal" + "github.com/radondb/radondb-mysql-kubernetes/utils" //+kubebuilder:scaffold:imports ) @@ -58,6 +60,7 @@ func init() { utilruntime.Must(mysqlv1alpha1.AddToScheme(scheme)) utilruntime.Must(mysqlv1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme + utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) } func main() { @@ -171,10 +174,12 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } - + // check crds + utils.RunUpdeteCRD(mgr.GetClient(), &setupLog) setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } + } diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1eea4289..f82cc575 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -6,6 +6,16 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch - apiGroups: - apps resources: diff --git a/config/samples/mysql_v1beta1_mysqlcluster.yaml b/config/samples/mysql_v1beta1_mysqlcluster.yaml index 7ea48068..0a600e3b 100644 --- a/config/samples/mysql_v1beta1_mysqlcluster.yaml +++ b/config/samples/mysql_v1beta1_mysqlcluster.yaml @@ -24,7 +24,7 @@ spec: # path: host image: percona/percona-server:8.0.25 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent logOpts: resources: requests: diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 6bcc1db0..a33bec90 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -55,6 +55,7 @@ type MysqlClusterReconciler struct { internal.XenonExecutor } +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=mysql.radondb.com,resources=mysqlclusters/finalizers,verbs=update diff --git a/utils/incluster.go b/utils/incluster.go index 113e3ae0..cfaeb18d 100644 --- a/utils/incluster.go +++ b/utils/incluster.go @@ -28,8 +28,12 @@ import ( "os/exec" "strconv" "strings" + "time" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -37,6 +41,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -207,3 +212,119 @@ func NewConfig() (*rest.Config, error) { return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( loader, &clientcmd.ConfigOverrides{}).ClientConfig() } + +func UpdateforCRD(crdName string, cli client.Client, log *logr.Logger) error { + // TODO: update CRD + + // MYNS=extension-dmp + // CRD1=mysqlclusters.mysql.radondb.com + // CRD2=backups.mysql.radondb.com + // SEC=radondb-mysql-webhook-certs + // CERT=$(kubectl -n $MYNS get secrets $SEC -ojsonpath='{.data.tls\.crt}') + // kubectl patch CustomResourceDefinition $CRD1 --type=merge -p '{"spec":{"conversion":{"webhook":{"clientConfig":{"caBundle":"'$CERT'","service":{"namespace":"'$MYNS'"}}}}}}' + // kubectl patch CustomResourceDefinition $CRD2 --type=merge -p '{"spec":{"conversion":{"webhook":{"clientConfig":{"caBundle":"'$CERT'","service":{"namespace":"'$MYNS'"}}}}}}' + // echo $CERT + // fetch a secret in dmp-extension namespace which is named radondb-mysql-webhook-certs + // 1. first get os environment value MY_NAMESPACE, if not set, use default namespace ,"dmp-extension" + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ns := os.Getenv("MY_NAMESPACE") + if len(ns) == 0 { + ns = "extension-dmp" + } + //2. get os environment value CERT_NAME, if not set, use default namespace "radondb-mysql-webhook-certs" + certName := os.Getenv("CERT_NAME") + if len(certName) == 0 { + certName = "radondb-mysql-webhook-certs" + } + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: certName, + Namespace: ns, + }, + } + err := cli.Get(ctx, types.NamespacedName{Name: secret.Name, Namespace: secret.Namespace}, secret) + // if err is not found, return error + if errors.IsNotFound(err) { + return fmt.Errorf("secret %s not found", certName) + } + cert := secret.Data["tls.crt"] + + //fetch the CustomResourceDefinition ,which name is mysqlclusters.mysql.radondb.com + // 创建 CRD 实例 + //apiextensionsv1.AddToScheme(scheme) + //CustomResourceDefinition + crd := &apiextensionsv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + } + // 使用客户端获取 CRD 资源 + errCRD := cli.Get(ctx, types.NamespacedName{Name: crdName}, crd) + if errCRD != nil { + return errCRD + } + hasBetaVersion := false + for _, v := range crd.Spec.Versions { + if v.Name == "v1beta1" { + hasBetaVersion = true + } + } + if !hasBetaVersion { + return fmt.Errorf("has not v1beta1 version") + } + + oldCrd := crd.DeepCopy() + // if CustomResourceConversion's CABundle of Webhook is not equal to cert, update it + if oldCrd.Spec.Conversion == nil || oldCrd.Spec.Conversion.Webhook == nil || oldCrd.Spec.Conversion.Webhook.ClientConfig == nil || + oldCrd.Spec.Conversion.Webhook.ClientConfig.CABundle == nil || !bytes.Equal(oldCrd.Spec.Conversion.Webhook.ClientConfig.CABundle, cert) { + log.Info("covert crd", "value", !bytes.Equal(oldCrd.Spec.Conversion.Webhook.ClientConfig.CABundle, cert)) + crd.Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + CABundle: []byte(cert), + Service: &apiextensionsv1.ServiceReference{ + Namespace: ns, + }, + }, + ConversionReviewVersions: []string{"v1"}, + }, + } + } else { + return nil + } + errCRD = cli.Patch(ctx, crd, client.MergeFrom(oldCrd)) + if errCRD != nil { + return errCRD + } + + return nil +} + +func RunUpdeteCRD(cli client.Client, log *logr.Logger) { + go func() { + // Just run in the first 500 seconds,almost eight minutes, because the crd webhook's CABundle is not correct just in the DMP reinstall period + // if this process failed, just need to restart the operetor pod + for i := 0; i < 100; i++ { + time.Sleep(time.Second * 5) + err := UpdateforCRD("mysqlclusters.mysql.radondb.com", cli, log) + if err != nil { + log.Info("update CRD failed", "error", err) + } + err = UpdateforCRD("backups.mysql.radondb.com", cli, log) + if err != nil { + log.Info("update CRD failed", "error", err) + } + } + log.Info("check the crd about 8 minutes, now exit.") + }() +}