Add mysql-configmap-history-limit flags (#733)

* Add mysql-configmap-history-limit flags Signed-off-by: d-kuro <kurosawa7620@gmail.com>
cybozu-go · Sep 19, 2024 · 2eb19ff · 2eb19ff
1 parent 742ccbc
commit 2eb19ff
Show file tree

Hide file tree

Showing 5 changed files with 132 additions and 94 deletions.
diff --git a/cmd/moco-controller/cmd/root.go b/cmd/moco-controller/cmd/root.go
@@ -23,23 +23,24 @@ var (
 )
 
 var config struct {
-	metricsAddr             string
-	probeAddr               string
-	pprofAddr               string
-	leaderElectionID        string
-	webhookAddr             string
-	certDir                 string
-	grpcCertDir             string
-	agentImage              string
-	backupImage             string
-	fluentBitImage          string
-	exporterImage           string
-	pvcSyncAnnotationKeys   []string
-	pvcSyncLabelKeys        []string
-	interval                time.Duration
-	maxConcurrentReconciles int
-	qps                     int
-	zapOpts                 zap.Options
+	metricsAddr                string
+	probeAddr                  string
+	pprofAddr                  string
+	leaderElectionID           string
+	webhookAddr                string
+	certDir                    string
+	grpcCertDir                string
+	agentImage                 string
+	backupImage                string
+	fluentBitImage             string
+	exporterImage              string
+	pvcSyncAnnotationKeys      []string
+	pvcSyncLabelKeys           []string
+	interval                   time.Duration
+	maxConcurrentReconciles    int
+	mySQLConfigMapHistoryLimit int
+	qps                        int
+	zapOpts                    zap.Options
 }
 
 func init() {
@@ -109,6 +110,7 @@ func init() {
 	fs.StringSliceVar(&config.pvcSyncLabelKeys, "pvc-sync-label-keys", []string{}, "The keys of labels from MySQLCluster's volumeClaimTemplates to be synced to the PVC")
 	fs.DurationVar(&config.interval, "check-interval", 1*time.Minute, "Interval of cluster maintenance")
 	fs.IntVar(&config.maxConcurrentReconciles, "max-concurrent-reconciles", 8, "The maximum number of concurrent reconciles which can be run")
+	fs.IntVar(&config.mySQLConfigMapHistoryLimit, "mysql-configmap-history-limit", 10, "The maximum number of MySQLConfigMap's history to be kept")
 	// The default QPS is 20.
 	// https://github.com/kubernetes-sigs/controller-runtime/blob/a26de2d610c3cf4b2a02688534aaf5a65749c743/pkg/client/config/config.go#L84-L85
 	fs.IntVar(&config.qps, "apiserver-qps-throttle", 20, "The maximum QPS to the API server.")

diff --git a/cmd/moco-controller/cmd/run.go b/cmd/moco-controller/cmd/run.go
@@ -102,18 +102,19 @@ func subMain(ns, addr string, port int) error {
 	defer clusterMgr.StopAll()
 
 	if err = (&controllers.MySQLClusterReconciler{
-		Client:                  mgr.GetClient(),
-		Scheme:                  mgr.GetScheme(),
-		Recorder:                mgr.GetEventRecorderFor("moco-controller"),
-		AgentImage:              config.agentImage,
-		BackupImage:             config.backupImage,
-		FluentBitImage:          config.fluentBitImage,
-		ExporterImage:           config.exporterImage,
-		SystemNamespace:         ns,
-		PVCSyncAnnotationKeys:   config.pvcSyncAnnotationKeys,
-		PVCSyncLabelKeys:        config.pvcSyncLabelKeys,
-		ClusterManager:          clusterMgr,
-		MaxConcurrentReconciles: config.maxConcurrentReconciles,
+		Client:                     mgr.GetClient(),
+		Scheme:                     mgr.GetScheme(),
+		Recorder:                   mgr.GetEventRecorderFor("moco-controller"),
+		AgentImage:                 config.agentImage,
+		BackupImage:                config.backupImage,
+		FluentBitImage:             config.fluentBitImage,
+		ExporterImage:              config.exporterImage,
+		SystemNamespace:            ns,
+		PVCSyncAnnotationKeys:      config.pvcSyncAnnotationKeys,
+		PVCSyncLabelKeys:           config.pvcSyncLabelKeys,
+		ClusterManager:             clusterMgr,
+		MaxConcurrentReconciles:    config.maxConcurrentReconciles,
+		MySQLConfigMapHistoryLimit: config.mySQLConfigMapHistoryLimit,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "MySQLCluster")
 		return err

diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go
@@ -8,6 +8,7 @@ import (
 	"hash/fnv"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -136,17 +137,18 @@ func apply[S any, T clientObjectConstraint[S], U any](ctx context.Context, r cli
 // MySQLClusterReconciler reconciles a MySQLCluster object
 type MySQLClusterReconciler struct {
 	client.Client
-	Scheme                  *runtime.Scheme
-	Recorder                record.EventRecorder
-	AgentImage              string
-	BackupImage             string
-	FluentBitImage          string
-	ExporterImage           string
-	SystemNamespace         string
-	PVCSyncAnnotationKeys   []string
-	PVCSyncLabelKeys        []string
-	ClusterManager          clustering.ClusterManager
-	MaxConcurrentReconciles int
+	Scheme                     *runtime.Scheme
+	Recorder                   record.EventRecorder
+	AgentImage                 string
+	BackupImage                string
+	FluentBitImage             string
+	ExporterImage              string
+	SystemNamespace            string
+	PVCSyncAnnotationKeys      []string
+	PVCSyncLabelKeys           []string
+	ClusterManager             clustering.ClusterManager
+	MaxConcurrentReconciles    int
+	MySQLConfigMapHistoryLimit int
 }
 
 //+kubebuilder:rbac:groups=moco.cybozu.com,resources=mysqlclusters,verbs=get;list;watch;update;patch
@@ -506,11 +508,22 @@ func (r *MySQLClusterReconciler) reconcileV1MyCnf(ctx context.Context, req ctrl.
 	if err := r.List(ctx, cms, client.InNamespace(cluster.Namespace)); err != nil {
 		return nil, err
 	}
-	for _, old := range cms.Items {
+
+	// Sort the ConfigMapList by creation timestamp in descending order
+	sort.Slice(cms.Items, func(i, j int) bool {
+		return cms.Items[i].CreationTimestamp.Time.After(cms.Items[j].CreationTimestamp.Time)
+	})
+
+	for i, old := range cms.Items {
+		if i < r.MySQLConfigMapHistoryLimit {
+			continue
+		}
+
 		if strings.HasPrefix(old.Name, prefix) && old.Name != cmName {
 			if err := r.Delete(ctx, &old); err != nil {
 				return nil, fmt.Errorf("failed to delete old my.cnf configmap %s/%s: %w", old.Namespace, old.Name, err)
 			}
+			log.Info("deleted old my.cnf configmap", "configMapName", old.Name)
 		}
 	}
 

diff --git a/controllers/mysqlcluster_controller_test.go b/controllers/mysqlcluster_controller_test.go
@@ -122,15 +122,16 @@ var _ = Describe("MySQLCluster reconciler", func() {
 			clusters: make(map[string]struct{}),
 		}
 		mysqlr := &MySQLClusterReconciler{
-			Client:          mgr.GetClient(),
-			Scheme:          scheme,
-			Recorder:        mgr.GetEventRecorderFor("moco-controller"),
-			SystemNamespace: testMocoSystemNamespace,
-			ClusterManager:  mockMgr,
-			AgentImage:      testAgentImage,
-			BackupImage:     testBackupImage,
-			FluentBitImage:  testFluentBitImage,
-			ExporterImage:   testExporterImage,
+			Client:                     mgr.GetClient(),
+			Scheme:                     scheme,
+			Recorder:                   mgr.GetEventRecorderFor("moco-controller"),
+			SystemNamespace:            testMocoSystemNamespace,
+			ClusterManager:             mockMgr,
+			AgentImage:                 testAgentImage,
+			BackupImage:                testBackupImage,
+			FluentBitImage:             testFluentBitImage,
+			ExporterImage:              testExporterImage,
+			MySQLConfigMapHistoryLimit: 2,
 		}
 		err = mysqlr.SetupWithManager(mgr)
 		Expect(err).ToNot(HaveOccurred())
@@ -422,19 +423,28 @@ var _ = Describe("MySQLCluster reconciler", func() {
 		err = k8sClient.Update(ctx, cluster)
 		Expect(err).NotTo(HaveOccurred())
 
-		oldName := cm.Name
 		Eventually(func() error {
 			cms := &corev1.ConfigMapList{}
 			if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil {
 				return err
 			}
 
-			var mycnfCMs []*corev1.ConfigMap
-			for i, cm := range cms.Items {
-				if cm.Name == oldName {
+			var mycnfCount int
+			prefix := cluster.PrefixedName() + "."
+			for _, cm := range cms.Items {
+				if !strings.HasPrefix(cm.Name, prefix) {
 					continue
 				}
-				if strings.HasPrefix(cm.Name, "moco-test.") {
+				mycnfCount++
+			}
+
+			if mycnfCount != 2 {
+				return fmt.Errorf("the number of config maps is not history limits: %d", len(cms.Items))
+			}
+
+			var mycnfCMs []*corev1.ConfigMap
+			for i, cm := range cms.Items {
+				if strings.HasPrefix(cm.Name, "moco-test.") && strings.Contains(cm.Data["my.cnf"], "foo = bar") {
 					mycnfCMs = append(mycnfCMs, &cms.Items[i])
 				}
 			}
@@ -458,19 +468,28 @@ var _ = Describe("MySQLCluster reconciler", func() {
 		err = k8sClient.Update(ctx, userCM)
 		Expect(err).NotTo(HaveOccurred())
 
-		oldName = cm.Name
 		Eventually(func() error {
 			cms := &corev1.ConfigMapList{}
 			if err := k8sClient.List(ctx, cms, client.InNamespace("test")); err != nil {
 				return err
 			}
 
-			var mycnfCMs []*corev1.ConfigMap
-			for i, cm := range cms.Items {
-				if cm.Name == oldName {
+			var mycnfCount int
+			prefix := cluster.PrefixedName() + "."
+			for _, cm := range cms.Items {
+				if !strings.HasPrefix(cm.Name, prefix) {
 					continue
 				}
-				if strings.HasPrefix(cm.Name, "moco-test.") {
+				mycnfCount++
+			}
+
+			if mycnfCount != 2 {
+				return fmt.Errorf("the number of config maps is not history limits: %d", len(cms.Items))
+			}
+
+			var mycnfCMs []*corev1.ConfigMap
+			for i, cm := range cms.Items {
+				if strings.HasPrefix(cm.Name, "moco-test.") && strings.Contains(cm.Data["my.cnf"], "foo = baz") {
 					mycnfCMs = append(mycnfCMs, &cms.Items[i])
 				}
 			}

diff --git a/docs/moco-controller.md b/docs/moco-controller.md
@@ -12,38 +12,41 @@
 
 ```
 Flags:
-      --add_dir_header                    If true, adds the file directory to the header of the log messages
-      --agent-image string                The image of moco-agent sidecar container
-      --alsologtostderr                   log to standard error as well as files (no effect when -logtostderr=true)
-      --apiserver-qps-throttle int        The maximum QPS to the API server. (default 20)
-      --backup-image string               The image of moco-backup container
-      --cert-dir string                   webhook certificate directory
-      --check-interval duration           Interval of cluster maintenance (default 1m0s)
-      --fluent-bit-image string           The image of fluent-bit sidecar container
-      --grpc-cert-dir string              gRPC certificate directory (default "/grpc-cert")
-      --health-probe-addr string          Listen address for health probes (default ":8081")
-  -h, --help                              help for moco-controller
-      --leader-election-id string         ID for leader election by controller-runtime (default "moco")
-      --log_backtrace_at traceLocation    when logging hits line file:N, emit a stack trace (default :0)
-      --log_dir string                    If non-empty, write log files in this directory (no effect when -logtostderr=true)
-      --log_file string                   If non-empty, use this log file (no effect when -logtostderr=true)
-      --log_file_max_size uint            Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
-      --logtostderr                       log to standard error instead of files (default true)
-      --max-concurrent-reconciles int     The maximum number of concurrent reconciles which can be run (default 8)
-      --metrics-addr string               Listen address for metric endpoint (default ":8080")
-      --mysqld-exporter-image string      The image of mysqld_exporter sidecar container
-      --one_output                        If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
-      --pprof-addr string                 Listen address for pprof endpoints. pprof is disabled by default
-      --skip_headers                      If true, avoid header prefixes in the log messages
-      --skip_log_headers                  If true, avoid headers when opening log files (no effect when -logtostderr=true)
-      --stderrthreshold severity          logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=false) (default 2)
-  -v, --v Level                           number for the log level verbosity
-      --version                           version for moco-controller
-      --vmodule moduleSpec                comma-separated list of pattern=N settings for file-filtered logging
-      --webhook-addr string               Listen address for the webhook endpoint (default ":9443")
-      --zap-devel                         Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error)
-      --zap-encoder encoder               Zap log encoding (one of 'json' or 'console')
-      --zap-log-level level               Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity
-      --zap-stacktrace-level level        Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').
-      --zap-time-encoding time-encoding   Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.
+      --add_dir_header                      If true, adds the file directory to the header of the log messages
+      --agent-image string                  The image of moco-agent sidecar container (default "ghcr.io/cybozu-go/moco-agent:0.12.1")
+      --alsologtostderr                     log to standard error as well as files (no effect when -logtostderr=true)
+      --apiserver-qps-throttle int          The maximum QPS to the API server. (default 20)
+      --backup-image string                 The image of moco-backup container (default "ghcr.io/cybozu-go/moco-backup:0.23.2")
+      --cert-dir string                     webhook certificate directory
+      --check-interval duration             Interval of cluster maintenance (default 1m0s)
+      --fluent-bit-image string             The image of fluent-bit sidecar container (default "ghcr.io/cybozu-go/moco/fluent-bit:3.0.2.1")
+      --grpc-cert-dir string                gRPC certificate directory (default "/grpc-cert")
+      --health-probe-addr string            Listen address for health probes (default ":8081")
+  -h, --help                                help for moco-controller
+      --leader-election-id string           ID for leader election by controller-runtime (default "moco")
+      --log_backtrace_at traceLocation      when logging hits line file:N, emit a stack trace (default :0)
+      --log_dir string                      If non-empty, write log files in this directory (no effect when -logtostderr=true)
+      --log_file string                     If non-empty, use this log file (no effect when -logtostderr=true)
+      --log_file_max_size uint              Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
+      --logtostderr                         log to standard error instead of files (default true)
+      --max-concurrent-reconciles int       The maximum number of concurrent reconciles which can be run (default 8)
+      --metrics-addr string                 Listen address for metric endpoint (default ":8080")
+      --mysql-configmap-history-limit int   The maximum number of MySQLConfigMap's history to be kept (default 10)
+      --mysqld-exporter-image string        The image of mysqld_exporter sidecar container (default "ghcr.io/cybozu-go/moco/mysqld_exporter:0.15.1.2")
+      --one_output                          If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
+      --pprof-addr string                   Listen address for pprof endpoints. pprof is disabled by default
+      --pvc-sync-annotation-keys strings    The keys of annotations from MySQLCluster's volumeClaimTemplates to be synced to the PVC
+      --pvc-sync-label-keys strings         The keys of labels from MySQLCluster's volumeClaimTemplates to be synced to the PVC
+      --skip_headers                        If true, avoid header prefixes in the log messages
+      --skip_log_headers                    If true, avoid headers when opening log files (no effect when -logtostderr=true)
+      --stderrthreshold severity            logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=true) (default 2)
+  -v, --v Level                             number for the log level verbosity
+      --version                             version for moco-controller
+      --vmodule moduleSpec                  comma-separated list of pattern=N settings for file-filtered logging
+      --webhook-addr string                 Listen address for the webhook endpoint (default ":9443")
+      --zap-devel                           Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error)
+      --zap-encoder encoder                 Zap log encoding (one of 'json' or 'console')
+      --zap-log-level level                 Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', or any integer value > 0 which corresponds to custom debug levels of increasing verbosity
+      --zap-stacktrace-level level          Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').
+      --zap-time-encoding time-encoding     Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.
 ```