diff --git a/api/v1alpha1/driver_types.go b/api/v1alpha1/driver_types.go index 1084c1d3..62231764 100644 --- a/api/v1alpha1/driver_types.go +++ b/api/v1alpha1/driver_types.go @@ -33,8 +33,10 @@ const ( MonthlyPeriod PeriodicityType = "monthly" ) +// +kubebuilder:validation:XValidation:message="Either maxLogSize or periodicity must be set",rule="(has(self.maxLogSize)) || (has(self.periodicity))" type LogRotationSpec struct { // MaxFiles is the number of logrtoate files + // Default to 7 //+kubebuilder:validation:Optional MaxFiles int `json:"maxFiles,omitempty"` @@ -48,6 +50,7 @@ type LogRotationSpec struct { Periodicity PeriodicityType `json:"periodicity,omitempty"` // LogHostPath is the prefix directory path for the csi log files + // Default to /var/lib/cephcsi //+kubebuilder:validation:Optional LogHostPath string `json:"logHostPath,omitempty"` } diff --git a/config/crd/bases/csi.ceph.io_drivers.yaml b/config/crd/bases/csi.ceph.io_drivers.yaml index 24e904cd..e592d853 100644 --- a/config/crd/bases/csi.ceph.io_drivers.yaml +++ b/config/crd/bases/csi.ceph.io_drivers.yaml @@ -3542,11 +3542,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path for - the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3565,6 +3568,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml index deb9e290..109d0a53 100644 --- a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml +++ b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml @@ -3581,11 +3581,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path - for the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3605,6 +3608,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/internal/controller/defaults.go b/internal/controller/defaults.go index 10ea404c..19ed50a4 100644 --- a/internal/controller/defaults.go +++ b/internal/controller/defaults.go @@ -38,8 +38,10 @@ var imageDefaults = map[string]string{ } const ( - defaultGRrpcTimeout = 150 - defaultKubeletDirPath = "/var/lib/kubelet" + defaultGRrpcTimeout = 150 + defaultKubeletDirPath = "/var/lib/kubelet" + defaultLogHostPath = "/var/lib/cephcsi" + defaultLogRotateMaxFiles = 7 ) var defaultLeaderElection = csiv1a1.LeaderElectionSpec{ diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index b766bf6f..c4544c88 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -230,6 +230,7 @@ func (r *driverReconcile) reconcile() error { // the desired state defined on the driver object errChan := utils.RunConcurrently( r.reconcileCsiConfigMap, + r.reconcileLogRotateConfigMap, r.reconcileK8sCsiDriver, r.reconcileControllerPluginDeployment, r.reconcileNodePluginDeamonSet, @@ -327,6 +328,63 @@ func (r *driverReconcile) LoadAndValidateDesiredState() error { return nil } +func (r *driverReconcile) reconcileLogRotateConfigMap() error { + logRotateConfigmap := &corev1.ConfigMap{} + logRotateConfigmap.Name = utils.LogRotateConfigMapName(r.driver.Name) + logRotateConfigmap.Namespace = r.driver.Namespace + + log := r.log.WithValues("logRotateConfigMap", logRotateConfigmap.Name) + log.Info("Reconciling logRotate configmap") + + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + if logRotationSpec != nil { + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, logRotateConfigmap, func() error { + if _, err := utils.ToggleOwnerReference(true, logRotateConfigmap, &r.driver, r.Scheme); err != nil { + log.Error(err, "Failed adding an owner reference on the LogRotate config map") + return err + } + if logRotationSpec.Periodicity != "" && logRotationSpec.MaxLogSize.IsZero() { + err := fmt.Errorf("invalid Log.Rotation spec") + log.Error(err, "Either \"maxLogSize\" or \"periodicity\" fields must be set") + return err + } + maxFiles := cmp.Or(logRotationSpec.MaxFiles, defaultLogRotateMaxFiles) + cronLogRotateSettings := []string{ + "\tmissingok", + "\tcompress", + "\tcopytruncate", + "\tnotifempty", + fmt.Sprintf("\trotate %d", maxFiles), + } + if logRotationSpec.Periodicity != "" { + periodicity := "\t" + string(logRotationSpec.Periodicity) + cronLogRotateSettings = append(cronLogRotateSettings, periodicity) + } + if logRotationSpec.MaxLogSize.String() != "0" { + maxSize := fmt.Sprintf("\tmaxsize %s", logRotationSpec.MaxLogSize.String()) + cronLogRotateSettings = append(cronLogRotateSettings, maxSize) + } + logRotateConfigmap.Data = map[string]string{ + "csi": fmt.Sprintf( + "/csi-logs/*.log {\n%s\n}\n", + strings.Join(cronLogRotateSettings, "\n"), + ), + } + return nil + }) + + logCreateOrUpdateResult(log, "LogRotateConfigMap", logRotateConfigmap, opResult, err) + return err + } else { + // Remove the logrotate configmap if logrotate setting is removed from the driver's spec + if err := r.Delete(r.ctx, logRotateConfigmap); client.IgnoreNotFound(err) != nil { + log.Error(err, "Unable to delete LogRotate configmap") + return err + } + return nil + } +} + func (r *driverReconcile) reconcileCsiConfigMap() error { csiConfigMap := &corev1.ConfigMap{} csiConfigMap.Name = utils.CsiConfigVolume.Name @@ -454,6 +512,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { logVerbosity := ptr.Deref(r.driver.Spec.Log, csiv1a1.LogSpec{}).Verbosity forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient snPolicy := cmp.Or(r.driver.Spec.SnapshotPolicy, csiv1a1.VolumeSnapshotSnapshotPolicy) + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + logRotationEnabled := logRotationSpec != nil + securityContext := utils.If(logRotationEnabled, &corev1.SecurityContext{Privileged: ptr.To(true)}, nil) leaderElectionSettingsArg := []string{ utils.LeaderElectionNamespaceContainerArg(r.driver.Namespace), @@ -487,6 +549,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: utils.DeleteZeroValues( []string{ utils.TypeContainerArg(string(r.driverType)), @@ -504,6 +567,13 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsEndpointContainerArg, "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -534,6 +604,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -654,6 +727,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, + SecurityContext: securityContext, Args: utils.DeleteZeroValues( append( slices.Clone(leaderElectionSettingsArg), @@ -664,6 +738,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, utils.NamespaceContainerArg, + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), ), ), Ports: []corev1.ContainerPort{ @@ -675,9 +752,15 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodNameEnvVar, utils.PodNamespaceEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.SocketDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.SocketDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -741,7 +824,22 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), }) } - + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{"/bin/bash", "-c"}, + Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -767,6 +865,13 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { volumes, utils.KmsConfigVolume(&r.driver.Spec.Encryption.ConfigMapRef)) } + if logRotationEnabled { + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, deploy.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -807,6 +912,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { topology := r.isRdbDriver() && pluginSpec.Topology != nil domainLabels := cmp.Or(pluginSpec.Topology, &csiv1a1.TopologySpec{}).DomainLabels + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + logRotationEnabled := logRotationSpec != nil daemonSet.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -884,6 +992,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.DomainLabelsContainerArg(domainLabels), "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -912,6 +1027,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -965,6 +1083,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Args: utils.DeleteZeroValues( []string{ utils.CsiAddonsNodeIdContainerArg, + utils.NodeIdContainerArg, utils.LogVerbosityContainerArg(logVerbosity), utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, @@ -972,6 +1091,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.NamespaceContainerArg, utils.PodUidContainerArg, utils.StagingPathContainerArg(kubeletDirPath), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), }, ), Ports: []corev1.ContainerPort{ @@ -983,9 +1105,15 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodNamespaceEnvVar, utils.PodUidEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.PluginDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.PluginDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -1026,6 +1154,22 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }) } } + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + Command: []string{"/bin/bash", "-c"}, + Args: []string{`while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done`}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -1067,6 +1211,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.OidcTokenVolume, ) } + if logRotationEnabled { + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, daemonSet.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -1273,6 +1424,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if src.ControllerPlugin != nil { @@ -1328,6 +1482,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if dest.AttachRequired == nil { diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 0df4ae77..4ab0521f 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -37,6 +37,9 @@ const ( CsiConfigMapConfigKey = "config.json" CsiConfigMapMappingKey = "cluster-mapping.json" + + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" ) // Ceph CSI common volumes @@ -182,6 +185,29 @@ func PodsMountDirVolume(kubeletDirPath string) corev1.Volume { }, } } +func LogsDirVolume(logHostPath, pluginName string) corev1.Volume { + return corev1.Volume{ + Name: logsDirVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: fmt.Sprintf("%s/%s", logHostPath, pluginName), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + } +} +func LogRotateDirVolumeName(driverName string) corev1.Volume { + return corev1.Volume{ + Name: logRotateDirVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: LogRotateConfigMapName(driverName), + }, + }, + }, + } +} // Ceph CSI common volume Mounts var SocketDirVolumeMount = corev1.VolumeMount{ @@ -236,6 +262,14 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{ MountPath: "/etc/selinux", ReadOnly: true, } +var LogsDirVolumeMount = corev1.VolumeMount{ + Name: logsDirVolumeName, + MountPath: "/csi-logs", +} +var LogRotateDirVolumeMount = corev1.VolumeMount{ + Name: logRotateDirVolumeName, + MountPath: "/logrotate-config", +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{ @@ -335,10 +369,15 @@ var ImmediateTopologyContainerArg = "--immediate-topology=false" var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true" var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true" var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true" +var LogToStdErrContainerArg = "--logtostderr=false" +var AlsoLogToStdErrContainerArg = "--alsologtostderr=true" func LogVerbosityContainerArg(level int) string { return fmt.Sprintf("--v=%d", Clamp(level, 0, 5)) } +func LogFileContainerArg(containerName string) string { + return fmt.Sprintf("--log_file=csi-logs/%s.log", containerName) +} func TypeContainerArg(t string) string { switch t { case "rbd", "cephfs", "nfs", "controller", "liveness": @@ -380,6 +419,9 @@ func KubeletRegistrationPathContainerArg(kubeletDirPath string, driverName strin func StagingPathContainerArg(kubeletDirPath string) string { return fmt.Sprintf("--stagingpath=%s/plugins/kubernetes.io/csi/", kubeletDirPath) } +func LogRotateConfigMapName(driverName string) string { + return fmt.Sprintf("%s-logrotate-config", driverName) +} func KernelMountOptionsContainerArg(options map[string]string) string { return If( len(options) > 0,