Merge pull request rook#14396 from parth-gr/csi-logrotate-file

csi: add log rotation for csi cephfs nfs pod containers
red-hat-storage · Jul 12, 2024 · da73421 · da73421
2 parents 76d2886 + 4ca8b12
commit da73421
Show file tree

Hide file tree

Showing 10 changed files with 159 additions and 27 deletions.
diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go
@@ -224,7 +224,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 
 	// if at least one cephcluster is present update the csi lograte sidecar
 	// with the first listed ceph cluster specs with logrotate enabled
-	setCSILogrotateParams(cephClusters.Items)
+	r.setCSILogrotateParams(cephClusters.Items)
 
 	err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
 	if err != nil {
@@ -309,7 +309,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
 	return reconcileResult, nil
 }
 
-func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
+func (r *ReconcileCSI) setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
 	logger.Debug("set logrotate values in csi param")
 	spec := cephClustersItems[0].Spec
 	for _, cluster := range cephClustersItems {
@@ -318,9 +318,9 @@ func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
 			break
 		}
 	}
-	CSIParam.CsiLogDirPath = spec.DataDirHostPath
+	csiRootPath = spec.DataDirHostPath
 	if spec.DataDirHostPath == "" {
-		CSIParam.CsiLogDirPath = k8sutil.DataDir
+		csiRootPath = k8sutil.DataDir
 	}
 
 	CSIParam.CSILogRotation = spec.LogCollector.Enabled

diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	_ "embed"
 	"fmt"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -52,7 +53,7 @@ type Param struct {
 	ResizerImage                             string
 	DriverNamePrefix                         string
 	KubeletDirPath                           string
-	CsiLogDirPath                            string
+	CsiLogRootPath                           string
 	ForceCephFSKernelClient                  string
 	CephFSKernelMountOptions                 string
 	CephFSPluginUpdateStrategy               string
@@ -99,9 +100,9 @@ type Param struct {
 	CSINFSPodLabels                          map[string]string
 	CSIRBDPodLabels                          map[string]string
 	CSILogRotation                           bool
+	CsiComponentName                         string
 	CSILogRotationMaxSize                    string
 	CSILogRotationPeriod                     string
-	CSILogFolder                             string
 	Privileged                               bool
 }
 
@@ -188,6 +189,8 @@ var (
 	LogrotateTemplatePath string
 
 	holderEnabled bool
+
+	csiRootPath string
 )
 
 const (
@@ -288,6 +291,8 @@ const (
 	rbdDriverSuffix       = "rbd.csi.ceph.com"
 	cephFSDriverSuffix    = "cephfs.csi.ceph.com"
 	nfsDriverSuffix       = "nfs.csi.ceph.com"
+	nodePlugin            = "node-plugin"
+	controllerPlugin      = "controller-plugin"
 )
 
 func CSIEnabled() bool {
@@ -359,7 +364,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 			return errors.Wrap(err, "failed to load rbdplugin template")
 		}
 		if tp.CSILogRotation {
-			tp.CSILogFolder = "rbd-plugin"
+			tp.CsiComponentName = nodePlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, RBDDriverName)
 			applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
 		}
 
@@ -368,7 +374,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
 		if tp.CSILogRotation {
-			tp.CSILogFolder = "rbd-provisioner"
+			tp.CsiComponentName = controllerPlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, RBDDriverName)
 			applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
 		}
 
@@ -394,11 +401,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load CephFS plugin template")
 		}
+		if tp.CSILogRotation {
+			tp.CsiComponentName = nodePlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, CephFSDriverName)
+			applyLogrotateSidecar(&cephfsPlugin.Spec.Template, "csi-cephfs-daemonset-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
+		if tp.CSILogRotation {
+			tp.CsiComponentName = controllerPlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, CephFSDriverName)
+			applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp)
+		}
+
 		// Create service if either liveness or GRPC metrics are enabled.
 		if CSIParam.EnableLiveness {
 			cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp)
@@ -422,11 +440,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if err != nil {
 			return errors.Wrap(err, "failed to load nfs plugin template")
 		}
+		if tp.CSILogRotation {
+			tp.CsiComponentName = nodePlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, NFSDriverName)
+			applyLogrotateSidecar(&nfsPlugin.Spec.Template, "csi-nfs-daemonset-log-collector", LogrotateTemplatePath, tp)
+		}
 
 		nfsProvisionerDeployment, err = templateToDeployment("nfs-provisioner", NFSProvisionerDepTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load nfs provisioner deployment template")
 		}
+		if tp.CSILogRotation {
+			tp.CsiComponentName = controllerPlugin
+			tp.CsiLogRootPath = path.Join(csiRootPath, NFSDriverName)
+			applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp)
+		}
+
 		enabledDrivers = append(enabledDrivers, driverDetails{
 			name:           NFSDriverShortName,
 			fullName:       NFSDriverName,

diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
@@ -119,6 +119,11 @@ spec:
             - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             - "--pidlimit=-1"
             - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log"
+            {{ end }}
             {{ if .CSIEnableMetadata }}
             - "--setmetadata={{ .CSIEnableMetadata }}"
             {{ end }}
@@ -166,6 +171,10 @@ spec:
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+              name: csi-log
+            {{ end }}
             {{ if .MountCustomCephConf }}
             - name: ceph-config
               mountPath: /etc/ceph/ceph.conf
@@ -191,6 +200,11 @@ spec:
             - "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
             - "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
+            {{ end }}
           ports:
             - containerPort: {{ .CSIAddonsPort }}
           env:
@@ -220,6 +234,10 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+              name: csi-log
+            {{ end }}
         {{ end }}
         {{ if .EnableLiveness }}
         - name: liveness-prometheus
@@ -267,6 +285,15 @@ spec:
           emptyDir: {
             medium: "Memory"
           }
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .MountCustomCephConf }}
         - name: ceph-config
           configMap:

diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
@@ -72,6 +72,11 @@ spec:
             - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
             - "--pidlimit=-1"
             - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log"
+            {{ end }}
             {{ if .CephFSKernelMountOptions }}
             - "--kernelmountoptions={{ .CephFSKernelMountOptions }}"
             {{ end }}
@@ -115,6 +120,10 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+              name: csi-log
+            {{ end }}
             {{ if .EnablePluginSelinuxHostMount }}
             -  name: etc-selinux
                mountPath: /etc/selinux
@@ -198,6 +207,15 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .EnablePluginSelinuxHostMount }}
         - name: etc-selinux
           hostPath:

diff --git a/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml b/pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml
@@ -1,18 +1,18 @@
 args:
   - |
     echo "Starting the csi-logrotate-sidecar"
-    mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }};
-    echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log {
+    mkdir -p {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}
+    echo '{{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/*.log {
         {{ .CSILogRotationPeriod }}
         missingok
         rotate 7
         compress
         copytruncate
         notifempty
-    }' >  {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
-    echo "File creation container completed";
+    }' >  {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi
+    echo "File creation container completed"
 
-    LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
+    LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi
     LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
     if [ "$LOG_MAX_SIZE" != "0" ]; then
       sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
@@ -29,7 +29,7 @@ image: {{ .CSIPluginImage }}
 imagePullPolicy: IfNotPresent
 name: log-collector
 volumeMounts:
-  - mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}
+  - mountPath: {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}
     name: csi-logs-logrotate
-  - mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }}
+  - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
     name: csi-log
diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml
@@ -113,6 +113,11 @@ spec:
             - "--controllerserver=true"
             - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
             - "--pidlimit=-1"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log"
+            {{ end }}
           env:
             - name: POD_IP
               valueFrom:
@@ -147,6 +152,10 @@ spec:
               mountPath: /etc/ceph-csi-config/
             - name: keys-tmp-dir
               mountPath: /tmp/csi/keys
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+              name: csi-log
+            {{ end }}
             {{ if .MountCustomCephConf }}
             - name: ceph-config
               mountPath: /etc/ceph/ceph.conf
@@ -176,6 +185,15 @@ spec:
           emptyDir: {
             medium: "Memory"
           }
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .MountCustomCephConf }}
         - name: ceph-config
           configMap:

diff --git a/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml b/pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml
@@ -66,6 +66,11 @@ spec:
             - "--nodeserver=true"
             - "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
             - "--pidlimit=-1"
+            {{ if .CSILogRotation }}
+            - "--logtostderr=false"
+            - "--alsologtostderr=true"
+            - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log"
+            {{ end }}
           env:
           - name: NODE_ID
             valueFrom:
@@ -98,6 +103,10 @@ spec:
               mountPath: /tmp/csi/keys
             - name: host-run-mount
               mountPath: /run/mount
+            {{ if .CSILogRotation }}
+            - mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+              name: csi-log
+            {{ end }}
             {{ if .EnablePluginSelinuxHostMount }}
             -  name: etc-selinux
                mountPath: /etc/selinux
@@ -147,6 +156,15 @@ spec:
         - name: host-run-mount
           hostPath:
             path: /run/mount
+        {{ if .CSILogRotation }}
+        - name: csi-log
+          hostPath:
+            path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
+            type: DirectoryOrCreate
+        - name: csi-logs-logrotate
+          emptyDir:
+            type: DirectoryOrCreate
+        {{ end }}
         {{ if .EnablePluginSelinuxHostMount }}
         - name: etc-selinux
           hostPath: