feat(metrics): add volume filesystem read only state metrics

ref: longhorn/longhorn 8508 Signed-off-by: Jack Lin <jack.lin@suse.com>
james-munson · Sep 18, 2024 · f3d710b · f3d710b
1 parent 86a4416
commit f3d710b
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 24 deletions.
diff --git a/controller/volume_controller.go b/controller/volume_controller.go
@@ -1491,25 +1491,10 @@ func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume
 	if v.Status.State == longhorn.VolumeStateAttached && e.Status.CurrentState == longhorn.InstanceStateRunning {
 		fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly)
 
-		isPVMountOptionReadOnly := false
-		kubeStatus := v.Status.KubernetesStatus
-		if kubeStatus.PVName != "" {
-			pv, err := c.ds.GetPersistentVolumeRO(kubeStatus.PVName)
-			if err != nil {
-				if apierrors.IsNotFound(err) {
-					return
-				}
-				log.WithError(err).Warnf("Failed to get PV when checking the mount option of the volume")
-				return
-			}
-			if pv != nil {
-				for _, opt := range pv.Spec.MountOptions {
-					if opt == "ro" {
-						isPVMountOptionReadOnly = true
-						break
-					}
-				}
-			}
+		isPVMountOptionReadOnly, err := c.ds.IsPVMountOptionReadOnly(v)
+		if err != nil {
+			log.WithError(err).Warn("Failed to check if volume's PV mount option is read only")
+			return
 		}
 
 		if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {

diff --git a/datastore/longhorn.go b/datastore/longhorn.go
@@ -5556,3 +5556,23 @@ func (s *DataStore) GetOneBackingImageReadyNodeDisk(backingImage *longhorn.Backi
 
 	return nil, "", fmt.Errorf("failed to find one ready backing image %v", backingImage.Name)
 }
+
+func (s *DataStore) IsPVMountOptionReadOnly(volume *longhorn.Volume) (bool, error) {
+	kubeStatus := volume.Status.KubernetesStatus
+	if kubeStatus.PVName == "" {
+		return false, nil
+	}
+
+	pv, err := s.GetPersistentVolumeRO(kubeStatus.PVName)
+	if err != nil {
+		return false, err
+	}
+	if pv != nil {
+		for _, opt := range pv.Spec.MountOptions {
+			if opt == "ro" {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}
diff --git a/metrics_collector/volume_collector.go b/metrics_collector/volume_collector.go
@@ -2,13 +2,15 @@ package metricscollector
 
 import (
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
-	"github.com/prometheus/client_golang/prometheus"
+	imtypes "github.com/longhorn/longhorn-instance-manager/pkg/types"
 
 	"github.com/longhorn/longhorn-manager/controller"
 	"github.com/longhorn/longhorn-manager/datastore"
 	"github.com/longhorn/longhorn-manager/engineapi"
+	"github.com/longhorn/longhorn-manager/types"
 	"github.com/longhorn/longhorn-manager/util"
 
 	longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
@@ -19,10 +21,11 @@ type VolumeCollector struct {
 
 	proxyConnCounter util.Counter
 
-	capacityMetric   metricInfo
-	sizeMetric       metricInfo
-	stateMetric      metricInfo
-	robustnessMetric metricInfo
+	capacityMetric           metricInfo
+	sizeMetric               metricInfo
+	stateMetric              metricInfo
+	robustnessMetric         metricInfo
+	fileSystemReadOnlyMetric metricInfo
 
 	volumePerfMetrics
 }
@@ -48,6 +51,16 @@ func NewVolumeCollector(
 		proxyConnCounter: util.NewAtomicCounter(),
 	}
 
+	vc.fileSystemReadOnlyMetric = metricInfo{
+		Desc: prometheus.NewDesc(
+			prometheus.BuildFQName(longhornName, subsystemVolume, "file_system_read_only"),
+			"Volume whose mount point is in read-only mode",
+			[]string{nodeLabel, volumeLabel, pvcLabel, pvcNamespaceLabel},
+			nil,
+		),
+		Type: prometheus.GaugeValue,
+	}
+
 	vc.capacityMetric = metricInfo{
 		Desc: prometheus.NewDesc(
 			prometheus.BuildFQName(longhornName, subsystemVolume, "capacity_bytes"),
@@ -156,6 +169,7 @@ func (vc *VolumeCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- vc.sizeMetric.Desc
 	ch <- vc.stateMetric.Desc
 	ch <- vc.robustnessMetric.Desc
+	ch <- vc.fileSystemReadOnlyMetric.Desc
 }
 
 func (vc *VolumeCollector) Collect(ch chan<- prometheus.Metric) {
@@ -214,6 +228,18 @@ func (vc *VolumeCollector) collectMetrics(ch chan<- prometheus.Metric, v *longho
 	ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.iopsMetrics.write.Desc, vc.volumePerfMetrics.iopsMetrics.write.Type, float64(vc.getVolumeWriteIOPS(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
 	ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.read.Desc, vc.volumePerfMetrics.latencyMetrics.read.Type, float64(vc.getVolumeReadLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
 	ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.write.Desc, vc.volumePerfMetrics.latencyMetrics.write.Type, float64(vc.getVolumeWriteLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
+
+	fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly)
+	isPVMountOptionReadOnly, err := vc.ds.IsPVMountOptionReadOnly(v)
+	if err != nil {
+		vc.logger.WithError(err).Warn("Failed to check if volume's PV mount option is read only during metric collection")
+		return
+	}
+
+	if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {
+		ch <- prometheus.MustNewConstMetric(vc.fileSystemReadOnlyMetric.Desc, vc.fileSystemReadOnlyMetric.Type, float64(1), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
+	}
+
 }
 
 func (vc *VolumeCollector) getEngineClientProxy(engine *longhorn.Engine) (c engineapi.EngineClientProxy, err error) {