diff --git a/controller/volume_controller.go b/controller/volume_controller.go index b1dbdc92a6..83f4b041e9 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -1491,25 +1491,10 @@ func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume if v.Status.State == longhorn.VolumeStateAttached && e.Status.CurrentState == longhorn.InstanceStateRunning { fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly) - isPVMountOptionReadOnly := false - kubeStatus := v.Status.KubernetesStatus - if kubeStatus.PVName != "" { - pv, err := c.ds.GetPersistentVolumeRO(kubeStatus.PVName) - if err != nil { - if apierrors.IsNotFound(err) { - return - } - log.WithError(err).Warnf("Failed to get PV when checking the mount option of the volume") - return - } - if pv != nil { - for _, opt := range pv.Spec.MountOptions { - if opt == "ro" { - isPVMountOptionReadOnly = true - break - } - } - } + isPVMountOptionReadOnly, err := c.ds.IsPVMountOptionReadOnly(v) + if err != nil { + log.WithError(err).Warn("Failed to check if volume's PV mount option is read only") + return } if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly { diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 5a9479ca99..d7cd2b6b37 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -5556,3 +5556,23 @@ func (s *DataStore) GetOneBackingImageReadyNodeDisk(backingImage *longhorn.Backi return nil, "", fmt.Errorf("failed to find one ready backing image %v", backingImage.Name) } + +func (s *DataStore) IsPVMountOptionReadOnly(volume *longhorn.Volume) (bool, error) { + kubeStatus := volume.Status.KubernetesStatus + if kubeStatus.PVName == "" { + return false, nil + } + + pv, err := s.GetPersistentVolumeRO(kubeStatus.PVName) + if err != nil { + return false, err + } + if pv != nil { + for _, opt := range pv.Spec.MountOptions { + if opt == "ro" { + return true, nil + } + } + } + return false, nil +} diff --git a/metrics_collector/volume_collector.go b/metrics_collector/volume_collector.go index bc740af6f7..df304fdd45 100644 --- a/metrics_collector/volume_collector.go +++ b/metrics_collector/volume_collector.go @@ -2,13 +2,15 @@ package metricscollector import ( "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" - "github.com/prometheus/client_golang/prometheus" + imtypes "github.com/longhorn/longhorn-instance-manager/pkg/types" "github.com/longhorn/longhorn-manager/controller" "github.com/longhorn/longhorn-manager/datastore" "github.com/longhorn/longhorn-manager/engineapi" + "github.com/longhorn/longhorn-manager/types" "github.com/longhorn/longhorn-manager/util" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" @@ -19,10 +21,11 @@ type VolumeCollector struct { proxyConnCounter util.Counter - capacityMetric metricInfo - sizeMetric metricInfo - stateMetric metricInfo - robustnessMetric metricInfo + capacityMetric metricInfo + sizeMetric metricInfo + stateMetric metricInfo + robustnessMetric metricInfo + fileSystemReadOnlyMetric metricInfo volumePerfMetrics } @@ -48,6 +51,16 @@ func NewVolumeCollector( proxyConnCounter: util.NewAtomicCounter(), } + vc.fileSystemReadOnlyMetric = metricInfo{ + Desc: prometheus.NewDesc( + prometheus.BuildFQName(longhornName, subsystemVolume, "file_system_read_only"), + "Volume whose mount point is in read-only mode", + []string{nodeLabel, volumeLabel, pvcLabel, pvcNamespaceLabel}, + nil, + ), + Type: prometheus.GaugeValue, + } + vc.capacityMetric = metricInfo{ Desc: prometheus.NewDesc( prometheus.BuildFQName(longhornName, subsystemVolume, "capacity_bytes"), @@ -156,6 +169,7 @@ func (vc *VolumeCollector) Describe(ch chan<- *prometheus.Desc) { ch <- vc.sizeMetric.Desc ch <- vc.stateMetric.Desc ch <- vc.robustnessMetric.Desc + ch <- vc.fileSystemReadOnlyMetric.Desc } func (vc *VolumeCollector) Collect(ch chan<- prometheus.Metric) { @@ -214,6 +228,18 @@ func (vc *VolumeCollector) collectMetrics(ch chan<- prometheus.Metric, v *longho ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.iopsMetrics.write.Desc, vc.volumePerfMetrics.iopsMetrics.write.Type, float64(vc.getVolumeWriteIOPS(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace) ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.read.Desc, vc.volumePerfMetrics.latencyMetrics.read.Type, float64(vc.getVolumeReadLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace) ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.write.Desc, vc.volumePerfMetrics.latencyMetrics.write.Type, float64(vc.getVolumeWriteLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace) + + fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly) + isPVMountOptionReadOnly, err := vc.ds.IsPVMountOptionReadOnly(v) + if err != nil { + vc.logger.WithError(err).Warn("Failed to check if volume's PV mount option is read only during metric collection") + return + } + + if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly { + ch <- prometheus.MustNewConstMetric(vc.fileSystemReadOnlyMetric.Desc, vc.fileSystemReadOnlyMetric.Type, float64(1), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace) + } + } func (vc *VolumeCollector) getEngineClientProxy(engine *longhorn.Engine) (c engineapi.EngineClientProxy, err error) {