diff --git a/csi/node_server.go b/csi/node_server.go index 1790b70025..6679601c48 100644 --- a/csi/node_server.go +++ b/csi/node_server.go @@ -126,7 +126,7 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if volume.State != string(longhorn.VolumeStateAttached) || volume.Controllers[0].Endpoint == "" { log.Infof("Volume %v hasn't been attached yet, unmounting potential mount point %v", volumeID, targetPath) if err := unmount(targetPath, mounter); err != nil { - log.WithError(err).Warn("Failed to unmount") + log.WithError(err).Warnf("Failed to unmount: %v", targetPath) } return nil, status.Errorf(codes.InvalidArgument, "volume %s hasn't been attached yet", volumeID) } diff --git a/csi/util.go b/csi/util.go index 5840cff234..a34d15760c 100644 --- a/csi/util.go +++ b/csi/util.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "strconv" "strings" @@ -29,6 +30,8 @@ const ( defaultStaleReplicaTimeout = 2880 defaultForceUmountTimeout = 30 * time.Second + + tempTestMountPointValidStatusFile = "/.longhornmountpointesttstatus" ) // NewForcedParamsExec creates a osExecutor that allows for adding additional params to later occurring Run calls @@ -239,6 +242,38 @@ func parseJSONRecurringJobs(jsonRecurringJobs string) ([]longhornclient.Recurrin return recurringJobs, nil } +func syncMountPointDirectory(targetPath string) error { + d, err := os.OpenFile(targetPath, os.O_SYNC, 0750) + if err != nil { + return err + } + defer d.Close() + + if _, err := d.Readdirnames(1); err != nil { + if !errors.Is(err, io.EOF) { + return err + } + } + + // it would not always return `Input/Output Error` or `read-only file system` errors if we only use ReadDir() or Readdirnames() without an I/O operation + // an I/O operation will make the targetPath mount point invalid immediately + tempFile := targetPath + tempTestMountPointValidStatusFile + f, err := os.OpenFile(tempFile, os.O_CREATE|os.O_RDWR|os.O_SYNC|os.O_TRUNC, 0666) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + logrus.WithError(err).Warnf("Failed to close %v", tempFile) + } + if err := os.Remove(tempFile); err != nil { + logrus.WithError(err).Warnf("Failed to remove %v", tempFile) + } + }() + + return f.Sync() +} + // ensureMountPoint evaluates whether a path is a valid mountPoint // in case the targetPath does not exists it will create a path and return false // in case where the mount point exists but is corrupt, the mount point will be cleaned up and a error is returned @@ -252,9 +287,9 @@ func ensureMountPoint(targetPath string, mounter mount.Interface) (bool, error) IsCorruptedMnt := mount.IsCorruptedMnt(err) if !IsCorruptedMnt { - logrus.Debugf("Mount point %v try reading dir to make sure it's healthy", targetPath) - if _, err := os.ReadDir(targetPath); err != nil { - logrus.Debugf("Mount point %v was identified as corrupt by ReadDir", targetPath) + logrus.Debugf("Mount point %v try opening and syncing dir to make sure it's healthy", targetPath) + if err := syncMountPointDirectory(targetPath); err != nil { + logrus.WithError(err).Warnf("Mount point %v was identified as corrupt by opening and syncing", targetPath) IsCorruptedMnt = true } }