Add unmount preflight checks

It is meant to detect if there are busy vnodes
in the mount, like that of a lower mountpoint
existing (is a snapshot mounted inside) and if
so, return EBUSY instead of hanging in unmount.

Signed-off-by: Jorgen Lundman <lundman@lundman.net>

include/os/windows/spl/sys/vnode.h

@@ -527,6 +527,7 @@ int vnode_fileobject_add(vnode_t *vp, void *fo);
 int vnode_fileobject_remove(vnode_t *vp, void *fo);
 int vnode_fileobject_empty(vnode_t *vp, int locked);
 int vnode_fileobject_member(vnode_t *vp, void *fo);
+int vnode_umount_preflight(struct mount *, struct vnode *, int);
 
 void vnode_lock(vnode_t *vp);
 void vnode_unlock(vnode_t *vp);

module/os/windows/spl/spl-vnode.c

@@ -1573,11 +1573,16 @@ vflush(struct mount *mp, struct vnode *skipvp, int flags)
 
 			// If we aren't FORCE and asked to SKIPROOT, and node
 			// is MARKROOT, then go to next.
-			if (!(flags & FORCECLOSE))
+			if (!(flags & FORCECLOSE)) {
 				if ((flags & SKIPROOT))
 					if (rvp->v_flags & VNODE_MARKROOT)
 						continue;
-
+#if 0 // when we use SYSTEM vnodes
+				if ((flags & SKIPSYSTEM))
+					if (rvp->v_flags & VNODE_MARKSYSTEM)
+						continue;
+#endif
+			}
 			// We are to remove this node, even if ROOT - unmark it.
 			mutex_exit(&vnode_all_list_lock);
 
@@ -1715,6 +1720,56 @@ vflush(struct mount *mp, struct vnode *skipvp, int flags)
 	return (0);
 }
 
+int
+vnode_umount_preflight(struct mount *mp, struct vnode *skipvp, int flags)
+{
+	struct vnode *rvp;
+	int Status;
+
+	dprintf("%s start\n", __func__);
+
+	mutex_enter(&vnode_all_list_lock);
+
+	for (rvp = list_head(&vnode_all_list);
+	    rvp;
+	    rvp = list_next(&vnode_all_list, rvp)) {
+
+		// skip vnodes not belonging to this mount
+		if (mp && rvp->v_mount != mp)
+			continue;
+
+		if (vnode_isdir(rvp))
+			continue;
+
+		if (rvp == skipvp)
+			continue;
+
+		if (!(flags & FORCECLOSE)) {
+			if ((flags & SKIPROOT))
+				if (rvp->v_flags & VNODE_MARKROOT)
+					continue;
+#if 0 // when we use SYSTEM vnodes
+			if ((flags & SKIPSYSTEM))
+				if (rvp->v_flags & VNODE_MARKSYSTEM)
+					continue;
+#endif
+		}
+
+		if (rvp->v_usecount != 0) {
+			mutex_exit(&vnode_all_list_lock);
+			return (EBUSY);
+		}
+#if 0
+		else if (rvp->v_iocount > 0) {
+			// macOS waits upto 3s here and tries again.
+		}
+#endif
+	} // for all vnodes
+
+	mutex_exit(&vnode_all_list_lock);
+	return (0);
+}
+
 /*
  * Set the Windows SecurityPolicy
  */

module/os/windows/zfs/zfs_vnops_windows_mount.c

@@ -1496,6 +1496,14 @@ zfs_windows_unmount(zfs_cmd_t *zc)
 		NTSTATUS ntstatus;
 		ASSERT(zmo->type == MOUNT_TYPE_VCB);
 
+		// As part of unmount-preflight, we call vflush()
+		// as it will indicate if we should return EBUSY.
+		if (vnode_umount_preflight(zmo, NULL,
+		    SKIPROOT|SKIPSYSTEM|SKIPSWAP)) {
+			vfs_unbusy(zmo);
+			return (SET_ERROR(EBUSY));
+		}
+
 		// getzfsvfs() grabs a READER lock,
 		// convert it to WRITER, and wait for it.
 		vfs_busy(zmo, LK_UPGRADE);