fuse: re-validate the DLM grant after waiting on the coherency gate

fs/fuse/dir.c

@@ -2100,18 +2100,35 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		WARN_ON(!(attr->ia_valid & ATTR_SIZE));
 		WARN_ON(attr->ia_size != 0);
 		if (fc->atomic_o_trunc) {
+			struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
 			/*
 			 * No need to send request to userspace, since actual
 			 * truncation has already been done by OPEN.  But still
 			 * need to truncate page cache.
+			 *
+			 * Revoke and drop under the coherency gate write side,
+			 * like the NOTIFY invalidate path: a gate reader that
+			 * already re-validated its grant must not have the
+			 * lock tree and the cache yanked mid-hold, or it
+			 * would repopulate the truncated range trusting a
+			 * grant that no longer exists.  Waiting for gate
+			 * readers here is safe: we hold i_rwsem exclusive, so
+			 * no gate holder can be waiting on it (the write path
+			 * takes i_rwsem before the gate, the read path never
+			 * takes it).
 			 */
+			if (wb_sem)
+				percpu_down_write(wb_sem);
 			if (fc->dlm && fc->writeback_cache)
 				fuse_dlm_cache_release_locks(fi);
 			spin_lock(&fi->lock);
 			fi->server_size = 0;
 			i_size_write(inode, 0);
 			spin_unlock(&fi->lock);
 			truncate_pagecache(inode, 0);
+			if (wb_sem)
+				percpu_up_write(wb_sem);
 			goto out;
 		}
 		file = NULL;
@@ -2222,11 +2239,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	 */
 	if ((is_truncate || !is_wb) &&
 	    S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) {
+		struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
+		/*
+		 * Revoke and drop under the coherency gate write side; see
+		 * the atomic-O_TRUNC branch above.  i_rwsem is held
+		 * exclusive here as well (setattr), so waiting out gate
+		 * readers cannot deadlock.
+		 */
+		if (wb_sem)
+			percpu_down_write(wb_sem);
 		if (fc->dlm && fc->writeback_cache)
 			fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1);
 
 		truncate_pagecache(inode, outarg.attr.size);
 		invalidate_inode_pages2(mapping);
+		if (wb_sem)
+			percpu_up_write(wb_sem);
 	}
 
 	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);

fs/fuse/file.c

@@ -1260,6 +1260,12 @@ static void fuse_readahead(struct readahead_control *rac)
 
 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to);
 
+/*
+ * Bound on re-requesting a revoked DLM grant before a cached read is
+ * served unlocked; see fuse_cache_read_iter().
+ */
+#define FUSE_DLM_READ_RETRIES 3
+
 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct file *file = iocb->ki_filp;
@@ -1268,6 +1274,7 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	struct fuse_inode *fi = get_fuse_inode(inode);
 	struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
 	ssize_t res;
+	int lock_err = 0;
 
 	/*
 	 * In auto invalidate mode, always update attributes on read.
@@ -1285,8 +1292,9 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	/* if we have dlm support acquire a read lock for the area
 	 * we are reading from. */
 	if (fc->writeback_cache && fc->dlm)
-		fuse_get_dlm_lock(file, iocb->ki_pos,
-				  iov_iter_count(to), FUSE_PAGE_LOCK_READ);
+		lock_err = fuse_get_dlm_lock(file, iocb->ki_pos,
+					     iov_iter_count(to),
+					     FUSE_PAGE_LOCK_READ);
 
 	/*
 	 * Fence the cache-serving read against a NOTIFY invalidate so we never
@@ -1298,11 +1306,44 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	 * wb_sem is NULL on non-writeback+dlm mounts (gate inactive).
 	 */
 	if (wb_sem) {
+		int tries = FUSE_DLM_READ_RETRIES;
+
+retry:
 		percpu_down_read(wb_sem);
 		if (fuse_inode_force_dio(inode)) {
 			percpu_up_read(wb_sem);
 			return fuse_direct_read_iter(iocb, to);
 		}
+		/*
+		 * The DLM lock was requested before entering the gate, and
+		 * the NOTIFY invalidate we may just have waited on revokes
+		 * locks under the gate write side.  Re-check the grant here
+		 * and re-request with the gate dropped, so a
+		 * FUSE_DLM_WB_LOCK round trip never parks a pending
+		 * invalidate behind our own gate hold.  Once the check
+		 * passes the lock cannot go away for the rest of the gate
+		 * hold.  A failed or unrecorded request falls through
+		 * unlocked, as before: the retry is taken even then (the
+		 * latch must be re-checked under the re-entered gate), so
+		 * lock_err has to stay sticky across it -- seeded by the
+		 * pre-gate request above -- or a grant that failed would
+		 * be re-requested forever.  The retry is also bounded: a
+		 * remote writer can revoke each successful grant before
+		 * the gate is re-entered, and a reader-only inode has no
+		 * force-DIO latch to end such a storm, so after
+		 * FUSE_DLM_READ_RETRIES re-requests the read is served
+		 * unlocked rather than looping without bound.
+		 */
+		if (!lock_err && fc->dlm && tries-- > 0 &&
+		    !fuse_dlm_lock_is_held(fi, iocb->ki_pos,
+					   iov_iter_count(to),
+					   FUSE_PAGE_LOCK_READ)) {
+			percpu_up_read(wb_sem);
+			lock_err = fuse_get_dlm_lock(file, iocb->ki_pos,
+						     iov_iter_count(to),
+						     FUSE_PAGE_LOCK_READ);
+			goto retry;
+		}
 	}
 
 	res = generic_file_read_iter(iocb, to);
@@ -1807,6 +1848,26 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive)
 		inode_unlock_shared(inode);
 }
 
+/*
+ * Request the DLM write lock covering a cached write.  -ENOSYS cleared
+ * fc->dlm: the server has no DLM, proceed as a plain cached write.  Any
+ * other failure means the cache would be dirtied without DLM coverage -
+ * the caller must fail the write instead.  A granted-but-unrecorded
+ * lock (positive return) is covered cluster-wide; proceed, but flag it
+ * so the in-gate re-validation skips a check an invisible grant could
+ * never pass.
+ */
+static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len,
+				  bool *unrecorded)
+{
+	int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE);
+
+	if (err < 0 && err != -ENOSYS)
+		return err;
+	*unrecorded = err > 0;
+	return 0;
+}
+
 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
@@ -1821,14 +1882,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	bool writeback = false;
 	bool wb_guard = false;
 	bool exclusive = true;
+	bool dlm_unrecorded = false;
+	loff_t dlm_pos = 0;
+	size_t dlm_len = 0;
 
-	/*
-	 * The inode may have been latched into forced direct IO -- by a
-	 * NOTIFY_INVAL_INODE arriving while this inode is open for writing here
-	 * -- after this write was routed to the cached path but before it took
-	 * any lock.  Re-route to the direct path (before taking a DLM lock) so
-	 * we do not repopulate the page cache the latch just dropped.
-	 */
 	if (fuse_inode_force_dio(inode))
 		return fuse_direct_write_iter(iocb, from);
 
@@ -1840,61 +1897,78 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 			return err;
 
 		if (!fc->handle_killpriv_v2 ||
-		    !setattr_should_drop_suidgid(idmap, file_inode(file))) {
+		    !setattr_should_drop_suidgid(idmap, file_inode(file)))
 			writeback = true;
-
-			/*
-			* If we have dlm support acquire the lock for the area
-			* we are writing into.
-			* dlm lock is only needed as the write is cached and the
-			* fuse server is not notified otherwise
-			*/
-			if (fc->dlm) {
-				/*
-				* Note that a file opened with O_APPEND will have
-				* relative values in ki_pos. This code is here for
-				* convenience and for libfuse overlay test.
-				* Filesystems should handle O_APPEND with 'direct io'
-				* to additionally get the performance benefits of
-				* 'parallel direct writes'.
-				*/
-				loff_t pos = file->f_flags & O_APPEND ?
-						i_size_read(inode) + iocb->ki_pos :
-						iocb->ki_pos;
-				size_t length = iov_iter_count(from);
-
-				fuse_get_dlm_lock(file, pos, length,
-					          FUSE_PAGE_LOCK_WRITE);
-			}
-		}
 	}
 
 	exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback);
+
+	/*
+	 * Request the DLM write lock before taking i_rwsem: the request is
+	 * an unbounded cluster round trip, and holding the writer-priority
+	 * rwsem across it would park a truncate -- and behind it every
+	 * later writer -- for the duration.  The grant-to-use window this
+	 * leaves open is closed by the in-gate re-validation below.  Only
+	 * the append case must wait for the lock: its range depends on
+	 * i_size, which is stable only under the exclusive inode lock.
+	 */
+	if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) {
+		dlm_pos = iocb->ki_pos;
+		dlm_len = iov_iter_count(from);
+
+		err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len,
+					     &dlm_unrecorded);
+		if (err)
+			return err;
+	}
+
 	if (exclusive)
 		inode_lock(inode);
 	else
 		inode_lock_shared(inode);
 
-	/*
-	 * The forced-direct-IO latch feature is active under writeback+dlm;
-	 * hold the coherency gate (wb_inval_rwsem) for read across the
-	 * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes
-	 * the write side (blocking, with priority) around its invalidate + latch
-	 * set -- cannot strand the folios we are about to write.  Re-check the
-	 * latch under it (it may have been set while we blocked on the inode
-	 * lock) and re-route to the direct path if set.  Taken before
-	 * task_io_account_write() so a re-route is not double-counted; the DLM
-	 * write lock taken above is harmless as the direct path does its own
-	 * server coordination.
-	 */
+	/* note that this small code dup will save us a lot of headache later
+	 * when appends are done concurrently without using parallel direct writes */
+	if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) {
+		/*
+		 * An append write lands at the current EOF no matter what
+		 * ki_pos holds: generic_write_checks() rewrites ki_pos to
+		 * i_size for IOCB_APPEND, and i_size is stable here because
+		 * append writes hold the inode lock exclusive.  Lock where
+		 * the data will land.
+		 */
+		dlm_pos = i_size_read(inode);
+		dlm_len = iov_iter_count(from);
+
+		err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len,
+					     &dlm_unrecorded);
+		if (err)
+			goto out;
+	}
+
 	wb_guard = !!wb_sem;
 	if (wb_guard) {
+retry:
 		percpu_down_read(wb_sem);
 		if (fuse_inode_force_dio(inode)) {
 			percpu_up_read(wb_sem);
 			fuse_cache_wr_unlock(inode, exclusive);
 			return fuse_direct_write_iter(iocb, from);
 		}
+		if (writeback && fc->dlm && !dlm_unrecorded &&
+		    !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len,
+					   FUSE_PAGE_LOCK_WRITE)) {
+			percpu_up_read(wb_sem);
+			err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len,
+						     &dlm_unrecorded);
+			if (err) {
+				/* The gate is already dropped; funnel the
+				 * failure through the one audited exit. */
+				wb_guard = false;
+				goto out;
+			}
+			goto retry;
+		}
 	}
 
 	err = count = generic_write_checks(iocb, from);