Backport performance changes from Ubuntu HWE 6.17

fs/fuse/dev_uring.c

@@ -1461,6 +1461,26 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd,
 	fuse_uring_send(ent, cmd, err, issue_flags);
 }
 
+/*
+ * The request was already copied to the ring buffer in the submitter's
+ * context, only the io_uring cmd completion is left to do.
+ * io_uring_cmd_done() must not run in the submitter's context as it would
+ * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex
+ * the ring task holds across its whole submission path and frequently gets
+ * preempted under while the just-woken submitter runs.
+ */
+static void fuse_uring_send_prepared_in_task(struct io_uring_cmd *cmd,
+					     unsigned int issue_flags)
+{
+	struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);
+	int err = 0;
+
+	if (unlikely(issue_flags & IO_URING_F_TASK_DEAD))
+		err = -ECANCELED;
+
+	fuse_uring_send(ent, cmd, err, issue_flags);
+}
+
 static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring,
 						       bool background)
 {
@@ -1560,7 +1580,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg)
 						 IO_URING_F_UNLOCKED);
 			return;
 		}
-		fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED);
+		uring_cmd_set_ring_ent(cmd, ent);
+		io_uring_cmd_complete_in_task(cmd,
+					      fuse_uring_send_prepared_in_task);
 	}
 }
 

fs/fuse/dir.c

@@ -2016,6 +2016,12 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff)
 		inarg.valid |= FATTR_FH;
 		inarg.fh = ff->fh;
 	}
+	/*
+	 * This is ->write_inode() flushing times the kernel owns locally, not
+	 * a userspace utimes(); let the server tell the two apart.
+	 */
+	if (fm->fc->setattr_writeback)
+		inarg.valid |= FATTR_WRITEBACK;
 	fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg);
 
 	return fuse_simple_request(fm, &args);
@@ -2076,15 +2082,35 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		WARN_ON(!(attr->ia_valid & ATTR_SIZE));
 		WARN_ON(attr->ia_size != 0);
 		if (fc->atomic_o_trunc) {
+			struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
 			/*
 			 * No need to send request to userspace, since actual
 			 * truncation has already been done by OPEN.  But still
 			 * need to truncate page cache.
+			 *
+			 * Revoke and drop under the coherency gate write side,
+			 * like the NOTIFY invalidate path: a gate reader that
+			 * already re-validated its grant must not have the
+			 * lock tree and the cache yanked mid-hold, or it
+			 * would repopulate the truncated range trusting a
+			 * grant that no longer exists.  Waiting for gate
+			 * readers here is safe: we hold i_rwsem exclusive, so
+			 * no gate holder can be waiting on it (the write path
+			 * takes i_rwsem before the gate, the read path never
+			 * takes it).
 			 */
+			if (wb_sem)
+				percpu_down_write(wb_sem);
 			if (fc->dlm && fc->writeback_cache)
 				fuse_dlm_cache_release_locks(fi);
+			spin_lock(&fi->lock);
+			fi->server_size = 0;
 			i_size_write(inode, 0);
+			spin_unlock(&fi->lock);
 			truncate_pagecache(inode, 0);
+			if (wb_sem)
+				percpu_up_write(wb_sem);
 			goto out;
 		}
 		file = NULL;
@@ -2175,6 +2201,13 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	/* see the comment in fuse_change_attributes() */
 	if (!is_wb || is_truncate)
 		i_size_write(inode, outarg.attr.size);
+	/*
+	 * A truncate settles the size on the server; only shrink the
+	 * server-materialized bound: growing just exposes zeros, which the
+	 * bound need not cover (see fuse_write_begin()).
+	 */
+	if (is_truncate && (loff_t) outarg.attr.size < fi->server_size)
+		fi->server_size = outarg.attr.size;
 
 	if (is_truncate) {
 		/* NOTE: this may release/reacquire fi->lock */
@@ -2188,11 +2221,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	 */
 	if ((is_truncate || !is_wb) &&
 	    S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) {
+		struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+
+		/*
+		 * Revoke and drop under the coherency gate write side; see
+		 * the atomic-O_TRUNC branch above.  i_rwsem is held
+		 * exclusive here as well (setattr), so waiting out gate
+		 * readers cannot deadlock.
+		 */
+		if (wb_sem)
+			percpu_down_write(wb_sem);
 		if (fc->dlm && fc->writeback_cache)
 			fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1);
 
 		truncate_pagecache(inode, outarg.attr.size);
 		invalidate_inode_pages2(mapping);
+		if (wb_sem)
+			percpu_up_write(wb_sem);
 	}
 
 	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);