diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 6b2de8b3e4ec8e..1d372cede3ee69 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1461,6 +1461,26 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd, fuse_uring_send(ent, cmd, err, issue_flags); } +/* + * The request was already copied to the ring buffer in the submitter's + * context, only the io_uring cmd completion is left to do. + * io_uring_cmd_done() must not run in the submitter's context as it would + * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex + * the ring task holds across its whole submission path and frequently gets + * preempted under while the just-woken submitter runs. + */ +static void fuse_uring_send_prepared_in_task(struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + int err = 0; + + if (unlikely(issue_flags & IO_URING_F_TASK_DEAD)) + err = -ECANCELED; + + fuse_uring_send(ent, cmd, err, issue_flags); +} + static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { @@ -1560,7 +1580,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) IO_URING_F_UNLOCKED); return; } - fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED); + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, + fuse_uring_send_prepared_in_task); } } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 3ec99cee3ef62d..99af2777432201 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2016,6 +2016,12 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid |= FATTR_FH; inarg.fh = ff->fh; } + /* + * This is ->write_inode() flushing times the kernel owns locally, not + * a userspace utimes(); let the server tell the two apart. + */ + if (fm->fc->setattr_writeback) + inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); return fuse_simple_request(fm, &args); @@ -2076,15 +2082,35 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. + * + * Revoke and drop under the coherency gate write side, + * like the NOTIFY invalidate path: a gate reader that + * already re-validated its grant must not have the + * lock tree and the cache yanked mid-hold, or it + * would repopulate the truncated range trusting a + * grant that no longer exists. Waiting for gate + * readers here is safe: we hold i_rwsem exclusive, so + * no gate holder can be waiting on it (the write path + * takes i_rwsem before the gate, the read path never + * takes it). */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); + spin_lock(&fi->lock); + fi->server_size = 0; i_size_write(inode, 0); + spin_unlock(&fi->lock); truncate_pagecache(inode, 0); + if (wb_sem) + percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2175,6 +2201,13 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /* see the comment in fuse_change_attributes() */ if (!is_wb || is_truncate) i_size_write(inode, outarg.attr.size); + /* + * A truncate settles the size on the server; only shrink the + * server-materialized bound: growing just exposes zeros, which the + * bound need not cover (see fuse_write_begin()). + */ + if (is_truncate && (loff_t) outarg.attr.size < fi->server_size) + fi->server_size = outarg.attr.size; if (is_truncate) { /* NOTE: this may release/reacquire fi->lock */ @@ -2188,11 +2221,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + + /* + * Revoke and drop under the coherency gate write side; see + * the atomic-O_TRUNC branch above. i_rwsem is held + * exclusive here as well (setattr), so waiting out gate + * readers cannot deadlock. + */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); + if (wb_sem) + percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index cf1f528914c208..a9198a7eb283e2 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -333,6 +333,7 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); + fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); file_update_time(file); @@ -410,6 +411,18 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, if (likely(fi)) { spin_lock(&fi->lock); list_del(&ff->write_entry); + /* + * Leave forced direct IO mode once the last writer is gone: with + * no local writer left there is no cached-write contention with + * the remote modifier that triggered the switch. Restore + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + } spin_unlock(&fi->lock); } spin_lock(&fc->lock); @@ -448,9 +461,24 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_release_args *ra = &ff->args->release_args; int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; + bool was_force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); fuse_prepare_release(fi, ff, open_flags, opcode, false); + /* + * If this release dropped the last writer, fuse_prepare_release() + * cleared the forced-direct-IO latch (under fi->lock). Drop any clean + * folios a read racing the latch may have repopulated so they cannot be + * served stale once caching mode resumes. No inode lock or + * wb_inval_rwsem: release may run on the fuse server thread (async fput + * from aio completion), where blocking on a contended inode lock could + * stall the connection. Writes were routed direct while latched, so + * only clean folios exist and this invalidate is server-free; the last + * writer is gone, so no forced-dio writer can race the drop. + */ + if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) + invalidate_inode_pages2(inode->i_mapping); + if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); @@ -947,7 +975,19 @@ static int fuse_do_readpage(struct file *file, struct page *page) fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); res = fuse_simple_request(fm, &ia.ap.args); if (res < 0) { - if (res == -EAGAIN) + /* + * The DLM subsystem refuses a READ whose grant would deadlock + * against a conflicting lock this node already holds. Ask the + * caller to drop the page and retry so the conflicting holder + * can drain first. + * + * -EDEADLK is the self-describing code the server reports; + * -EAGAIN is the legacy spelling, still accepted so an older + * daemon keeps working. Only a DLM connection may make that + * claim -- elsewhere -EAGAIN is a plain error from the server + * and must be passed through. + */ + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) res = AOP_TRUNCATED_PAGE; return res; } @@ -1096,10 +1136,23 @@ static void fuse_readahead(struct readahead_control *rac) } } +static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); + +/* + * Bound on re-requesting a revoked DLM grant before a cached read is + * served unlocked; see fuse_cache_read_iter(). + */ +#define FUSE_DLM_READ_RETRIES 3 + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { - struct inode *inode = iocb->ki_filp->f_mapping->host; + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + ssize_t res; + int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1114,7 +1167,69 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } - return generic_file_read_iter(iocb, to); + /* if we have dlm support acquire a read lock for the area + * we are reading from. */ + if (fc->writeback_cache && fc->dlm) + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + + /* + * Fence the cache-serving read against a NOTIFY invalidate so we never + * hand back a folio the server has just superseded. The gate read side + * is per-CPU cheap; the NOTIFY holds the write side with priority. + * Re-check the forced-DIO latch under it: if a storm latched us while we + * waited on a pending writer, reroute to direct like the buffered write + * path, so we do not repopulate the cache the latch just dropped. + * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + */ + if (wb_sem) { + int tries = FUSE_DLM_READ_RETRIES; + +retry: + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + return fuse_direct_read_iter(iocb, to); + } + /* + * The DLM lock was requested before entering the gate, and + * the NOTIFY invalidate we may just have waited on revokes + * locks under the gate write side. Re-check the grant here + * and re-request with the gate dropped, so a + * FUSE_DLM_WB_LOCK round trip never parks a pending + * invalidate behind our own gate hold. Once the check + * passes the lock cannot go away for the rest of the gate + * hold. A failed or unrecorded request falls through + * unlocked, as before: the retry is taken even then (the + * latch must be re-checked under the re-entered gate), so + * lock_err has to stay sticky across it -- seeded by the + * pre-gate request above -- or a grant that failed would + * be re-requested forever. The retry is also bounded: a + * remote writer can revoke each successful grant before + * the gate is re-entered, and a reader-only inode has no + * force-DIO latch to end such a storm, so after + * FUSE_DLM_READ_RETRIES re-requests the read is served + * unlocked rather than looping without bound. + */ + if (!lock_err && fc->dlm && tries-- > 0 && + !fuse_dlm_lock_is_held(fi, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ)) { + percpu_up_read(wb_sem); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + goto retry; + } + } + + res = generic_file_read_iter(iocb, to); + + if (wb_sem) + percpu_up_read(wb_sem); + + return res; } static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, @@ -1186,6 +1301,15 @@ bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); + if (written > 0 && S_ISREG(inode->i_mode)) { + /* + * The server acknowledged data up to @pos, keep the + * server-materialized bound in sync for the expansion + * zero-fill in fuse_write_begin(). + */ + if (pos > fi->server_size) + fi->server_size = pos; + } if (written > 0 && pos > inode->i_size) { i_size_write(inode, pos); ret = true; @@ -1385,13 +1509,6 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) return res; } -static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter) -{ - struct inode *inode = file_inode(iocb->ki_filp); - - return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode); -} - /* * @return true if an exclusive lock for direct IO writes is needed */ @@ -1401,9 +1518,15 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); + bool force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); - /* Server side has to advise that it supports parallel dio writes. */ - if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) + /* + * Server side has to advise that it supports parallel dio writes. + * When the inode is latched into forced direct IO, parallel writes are + * used unconditionally: the page cache has been flushed and is bypassed + * for this inode. + */ + if (!force_dio && !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) return true; /* @@ -1414,18 +1537,14 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from return true; /* shared locks are not allowed with parallel page cache IO */ - if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) - return true; - - /* Parallel dio beyond EOF is not supported, at least for now. */ - if (fuse_io_past_eof(iocb, from)) + if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) return true; return false; } static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, - bool *exclusive) + bool *exclusive, bool *uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1439,19 +1558,20 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * New parallal dio allowed only if inode is not in caching * mode and denies new opens in caching mode. This check * should be performed only after taking shared inode lock. - * Previous past eof check was without inode lock and might - * have raced, so check it again. */ - if (fuse_io_past_eof(iocb, from) || - fuse_inode_uncached_io_start(fi, NULL) != 0) { - inode_unlock_shared(inode); - inode_lock(inode); - *exclusive = true; + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) { + if (fuse_inode_uncached_io_start(fi, NULL) != 0) { + inode_unlock_shared(inode); + inode_lock(inode); + *exclusive = true; + } else { + *uncached = true; + } } } } -static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) +static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1459,12 +1579,73 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) if (exclusive) { inode_unlock(inode); } else { - /* Allow opens in caching mode after last parallel dio end */ - fuse_inode_uncached_io_end(fi); + if (uncached) + fuse_inode_uncached_io_end(fi); inode_unlock_shared(inode); } } +static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); + +/* + * @return true if an exclusive inode lock is needed for a cached (buffered) + * write. + * + * Buffered writes normally hold the inode rwsem exclusively, serialising all + * writers even on disjoint ranges. The DLM-serialised writeback path is + * the exception: the DLM already excludes cluster-wide, and i_size is committed + * under fi->lock rather than the inode rwsem (see fuse_write_end()), so + * disjoint writers (MPI-IO / IOR) may share the lock. Mirrors + * fuse_dio_wr_exclusive_lock() for the direct path. + */ +static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool writeback) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_conn *fc = get_fuse_conn(inode); + + /* Only the DLM-serialised writeback path relaxes the lock. */ + if (!fc->dlm || !writeback) + return true; + + /* O_DIRECT writes fall back to generic_file_direct_write(). */ + if (iocb->ki_flags & IOCB_DIRECT) + return true; + + /* Append needs the eventual EOF - always needs an exclusive lock. */ + if (iocb->ki_flags & IOCB_APPEND) + return true; + + return false; +} + +static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) +{ + if (exclusive) + inode_unlock(inode); + else + inode_unlock_shared(inode); +} + +/* + * Request the DLM write lock covering a cached write. -ENOSYS cleared + * fc->dlm: the server has no DLM, proceed as a plain cached write. Any + * other failure means the cache would be dirtied without DLM coverage - + * the caller must fail the write instead. A granted-but-unrecorded + * lock (positive return) is covered cluster-wide; proceed, but flag it + * so the in-gate re-validation skips a check an invisible grant could + * never pass. + */ +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, + bool *unrecorded) +{ + int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + + if (err < 0 && err != -ENOSYS) + return err; + *unrecorded = err > 0; + return 0; +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1474,32 +1655,203 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + bool wb_guard = false; + bool exclusive = true; + bool dlm_unrecorded = false; + loff_t dlm_pos = 0; + size_t dlm_len = 0; + + /* + * The inode may have been latched into forced direct IO -- by a + * NOTIFY_INVAL_INODE arriving while this inode is open for writing here + * -- after this write was routed to the cached path but before it took + * any lock. Re-route to the direct path (before taking a DLM lock) so + * we do not repopulate the page cache the latch just dropped. + */ + if (fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); if (fc->writeback_cache) { - /* Update size (EOF optimization) and mode (SUID clearing) */ - err = fuse_update_attributes(mapping->host, file, - STATX_SIZE | STATX_MODE); + /* Update mode for SUID clearing, and also update size if the file + * is opened with O_APPEND mode. + */ + u32 request_mask = (file->f_flags & O_APPEND) ? + (STATX_SIZE | STATX_MODE) : STATX_MODE; + err = fuse_update_attributes(mapping->host, file, request_mask); if (err) return err; - if (fc->handle_killpriv_v2 && - setattr_should_drop_suidgid(idmap, - file_inode(file))) { + /* + * A write that drops suid/sgid goes down the writethrough + * branch, which holds i_rwsem exclusive. + * + * With handle_killpriv_v2 that is because the server does the + * killing from the WRITE itself. Without it, fuse_setattr() + * has to ask the server, and fuse_do_setattr() freezes + * writepages around that SETATTR: fuse_set_nowrite() asserts + * BUG_ON(fi->writectr < 0), which assumes an exclusive + * i_rwsem, and the DLM-relaxed buffered write path below holds + * it only shared. Two writers can both see the bits set + * before either has cleared them, and the second one would + * then oops inside spin_lock(&fi->lock). + * + * Only the DLM path needs the detour: everywhere else the + * buffered write already holds i_rwsem exclusive, so the two + * writers cannot overlap in the first place. + * + * The bits are read without the inode lock here, so a server + * attribute update can still set them between this test and + * file_remove_privs(). That leaves the same race, but only + * for writers whose mode changed underneath them, rather than + * for every write to a suid file. + */ + if ((fc->handle_killpriv_v2 || fc->dlm) && + setattr_should_drop_suidgid(idmap, file_inode(file))) goto writethrough; + + exclusive = fuse_cache_wr_exclusive_lock(iocb, true); + + /* + * Request the DLM write lock before taking i_rwsem: the request + * is an unbounded cluster round trip, and holding the + * writer-priority rwsem across it would park a truncate -- and + * behind it every later writer -- for the duration. The + * grant-to-use window this leaves open is closed by the in-gate + * re-validation below. Only the append case must wait for the + * lock: its range depends on i_size, which is stable only under + * the exclusive inode lock. + */ + if (fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + dlm_pos = iocb->ki_pos; + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + return err; + + /* + * The request above may have found that the server has + * no DLM at all, in which case it cleared fc->dlm. The + * relaxed shared lock was chosen just before, while + * fc->dlm still read 1, and it is only sound under DLM: + * nothing else excludes a concurrent writer on a + * disjoint range, and the buffered write path no longer + * serialises them itself. Re-decide now, while no lock + * is held yet. + */ + exclusive = fuse_cache_wr_exclusive_lock(iocb, true); } - /* if we have dlm support acquire the lock for the area - * we are writing into */ - if (fc->dlm) { - /* note that a file opened with O_APPEND will have relative values - * in ki_pos. This code is here for convenience and for libfuse overlay test. - * Filesystems should handle O_APPEND with 'direct io' to additionally - * get the performance benefits of 'parallel direct writes'. */ - loff_t pos = file->f_flags & O_APPEND ? i_size_read(inode) + iocb->ki_pos : iocb->ki_pos; - size_t length = iov_iter_count(from); - fuse_get_dlm_write_lock(file, pos, length); + /* + * Open-code generic_file_write_iter() so that the coherency + * gate can be held for read across the page-cache dirtying: a + * concurrent NOTIFY_INVAL_INODE -- which takes the write side + * of that gate (blocking, with priority) around its invalidate + * + latch set -- must not be able to strand the folios we are + * about to dirty. Re-check the latch under it (it may have been + * set while we blocked on the inode lock) and re-route to the + * direct path if it is now set; the DLM write lock taken above + * is harmless there, as the direct path does its own server + * coordination. wb_sem is NULL on mounts where the gate is + * inactive. + */ + if (exclusive) + inode_lock(inode); + else + inode_lock_shared(inode); + + /* note that this small code dup will save us a lot of headache later + * when appends are done concurrently without using parallel direct writes */ + if (fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + /* + * An append write lands at the current EOF no matter + * what ki_pos holds: generic_write_checks() rewrites + * ki_pos to i_size for IOCB_APPEND, and i_size is + * stable here because append writes hold the inode lock + * exclusive. Lock where the data will land. + */ + dlm_pos = i_size_read(inode); + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + goto wb_out; } - return generic_file_write_iter(iocb, from); + + written = generic_write_checks(iocb, from); + if (written <= 0) + goto wb_out; + + /* + * Kill suid/sgid and stamp the timestamps here, before the + * gate, instead of leaving them to + * __generic_file_write_iter(). file_remove_privs() is the one + * that reaches the server: without handle_killpriv[_v2] + * fuse_setattr() kills the bits by asking it (a FUSE_GETATTR to + * refresh the mode, then a FUSE_SETATTR, which for a writeback + * inode first flushes and freezes writepages), and + * security_inode_killpriv() can drop the capability xattr with + * another round trip. A server may have to invalidate this + * inode from inside such a handler; its NOTIFY_INVAL_INODE then + * blocks in percpu_down_write() draining a gate reader that is + * itself waiting for the reply. Nothing held under the gate may + * wait for the server. file_update_time() only marks the inode + * dirty, but stays next to it to keep the VFS order. + */ + err = file_remove_privs(file); + if (!err) + err = file_update_time(file); + if (err) { + written = err; + goto wb_out; + } + + if (wb_sem) { + wb_guard = true; +retry: + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + fuse_cache_wr_unlock(inode, exclusive); + return fuse_direct_write_iter(iocb, from); + } + if (fc->dlm && !dlm_unrecorded && + !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, + FUSE_PAGE_LOCK_WRITE)) { + percpu_up_read(wb_sem); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, + dlm_len, + &dlm_unrecorded); + if (err) { + /* The gate is already dropped; funnel + * the failure through the one audited + * exit. */ + wb_guard = false; + goto wb_out; + } + goto retry; + } + } + if (iocb->ki_flags & IOCB_DIRECT) { + written = generic_file_direct_write(iocb, from); + if (written < 0 || !iov_iter_count(from)) + goto wb_out; + written = direct_write_fallback(iocb, from, written, + generic_perform_write(iocb, from)); + } else { + written = generic_perform_write(iocb, from); + } +wb_out: + if (wb_guard) + percpu_up_read(wb_sem); + fuse_cache_wr_unlock(inode, exclusive); + if (written > 0) + written = generic_write_sync(iocb, written); + return written ? written : err; } writethrough: @@ -1509,8 +1861,14 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err <= 0) goto out; - task_io_account_write(count); - + /* + * Kill suid/sgid and stamp the timestamps before entering the gate, + * for the reason given in the writeback branch: file_remove_privs() + * can issue a request, and a request must never be waited for under + * the gate. They run before the forced-DIO re-route below, so a + * re-routed write repeats them; neither has anything left to do the + * second time. + */ err = file_remove_privs(file); if (err) goto out; @@ -1519,6 +1877,29 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) goto out; + /* + * The killpriv fallback lands here with the writeback cache still on, + * so it populates the page cache too and needs the same guard as the + * writeback branch above: hold the coherency gate for read across the + * page-cache population and re-check the latch under it, so a + * concurrent NOTIFY_INVAL_INODE cannot have the cache repopulated + * behind the invalidate it just did. Still taken before + * task_io_account_write() so a re-route is not double-counted. + * wb_sem is NULL on mounts where the gate is inactive, and such a + * connection never latches either. + */ + wb_guard = !!wb_sem; + if (wb_guard) { + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); + } + } + + task_io_account_write(count); + if (iocb->ki_flags & IOCB_DIRECT) { written = generic_file_direct_write(iocb, from); if (written < 0 || !iov_iter_count(from)) @@ -1529,6 +1910,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from); } out: + if (wb_guard) + percpu_up_read(wb_sem); inode_unlock(inode); if (written > 0) written = generic_write_sync(iocb, written); @@ -1742,14 +2125,16 @@ static ssize_t __fuse_direct_read(struct fuse_io_priv *io, return res; } -static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); +static ssize_t __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, + bool exclusive); static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) { ssize_t res; if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { - res = fuse_direct_IO(iocb, to); + /* exclusive is unused on reads; rollback is write-only */ + res = __fuse_direct_IO(iocb, to, true); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -1765,15 +2150,16 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); struct address_space *mapping = inode->i_mapping; loff_t pos = iocb->ki_pos; + bool exclusive = false; + bool uncached = false; ssize_t res; - bool exclusive; - fuse_dio_lock(iocb, from, &exclusive); + fuse_dio_lock(iocb, from, &exclusive, &uncached); res = generic_write_checks(iocb, from); if (res > 0) { task_io_account_write(res); if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { - res = fuse_direct_IO(iocb, from); + res = __fuse_direct_IO(iocb, from, exclusive); } else { res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); @@ -1790,7 +2176,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) (pos + res - 1) >> PAGE_SHIFT); } } - fuse_dio_unlock(iocb, exclusive); + fuse_dio_unlock(iocb, exclusive, uncached); return res; } @@ -1808,7 +2194,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_read_iter(iocb, to); else if (fuse_file_passthrough(ff)) return fuse_passthrough_read_iter(iocb, to); @@ -1829,7 +2215,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); else if (fuse_file_passthrough(ff)) return fuse_passthrough_write_iter(iocb, from); @@ -1876,14 +2262,6 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } -static void fuse_writepage_finish_stat(struct inode *inode, struct page *page) -{ - struct backing_dev_info *bdi = inode_to_bdi(inode); - - dec_wb_stat(&bdi->wb, WB_WRITEBACK); - wb_writeout_inc(&bdi->wb); -} - static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -1891,10 +2269,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) struct fuse_inode *fi = get_fuse_inode(inode); int i; - for (i = 0; i < ap->num_pages; i++) { - fuse_writepage_finish_stat(inode, ap->pages[i]); + for (i = 0; i < ap->num_pages; i++) end_page_writeback(ap->pages[i]); - } wake_up(&fi->page_waitq); } @@ -1988,6 +2364,20 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, if (!fc->writeback_cache) fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); spin_lock(&fi->lock); + if (!error) { + struct fuse_write_in *inarg = &wpa->ia.write.in; + + /* + * The server acknowledged this writeback, so data up to the + * end of the request is materialized on the server. Advance + * the bound before the pages end writeback below, i.e. before + * they can go clean and be reclaimed, so that + * fuse_write_begin() can never zero-fill a reclaimed range + * the server holds data in. + */ + if ((loff_t) (inarg->offset + inarg->size) > fi->server_size) + fi->server_size = inarg->offset + inarg->size; + } fi->writectr--; fuse_writepage_finish(wpa); spin_unlock(&fi->lock); @@ -2076,14 +2466,11 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, uint32_t page_index) { - struct inode *inode = folio->mapping->host; struct fuse_args_pages *ap = &wpa->ia.ap; ap->pages[page_index] = &folio->page; ap->descs[page_index].offset = 0; ap->descs[page_index].length = PAGE_SIZE; - - inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); } static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, @@ -2272,7 +2659,6 @@ static int fuse_writepages_fill(struct folio *folio, ap->descs[ap->num_pages].length = PAGE_SIZE; ap->pages[ap->num_pages] = &folio->page; ap->num_pages++; - inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); err = 0; if (!data->wpa) { @@ -2327,10 +2713,12 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, struct fuse_conn *fc = get_fuse_conn(file_inode(file)); struct folio *folio; loff_t fsize; - int err = -ENOMEM; + int err; WARN_ON(!fc->writeback_cache); +retry: + err = -ENOMEM; folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, mapping_gfp_mask(mapping)); if (IS_ERR(folio)) @@ -2351,6 +2739,43 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, folio_zero_segment(folio, 0, off); goto success; } + + /* + * The folio is inside i_size but may still sit in a range the server + * holds no data for: a shared-lock writer extends i_size past regions + * it has not written yet (see fuse_write_end()), and every such folio + * would otherwise be read back from the server although it cannot + * contain data. Zero-fill locally instead when the server is known to + * hold nothing in the range and we hold the DLM write lock covering + * it: + * + * - fi->server_size bounds the data materialized on the server + * (writeback and direct write acknowledgements, server + * attributes), + * - local data not yet acknowledged sits in uptodate folios, which + * are already handled above, + * - the page-granular DLM write lock excludes data written by other + * nodes, re-checked against the live lock tree so a revoked lock + * falls back to reading. + */ + if (fc->dlm) { + struct fuse_inode *fi = get_fuse_inode(mapping->host); + loff_t fpos = folio_pos(folio); + size_t fsz = folio_size(folio); + bool hole; + + spin_lock(&fi->lock); + hole = fpos >= fi->server_size; + spin_unlock(&fi->lock); + + if (hole && fuse_dlm_range_is_locked(fi, fpos, fpos + fsz - 1, + FUSE_PAGE_LOCK_WRITE)) { + folio_zero_range(folio, 0, fsz); + folio_mark_uptodate(folio); + goto success; + } + } + err = fuse_do_readpage(file, &folio->page); if (err) goto cleanup; @@ -2361,6 +2786,18 @@ static int fuse_write_begin(struct file *file, struct address_space *mapping, cleanup: folio_unlock(folio); folio_put(folio); + /* + * The DLM refused the read because a conflicting lock is held, and + * fuse_do_readpage() asked for the page to be dropped and the read + * retried. ->write_begin() has no way to pass that request on: + * generic_perform_write() only inspects negative returns, so an + * AOP_TRUNCATED_PAGE escaping here would be taken for success and + * the uninitialised *foliop dereferenced. Retry here instead, now + * that the folio lock the revoke was waiting for has been dropped. + * Each pass costs a server round trip, which throttles the loop. + */ + if (err == AOP_TRUNCATED_PAGE) + goto retry; error: return err; } @@ -2384,8 +2821,28 @@ static int fuse_write_end(struct file *file, struct address_space *mapping, folio_mark_uptodate(folio); } - if (pos > inode->i_size) - i_size_write(inode, pos); + /* + * On the DLM writeback path the inode rwsem may be held shared (see + * fuse_cache_wr_exclusive_lock()), so i_size is no longer serialised + * by it. Commit the extension monotonically under fi->lock -- the + * same way fuse_write_update_attr() does on the direct path -- rather + * than by an unlocked read-modify-write two concurrent extenders could + * lose an update to. + * + * Growing i_size just behind the write cursor, rather than claiming + * the whole extension up front, also keeps fuse_write_begin()'s + * beyond-EOF optimisation effective: folios wholly past EOF are zeroed + * locally instead of sending the server a read-modify-write READ for + * data that does not exist yet. + */ + if (pos > inode->i_size) { + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + if (pos > inode->i_size) + i_size_write(inode, pos); + spin_unlock(&fi->lock); + } folio_mark_dirty(folio); @@ -2426,7 +2883,7 @@ static void fuse_vma_close(struct vm_area_struct *vma) /** * Request a DLM lock from the FUSE server. * - * This routine is similar to fuse_get_dlm_write_lock(), but it + * This routine is similar to fuse_get_dlm_lock(), but it * does not cache the DLM lock in the kernel. */ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t length) @@ -2547,6 +3004,29 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) else if (fuse_inode_backing(get_fuse_inode(inode))) return -ENODEV; + /* + * If the inode was latched into forced direct IO after a remote-modify + * notification, a mapping needs the page cache, so revert to caching + * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * under mmap_lock and the buffered write path holds both across a fault + * on the user buffer (which takes mmap_lock), so taking either here + * would invert lock order (ABBA). Clearing the latch and dropping the + * cache is sufficient -- writers re-check the latch and route to cached + * IO once it is clear, and in-flight parallel dio drains itself. Cached + * opens frozen while latched are still counted in iocachectr, so restore + * FUSE_I_CACHE_IO_MODE for them. + */ + if (fuse_inode_force_dio(inode)) { + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + invalidate_inode_pages2(file->f_mapping); + } + /* * FOPEN_DIRECT_IO handling is special compared to O_DIRECT, * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. @@ -2975,7 +3455,7 @@ static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) } static ssize_t -fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +__fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) { DECLARE_COMPLETION_ONSTACK(wait); ssize_t ret = 0; @@ -3069,14 +3549,27 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (iov_iter_rw(iter) == WRITE) { fuse_write_update_attr(inode, pos, ret); - /* For extending writes we already hold exclusive lock */ - if (ret < 0 && offset + count > i_size) + /* + * Whole-file rollback is only safe under an exclusive lock. + * Parallel writers commit i_size only on success (nothing to + * undo); the server owns failed-extend cleanup. + */ + if (exclusive && ret < 0 && offset + count > i_size) fuse_do_truncate(file); } return ret; } +static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + /* + * Only reached via generic_file_direct_write/read() + * (caching-mode O_DIRECT), which holds the inode lock exclusively. + */ + return __fuse_direct_IO(iocb, iter, true); +} + static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) { int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); @@ -3351,6 +3844,7 @@ static const struct address_space_operations fuse_file_aops = { void fuse_init_file_inode(struct inode *inode, unsigned int flags) { struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); inode->i_fop = &fuse_file_operations; inode->i_data.a_ops = &fuse_file_aops; @@ -3360,8 +3854,28 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fuse_dlm_cache_init(fi); fi->writectr = 0; fi->iocachectr = 0; + fi->server_size = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); + /* + * Coherency gate for the forced-direct-IO feature; only writeback+dlm + * regular files need it. A percpu_rw_semaphore embeds per-CPU state, + * so allocate it out of line and only when the mount can use it rather + * than paying it on every inode. On failure leave it NULL: the gate + * stays inactive (best-effort invalidate) and the inode is still usable. + */ + fi->wb_inval_rwsem = NULL; + if (fc->writeback_cache && fc->dlm) { + struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); + + if (sem && percpu_init_rwsem(sem)) { + kfree(sem); + sem = NULL; + } + fi->wb_inval_rwsem = sem; + } + fi->notify_stamp = jiffies; + fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index d765dd8018cc6a..bc6dbae2d5aeb0 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -63,6 +63,7 @@ int fuse_dlm_cache_init(struct fuse_inode *inode) init_rwsem(&cache->lock); cache->ranges = RB_ROOT_CACHED; + cache->revoke_gen = 0; return 0; } @@ -84,6 +85,7 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /* Release all locks */ down_write(&cache->lock); + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); while ((node = rb_first_cached(&cache->ranges)) != NULL) { range = rb_entry(node, struct fuse_dlm_range, rb); fuse_page_it_remove(range, &cache->ranges); @@ -120,26 +122,25 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *range, *next; - struct rb_node *node; + uint64_t first = start ? start - 1 : start; + uint64_t last = end < U64_MAX ? end + 1 : end; if (!cache) return; - /* Find the first range that might need merging */ - range = NULL; - node = rb_first_cached(&cache->ranges); - while (node) { - range = rb_entry(node, struct fuse_dlm_range, rb); - if (range->end >= start - 1) - break; - node = rb_next(node); - } - - if (!range || range->start > end + 1) - return; + /* + * Find the first range that might need merging. Directly adjacent + * ranges can merge, hence the region is widened by one unit to each + * side (saturating at the type bounds). This must stay an + * interval-tree lookup: the tree holds every cached grant of the + * inode and strided writers grow it for the lifetime of the file, + * so seeding the merge by walking from the tree minimum would make + * every new grant cost a full scan. + */ + range = fuse_page_it_iter_first(&cache->ranges, first, last); /* Try to merge ranges in and around the specified region */ - while (range && range->start <= end + 1) { + while (range && range->start <= last) { /* Get next range before we potentially modify the tree */ next = NULL; if (rb_next(&range->rb)) { @@ -150,11 +151,11 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, /* Try to merge with next range if adjacent and same mode */ if (next && range->mode == next->mode && range->end + 1 == next->start) { - /* Merge ranges */ - range->end = next->end; - - /* Remove next from tree */ + /* Merge ranges: re-insert so __subtree_end is updated */ fuse_page_it_remove(next, &cache->ranges); + fuse_page_it_remove(range, &cache->ranges); + range->end = next->end; + fuse_page_it_insert(range, &cache->ranges); kfree(next); /* Continue with the same range */ @@ -167,11 +168,13 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, } /** - * fuse_dlm_lock_range - Lock a range of pages + * __fuse_dlm_lock_range - Lock a range of pages * @cache: The page cache * @start: Start page offset * @end: End page offset * @mode: Lock mode (read or write) + * @genp: If non-NULL, the revocation generation sampled before the grant + * was requested; recording fails with -EAGAIN if it has moved * * Add a locked range on the specified range of pages. * If parts of the range are already locked, only add the remaining parts. @@ -182,12 +185,14 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + const uint64_t *genp) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; int lock_mode; + bool covered_to_end = false; int ret = 0; LIST_HEAD(to_lock); LIST_HEAD(to_upgrade); @@ -202,6 +207,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, down_write(&cache->lock); + /* + * A revoke was processed after @genp was sampled; the grant this + * record carries may be the very one it targeted (a revoke of a + * not-yet-recorded grant removes nothing and would never be + * retried). Refuse, the caller re-requests. + */ + if (genp && cache->revoke_gen != *genp) { + up_write(&cache->lock); + return -EAGAIN; + } + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -233,14 +249,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, } /* Move current_start past this range */ - current_start = max(current_start, range->end + 1); + if (range->end >= end) + covered_to_end = true; + else + current_start = max(current_start, range->end + 1); /* Move to next range */ range = next; } /* If there's a gap after the last range to the end, extend the range */ - if (current_start <= end) { + if (!covered_to_end && current_start <= end) { new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); if (!new_range) { ret = -ENOMEM; @@ -294,6 +313,35 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, return ret; } +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, NULL); +} + +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, &gen); +} + +/** + * fuse_dlm_revoke_gen - sample the revocation generation + * @inode: the fuse inode + * + * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The + * reply and a NOTIFY revoke can be serviced on different threads, so a + * revoke may be processed between the reply arriving and its grant + * being recorded. fuse_dlm_lock_range_gen() re-checks the generation + * under the cache lock and refuses to record a grant such a revoke may + * have already killed. + */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +{ + return READ_ONCE(inode->dlm_locked_areas.revoke_gen); +} + /** * fuse_dlm_punch_hole - Punch a hole in a locked range * @cache: The page cache @@ -322,13 +370,17 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, /* If the hole is at the beginning of the range */ if (start == range->start) { + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } /* If the hole is at the end of the range */ if (end == range->end) { + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } @@ -362,8 +414,12 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, * @start: Start page offset * @end: End page offset * - * Release locks on the specified range of pages. - * Note that if start and end are set to zero the cache is destroyed. + * Release locks on the specified range of pages. An inverted range is + * rejected rather than silently removing nothing: the callers revoke + * coverage, and a revoke that quietly keeps the grant alive would let + * the re-validating IO paths trust a lock the server has taken away. + * To drop every grant use fuse_dlm_cache_release_locks() (there is no + * in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ @@ -374,16 +430,19 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, struct fuse_dlm_range *range, *next; int ret = 0; - if (!cache) + if (!cache || start > end) return -EINVAL; - if (start == 0 && end == 0) { - fuse_dlm_cache_release_locks(inode); - return 0; - } - down_write(&cache->lock); + /* + * Unconditional, even when nothing overlaps: the revoke racing + * with an in-flight grant finds an empty tree precisely because + * the grant is not recorded yet, and the bump is what makes the + * recording side notice (see fuse_dlm_lock_range_gen()). + */ + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -400,10 +459,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, break; } else if (start > range->start) { /* Adjust the end of the range */ + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); } else if (end < range->end) { /* Adjust the start of the range */ + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); } else { /* Complete overlap, remove the range */ fuse_page_it_remove(range, &cache->ranges); @@ -454,9 +517,16 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, /* Check if the entire range is covered */ while (range && current_start <= end) { - /* If we're checking for a specific mode, verify it matches */ - if (lock_mode && range->mode != lock_mode) { - /* Wrong lock mode */ + /* + * The held lock must be at least as strong as the one + * requested. A WRITE lock (exclusive) satisfies a READ + * request, so only treat the range as uncovered when the + * held mode is weaker than what we ask for. This avoids + * re-requesting a READ lock for a range we already hold + * a WRITE lock on (e.g. read-after-write). + */ + if (lock_mode && range->mode < lock_mode) { + /* Held lock is weaker than requested */ up_read(&cache->lock); return false; } @@ -468,6 +538,12 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return false; } + /* Covered through the end of the requested range? */ + if (range->end >= end) { + up_read(&cache->lock); + return true; + } + /* Move current_start past this range */ current_start = range->end + 1; @@ -487,42 +563,127 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, } /** - * request a dlm lock from the fuse server + * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere + * @fi: the fuse inode + * + * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully + * covered, this asks whether any part of the file is held exclusively. A + * client that holds a write grant may be sitting on dirty page cache the + * server has not seen, so its mtime and ctime run ahead of anything the + * server can report. + * + * Return: true if at least one recorded range is held for write */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length) +bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) +{ + struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; + struct fuse_dlm_range *range; + bool held = false; + + down_read(&cache->lock); + for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; + range = fuse_page_it_iter_next(range, 0, U64_MAX)) { + if (range->mode == FUSE_PCACHE_LK_WRITE) { + held = true; + break; + } + } + up_read(&cache->lock); + + return held; +} + +/** + * fuse_dlm_lock_is_held - check that a byte range is covered by a granted lock + * @fi: the fuse inode + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Re-validation helper for fuse_get_dlm_lock() callers: checks the same + * page-aligned range a fuse_get_dlm_lock() call with these arguments + * requests, against the live lock tree. + */ +bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); + + /* + * An empty range needs no coverage. Reporting it held keeps the + * re-validating IO paths from re-requesting a lock the tree can + * never show (the page-aligned end would invert below). + */ + if (!length) + return true; + + return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); +} + +/** + * fuse_get_dlm_lock - request a dlm lock from the fuse server + * @file: the file being accessed + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Return: 0 when the range is covered by a recorded grant on return, + * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but + * recording it failed (covered cluster-wide, invisible to + * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers + * re-validating the grant must not re-request on a nonzero return or + * they would spin. + */ +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; - uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); - - /* note that the offset and length don't have to be page aligned here - * but since we only get here on writeback caching we will send out - * page aligned requests */ - offset &= PAGE_MASK; FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; + uint64_t gen; int err; + /* An empty range needs no lock. */ + if (!length) + return 0; + +restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care - * of any races in lock requests */ - if (fuse_dlm_range_is_locked(fi, offset, - end, FUSE_PAGE_LOCK_WRITE)) - return; /* we already have this area locked */ + * of any races in lock requests. + * The early exit uses the same helper the callers re-validate + * with, so this check and a later fuse_dlm_lock_is_held() can + * never disagree about what counts as covered. */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) + return 0; /* we already have this area locked */ + + /* + * Sample the revocation generation before the request leaves. + * The reply and a NOTIFY revoke are serviced on different + * threads, so a revoke aimed at the grant this request returns + * can be processed before the grant is recorded below -- + * recording it anyway would resurrect a dead grant that no later + * NOTIFY will ever remove. + */ + gen = fuse_dlm_revoke_gen(fi); memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.start = offset; - inarg.end = end; - inarg.type = FUSE_DLM_LOCK_WRITE; + /* note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests */ + inarg.start = offset & PAGE_MASK; + inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? + FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; args.opcode = FUSE_DLM_WB_LOCK; args.nodeid = get_node_id(inode); @@ -536,23 +697,49 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, if (err == -ENOSYS) { /* fuse server does not support dlm, save the info */ fc->dlm = 0; - return; + return err; } if (err) - return; - else - if (inarg.start < outarg.start || - inarg.end > outarg.end) { - /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", - inarg.start, inarg.end, outarg.start, outarg.end); - fuse_abort_conn(fc); - return; - } else { - /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, outarg.start, - outarg.end, - FUSE_PAGE_LOCK_WRITE); - } + return err; + + if (inarg.start < outarg.start || inarg.end > outarg.end) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); + fuse_abort_conn(fc); + return -EIO; + } + + /* + * The server granted the lock; record it so + * fuse_dlm_lock_is_held() sees it. + */ + err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + if (err == -EAGAIN) { + /* + * A revoke was processed while the request was in flight; + * the grant may already be dead, so re-request instead of + * recording it. Retry until a grant survives long enough to + * be recorded: giving up here would hand the caller an error + * for a range no one else holds, and the write path turns + * that into a failed write. Each pass makes a fresh server + * round trip, so a revoke storm throttles this loop rather + * than spinning it. + */ + goto restart; + } + + /* + * A failure to record (small-allocation -ENOMEM) does not undo + * the grant: coverage exists cluster-wide, only the local + * bookkeeping is missing. Report that as + * FUSE_DLM_GRANT_UNRECORDED so callers neither fail an IO that + * is actually covered nor keep re-requesting a grant that will + * not become visible. + */ + if (err) + return FUSE_DLM_GRANT_UNRECORDED; + + return 0; } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 438d31d28b666e..30fdbb26bd3daf 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -17,12 +17,28 @@ struct fuse_inode; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * fuse_get_dlm_lock() result: the server granted the lock but recording + * it locally failed, leaving the grant invisible to + * fuse_dlm_lock_is_held(). The IO is covered cluster-wide; the caller + * must proceed without re-validating (a re-request would spin) instead + * of failing the IO. + */ +#define FUSE_DLM_GRANT_UNRECORDED 1 + /* Page cache lock manager */ struct fuse_dlm_cache { /* Lock protecting the tree */ struct rw_semaphore lock; /* Interval tree of locked ranges */ struct rb_root_cached ranges; + /* + * Bumped under @lock by every revocation + * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); + * lets fuse_get_dlm_lock() order recording a reply's grant + * against revokes processed while the reply was in flight. + */ + uint64_t revoke_gen; }; /* Initialize a page cache lock manager */ @@ -35,6 +51,14 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen); + +/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); + /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); @@ -43,8 +67,15 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); -/* this is the interface to the filesystem */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length); +/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ +bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); + +/* Is any part of the file held for write? */ +bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); + +/* This is the interface to the filesystem */ +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); #endif /* _FS_FUSE_DLM_CACHE_H */ diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 3b3897f7ddfc80..c6e9791105671d 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +107,20 @@ struct dlm_locked_area size_t size; }; +/* + * Force-DIO switch trigger: an exponentially weighted moving average of the + * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for + * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a + * remote writer streaming invalidations -- and the file is open for writing + * here, it is latched into direct IO. These are the source-level (not + * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, + * seeded and capped at SEED so it takes a short burst rather than a single + * notify to trip. + */ +#define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) +#define FUSE_NOTIFY_EWMA_SHIFT 2 +#define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) + /** FUSE inode */ struct fuse_inode { /** Inode data */ @@ -173,6 +188,57 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; + + /* + * Server-materialized size: an upper bound for how far + * the server holds file data. Seeded from + * server-reported attributes, advanced when the server + * acknowledges data (writeback completion, + * fuse_write_update_attr()), lowered again on + * truncate. A read-modify-write of a folio starting + * at or past this bound needs no READ request under a + * held DLM write lock: the server has no data there + * (see fuse_write_begin()). Protected by fi->lock. + */ + loff_t server_size; + + /* + * Per-inode read/write coherency gate for the + * forced-direct-IO feature. Cache-serving buffered reads + * and buffered writes hold it for read; being a + * percpu_rw_semaphore the read side is per-CPU cheap and + * scales on a shared file. The NOTIFY invalidate + * (fuse_reverse_inval_inode()) holds it for write, which + * BLOCKS so the coherency notify has priority: it fences + * cache-serving reads (and buffered writes) out for the + * whole invalidate, so no folio a remote modify has + * superseded is ever handed back. + * + * The write side may run on the server thread delivering + * the notify, so a blocking writer is safe only under a + * server that services request replies on threads other + * than the one delivering the notify (see the NOTIFY site). + * + * Allocated out of line only for writeback+dlm regular + * files (it shares storage with the readdir-cache union + * arm); NULL on other mounts and on allocation failure, + * where the gate is inactive and the invalidate falls back + * to best-effort. + */ + struct percpu_rw_semaphore *wb_inval_rwsem; + + /* + * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations + * for this whole file: notify_stamp is the jiffies of + * the last one, notify_interval_ewma the EWMA of the + * inter-arrival interval (jiffies, scaled by + * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short + * average interval) with a local writer latches the + * inode into direct IO. Protected by fi->lock; regular + * files only (shares the readdir-cache union arm). + */ + unsigned long notify_stamp; + unsigned int notify_interval_ewma; }; /* readdir cache (directory only) */ @@ -238,6 +304,14 @@ enum { FUSE_I_BTIME, /* Wants or already has page cache IO */ FUSE_I_CACHE_IO_MODE, + /* + * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file + * was open for writing here, so another (remote) entity is modifying it + * concurrently. Reads and writes are routed direct (shared-lock + * parallel dio) until the last writer closes or the inode is mmapped. + * See fuse_reverse_inval_inode()/fuse_file_io_open(). + */ + FUSE_I_FORCE_DIO, }; struct fuse_conn; @@ -766,6 +840,9 @@ struct fuse_conn { /* expire inode entries when doing inode invalidation */ unsigned expire_inode_entries:1; + /* mark writeback-initiated SETATTR requests with FATTR_WRITEBACK */ + unsigned setattr_writeback:1; + /* * The following bitfields are only for optimization purposes * and hence races in setting them will not cause malfunction @@ -1556,6 +1633,12 @@ void fuse_inode_uncached_io_end(struct fuse_inode *fi); int fuse_file_io_open(struct file *file, struct inode *inode); void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); +/* Inode latched into forced direct IO after a remote-modify notification */ +static inline bool fuse_inode_force_dio(struct inode *inode) +{ + return test_bit(FUSE_I_FORCE_DIO, &get_fuse_inode(inode)->state); +} + /* file.c */ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, struct inode *inode, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 88c377e95bf569..b4b4cec2d65671 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -36,6 +36,19 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); +/* + * Gate for the notify-driven direct-IO latch (see + * fuse_reverse_inval_inode()): when a remote writer keeps invalidating a + * file that is also open for writing here, the inode is switched to + * direct IO until its last writer closes. Off by default -- it trades + * the writeback cache away for the duration, which only pays off on + * workloads that actually see such storms. + */ +static bool __read_mostly enable_notify_dio; +module_param(enable_notify_dio, bool, 0644); +MODULE_PARM_DESC(enable_notify_dio, + "Latch a contended inode to direct IO on an invalidation notify storm"); + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); @@ -195,6 +208,23 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } + + /* + * Free the coherency gate here rather than in ->free_inode: that runs + * from an RCU callback, where percpu_free_rwsem() may sleep in + * rcu_sync_dtor() if the write side has not fully quiesced. No user + * can remain by eviction time: gate readers hold a file reference and + * a concurrent notify holds an inode reference. wb_inval_rwsem lives + * in the regular-file union arm and is only ever allocated for regular + * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked + * regular files still own a gate); a directory's overlapping + * readdir-cache fields must not be misread. + */ + if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { + percpu_free_rwsem(fi->wb_inval_rwsem); + kfree(fi->wb_inval_rwsem); + fi->wb_inval_rwsem = NULL; + } } static int fuse_reconfigure(struct fs_context *fsc) @@ -463,12 +493,74 @@ u32 fuse_get_cache_mask(struct inode *inode) { struct fuse_conn *fc = get_fuse_conn(inode); - if (!fc->writeback_cache || !S_ISREG(inode->i_mode)) + if (!fc->writeback_cache || !S_ISREG(inode->i_mode) || fc->dlm) return 0; return STATX_MTIME | STATX_CTIME | STATX_SIZE; } +/* + * Which cached attributes survive a server reply. + * + * Without DLM this is fuse_get_cache_mask(): with the writeback cache on, + * writes update mtime and ctime and may extend i_size locally, the server + * knows about none of it, so the cached values win. + * + * With DLM the server is the authority (fuse_get_cache_mask() returns 0), + * because another node may have changed the file behind us and only the + * server can say so. That holds for the parts of the file we do not own. A + * write grant means no other node can touch the range until we are revoked, + * so anything the server reports about it is at best as new as what we have, + * and older if we still have unwritten data there. Keep the cached values + * for exactly what the grant covers: + * + * - size, when the server reports less than i_size and the tail it does not + * know about, [srv_size, i_size), is entirely under a write grant. Taking + * the server's answer would shrink i_size and have truncate_pagecache() + * throw the unwritten tail away. + * - mtime and ctime, while a write grant covers unwritten data: our writes + * have stamped them locally and the server's stamps predate them. Only + * while the cache is actually dirty, not for as long as the grant lives: + * a grant is held until it is revoked or the inode is evicted, and past + * the writeback the server's stamps are the newer ones. Keeping ours + * beyond that would hide a remote chown or chmod indefinitely. + * + * A remote truncate cannot slip through. It has to revoke the grant first, + * and the revoke launders the tail and drops the grant, so by the time the + * smaller size is reported neither check holds and the server's answer is + * applied as usual. A grant the server made but that could not be recorded + * (FUSE_DLM_GRANT_UNRECORDED) is invisible to the lock tree and falls back to + * trusting the server, as before. + * + * Must be called without fi->lock: the lock tree query sleeps. + */ +static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, + bool have_size) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + u32 cache_mask = fuse_get_cache_mask(inode); + loff_t size = i_size_read(inode); + + if (cache_mask || !fc->dlm || !fc->writeback_cache || + !S_ISREG(inode->i_mode)) + return cache_mask; + + if (!fuse_dlm_write_grant_exists(fi)) + return cache_mask; + + if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || + mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) + cache_mask |= STATX_MTIME | STATX_CTIME; + + if (have_size && size > (loff_t) attr->size && + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE)) + cache_mask |= STATX_SIZE; + + return cache_mask; +} + static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u64 attr_version, u64 evict_ctr) @@ -479,15 +571,13 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr loff_t oldsize; struct timespec64 old_mtime; bool have_size = !sx || (sx->mask & STATX_SIZE); + u64 srv_size; + + cache_mask = fuse_attr_cache_mask(inode, attr, have_size); spin_lock(&fi->lock); + srv_size = attr->size; - /* - * In case of writeback_cache enabled, writes update mtime, ctime and - * may update i_size. In these cases trust the cached value in the - * inode. - */ - cache_mask = fuse_get_cache_mask(inode); if (cache_mask & STATX_SIZE) attr->size = i_size_read(inode); @@ -506,6 +596,18 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr return; } + /* + * srv_size is the size the server reported before the writeback + * cache_mask above replaced attr->size with the local value. It + * bounds how far the server can hold data, letting the buffered write + * path zero-fill expansion read-modify-writes instead of sending READ + * requests, see fuse_write_begin(). Only ever grow it here: stale + * attributes were rejected above and truncation lowers it directly. + */ + if (have_size && S_ISREG(inode->i_mode) && + (loff_t) srv_size > fi->server_size) + fi->server_size = srv_size; + old_mtime = inode_get_mtime(inode); fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, evict_ctr); @@ -522,11 +624,16 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr spin_unlock(&fi->lock); /* - * Only do page cache invalidation when cache_mask is not set - * (writeback_cache disabled) AND the relevant attributes (SIZE/MTIME) - * were actually returned by the server. + * Only do page cache invalidation when the size was not served from + * the cache (writeback_cache disabled, or no grant covering the tail) + * AND the relevant attributes (SIZE/MTIME) were actually returned by + * the server. This has to key off STATX_SIZE alone: i_size_write() + * above took the server's size for any mask without that bit, and the + * cache has to be truncated to match it. The mtime branch neutralises + * itself when STATX_MTIME is set, since attr->mtime then holds the + * value old_mtime was read from. */ - if (!cache_mask && S_ISREG(inode->i_mode)) { + if (!(cache_mask & STATX_SIZE) && S_ISREG(inode->i_mode)) { bool inval = false; bool have_mtime = !sx || (sx->mask & STATX_MTIME); @@ -745,9 +852,92 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode + * moving average of the notification inter-arrival interval and report whether + * the file is now "hot" -- notifications are arriving fast enough (short + * average interval) that a remote writer is repeatedly invalidating it. The + * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped + * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average + * and cannot overflow the accumulator. Must be called under fi->lock; called + * for every data invalidation so the average stays current even while no local + * writer is open. + */ +static bool fuse_notify_inval_hot(struct fuse_inode *fi) +{ + unsigned long now = jiffies; + unsigned long sample; + unsigned int avg; + + sample = min_t(unsigned long, now - fi->notify_stamp, + FUSE_NOTIFY_EWMA_SEED); + fi->notify_stamp = now; + + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + fi->notify_interval_ewma += sample - + (fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT); + avg = fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT; + + return avg < FUSE_NOTIFY_DIO_INTERVAL; +} + +/* + * Revoke the DLM grants backing an invalidated byte range. Grants are + * recorded page-aligned, so widen the revoke to page boundaries: dropping + * more than the server invalidated only costs a re-request, dropping less + * would leave a stale grant that fuse_dlm_lock_is_held() keeps trusting. + * len <= 0 means "invalidate to EOF" (see fuse_notify_inval_inode()) and + * revokes through U64_MAX -- it must not become an inverted range, which + * fuse_dlm_unlock_range() rejects without removing anything. + */ +static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, + loff_t len) +{ + uint64_t start = (uint64_t)offset & PAGE_MASK; + uint64_t end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + + fuse_dlm_unlock_range(fi, start, end); +} + +/* + * Drop a page-cache range on behalf of a NOTIFY invalidate. + * + * invalidate_inode_pages2_range() waits out folios under writeback and + * launders dirty ones, both of which need a FUSE_WRITE reply. While + * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, + * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() + * parks the request on fi->queued_writes until fuse_release_nowrite(). A + * server that revokes from inside the handler it is revoking for then + * deadlocks against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. + * + * So while frozen use invalidate_mapping_pages(), which skips dirty and + * under-writeback folios and never blocks. The stale clean folios still + * go, and the freezes that span a request drop the cache themselves once + * they complete: fuse_do_setattr() invalidates the mapping after releasing + * the freeze, the O_TRUNC open path calls truncate_pagecache(). + */ +static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, + pgoff_t end) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + bool frozen; + + spin_lock(&fi->lock); + frozen = fi->writectr < 0; + spin_unlock(&fi->lock); + + if (frozen) + invalidate_mapping_pages(inode->i_mapping, start, end); + else + invalidate_inode_pages2_range(inode->i_mapping, start, end); +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { + struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; pgoff_t pg_start; @@ -784,18 +974,117 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - if (fc->dlm && fc->writeback_cache) - /* Invalidate the range exactly as the fuse server requested - * except for the case where it sends -1. - * Note that this can lead to some inconsistencies if - * the fuse server sends unaligned data */ - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); - - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + /* + * A data invalidation means another (remote) entity is modifying + * the file. Two things happen here: + * + * 1. Coherency. Drop the affected page-cache range so no local + * read returns a folio the remote modify has superseded. This + * runs under the write side of the per-inode coherency gate + * (wb_inval_rwsem), which fences cache-serving buffered reads + * and buffered writes out for the whole invalidate. Unlike the + * old best-effort trylock this BLOCKS -- the notify has + * priority: percpu_down_write() parks new gate readers, drains + * in-flight ones, then invalidates. A blocking writer here is + * safe only under a server that services request replies on + * threads other than the one delivering this notify: the write + * side waits for gate readers to drain, and a cache-miss read + * holds the read side across its FUSE_READ round-trip. redfs' + * dlm server provides that contract; a server that cannot must + * not enable writeback+dlm. + * + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under + * fi->lock, updated for every data invalidation) of how fast + * these arrive; when they come in a rapid stream -- a remote + * writer repeatedly invalidating -- and the inode is also open + * for writing here, latch it into direct IO until the last + * writer closes or it is mmapped. When latched, drop the whole + * mapping rather than just the notified range, or dirty folios + * outside it would be invisible to the forced direct reads + * (stale read / lost write). Latching is opt-in via the + * enable_notify_dio module parameter and off by default; the + * average is kept up to date either way, so enabling it at + * runtime takes effect on the next storm rather than after a + * warm-up. Clearing it at runtime stops new latches but lets + * already-latched inodes run out on the usual exits (last + * writer closes, or mmap). + * + * The gate (and the average) exist only for writeback+dlm regular + * files; elsewhere wb_sem is NULL and the invalidate runs + * unserialized (best-effort), as before. An mmapped inode + * keeps the gate -- fuse_cache_read_iter() and + * fuse_cache_write_iter() enter it unconditionally and rely + * on the revoke staying fenced -- but is never latched: + * a mapping needs the page cache, and fuse_file_mmap() + * reverts any latch it races with. + */ + if (S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi)) + wb_sem = fi->wb_inval_rwsem; + + if (wb_sem) { + bool hot, has_writer, latched = false; + + spin_lock(&fi->lock); + hot = fuse_notify_inval_hot(fi); + has_writer = !list_empty(&fi->write_files); + spin_unlock(&fi->lock); + + /* + * Priority write side: park new gate readers, + * drain in-flight ones, then invalidate. Blocks + * (unlike the old trylock) -- see the contract in + * the comment above. + */ + percpu_down_write(wb_sem); + + /* + * Revoke the DLM lock range under the gate write + * side, atomically with the page drop: gate readers + * re-validate their grant right after entering, and + * a grant that passed that check must stay visible + * for their whole gate hold. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_revoke_inval_range(fi, offset, len); + + if (enable_notify_dio && hot && has_writer && + !mapping_mapped(inode->i_mapping) && + !fuse_inode_force_dio(inode)) { + spin_lock(&fi->lock); + if (!list_empty(&fi->write_files)) { + set_bit(FUSE_I_FORCE_DIO, &fi->state); + latched = true; + } + spin_unlock(&fi->lock); + } + + /* + * Latched: drop the whole mapping (dirty folios + * outside the notified range would be invisible to + * the forced direct reads). Otherwise just the + * notified range. + */ + if (fuse_inode_force_dio(inode)) + fuse_notify_invalidate_range(inode, 0, -1); + else + fuse_notify_invalidate_range(inode, pg_start, + pg_end); + + percpu_up_write(wb_sem); + + if (latched) + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); + } else { + /* No gate on this inode (DAX, backing, non-regular, + * or the gate allocation failed): drop the lock + * range unserialized (best-effort), as before. */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_revoke_inval_range(fi, offset, len); + fuse_notify_invalidate_range(inode, pg_start, pg_end); + } } iput(inode); return 0; @@ -1643,6 +1932,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->inval_inode_entries = 1; if (flags & FUSE_EXPIRE_INODE_ENTRY) fc->expire_inode_entries = 1; + if (flags & FUSE_SETATTR_WRITEBACK) + fc->setattr_writeback = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1693,7 +1984,8 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY | - FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q; + FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q | + FUSE_SETATTR_WRITEBACK; #ifdef CONFIG_FUSE_DAX if (fm->fc->dax) flags |= FUSE_MAP_ALIGNMENT; @@ -1781,9 +2073,7 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) if (err) return err; - /* fuse does it's own writeback accounting */ - sb->s_bdi->capabilities &= ~BDI_CAP_WRITEBACK_ACCT; - sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT; + sb->s_bdi->capabilities &= ~BDI_CAP_STRICTLIMIT; /* * For a single fuse filesystem use max 1% of dirty + diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c index c99e285f3183ef..301294c8fb25f0 100644 --- a/fs/fuse/iomode.c +++ b/fs/fuse/iomode.c @@ -233,6 +233,16 @@ int fuse_file_io_open(struct file *file, struct inode *inode) !(ff->open_flags & FOPEN_PASSTHROUGH)) return 0; + /* + * The inode was latched into direct IO after a remote-modify + * notification arrived while it was open for writing here. Open this + * file uncached as well so its IO is routed direct and it does not + * re-enter caching mode. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !(ff->open_flags & FOPEN_PASSTHROUGH)) + return 0; + if (ff->open_flags & FOPEN_PASSTHROUGH) err = fuse_file_passthrough_open(inode, file); else diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 7902f034908cb8..d814a339442ff2 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -360,6 +360,17 @@ struct fuse_file_lock { #define FATTR_LOCKOWNER (1 << 9) #define FATTR_CTIME (1 << 10) #define FATTR_KILL_SUIDGID (1 << 11) +/* + * Not an attribute selector: marks the request as a kernel-initiated + * writeback of locally owned attributes rather than a userspace-initiated + * change. Only sent if the server negotiated FUSE_SETATTR_WRITEBACK. + * + * The bit is deliberately far above the sequentially allocated FATTR_* + * range: libfuse mirrors these bits into its own FUSE_SET_ATTR_* space, + * which has its own allocations from bit 12 upwards, and only a value that + * is free on both sides can be passed through without translation. + */ +#define FATTR_WRITEBACK (1 << 30) /** * Flags returned by the OPEN request @@ -441,6 +452,8 @@ struct fuse_file_lock { * optimal io-size alignment * FUSE_URING_REDUCED_Q: Client (kernel) supports less queues - Server is free * to register between 1 and nr-core io-uring queues + * FUSE_SETATTR_WRITEBACK: kernel marks writeback-initiated SETATTR requests + * with FATTR_WRITEBACK */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -490,6 +503,7 @@ struct fuse_file_lock { #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_ALIGN_PG_ORDER (1ULL << 50) +#define FUSE_SETATTR_WRITEBACK (1ULL << 58) #define FUSE_URING_REDUCED_Q (1ULL << 59) #define FUSE_INVAL_INODE_ENTRY (1ULL << 60) #define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61)