From 8113aa91360a013ebe00763bb0823b5a41b11c4d Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 3 Jan 2023 19:37:49 +0800 Subject: [PATCH 001/265] fs: dlm: fix return value check in dlm_memory_init() It should check 'cb_cache', after calling kmem_cache_create("dlm_cb"). Fixes: 61bed0baa4db ("fs: dlm: use a non-static queue for callbacks") Signed-off-by: Yang Yingliang Signed-off-by: David Teigland --- fs/dlm/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c index eb7a08641fcf..cdbaa452fc05 100644 --- a/fs/dlm/memory.c +++ b/fs/dlm/memory.c @@ -51,7 +51,7 @@ int __init dlm_memory_init(void) cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback), __alignof__(struct dlm_callback), 0, NULL); - if (!rsb_cache) + if (!cb_cache) goto cb; return 0; From aa7f4a21f6e5b0fa256b4de0924fea506ad801ef Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 4 Jan 2023 16:38:05 -0800 Subject: [PATCH 002/265] fs/dlm: Remove "select SRCU" Now that the SRCU Kconfig option is unconditionally selected, there is no longer any point in selecting it. Therefore, remove the "select SRCU" Kconfig statements. Signed-off-by: Paul E. McKenney Signed-off-by: David Teigland --- fs/dlm/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/dlm/Kconfig b/fs/dlm/Kconfig index 1105ce3c80cb..b3b86dbdc187 100644 --- a/fs/dlm/Kconfig +++ b/fs/dlm/Kconfig @@ -4,7 +4,6 @@ menuconfig DLM depends on INET depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n) select IP_SCTP - select SRCU help A general purpose distributed lock manager for kernel or userspace applications. From 2ad9bd8332ac1bc072cc7d785e7cb09d6ad36f4c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 5 Jan 2023 11:07:30 -0800 Subject: [PATCH 003/265] iov: add import_ubuf() Like import_single_range(), but for ITER_UBUF. Signed-off-by: Jens Axboe Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig --- include/linux/uio.h | 1 + lib/iov_iter.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/linux/uio.h b/include/linux/uio.h index 9f158238edba..73b1d5d1e4f1 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -346,6 +346,7 @@ ssize_t __import_iovec(int type, const struct iovec __user *uvec, struct iov_iter *i, bool compat); int import_single_range(int type, void __user *buf, size_t len, struct iovec *iov, struct iov_iter *i); +int import_ubuf(int type, void __user *buf, size_t len, struct iov_iter *i); static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, void __user *buf, size_t count) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f9a3ff37ecd1..f036760b66e2 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1877,6 +1877,17 @@ int import_single_range(int rw, void __user *buf, size_t len, } EXPORT_SYMBOL(import_single_range); +int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) +{ + if (len > MAX_RW_COUNT) + len = MAX_RW_COUNT; + if (unlikely(!access_ok(buf, len))) + return -EFAULT; + + iov_iter_ubuf(i, rw, buf, len); + return 0; +} + /** * iov_iter_restore() - Restore a &struct iov_iter to the same state as when * iov_iter_save_state() was called. From 4b61152e107a95bc0a73d84072dbd75cb97689e3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 5 Jan 2023 11:07:31 -0800 Subject: [PATCH 004/265] io_uring: switch network send/recv to ITER_UBUF This is more efficient than iter_iov. Signed-off-by: Jens Axboe [merged to 6.2] Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig --- io_uring/net.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/io_uring/net.c b/io_uring/net.c index fbc34a7c2743..0fb9dcab20b7 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -181,7 +181,7 @@ static int io_setup_async_msg(struct io_kiocb *req, if (async_msg->msg.msg_name) async_msg->msg.msg_name = &async_msg->addr; /* if were using fast_iov, set it to the new one */ - if (!kmsg->free_iov) { + if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) { size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; } @@ -344,7 +344,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) struct sockaddr_storage __address; struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; - struct iovec iov; struct socket *sock; unsigned flags; int min_ret = 0; @@ -378,7 +377,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(!sock)) return -ENOTSOCK; - ret = import_single_range(ITER_SOURCE, sr->buf, sr->len, &iov, &msg.msg_iter); + ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &msg.msg_iter); if (unlikely(ret)) return ret; @@ -764,10 +763,7 @@ retry_multishot: } } - kmsg->fast_iov[0].iov_base = buf; - kmsg->fast_iov[0].iov_len = len; - iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, kmsg->fast_iov, 1, - len); + iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len); } flags = sr->msg_flags; @@ -835,7 +831,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct socket *sock; - struct iovec iov; unsigned int cflags; unsigned flags; int ret, min_ret = 0; @@ -863,7 +858,7 @@ retry_multishot: sr->buf = buf; } - ret = import_single_range(ITER_DEST, sr->buf, len, &iov, &msg.msg_iter); + ret = import_ubuf(ITER_DEST, sr->buf, len, &msg.msg_iter); if (unlikely(ret)) goto out_free; @@ -1074,7 +1069,6 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) struct sockaddr_storage __address; struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; - struct iovec iov; struct socket *sock; unsigned msg_flags; int ret, min_ret = 0; @@ -1116,8 +1110,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) msg.sg_from_iter = io_sg_from_iter; } else { io_notif_set_extended(zc->notif); - ret = import_single_range(ITER_SOURCE, zc->buf, zc->len, &iov, - &msg.msg_iter); + ret = import_ubuf(ITER_SOURCE, zc->buf, zc->len, &msg.msg_iter); if (unlikely(ret)) return ret; ret = io_notif_account_mem(zc->notif, zc->len); From 1e23db450cff5f0410480137041181d1514bda2a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 5 Jan 2023 11:07:32 -0800 Subject: [PATCH 005/265] io_uring: use iter_ubuf for single range imports This is more efficient than iter_iov. Signed-off-by: Jens Axboe [merge to 6.2, minor fixes] Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig --- io_uring/rw.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index 8227af2e1c0f..436a2e064df6 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -391,7 +391,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, rw->len = sqe_len; } - ret = import_single_range(ddir, buf, sqe_len, s->fast_iov, iter); + ret = import_ubuf(ddir, buf, sqe_len, iter); if (ret) return ERR_PTR(ret); return NULL; @@ -450,7 +450,10 @@ static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) struct iovec iovec; ssize_t nr; - if (!iov_iter_is_bvec(iter)) { + if (iter_is_ubuf(iter)) { + iovec.iov_base = iter->ubuf + iter->iov_offset; + iovec.iov_len = iov_iter_count(iter); + } else if (!iov_iter_is_bvec(iter)) { iovec = iov_iter_iovec(iter); } else { iovec.iov_base = u64_to_user_ptr(rw->addr); @@ -495,7 +498,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, io->free_iovec = iovec; io->bytes_done = 0; /* can only be fixed buffers, no need to do anything */ - if (iov_iter_is_bvec(iter)) + if (iov_iter_is_bvec(iter) || iter_is_ubuf(iter)) return; if (!iovec) { unsigned iov_off = 0; From 4397a17c1dc53f436285f372432dd1aea44e7953 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 5 Jan 2023 11:07:33 -0800 Subject: [PATCH 006/265] iov_iter: move iter_ubuf check inside restore WARN io_uring is using iter_ubuf types for single vector requests. We expect state restore may happen for this type now, and it is already handled correctly, so suppress the warning. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe Reviewed-by: Christoph Hellwig --- lib/iov_iter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index f036760b66e2..d9b3332c8405 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1902,8 +1902,8 @@ int import_ubuf(int rw, void __user *buf, size_t len, struct iov_iter *i) */ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) { - if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && - !iov_iter_is_kvec(i) && !iter_is_ubuf(i)) + if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i) && + !iter_is_ubuf(i)) && !iov_iter_is_kvec(i)) return; i->iov_offset = state->iov_offset; i->count = state->count; From 3d2d7e61553dbcc8ba45201d8ae4f383742c8202 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 29 Sep 2022 16:34:45 +0200 Subject: [PATCH 007/265] udf: Define EFSCORRUPTED error code Similarly to other filesystems define EFSCORRUPTED error code for reporting internal filesystem corruption. Signed-off-by: Jan Kara --- fs/udf/udf_sb.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/udf/udf_sb.h b/fs/udf/udf_sb.h index 291b56dd011e..6bccff3c70f5 100644 --- a/fs/udf/udf_sb.h +++ b/fs/udf/udf_sb.h @@ -55,6 +55,8 @@ #define MF_DUPLICATE_MD 0x01 #define MF_MIRROR_FE_LOADED 0x02 +#define EFSCORRUPTED EUCLEAN + struct udf_meta_data { __u32 s_meta_file_loc; __u32 s_mirror_file_loc; From d16076d9b684b7c8d3ccbe9c33d5ea9fe8fcca09 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 3 Oct 2022 23:33:22 +0200 Subject: [PATCH 008/265] udf: New directory iteration code Add new support code for iterating directory entries. The code is also more carefully verifying validity of on-disk directory entries to avoid crashes on malicious media. Signed-off-by: Jan Kara --- fs/udf/directory.c | 395 +++++++++++++++++++++++++++++++++++++++++++++ fs/udf/udfdecl.h | 23 +++ 2 files changed, 418 insertions(+) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 16bcf2c6b8b3..2842c463725c 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -17,6 +17,401 @@ #include #include #include +#include +#include + +static int udf_verify_fi(struct udf_fileident_iter *iter) +{ + unsigned int len; + + if (iter->fi.descTag.tagIdent != cpu_to_le16(TAG_IDENT_FID)) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has entry at pos %llu with incorrect tag %x\n", + iter->dir->i_ino, (unsigned long long)iter->pos, + le16_to_cpu(iter->fi.descTag.tagIdent)); + return -EFSCORRUPTED; + } + len = udf_dir_entry_len(&iter->fi); + if (le16_to_cpu(iter->fi.lengthOfImpUse) & 3) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has entry at pos %llu with unaligned lenght of impUse field\n", + iter->dir->i_ino, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + /* + * This is in fact allowed by the spec due to long impUse field but + * we don't support it. If there is real media with this large impUse + * field, support can be added. + */ + if (len > 1 << iter->dir->i_blkbits) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has too big (%u) entry at pos %llu\n", + iter->dir->i_ino, len, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + if (iter->pos + len > iter->dir->i_size) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has entry past directory size at pos %llu\n", + iter->dir->i_ino, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + if (udf_dir_entry_len(&iter->fi) != + sizeof(struct tag) + le16_to_cpu(iter->fi.descTag.descCRCLength)) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has entry where CRC length (%u) does not match entry length (%u)\n", + iter->dir->i_ino, + (unsigned)le16_to_cpu(iter->fi.descTag.descCRCLength), + (unsigned)(udf_dir_entry_len(&iter->fi) - + sizeof(struct tag))); + return -EFSCORRUPTED; + } + return 0; +} + +static int udf_copy_fi(struct udf_fileident_iter *iter) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + int blksize = 1 << iter->dir->i_blkbits; + int err, off, len, nameoff; + + /* Skip copying when we are at EOF */ + if (iter->pos >= iter->dir->i_size) { + iter->name = NULL; + return 0; + } + if (iter->dir->i_size < iter->pos + sizeof(struct fileIdentDesc)) { + udf_err(iter->dir->i_sb, + "directory (ino %lu) has entry straddling EOF\n", + iter->dir->i_ino); + return -EFSCORRUPTED; + } + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + memcpy(&iter->fi, iinfo->i_data + iinfo->i_lenEAttr + iter->pos, + sizeof(struct fileIdentDesc)); + err = udf_verify_fi(iter); + if (err < 0) + return err; + iter->name = iinfo->i_data + iinfo->i_lenEAttr + iter->pos + + sizeof(struct fileIdentDesc) + + le16_to_cpu(iter->fi.lengthOfImpUse); + return 0; + } + + off = iter->pos & (blksize - 1); + len = min_t(int, sizeof(struct fileIdentDesc), blksize - off); + memcpy(&iter->fi, iter->bh[0]->b_data + off, len); + if (len < sizeof(struct fileIdentDesc)) + memcpy((char *)(&iter->fi) + len, iter->bh[1]->b_data, + sizeof(struct fileIdentDesc) - len); + err = udf_verify_fi(iter); + if (err < 0) + return err; + + /* Handle directory entry name */ + nameoff = off + sizeof(struct fileIdentDesc) + + le16_to_cpu(iter->fi.lengthOfImpUse); + if (off + udf_dir_entry_len(&iter->fi) <= blksize) { + iter->name = iter->bh[0]->b_data + nameoff; + } else if (nameoff >= blksize) { + iter->name = iter->bh[1]->b_data + (nameoff - blksize); + } else { + iter->name = iter->namebuf; + len = blksize - nameoff; + memcpy(iter->name, iter->bh[0]->b_data + nameoff, len); + memcpy(iter->name + len, iter->bh[1]->b_data, + iter->fi.lengthFileIdent - len); + } + return 0; +} + +/* Readahead 8k once we are at 8k boundary */ +static void udf_readahead_dir(struct udf_fileident_iter *iter) +{ + unsigned int ralen = 16 >> (iter->dir->i_blkbits - 9); + struct buffer_head *tmp, *bha[16]; + int i, num; + udf_pblk_t blk; + + if (iter->loffset & (ralen - 1)) + return; + + if (iter->loffset + ralen > (iter->elen >> iter->dir->i_blkbits)) + ralen = (iter->elen >> iter->dir->i_blkbits) - iter->loffset; + num = 0; + for (i = 0; i < ralen; i++) { + blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, + iter->loffset + i); + tmp = udf_tgetblk(iter->dir->i_sb, blk); + if (tmp && !buffer_uptodate(tmp) && !buffer_locked(tmp)) + bha[num++] = tmp; + else + brelse(tmp); + } + if (num) { + bh_readahead_batch(num, bha, REQ_RAHEAD); + for (i = 0; i < num; i++) + brelse(bha[i]); + } +} + +static struct buffer_head *udf_fiiter_bread_blk(struct udf_fileident_iter *iter) +{ + udf_pblk_t blk; + + udf_readahead_dir(iter); + blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, iter->loffset); + return udf_tread(iter->dir->i_sb, blk); +} + +/* + * Updates loffset to point to next directory block; eloc, elen & epos are + * updated if we need to traverse to the next extent as well. + */ +static int udf_fiiter_advance_blk(struct udf_fileident_iter *iter) +{ + iter->loffset++; + if (iter->loffset < iter->elen >> iter->dir->i_blkbits) + return 0; + + iter->loffset = 0; + if (udf_next_aext(iter->dir, &iter->epos, &iter->eloc, &iter->elen, 1) + != (EXT_RECORDED_ALLOCATED >> 30)) { + if (iter->pos == iter->dir->i_size) { + iter->elen = 0; + return 0; + } + udf_err(iter->dir->i_sb, + "extent after position %llu not allocated in directory (ino %lu)\n", + (unsigned long long)iter->pos, iter->dir->i_ino); + return -EFSCORRUPTED; + } + return 0; +} + +static int udf_fiiter_load_bhs(struct udf_fileident_iter *iter) +{ + int blksize = 1 << iter->dir->i_blkbits; + int off = iter->pos & (blksize - 1); + int err; + struct fileIdentDesc *fi; + + /* Is there any further extent we can map from? */ + if (!iter->bh[0] && iter->elen) { + iter->bh[0] = udf_fiiter_bread_blk(iter); + if (!iter->bh[0]) { + err = -ENOMEM; + goto out_brelse; + } + if (!buffer_uptodate(iter->bh[0])) { + err = -EIO; + goto out_brelse; + } + } + /* There's no next block so we are done */ + if (iter->pos >= iter->dir->i_size) + return 0; + /* Need to fetch next block as well? */ + if (off + sizeof(struct fileIdentDesc) > blksize) + goto fetch_next; + fi = (struct fileIdentDesc *)(iter->bh[0]->b_data + off); + /* Need to fetch next block to get name? */ + if (off + udf_dir_entry_len(fi) > blksize) { +fetch_next: + udf_fiiter_advance_blk(iter); + iter->bh[1] = udf_fiiter_bread_blk(iter); + if (!iter->bh[1]) { + err = -ENOMEM; + goto out_brelse; + } + if (!buffer_uptodate(iter->bh[1])) { + err = -EIO; + goto out_brelse; + } + } + return 0; +out_brelse: + brelse(iter->bh[0]); + brelse(iter->bh[1]); + iter->bh[0] = iter->bh[1] = NULL; + return err; +} + +int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, + loff_t pos) +{ + struct udf_inode_info *iinfo = UDF_I(dir); + int err = 0; + + iter->dir = dir; + iter->bh[0] = iter->bh[1] = NULL; + iter->pos = pos; + iter->elen = 0; + iter->epos.bh = NULL; + iter->name = NULL; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return udf_copy_fi(iter); + + if (inode_bmap(dir, iter->pos >> dir->i_blkbits, &iter->epos, + &iter->eloc, &iter->elen, &iter->loffset) != + (EXT_RECORDED_ALLOCATED >> 30)) { + if (pos == dir->i_size) + return 0; + udf_err(dir->i_sb, + "position %llu not allocated in directory (ino %lu)\n", + (unsigned long long)pos, dir->i_ino); + return -EFSCORRUPTED; + } + err = udf_fiiter_load_bhs(iter); + if (err < 0) + return err; + err = udf_copy_fi(iter); + if (err < 0) { + udf_fiiter_release(iter); + return err; + } + return 0; +} + +int udf_fiiter_advance(struct udf_fileident_iter *iter) +{ + unsigned int oldoff, len; + int blksize = 1 << iter->dir->i_blkbits; + int err; + + oldoff = iter->pos & (blksize - 1); + len = udf_dir_entry_len(&iter->fi); + iter->pos += len; + if (UDF_I(iter->dir)->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + if (oldoff + len >= blksize) { + brelse(iter->bh[0]); + iter->bh[0] = NULL; + /* Next block already loaded? */ + if (iter->bh[1]) { + iter->bh[0] = iter->bh[1]; + iter->bh[1] = NULL; + } else { + udf_fiiter_advance_blk(iter); + } + } + err = udf_fiiter_load_bhs(iter); + if (err < 0) + return err; + } + return udf_copy_fi(iter); +} + +void udf_fiiter_release(struct udf_fileident_iter *iter) +{ + iter->dir = NULL; + brelse(iter->bh[0]); + brelse(iter->bh[1]); + iter->bh[0] = iter->bh[1] = NULL; +} + +static void udf_copy_to_bufs(void *buf1, int len1, void *buf2, int len2, + int off, void *src, int len) +{ + int copy; + + if (off >= len1) { + off -= len1; + } else { + copy = min(off + len, len1) - off; + memcpy(buf1 + off, src, copy); + src += copy; + len -= copy; + off = 0; + } + if (len > 0) { + if (WARN_ON_ONCE(off + len > len2 || !buf2)) + return; + memcpy(buf2 + off, src, len); + } +} + +static uint16_t udf_crc_fi_bufs(void *buf1, int len1, void *buf2, int len2, + int off, int len) +{ + int copy; + uint16_t crc = 0; + + if (off >= len1) { + off -= len1; + } else { + copy = min(off + len, len1) - off; + crc = crc_itu_t(crc, buf1 + off, copy); + len -= copy; + off = 0; + } + if (len > 0) { + if (WARN_ON_ONCE(off + len > len2 || !buf2)) + return 0; + crc = crc_itu_t(crc, buf2 + off, len); + } + return crc; +} + +static void udf_copy_fi_to_bufs(char *buf1, int len1, char *buf2, int len2, + int off, struct fileIdentDesc *fi, + uint8_t *impuse, uint8_t *name) +{ + uint16_t crc; + int fioff = off; + int crcoff = off + sizeof(struct tag); + unsigned int crclen = udf_dir_entry_len(fi) - sizeof(struct tag); + + udf_copy_to_bufs(buf1, len1, buf2, len2, off, fi, + sizeof(struct fileIdentDesc)); + off += sizeof(struct fileIdentDesc); + if (impuse) + udf_copy_to_bufs(buf1, len1, buf2, len2, off, impuse, + le16_to_cpu(fi->lengthOfImpUse)); + off += le16_to_cpu(fi->lengthOfImpUse); + if (name) + udf_copy_to_bufs(buf1, len1, buf2, len2, off, name, + fi->lengthFileIdent); + + crc = udf_crc_fi_bufs(buf1, len1, buf2, len2, crcoff, crclen); + fi->descTag.descCRC = cpu_to_le16(crc); + fi->descTag.descCRCLength = cpu_to_le16(crclen); + fi->descTag.tagChecksum = udf_tag_checksum(&fi->descTag); + + udf_copy_to_bufs(buf1, len1, buf2, len2, fioff, fi, sizeof(struct tag)); +} + +void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + void *buf1, *buf2 = NULL; + int len1, len2 = 0, off; + int blksize = 1 << iter->dir->i_blkbits; + + off = iter->pos & (blksize - 1); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + buf1 = iinfo->i_data + iinfo->i_lenEAttr; + len1 = iter->dir->i_size; + } else { + buf1 = iter->bh[0]->b_data; + len1 = blksize; + if (iter->bh[1]) { + buf2 = iter->bh[1]->b_data; + len2 = blksize; + } + } + + udf_copy_fi_to_bufs(buf1, len1, buf2, len2, off, &iter->fi, impuse, + iter->name == iter->namebuf ? iter->name : NULL); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + mark_inode_dirty(iter->dir); + } else { + mark_buffer_dirty_inode(iter->bh[0], iter->dir); + if (iter->bh[1]) + mark_buffer_dirty_inode(iter->bh[1], iter->dir); + } + inode_inc_iversion(iter->dir); +} struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, struct udf_fileident_bh *fibh, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 7e258f15b8ef..22a8466e335c 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -86,6 +86,24 @@ extern const struct address_space_operations udf_aops; extern const struct address_space_operations udf_adinicb_aops; extern const struct address_space_operations udf_symlink_aops; +struct udf_fileident_iter { + struct inode *dir; /* Directory we are working with */ + loff_t pos; /* Logical position in a dir */ + struct buffer_head *bh[2]; /* Buffer containing 'pos' and possibly + * next buffer if entry straddles + * blocks */ + struct kernel_lb_addr eloc; /* Start of extent containing 'pos' */ + uint32_t elen; /* Length of extent containing 'pos' */ + sector_t loffset; /* Block offset of 'pos' within above + * extent */ + struct extent_position epos; /* Position after the above extent */ + struct fileIdentDesc fi; /* Copied directory entry */ + uint8_t *name; /* Pointer to entry name */ + uint8_t namebuf[UDF_NAME_LEN_CS0]; /* Storage for entry name in case + * the name is split between two blocks + */ +}; + struct udf_fileident_bh { struct buffer_head *sbh; struct buffer_head *ebh; @@ -243,6 +261,11 @@ extern udf_pblk_t udf_new_block(struct super_block *sb, struct inode *inode, uint16_t partition, uint32_t goal, int *err); /* directory.c */ +int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, + loff_t pos); +int udf_fiiter_advance(struct udf_fileident_iter *iter); +void udf_fiiter_release(struct udf_fileident_iter *iter); +void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse); extern struct fileIdentDesc *udf_fileident_read(struct inode *, loff_t *, struct udf_fileident_bh *, struct fileIdentDesc *, From 7cd7a36ab44d3e8c1dee7185ef407b9831a8220b Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 4 Oct 2022 15:00:15 +0200 Subject: [PATCH 009/265] udf: Convert udf_readdir() to new directory iteration Convert udf_readdir() to new directory iteration functions. Signed-off-by: Jan Kara --- fs/udf/dir.c | 148 ++++++++++----------------------------------------- 1 file changed, 27 insertions(+), 121 deletions(-) diff --git a/fs/udf/dir.c b/fs/udf/dir.c index be640f4b2f2c..212393b12c22 100644 --- a/fs/udf/dir.c +++ b/fs/udf/dir.c @@ -39,26 +39,13 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) { struct inode *dir = file_inode(file); - struct udf_inode_info *iinfo = UDF_I(dir); - struct udf_fileident_bh fibh = { .sbh = NULL, .ebh = NULL}; - struct fileIdentDesc *fi = NULL; - struct fileIdentDesc cfi; - udf_pblk_t block, iblock; loff_t nf_pos, emit_pos = 0; int flen; - unsigned char *fname = NULL, *copy_name = NULL; - unsigned char *nameptr; - uint16_t liu; - uint8_t lfi; - loff_t size = udf_ext0_offset(dir) + dir->i_size; - struct buffer_head *tmp, *bha[16]; - struct kernel_lb_addr eloc; - uint32_t elen; - sector_t offset; - int i, num, ret = 0; - struct extent_position epos = { NULL, 0, {0, 0} }; + unsigned char *fname = NULL; + int ret = 0; struct super_block *sb = dir->i_sb; bool pos_valid = false; + struct udf_fileident_iter iter; if (ctx->pos == 0) { if (!dir_emit_dot(file, ctx)) @@ -66,7 +53,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) ctx->pos = 1; } nf_pos = (ctx->pos - 1) << 2; - if (nf_pos >= size) + if (nf_pos >= dir->i_size) goto out; /* @@ -90,138 +77,57 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) goto out; } - if (nf_pos == 0) - nf_pos = udf_ext0_offset(dir); - - fibh.soffset = fibh.eoffset = nf_pos & (sb->s_blocksize - 1); - if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { - if (inode_bmap(dir, nf_pos >> sb->s_blocksize_bits, - &epos, &eloc, &elen, &offset) - != (EXT_RECORDED_ALLOCATED >> 30)) { - ret = -ENOENT; - goto out; - } - block = udf_get_lb_pblock(sb, &eloc, offset); - if ((++offset << sb->s_blocksize_bits) < elen) { - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (iinfo->i_alloc_type == - ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - } else { - offset = 0; - } - - if (!(fibh.sbh = fibh.ebh = udf_tread(sb, block))) { - ret = -EIO; - goto out; - } - - if (!(offset & ((16 >> (sb->s_blocksize_bits - 9)) - 1))) { - i = 16 >> (sb->s_blocksize_bits - 9); - if (i + offset > (elen >> sb->s_blocksize_bits)) - i = (elen >> sb->s_blocksize_bits) - offset; - for (num = 0; i > 0; i--) { - block = udf_get_lb_pblock(sb, &eloc, offset + i); - tmp = udf_tgetblk(sb, block); - if (tmp && !buffer_uptodate(tmp) && !buffer_locked(tmp)) - bha[num++] = tmp; - else - brelse(tmp); - } - if (num) { - bh_readahead_batch(num, bha, REQ_RAHEAD); - for (i = 0; i < num; i++) - brelse(bha[i]); - } - } - } - - while (nf_pos < size) { + for (ret = udf_fiiter_init(&iter, dir, nf_pos); + !ret && iter.pos < dir->i_size; + ret = udf_fiiter_advance(&iter)) { struct kernel_lb_addr tloc; - loff_t cur_pos = nf_pos; + udf_pblk_t iblock; - /* Update file position only if we got past the current one */ - if (nf_pos >= emit_pos) { - ctx->pos = (nf_pos >> 2) + 1; - pos_valid = true; - } - - fi = udf_fileident_read(dir, &nf_pos, &fibh, &cfi, &epos, &eloc, - &elen, &offset); - if (!fi) - goto out; /* Still not at offset where user asked us to read from? */ - if (cur_pos < emit_pos) + if (iter.pos < emit_pos) continue; - liu = le16_to_cpu(cfi.lengthOfImpUse); - lfi = cfi.lengthFileIdent; + /* Update file position only if we got past the current one */ + pos_valid = true; + ctx->pos = (iter.pos >> 2) + 1; - if (fibh.sbh == fibh.ebh) { - nameptr = udf_get_fi_ident(fi); - } else { - int poffset; /* Unpaded ending offset */ - - poffset = fibh.soffset + sizeof(struct fileIdentDesc) + liu + lfi; - - if (poffset >= lfi) { - nameptr = (char *)(fibh.ebh->b_data + poffset - lfi); - } else { - if (!copy_name) { - copy_name = kmalloc(UDF_NAME_LEN, - GFP_NOFS); - if (!copy_name) { - ret = -ENOMEM; - goto out; - } - } - nameptr = copy_name; - memcpy(nameptr, udf_get_fi_ident(fi), - lfi - poffset); - memcpy(nameptr + lfi - poffset, - fibh.ebh->b_data, poffset); - } - } - - if ((cfi.fileCharacteristics & FID_FILE_CHAR_DELETED) != 0) { + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) continue; } - if ((cfi.fileCharacteristics & FID_FILE_CHAR_HIDDEN) != 0) { + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_HIDDEN) { if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) continue; } - if (cfi.fileCharacteristics & FID_FILE_CHAR_PARENT) { + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_PARENT) { if (!dir_emit_dotdot(file, ctx)) - goto out; + goto out_iter; continue; } - flen = udf_get_filename(sb, nameptr, lfi, fname, UDF_NAME_LEN); + flen = udf_get_filename(sb, iter.name, + iter.fi.lengthFileIdent, fname, UDF_NAME_LEN); if (flen < 0) continue; - tloc = lelb_to_cpu(cfi.icb.extLocation); + tloc = lelb_to_cpu(iter.fi.icb.extLocation); iblock = udf_get_lb_pblock(sb, &tloc, 0); if (!dir_emit(ctx, fname, flen, iblock, DT_UNKNOWN)) - goto out; - } /* end while */ - - ctx->pos = (nf_pos >> 2) + 1; - pos_valid = true; + goto out_iter; + } + if (!ret) { + ctx->pos = (iter.pos >> 2) + 1; + pos_valid = true; + } +out_iter: + udf_fiiter_release(&iter); out: if (pos_valid) file->f_version = inode_query_iversion(dir); - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - brelse(epos.bh); kfree(fname); - kfree(copy_name); return ret; } From 57bda9fb169d689bff4108265a897d324b5fb8c3 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 17:59:12 +0200 Subject: [PATCH 010/265] udf: Convert udf_expand_dir_adinicb() to new directory iteration Convert udf_expand_dir_adinicb() to new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/inode.c | 66 ++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 34e416327dd4..4b2713799589 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -326,14 +326,12 @@ struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, udf_pblk_t newblock; struct buffer_head *dbh = NULL; struct kernel_lb_addr eloc; - uint8_t alloctype; struct extent_position epos; - - struct udf_fileident_bh sfibh, dfibh; - loff_t f_pos = udf_ext0_offset(inode); - int size = udf_ext0_offset(inode) + inode->i_size; - struct fileIdentDesc cfi, *sfi, *dfi; + uint8_t alloctype; struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_fileident_iter iter; + uint8_t *impuse; + int ret; if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) alloctype = ICBTAG_FLAG_AD_SHORT; @@ -361,38 +359,14 @@ struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, if (!dbh) return NULL; lock_buffer(dbh); - memset(dbh->b_data, 0x00, inode->i_sb->s_blocksize); + memcpy(dbh->b_data, iinfo->i_data, inode->i_size); + memset(dbh->b_data + inode->i_size, 0, + inode->i_sb->s_blocksize - inode->i_size); set_buffer_uptodate(dbh); unlock_buffer(dbh); - mark_buffer_dirty_inode(dbh, inode); - - sfibh.soffset = sfibh.eoffset = - f_pos & (inode->i_sb->s_blocksize - 1); - sfibh.sbh = sfibh.ebh = NULL; - dfibh.soffset = dfibh.eoffset = 0; - dfibh.sbh = dfibh.ebh = dbh; - while (f_pos < size) { - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - sfi = udf_fileident_read(inode, &f_pos, &sfibh, &cfi, NULL, - NULL, NULL, NULL); - if (!sfi) { - brelse(dbh); - return NULL; - } - iinfo->i_alloc_type = alloctype; - sfi->descTag.tagLocation = cpu_to_le32(*block); - dfibh.soffset = dfibh.eoffset; - dfibh.eoffset += (sfibh.eoffset - sfibh.soffset); - dfi = (struct fileIdentDesc *)(dbh->b_data + dfibh.soffset); - if (udf_write_fi(inode, sfi, dfi, &dfibh, sfi->impUse, - udf_get_fi_ident(sfi))) { - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - brelse(dbh); - return NULL; - } - } - mark_buffer_dirty_inode(dbh, inode); + /* Drop inline data, add block instead */ + iinfo->i_alloc_type = alloctype; memset(iinfo->i_data + iinfo->i_lenEAttr, 0, iinfo->i_lenAlloc); iinfo->i_lenAlloc = 0; eloc.logicalBlockNum = *block; @@ -403,10 +377,28 @@ struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, epos.block = iinfo->i_location; epos.offset = udf_file_entry_alloc_offset(inode); udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); - /* UniqueID stuff */ - brelse(epos.bh); mark_inode_dirty(inode); + + /* Now fixup tags in moved directory entries */ + for (ret = udf_fiiter_init(&iter, inode, 0); + !ret && iter.pos < inode->i_size; + ret = udf_fiiter_advance(&iter)) { + iter.fi.descTag.tagLocation = cpu_to_le32(*block); + if (iter.fi.lengthOfImpUse != cpu_to_le16(0)) + impuse = dbh->b_data + iter.pos + + sizeof(struct fileIdentDesc); + else + impuse = NULL; + udf_fiiter_write_fi(&iter, impuse); + } + /* + * We don't expect the iteration to fail as the directory has been + * already verified to be correct + */ + WARN_ON_ONCE(ret); + udf_fiiter_release(&iter); + return dbh; } From a27b2923de7efaa1da1e243fb80ff0fa432e4be0 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 18:10:37 +0200 Subject: [PATCH 011/265] udf: Move udf_expand_dir_adinicb() to its callsite There is just one caller of udf_expand_dir_adinicb(). Move the function to its caller into namei.c as it is more about directory handling than anything else anyway. Signed-off-by: Jan Kara --- fs/udf/inode.c | 82 ------------------------------------------------ fs/udf/namei.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/udf/udfdecl.h | 2 -- 3 files changed, 82 insertions(+), 84 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 4b2713799589..9655fc014c7a 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -320,88 +320,6 @@ int udf_expand_file_adinicb(struct inode *inode) return err; } -struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, - udf_pblk_t *block, int *err) -{ - udf_pblk_t newblock; - struct buffer_head *dbh = NULL; - struct kernel_lb_addr eloc; - struct extent_position epos; - uint8_t alloctype; - struct udf_inode_info *iinfo = UDF_I(inode); - struct udf_fileident_iter iter; - uint8_t *impuse; - int ret; - - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - alloctype = ICBTAG_FLAG_AD_SHORT; - else - alloctype = ICBTAG_FLAG_AD_LONG; - - if (!inode->i_size) { - iinfo->i_alloc_type = alloctype; - mark_inode_dirty(inode); - return NULL; - } - - /* alloc block, and copy data to it */ - *block = udf_new_block(inode->i_sb, inode, - iinfo->i_location.partitionReferenceNum, - iinfo->i_location.logicalBlockNum, err); - if (!(*block)) - return NULL; - newblock = udf_get_pblock(inode->i_sb, *block, - iinfo->i_location.partitionReferenceNum, - 0); - if (!newblock) - return NULL; - dbh = udf_tgetblk(inode->i_sb, newblock); - if (!dbh) - return NULL; - lock_buffer(dbh); - memcpy(dbh->b_data, iinfo->i_data, inode->i_size); - memset(dbh->b_data + inode->i_size, 0, - inode->i_sb->s_blocksize - inode->i_size); - set_buffer_uptodate(dbh); - unlock_buffer(dbh); - - /* Drop inline data, add block instead */ - iinfo->i_alloc_type = alloctype; - memset(iinfo->i_data + iinfo->i_lenEAttr, 0, iinfo->i_lenAlloc); - iinfo->i_lenAlloc = 0; - eloc.logicalBlockNum = *block; - eloc.partitionReferenceNum = - iinfo->i_location.partitionReferenceNum; - iinfo->i_lenExtents = inode->i_size; - epos.bh = NULL; - epos.block = iinfo->i_location; - epos.offset = udf_file_entry_alloc_offset(inode); - udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); - brelse(epos.bh); - mark_inode_dirty(inode); - - /* Now fixup tags in moved directory entries */ - for (ret = udf_fiiter_init(&iter, inode, 0); - !ret && iter.pos < inode->i_size; - ret = udf_fiiter_advance(&iter)) { - iter.fi.descTag.tagLocation = cpu_to_le32(*block); - if (iter.fi.lengthOfImpUse != cpu_to_le16(0)) - impuse = dbh->b_data + iter.pos + - sizeof(struct fileIdentDesc); - else - impuse = NULL; - udf_fiiter_write_fi(&iter, impuse); - } - /* - * We don't expect the iteration to fail as the directory has been - * already verified to be correct - */ - WARN_ON_ONCE(ret); - udf_fiiter_release(&iter); - - return dbh; -} - static int udf_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int create) { diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 7c95c549dd64..78bc4bbb7c54 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -326,6 +326,88 @@ static struct dentry *udf_lookup(struct inode *dir, struct dentry *dentry, return d_splice_alias(inode, dentry); } +static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, + udf_pblk_t *block, int *err) +{ + udf_pblk_t newblock; + struct buffer_head *dbh = NULL; + struct kernel_lb_addr eloc; + struct extent_position epos; + uint8_t alloctype; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_fileident_iter iter; + uint8_t *impuse; + int ret; + + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + alloctype = ICBTAG_FLAG_AD_SHORT; + else + alloctype = ICBTAG_FLAG_AD_LONG; + + if (!inode->i_size) { + iinfo->i_alloc_type = alloctype; + mark_inode_dirty(inode); + return NULL; + } + + /* alloc block, and copy data to it */ + *block = udf_new_block(inode->i_sb, inode, + iinfo->i_location.partitionReferenceNum, + iinfo->i_location.logicalBlockNum, err); + if (!(*block)) + return NULL; + newblock = udf_get_pblock(inode->i_sb, *block, + iinfo->i_location.partitionReferenceNum, + 0); + if (!newblock) + return NULL; + dbh = udf_tgetblk(inode->i_sb, newblock); + if (!dbh) + return NULL; + lock_buffer(dbh); + memcpy(dbh->b_data, iinfo->i_data, inode->i_size); + memset(dbh->b_data + inode->i_size, 0, + inode->i_sb->s_blocksize - inode->i_size); + set_buffer_uptodate(dbh); + unlock_buffer(dbh); + + /* Drop inline data, add block instead */ + iinfo->i_alloc_type = alloctype; + memset(iinfo->i_data + iinfo->i_lenEAttr, 0, iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + eloc.logicalBlockNum = *block; + eloc.partitionReferenceNum = + iinfo->i_location.partitionReferenceNum; + iinfo->i_lenExtents = inode->i_size; + epos.bh = NULL; + epos.block = iinfo->i_location; + epos.offset = udf_file_entry_alloc_offset(inode); + udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); + brelse(epos.bh); + mark_inode_dirty(inode); + + /* Now fixup tags in moved directory entries */ + for (ret = udf_fiiter_init(&iter, inode, 0); + !ret && iter.pos < inode->i_size; + ret = udf_fiiter_advance(&iter)) { + iter.fi.descTag.tagLocation = cpu_to_le32(*block); + if (iter.fi.lengthOfImpUse != cpu_to_le16(0)) + impuse = dbh->b_data + iter.pos + + sizeof(struct fileIdentDesc); + else + impuse = NULL; + udf_fiiter_write_fi(&iter, impuse); + } + /* + * We don't expect the iteration to fail as the directory has been + * already verified to be correct + */ + WARN_ON_ONCE(ret); + udf_fiiter_release(&iter); + + return dbh; +} + static struct fileIdentDesc *udf_add_entry(struct inode *dir, struct dentry *dentry, struct udf_fileident_bh *fibh, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 22a8466e335c..676fa2996ffe 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -169,8 +169,6 @@ static inline struct inode *udf_iget(struct super_block *sb, return __udf_iget(sb, ino, false); } extern int udf_expand_file_adinicb(struct inode *); -extern struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, - udf_pblk_t *block, int *err); extern struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, int create, int *err); extern int udf_setsize(struct inode *, loff_t); From 1c80afa04db39c98aebea9aabfafa37a208cdfee Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 18:28:33 +0200 Subject: [PATCH 012/265] udf: Implement searching for directory entry using new iteration code Implement searching for directory entry - udf_fiiter_find_entry() - using new directory iteration code. Reported-by: syzbot+69c9fdccc6dd08961d34@syzkaller.appspotmail.com Signed-off-by: Jan Kara --- fs/udf/namei.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 78bc4bbb7c54..145883d15c0f 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -140,6 +140,73 @@ int udf_write_fi(struct inode *inode, struct fileIdentDesc *cfi, return 0; } +/** + * udf_fiiter_find_entry - find entry in given directory. + * + * @dir: directory inode to search in + * @child: qstr of the name + * @iter: iter to use for searching + * + * This function searches in the directory @dir for a file name @child. When + * found, @iter points to the position in the directory with given entry. + * + * Returns 0 on success, < 0 on error (including -ENOENT). + */ +static int udf_fiiter_find_entry(struct inode *dir, const struct qstr *child, + struct udf_fileident_iter *iter) +{ + int flen; + unsigned char *fname = NULL; + struct super_block *sb = dir->i_sb; + int isdotdot = child->len == 2 && + child->name[0] == '.' && child->name[1] == '.'; + int ret; + + fname = kmalloc(UDF_NAME_LEN, GFP_NOFS); + if (!fname) + return -ENOMEM; + + for (ret = udf_fiiter_init(iter, dir, 0); + !ret && iter->pos < dir->i_size; + ret = udf_fiiter_advance(iter)) { + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) + continue; + } + + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_HIDDEN) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) + continue; + } + + if ((iter->fi.fileCharacteristics & FID_FILE_CHAR_PARENT) && + isdotdot) + goto out_ok; + + if (!iter->fi.lengthFileIdent) + continue; + + flen = udf_get_filename(sb, iter->name, + iter->fi.lengthFileIdent, fname, UDF_NAME_LEN); + if (flen < 0) { + ret = flen; + goto out_err; + } + + if (udf_match(flen, fname, child->len, child->name)) + goto out_ok; + } + if (!ret) + ret = -ENOENT; + +out_err: + udf_fiiter_release(iter); +out_ok: + kfree(fname); + + return ret; +} + /** * udf_find_entry - find entry in given directory. * From 200918b34d158cdaee531db7e0c80b92c57e66f1 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 18:36:21 +0200 Subject: [PATCH 013/265] udf: Convert udf_lookup() to use new directory iteration code Convert udf_lookup() to use udf_fiiter_find_entry() for looking up directory entries. Signed-off-by: Jan Kara --- fs/udf/namei.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 145883d15c0f..efc75cf5722d 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -366,25 +366,22 @@ static struct dentry *udf_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct inode *inode = NULL; - struct fileIdentDesc cfi; - struct udf_fileident_bh fibh; - struct fileIdentDesc *fi; + struct udf_fileident_iter iter; + int err; if (dentry->d_name.len > UDF_NAME_LEN) return ERR_PTR(-ENAMETOOLONG); - fi = udf_find_entry(dir, &dentry->d_name, &fibh, &cfi); - if (IS_ERR(fi)) - return ERR_CAST(fi); + err = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (err < 0 && err != -ENOENT) + return ERR_PTR(err); - if (fi) { + if (err == 0) { struct kernel_lb_addr loc; - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); + loc = lelb_to_cpu(iter.fi.icb.extLocation); + udf_fiiter_release(&iter); - loc = lelb_to_cpu(cfi.icb.extLocation); inode = udf_iget(dir->i_sb, &loc); if (IS_ERR(inode)) return ERR_CAST(inode); From 9b06fbef4202363d74bba5459ddd231db6d3b1af Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 18:48:45 +0200 Subject: [PATCH 014/265] udf: Convert udf_get_parent() to new directory iteration code Convert udf_get_parent() to use udf_fiiter_find_entry(). Signed-off-by: Jan Kara --- fs/udf/namei.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index efc75cf5722d..812786050617 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -1367,17 +1367,15 @@ static struct dentry *udf_get_parent(struct dentry *child) { struct kernel_lb_addr tloc; struct inode *inode = NULL; - struct fileIdentDesc cfi; - struct udf_fileident_bh fibh; + struct udf_fileident_iter iter; + int err; - if (!udf_find_entry(d_inode(child), &dotdot_name, &fibh, &cfi)) - return ERR_PTR(-EACCES); + err = udf_fiiter_find_entry(d_inode(child), &dotdot_name, &iter); + if (err) + return ERR_PTR(err); - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - - tloc = lelb_to_cpu(cfi.icb.extLocation); + tloc = lelb_to_cpu(iter.fi.icb.extLocation); + udf_fiiter_release(&iter); inode = udf_iget(child->d_sb, &tloc); if (IS_ERR(inode)) return ERR_CAST(inode); From afb525f466f9fdc140b975221cb43fbb5c59314e Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 19:10:02 +0200 Subject: [PATCH 015/265] udf: Convert empty_dir() to new directory iteration code Convert empty_dir() to new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 68 +++++++------------------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 812786050617..964ac7d4e274 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -869,69 +869,19 @@ out: static int empty_dir(struct inode *dir) { - struct fileIdentDesc *fi, cfi; - struct udf_fileident_bh fibh; - loff_t f_pos; - loff_t size = udf_ext0_offset(dir) + dir->i_size; - udf_pblk_t block; - struct kernel_lb_addr eloc; - uint32_t elen; - sector_t offset; - struct extent_position epos = {}; - struct udf_inode_info *dinfo = UDF_I(dir); + struct udf_fileident_iter iter; + int ret; - f_pos = udf_ext0_offset(dir); - fibh.soffset = fibh.eoffset = f_pos & (dir->i_sb->s_blocksize - 1); - - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - fibh.sbh = fibh.ebh = NULL; - else if (inode_bmap(dir, f_pos >> dir->i_sb->s_blocksize_bits, - &epos, &eloc, &elen, &offset) == - (EXT_RECORDED_ALLOCATED >> 30)) { - block = udf_get_lb_pblock(dir->i_sb, &eloc, offset); - if ((++offset << dir->i_sb->s_blocksize_bits) < elen) { - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - } else - offset = 0; - - fibh.sbh = fibh.ebh = udf_tread(dir->i_sb, block); - if (!fibh.sbh) { - brelse(epos.bh); - return 0; - } - } else { - brelse(epos.bh); - return 0; - } - - while (f_pos < size) { - fi = udf_fileident_read(dir, &f_pos, &fibh, &cfi, &epos, &eloc, - &elen, &offset); - if (!fi) { - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - brelse(epos.bh); - return 0; - } - - if (cfi.lengthFileIdent && - (cfi.fileCharacteristics & FID_FILE_CHAR_DELETED) == 0) { - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - brelse(epos.bh); + for (ret = udf_fiiter_init(&iter, dir, 0); + !ret && iter.pos < dir->i_size; + ret = udf_fiiter_advance(&iter)) { + if (iter.fi.lengthFileIdent && + !(iter.fi.fileCharacteristics & FID_FILE_CHAR_DELETED)) { + udf_fiiter_release(&iter); return 0; } } - - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - brelse(epos.bh); + udf_fiiter_release(&iter); return 1; } From 4cca7e3df7bea8661a0c2a70c0d250e9aa5cedb4 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 19:26:59 +0200 Subject: [PATCH 016/265] udf: Provide function to mark entry as deleted using new directory iteration code Provide function udf_fiiter_delete_entry() to mark directory entry as deleted using new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 964ac7d4e274..f38ed9c1b54d 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -711,6 +711,16 @@ out_ok: return fi; } +static void udf_fiiter_delete_entry(struct udf_fileident_iter *iter) +{ + iter->fi.fileCharacteristics |= FID_FILE_CHAR_DELETED; + + if (UDF_QUERY_FLAG(iter->dir->i_sb, UDF_FLAG_STRICT)) + memset(&iter->fi.icb, 0x00, sizeof(struct long_ad)); + + udf_fiiter_write_fi(iter, NULL); +} + static int udf_delete_entry(struct inode *inode, struct fileIdentDesc *fi, struct udf_fileident_bh *fibh, struct fileIdentDesc *cfi) From d11ffa8d3ec11fdb665f12f95d58d74673051a93 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 19:33:23 +0200 Subject: [PATCH 017/265] udf: Convert udf_rmdir() to new directory iteration code Convert udf_rmdir() to use new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index f38ed9c1b54d..703303562778 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -898,30 +898,23 @@ static int empty_dir(struct inode *dir) static int udf_rmdir(struct inode *dir, struct dentry *dentry) { - int retval; + int ret; struct inode *inode = d_inode(dentry); - struct udf_fileident_bh fibh; - struct fileIdentDesc *fi, cfi; + struct udf_fileident_iter iter; struct kernel_lb_addr tloc; - retval = -ENOENT; - fi = udf_find_entry(dir, &dentry->d_name, &fibh, &cfi); - if (IS_ERR_OR_NULL(fi)) { - if (fi) - retval = PTR_ERR(fi); + ret = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (ret) goto out; - } - retval = -EIO; - tloc = lelb_to_cpu(cfi.icb.extLocation); + ret = -EFSCORRUPTED; + tloc = lelb_to_cpu(iter.fi.icb.extLocation); if (udf_get_lb_pblock(dir->i_sb, &tloc, 0) != inode->i_ino) goto end_rmdir; - retval = -ENOTEMPTY; + ret = -ENOTEMPTY; if (!empty_dir(inode)) goto end_rmdir; - retval = udf_delete_entry(dir, fi, &fibh, &cfi); - if (retval) - goto end_rmdir; + udf_fiiter_delete_entry(&iter); if (inode->i_nlink != 2) udf_warn(inode->i_sb, "empty directory has nlink != 2 (%u)\n", inode->i_nlink); @@ -931,14 +924,11 @@ static int udf_rmdir(struct inode *dir, struct dentry *dentry) inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); mark_inode_dirty(dir); - + ret = 0; end_rmdir: - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - + udf_fiiter_release(&iter); out: - return retval; + return ret; } static int udf_unlink(struct inode *dir, struct dentry *dentry) From 6ec01a8020b54e278fecd1efe8603f8eb38fed84 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 5 Oct 2022 19:36:49 +0200 Subject: [PATCH 018/265] udf: Convert udf_unlink() to new directory iteration code Convert udf_unlink() to new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 703303562778..d0ffd2083519 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -933,24 +933,17 @@ out: static int udf_unlink(struct inode *dir, struct dentry *dentry) { - int retval; + int ret; struct inode *inode = d_inode(dentry); - struct udf_fileident_bh fibh; - struct fileIdentDesc *fi; - struct fileIdentDesc cfi; + struct udf_fileident_iter iter; struct kernel_lb_addr tloc; - retval = -ENOENT; - fi = udf_find_entry(dir, &dentry->d_name, &fibh, &cfi); - - if (IS_ERR_OR_NULL(fi)) { - if (fi) - retval = PTR_ERR(fi); + ret = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (ret) goto out; - } - retval = -EIO; - tloc = lelb_to_cpu(cfi.icb.extLocation); + ret = -EFSCORRUPTED; + tloc = lelb_to_cpu(iter.fi.icb.extLocation); if (udf_get_lb_pblock(dir->i_sb, &tloc, 0) != inode->i_ino) goto end_unlink; @@ -959,22 +952,16 @@ static int udf_unlink(struct inode *dir, struct dentry *dentry) inode->i_ino, inode->i_nlink); set_nlink(inode, 1); } - retval = udf_delete_entry(dir, fi, &fibh, &cfi); - if (retval) - goto end_unlink; + udf_fiiter_delete_entry(&iter); dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); inode_dec_link_count(inode); inode->i_ctime = dir->i_ctime; - retval = 0; - + ret = 0; end_unlink: - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - + udf_fiiter_release(&iter); out: - return retval; + return ret; } static int udf_symlink(struct user_namespace *mnt_userns, struct inode *dir, From f2844803404d9729f893e279ddea12678710e7fb Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 13:25:16 +0200 Subject: [PATCH 019/265] udf: Implement adding of dir entries using new iteration code Implement function udf_fiiter_add_entry() adding new directory entries using new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/directory.c | 57 +++++++++++++++++++++++ fs/udf/namei.c | 110 +++++++++++++++++++++++++++++++++++++++++++++ fs/udf/udfdecl.h | 2 + 3 files changed, 169 insertions(+) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 2842c463725c..be85d1b2a6cf 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -413,6 +413,63 @@ void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse) inode_inc_iversion(iter->dir); } +void udf_fiiter_update_elen(struct udf_fileident_iter *iter, uint32_t new_elen) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + int diff = new_elen - iter->elen; + + /* Skip update when we already went past the last extent */ + if (!iter->elen) + return; + iter->elen = new_elen; + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + iter->epos.offset -= sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + iter->epos.offset -= sizeof(struct long_ad); + udf_write_aext(iter->dir, &iter->epos, &iter->eloc, iter->elen, 1); + iinfo->i_lenExtents += diff; + mark_inode_dirty(iter->dir); +} + +/* Append new block to directory. @iter is expected to point at EOF */ +int udf_fiiter_append_blk(struct udf_fileident_iter *iter) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + int blksize = 1 << iter->dir->i_blkbits; + struct buffer_head *bh; + sector_t block; + uint32_t old_elen = iter->elen; + int err; + + if (WARN_ON_ONCE(iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB)) + return -EINVAL; + + /* Round up last extent in the file */ + udf_fiiter_update_elen(iter, ALIGN(iter->elen, blksize)); + + /* Allocate new block and refresh mapping information */ + block = iinfo->i_lenExtents >> iter->dir->i_blkbits; + bh = udf_bread(iter->dir, block, 1, &err); + if (!bh) { + udf_fiiter_update_elen(iter, old_elen); + return err; + } + if (inode_bmap(iter->dir, block, &iter->epos, &iter->eloc, &iter->elen, + &iter->loffset) != (EXT_RECORDED_ALLOCATED >> 30)) { + udf_err(iter->dir->i_sb, + "block %llu not allocated in directory (ino %lu)\n", + (unsigned long long)block, iter->dir->i_ino); + return -EFSCORRUPTED; + } + if (!(iter->pos & (blksize - 1))) { + brelse(iter->bh[0]); + iter->bh[0] = bh; + } else { + iter->bh[1] = bh; + } + return 0; +} + struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, struct udf_fileident_bh *fibh, struct fileIdentDesc *cfi, diff --git a/fs/udf/namei.c b/fs/udf/namei.c index d0ffd2083519..2e423c10dc99 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -472,6 +472,116 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, return dbh; } +static int udf_fiiter_add_entry(struct inode *dir, struct dentry *dentry, + struct udf_fileident_iter *iter) +{ + struct udf_inode_info *dinfo = UDF_I(dir); + int nfidlen, namelen = 0; + int ret; + int off, blksize = 1 << dir->i_blkbits; + udf_pblk_t block; + char name[UDF_NAME_LEN_CS0]; + + if (dentry) { + if (!dentry->d_name.len) + return -EINVAL; + namelen = udf_put_filename(dir->i_sb, dentry->d_name.name, + dentry->d_name.len, + name, UDF_NAME_LEN_CS0); + if (!namelen) + return -ENAMETOOLONG; + } + nfidlen = ALIGN(sizeof(struct fileIdentDesc) + namelen, UDF_NAME_PAD); + + for (ret = udf_fiiter_init(iter, dir, 0); + !ret && iter->pos < dir->i_size; + ret = udf_fiiter_advance(iter)) { + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { + if (udf_dir_entry_len(&iter->fi) == nfidlen) { + iter->fi.descTag.tagSerialNum = cpu_to_le16(1); + iter->fi.fileVersionNum = cpu_to_le16(1); + iter->fi.fileCharacteristics = 0; + iter->fi.lengthFileIdent = namelen; + iter->fi.lengthOfImpUse = cpu_to_le16(0); + memcpy(iter->namebuf, name, namelen); + iter->name = iter->namebuf; + return 0; + } + } + } + if (ret) { + udf_fiiter_release(iter); + return ret; + } + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && + blksize - udf_ext0_offset(dir) - iter->pos < nfidlen) { + struct buffer_head *retbh; + + udf_fiiter_release(iter); + /* + * FIXME: udf_expand_dir_adinicb does not need to return bh + * once other users are gone + */ + retbh = udf_expand_dir_adinicb(dir, &block, &ret); + if (!retbh) + return ret; + brelse(retbh); + ret = udf_fiiter_init(iter, dir, dir->i_size); + if (ret < 0) + return ret; + } + + /* Get blocknumber to use for entry tag */ + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + block = dinfo->i_location.logicalBlockNum; + } else { + block = iter->eloc.logicalBlockNum + + ((iter->elen - 1) >> dir->i_blkbits); + } + off = iter->pos & (blksize - 1); + if (!off) + off = blksize; + /* Entry fits into current block? */ + if (blksize - udf_ext0_offset(dir) - off >= nfidlen) + goto store_fi; + + ret = udf_fiiter_append_blk(iter); + if (ret) { + udf_fiiter_release(iter); + return ret; + } + + /* Entry will be completely in the new block? Update tag location... */ + if (!(iter->pos & (blksize - 1))) + block = iter->eloc.logicalBlockNum + + ((iter->elen - 1) >> dir->i_blkbits); +store_fi: + memset(&iter->fi, 0, sizeof(struct fileIdentDesc)); + if (UDF_SB(dir->i_sb)->s_udfrev >= 0x0200) + udf_new_tag((char *)(&iter->fi), TAG_IDENT_FID, 3, 1, block, + sizeof(struct tag)); + else + udf_new_tag((char *)(&iter->fi), TAG_IDENT_FID, 2, 1, block, + sizeof(struct tag)); + iter->fi.fileVersionNum = cpu_to_le16(1); + iter->fi.lengthFileIdent = namelen; + iter->fi.lengthOfImpUse = cpu_to_le16(0); + memcpy(iter->namebuf, name, namelen); + iter->name = iter->namebuf; + + dir->i_size += nfidlen; + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + dinfo->i_lenAlloc += nfidlen; + } else { + /* Truncate last extent to proper size */ + udf_fiiter_update_elen(iter, iter->elen - + (dinfo->i_lenExtents - dir->i_size)); + } + mark_inode_dirty(dir); + + return 0; +} + static struct fileIdentDesc *udf_add_entry(struct inode *dir, struct dentry *dentry, struct udf_fileident_bh *fibh, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 676fa2996ffe..e47b2f0c3e05 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -264,6 +264,8 @@ int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, int udf_fiiter_advance(struct udf_fileident_iter *iter); void udf_fiiter_release(struct udf_fileident_iter *iter); void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse); +void udf_fiiter_update_elen(struct udf_fileident_iter *iter, uint32_t new_elen); +int udf_fiiter_append_blk(struct udf_fileident_iter *iter); extern struct fileIdentDesc *udf_fileident_read(struct inode *, loff_t *, struct udf_fileident_bh *, struct fileIdentDesc *, From ef91f9998bece00cf7f82ad26177f910a7124b25 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 14:08:59 +0200 Subject: [PATCH 020/265] udf: Convert udf_add_nondir() to new directory iteration Convert udf_add_nondir() to new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 2e423c10dc99..788ac6c1642e 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -847,26 +847,23 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode) { struct udf_inode_info *iinfo = UDF_I(inode); struct inode *dir = d_inode(dentry->d_parent); - struct udf_fileident_bh fibh; - struct fileIdentDesc cfi, *fi; + struct udf_fileident_iter iter; int err; - fi = udf_add_entry(dir, dentry, &fibh, &cfi, &err); - if (unlikely(!fi)) { + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) { inode_dec_link_count(inode); discard_new_inode(inode); return err; } - cfi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); - cfi.icb.extLocation = cpu_to_lelb(iinfo->i_location); - *(__le32 *)((struct allocDescImpUse *)cfi.icb.impUse)->impUse = + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(iinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = cpu_to_le32(iinfo->i_unique & 0x00000000FFFFFFFFUL); - udf_write_fi(dir, &cfi, fi, &fibh, NULL, NULL); + udf_fiiter_write_fi(&iter, NULL); dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); + udf_fiiter_release(&iter); d_instantiate_new(dentry, inode); return 0; From 00bce6f792caccefa73daeaf9bde82d24d50037f Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 14:22:33 +0200 Subject: [PATCH 021/265] udf: Convert udf_mkdir() to new directory iteration code Convert udf_mkdir() to new directory iteration code. Signed-off-by: Jan Kara --- fs/udf/namei.c | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 788ac6c1642e..c012f2a43ba8 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -928,8 +928,7 @@ static int udf_mkdir(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode) { struct inode *inode; - struct udf_fileident_bh fibh; - struct fileIdentDesc cfi, *fi; + struct udf_fileident_iter iter; int err; struct udf_inode_info *dinfo = UDF_I(dir); struct udf_inode_info *iinfo; @@ -941,47 +940,42 @@ static int udf_mkdir(struct user_namespace *mnt_userns, struct inode *dir, iinfo = UDF_I(inode); inode->i_op = &udf_dir_inode_operations; inode->i_fop = &udf_dir_operations; - fi = udf_add_entry(inode, NULL, &fibh, &cfi, &err); - if (!fi) { - inode_dec_link_count(inode); + err = udf_fiiter_add_entry(inode, NULL, &iter); + if (err) { + clear_nlink(inode); discard_new_inode(inode); - goto out; + return err; } set_nlink(inode, 2); - cfi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); - cfi.icb.extLocation = cpu_to_lelb(dinfo->i_location); - *(__le32 *)((struct allocDescImpUse *)cfi.icb.impUse)->impUse = + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(dinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = cpu_to_le32(dinfo->i_unique & 0x00000000FFFFFFFFUL); - cfi.fileCharacteristics = + iter.fi.fileCharacteristics = FID_FILE_CHAR_DIRECTORY | FID_FILE_CHAR_PARENT; - udf_write_fi(inode, &cfi, fi, &fibh, NULL, NULL); - brelse(fibh.sbh); + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); mark_inode_dirty(inode); - fi = udf_add_entry(dir, dentry, &fibh, &cfi, &err); - if (!fi) { + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) { clear_nlink(inode); - mark_inode_dirty(inode); discard_new_inode(inode); - goto out; + return err; } - cfi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); - cfi.icb.extLocation = cpu_to_lelb(iinfo->i_location); - *(__le32 *)((struct allocDescImpUse *)cfi.icb.impUse)->impUse = + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(iinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = cpu_to_le32(iinfo->i_unique & 0x00000000FFFFFFFFUL); - cfi.fileCharacteristics |= FID_FILE_CHAR_DIRECTORY; - udf_write_fi(dir, &cfi, fi, &fibh, NULL, NULL); + iter.fi.fileCharacteristics |= FID_FILE_CHAR_DIRECTORY; + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); inc_nlink(dir); dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); d_instantiate_new(dentry, inode); - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); - err = 0; -out: - return err; + return 0; } static int empty_dir(struct inode *dir) From dbfb102d16fb780c84f41adbaeb7eac907c415dc Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 14:59:11 +0200 Subject: [PATCH 022/265] udf: Convert udf_link() to new directory iteration code Convert udf_link() to use new directory iteration code for adding entry into the directory. Signed-off-by: Jan Kara --- fs/udf/namei.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index c012f2a43ba8..1b3ee80484f1 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -1222,27 +1222,21 @@ static int udf_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) { struct inode *inode = d_inode(old_dentry); - struct udf_fileident_bh fibh; - struct fileIdentDesc cfi, *fi; + struct udf_fileident_iter iter; int err; - fi = udf_add_entry(dir, dentry, &fibh, &cfi, &err); - if (!fi) { + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) return err; - } - cfi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); - cfi.icb.extLocation = cpu_to_lelb(UDF_I(inode)->i_location); + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(UDF_I(inode)->i_location); if (UDF_SB(inode->i_sb)->s_lvid_bh) { - *(__le32 *)((struct allocDescImpUse *)cfi.icb.impUse)->impUse = + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = cpu_to_le32(lvid_get_unique_id(inode->i_sb)); } - udf_write_fi(dir, &cfi, fi, &fibh, NULL, NULL); - if (UDF_I(dir)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - mark_inode_dirty(dir); + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); - if (fibh.sbh != fibh.ebh) - brelse(fibh.ebh); - brelse(fibh.sbh); inc_nlink(inode); inode->i_ctime = current_time(inode); mark_inode_dirty(inode); From e9109a92d2a95889498bed3719cd2318892171a2 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 16:41:23 +0200 Subject: [PATCH 023/265] udf: Convert udf_rename() to new directory iteration code Convert udf_rename() to use new directory iteration code. Reported-by: syzbot+0eaad3590d65102b9391@syzkaller.appspotmail.com Reported-by: syzbot+b7fc73213bc2361ab650@syzkaller.appspotmail.com Signed-off-by: Jan Kara --- fs/udf/namei.c | 165 +++++++++++++++++++++++-------------------------- 1 file changed, 78 insertions(+), 87 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 1b3ee80484f1..b9eb76438b68 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -1257,78 +1257,68 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, { struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); - struct udf_fileident_bh ofibh, nfibh; - struct fileIdentDesc *ofi = NULL, *nfi = NULL, *dir_fi = NULL; - struct fileIdentDesc ocfi, ncfi; - struct buffer_head *dir_bh = NULL; - int retval = -ENOENT; + struct udf_fileident_iter oiter, niter, diriter; + bool has_diriter = false; + int retval; struct kernel_lb_addr tloc; - struct udf_inode_info *old_iinfo = UDF_I(old_inode); if (flags & ~RENAME_NOREPLACE) return -EINVAL; - ofi = udf_find_entry(old_dir, &old_dentry->d_name, &ofibh, &ocfi); - if (!ofi || IS_ERR(ofi)) { - if (IS_ERR(ofi)) - retval = PTR_ERR(ofi); - goto end_rename; + retval = udf_fiiter_find_entry(old_dir, &old_dentry->d_name, &oiter); + if (retval) + return retval; + + tloc = lelb_to_cpu(oiter.fi.icb.extLocation); + if (udf_get_lb_pblock(old_dir->i_sb, &tloc, 0) != old_inode->i_ino) { + retval = -ENOENT; + goto out_oiter; } - if (ofibh.sbh != ofibh.ebh) - brelse(ofibh.ebh); - - brelse(ofibh.sbh); - tloc = lelb_to_cpu(ocfi.icb.extLocation); - if (udf_get_lb_pblock(old_dir->i_sb, &tloc, 0) != old_inode->i_ino) - goto end_rename; - - nfi = udf_find_entry(new_dir, &new_dentry->d_name, &nfibh, &ncfi); - if (IS_ERR(nfi)) { - retval = PTR_ERR(nfi); - goto end_rename; - } - if (nfi && !new_inode) { - if (nfibh.sbh != nfibh.ebh) - brelse(nfibh.ebh); - brelse(nfibh.sbh); - nfi = NULL; - } if (S_ISDIR(old_inode->i_mode)) { - int offset = udf_ext0_offset(old_inode); - if (new_inode) { retval = -ENOTEMPTY; if (!empty_dir(new_inode)) - goto end_rename; + goto out_oiter; } - retval = -EIO; - if (old_iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - dir_fi = udf_get_fileident( - old_iinfo->i_data - - (old_iinfo->i_efe ? - sizeof(struct extendedFileEntry) : - sizeof(struct fileEntry)), - old_inode->i_sb->s_blocksize, &offset); - } else { - dir_bh = udf_bread(old_inode, 0, 0, &retval); - if (!dir_bh) - goto end_rename; - dir_fi = udf_get_fileident(dir_bh->b_data, - old_inode->i_sb->s_blocksize, &offset); + retval = udf_fiiter_find_entry(old_inode, &dotdot_name, + &diriter); + if (retval == -ENOENT) { + udf_err(old_inode->i_sb, + "directory (ino %lu) has no '..' entry\n", + old_inode->i_ino); + retval = -EFSCORRUPTED; } - if (!dir_fi) - goto end_rename; - tloc = lelb_to_cpu(dir_fi->icb.extLocation); + if (retval) + goto out_oiter; + has_diriter = true; + tloc = lelb_to_cpu(diriter.fi.icb.extLocation); if (udf_get_lb_pblock(old_inode->i_sb, &tloc, 0) != - old_dir->i_ino) - goto end_rename; + old_dir->i_ino) { + retval = -EFSCORRUPTED; + udf_err(old_inode->i_sb, + "directory (ino %lu) has parent entry pointing to another inode (%lu != %u)\n", + old_inode->i_ino, old_dir->i_ino, + udf_get_lb_pblock(old_inode->i_sb, &tloc, 0)); + goto out_oiter; + } } - if (!nfi) { - nfi = udf_add_entry(new_dir, new_dentry, &nfibh, &ncfi, - &retval); - if (!nfi) - goto end_rename; + + retval = udf_fiiter_find_entry(new_dir, &new_dentry->d_name, &niter); + if (retval && retval != -ENOENT) + goto out_oiter; + /* Entry found but not passed by VFS? */ + if (!retval && !new_inode) { + retval = -EFSCORRUPTED; + udf_fiiter_release(&niter); + goto out_oiter; + } + /* Entry not found? Need to add one... */ + if (retval) { + udf_fiiter_release(&niter); + retval = udf_fiiter_add_entry(new_dir, new_dentry, &niter); + if (retval) + goto out_oiter; } /* @@ -1341,14 +1331,26 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, /* * ok, that's it */ - ncfi.fileVersionNum = ocfi.fileVersionNum; - ncfi.fileCharacteristics = ocfi.fileCharacteristics; - memcpy(&(ncfi.icb), &(ocfi.icb), sizeof(ocfi.icb)); - udf_write_fi(new_dir, &ncfi, nfi, &nfibh, NULL, NULL); + niter.fi.fileVersionNum = oiter.fi.fileVersionNum; + niter.fi.fileCharacteristics = oiter.fi.fileCharacteristics; + memcpy(&(niter.fi.icb), &(oiter.fi.icb), sizeof(oiter.fi.icb)); + udf_fiiter_write_fi(&niter, NULL); + udf_fiiter_release(&niter); - /* The old fid may have moved - find it again */ - ofi = udf_find_entry(old_dir, &old_dentry->d_name, &ofibh, &ocfi); - udf_delete_entry(old_dir, ofi, &ofibh, &ocfi); + /* + * The old entry may have moved due to new entry allocation. Find it + * again. + */ + udf_fiiter_release(&oiter); + retval = udf_fiiter_find_entry(old_dir, &old_dentry->d_name, &oiter); + if (retval) { + udf_err(old_dir->i_sb, + "failed to find renamed entry again in directory (ino %lu)\n", + old_dir->i_ino); + } else { + udf_fiiter_delete_entry(&oiter); + udf_fiiter_release(&oiter); + } if (new_inode) { new_inode->i_ctime = current_time(new_inode); @@ -1359,13 +1361,13 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, mark_inode_dirty(old_dir); mark_inode_dirty(new_dir); - if (dir_fi) { - dir_fi->icb.extLocation = cpu_to_lelb(UDF_I(new_dir)->i_location); - udf_update_tag((char *)dir_fi, udf_dir_entry_len(dir_fi)); - if (old_iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - mark_inode_dirty(old_inode); - else - mark_buffer_dirty_inode(dir_bh, old_inode); + if (has_diriter) { + diriter.fi.icb.extLocation = + cpu_to_lelb(UDF_I(new_dir)->i_location); + udf_update_tag((char *)&diriter.fi, + udf_dir_entry_len(&diriter.fi)); + udf_fiiter_write_fi(&diriter, NULL); + udf_fiiter_release(&diriter); inode_dec_link_count(old_dir); if (new_inode) @@ -1375,22 +1377,11 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, mark_inode_dirty(new_dir); } } - - if (ofi) { - if (ofibh.sbh != ofibh.ebh) - brelse(ofibh.ebh); - brelse(ofibh.sbh); - } - - retval = 0; - -end_rename: - brelse(dir_bh); - if (nfi) { - if (nfibh.sbh != nfibh.ebh) - brelse(nfibh.ebh); - brelse(nfibh.sbh); - } + return 0; +out_oiter: + if (has_diriter) + udf_fiiter_release(&diriter); + udf_fiiter_release(&oiter); return retval; } From 1e0290d61a870ed61a6510863029939bbf6b0006 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 6 Oct 2022 16:44:53 +0200 Subject: [PATCH 024/265] udf: Remove old directory iteration code Remove old directory iteration code that is now unused. Signed-off-by: Jan Kara --- fs/udf/directory.c | 178 ---------------- fs/udf/namei.c | 505 --------------------------------------------- fs/udf/udfdecl.h | 22 -- 3 files changed, 705 deletions(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index be85d1b2a6cf..e7e8b30876d9 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -470,184 +470,6 @@ int udf_fiiter_append_blk(struct udf_fileident_iter *iter) return 0; } -struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, - struct udf_fileident_bh *fibh, - struct fileIdentDesc *cfi, - struct extent_position *epos, - struct kernel_lb_addr *eloc, uint32_t *elen, - sector_t *offset) -{ - struct fileIdentDesc *fi; - int i, num; - udf_pblk_t block; - struct buffer_head *tmp, *bha[16]; - struct udf_inode_info *iinfo = UDF_I(dir); - - fibh->soffset = fibh->eoffset; - - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - fi = udf_get_fileident(iinfo->i_data - - (iinfo->i_efe ? - sizeof(struct extendedFileEntry) : - sizeof(struct fileEntry)), - dir->i_sb->s_blocksize, - &(fibh->eoffset)); - if (!fi) - return NULL; - - *nf_pos += fibh->eoffset - fibh->soffset; - - memcpy((uint8_t *)cfi, (uint8_t *)fi, - sizeof(struct fileIdentDesc)); - - return fi; - } - - if (fibh->eoffset == dir->i_sb->s_blocksize) { - uint32_t lextoffset = epos->offset; - unsigned char blocksize_bits = dir->i_sb->s_blocksize_bits; - - if (udf_next_aext(dir, epos, eloc, elen, 1) != - (EXT_RECORDED_ALLOCATED >> 30)) - return NULL; - - block = udf_get_lb_pblock(dir->i_sb, eloc, *offset); - - (*offset)++; - - if ((*offset << blocksize_bits) >= *elen) - *offset = 0; - else - epos->offset = lextoffset; - - brelse(fibh->sbh); - fibh->sbh = fibh->ebh = udf_tread(dir->i_sb, block); - if (!fibh->sbh) - return NULL; - fibh->soffset = fibh->eoffset = 0; - - if (!(*offset & ((16 >> (blocksize_bits - 9)) - 1))) { - i = 16 >> (blocksize_bits - 9); - if (i + *offset > (*elen >> blocksize_bits)) - i = (*elen >> blocksize_bits)-*offset; - for (num = 0; i > 0; i--) { - block = udf_get_lb_pblock(dir->i_sb, eloc, - *offset + i); - tmp = udf_tgetblk(dir->i_sb, block); - if (tmp && !buffer_uptodate(tmp) && - !buffer_locked(tmp)) - bha[num++] = tmp; - else - brelse(tmp); - } - if (num) { - bh_readahead_batch(num, bha, REQ_RAHEAD); - for (i = 0; i < num; i++) - brelse(bha[i]); - } - } - } else if (fibh->sbh != fibh->ebh) { - brelse(fibh->sbh); - fibh->sbh = fibh->ebh; - } - - fi = udf_get_fileident(fibh->sbh->b_data, dir->i_sb->s_blocksize, - &(fibh->eoffset)); - - if (!fi) - return NULL; - - *nf_pos += fibh->eoffset - fibh->soffset; - - if (fibh->eoffset <= dir->i_sb->s_blocksize) { - memcpy((uint8_t *)cfi, (uint8_t *)fi, - sizeof(struct fileIdentDesc)); - } else if (fibh->eoffset > dir->i_sb->s_blocksize) { - uint32_t lextoffset = epos->offset; - - if (udf_next_aext(dir, epos, eloc, elen, 1) != - (EXT_RECORDED_ALLOCATED >> 30)) - return NULL; - - block = udf_get_lb_pblock(dir->i_sb, eloc, *offset); - - (*offset)++; - - if ((*offset << dir->i_sb->s_blocksize_bits) >= *elen) - *offset = 0; - else - epos->offset = lextoffset; - - fibh->soffset -= dir->i_sb->s_blocksize; - fibh->eoffset -= dir->i_sb->s_blocksize; - - fibh->ebh = udf_tread(dir->i_sb, block); - if (!fibh->ebh) - return NULL; - - if (sizeof(struct fileIdentDesc) > -fibh->soffset) { - int fi_len; - - memcpy((uint8_t *)cfi, (uint8_t *)fi, -fibh->soffset); - memcpy((uint8_t *)cfi - fibh->soffset, - fibh->ebh->b_data, - sizeof(struct fileIdentDesc) + fibh->soffset); - - fi_len = udf_dir_entry_len(cfi); - *nf_pos += fi_len - (fibh->eoffset - fibh->soffset); - fibh->eoffset = fibh->soffset + fi_len; - } else { - memcpy((uint8_t *)cfi, (uint8_t *)fi, - sizeof(struct fileIdentDesc)); - } - } - /* Got last entry outside of dir size - fs is corrupted! */ - if (*nf_pos > dir->i_size) - return NULL; - return fi; -} - -struct fileIdentDesc *udf_get_fileident(void *buffer, int bufsize, int *offset) -{ - struct fileIdentDesc *fi; - int lengthThisIdent; - uint8_t *ptr; - int padlen; - - if ((!buffer) || (!offset)) { - udf_debug("invalidparms, buffer=%p, offset=%p\n", - buffer, offset); - return NULL; - } - - ptr = buffer; - - if ((*offset > 0) && (*offset < bufsize)) - ptr += *offset; - fi = (struct fileIdentDesc *)ptr; - if (fi->descTag.tagIdent != cpu_to_le16(TAG_IDENT_FID)) { - udf_debug("0x%x != TAG_IDENT_FID\n", - le16_to_cpu(fi->descTag.tagIdent)); - udf_debug("offset: %d sizeof: %lu bufsize: %d\n", - *offset, (unsigned long)sizeof(struct fileIdentDesc), - bufsize); - return NULL; - } - if ((*offset + sizeof(struct fileIdentDesc)) > bufsize) - lengthThisIdent = sizeof(struct fileIdentDesc); - else - lengthThisIdent = sizeof(struct fileIdentDesc) + - fi->lengthFileIdent + le16_to_cpu(fi->lengthOfImpUse); - - /* we need to figure padding, too! */ - padlen = lengthThisIdent % UDF_NAME_PAD; - if (padlen) - lengthThisIdent += (UDF_NAME_PAD - padlen); - *offset = *offset + lengthThisIdent; - - return fi; -} - struct short_ad *udf_get_fileshortad(uint8_t *ptr, int maxoffset, uint32_t *offset, int inc) { diff --git a/fs/udf/namei.c b/fs/udf/namei.c index b9eb76438b68..ff9455b8c2fd 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -41,105 +41,6 @@ static inline int udf_match(int len1, const unsigned char *name1, int len2, return !memcmp(name1, name2, len1); } -int udf_write_fi(struct inode *inode, struct fileIdentDesc *cfi, - struct fileIdentDesc *sfi, struct udf_fileident_bh *fibh, - uint8_t *impuse, uint8_t *fileident) -{ - uint16_t crclen = fibh->eoffset - fibh->soffset - sizeof(struct tag); - uint16_t crc; - int offset; - uint16_t liu = le16_to_cpu(cfi->lengthOfImpUse); - uint8_t lfi = cfi->lengthFileIdent; - int padlen = fibh->eoffset - fibh->soffset - liu - lfi - - sizeof(struct fileIdentDesc); - int adinicb = 0; - - if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - adinicb = 1; - - offset = fibh->soffset + sizeof(struct fileIdentDesc); - - if (impuse) { - if (adinicb || (offset + liu < 0)) { - memcpy((uint8_t *)sfi->impUse, impuse, liu); - } else if (offset >= 0) { - memcpy(fibh->ebh->b_data + offset, impuse, liu); - } else { - memcpy((uint8_t *)sfi->impUse, impuse, -offset); - memcpy(fibh->ebh->b_data, impuse - offset, - liu + offset); - } - } - - offset += liu; - - if (fileident) { - if (adinicb || (offset + lfi < 0)) { - memcpy(sfi->impUse + liu, fileident, lfi); - } else if (offset >= 0) { - memcpy(fibh->ebh->b_data + offset, fileident, lfi); - } else { - memcpy(sfi->impUse + liu, fileident, -offset); - memcpy(fibh->ebh->b_data, fileident - offset, - lfi + offset); - } - } - - offset += lfi; - - if (adinicb || (offset + padlen < 0)) { - memset(sfi->impUse + liu + lfi, 0x00, padlen); - } else if (offset >= 0) { - memset(fibh->ebh->b_data + offset, 0x00, padlen); - } else { - memset(sfi->impUse + liu + lfi, 0x00, -offset); - memset(fibh->ebh->b_data, 0x00, padlen + offset); - } - - crc = crc_itu_t(0, (uint8_t *)cfi + sizeof(struct tag), - sizeof(struct fileIdentDesc) - sizeof(struct tag)); - - if (fibh->sbh == fibh->ebh) { - crc = crc_itu_t(crc, (uint8_t *)sfi->impUse, - crclen + sizeof(struct tag) - - sizeof(struct fileIdentDesc)); - } else if (sizeof(struct fileIdentDesc) >= -fibh->soffset) { - crc = crc_itu_t(crc, fibh->ebh->b_data + - sizeof(struct fileIdentDesc) + - fibh->soffset, - crclen + sizeof(struct tag) - - sizeof(struct fileIdentDesc)); - } else { - crc = crc_itu_t(crc, (uint8_t *)sfi->impUse, - -fibh->soffset - sizeof(struct fileIdentDesc)); - crc = crc_itu_t(crc, fibh->ebh->b_data, fibh->eoffset); - } - - cfi->descTag.descCRC = cpu_to_le16(crc); - cfi->descTag.descCRCLength = cpu_to_le16(crclen); - cfi->descTag.tagChecksum = udf_tag_checksum(&cfi->descTag); - - if (adinicb || (sizeof(struct fileIdentDesc) <= -fibh->soffset)) { - memcpy((uint8_t *)sfi, (uint8_t *)cfi, - sizeof(struct fileIdentDesc)); - } else { - memcpy((uint8_t *)sfi, (uint8_t *)cfi, -fibh->soffset); - memcpy(fibh->ebh->b_data, (uint8_t *)cfi - fibh->soffset, - sizeof(struct fileIdentDesc) + fibh->soffset); - } - - if (adinicb) { - mark_inode_dirty(inode); - } else { - if (fibh->sbh != fibh->ebh) - mark_buffer_dirty_inode(fibh->ebh, inode); - mark_buffer_dirty_inode(fibh->sbh, inode); - } - inode_inc_iversion(inode); - - return 0; -} - /** * udf_fiiter_find_entry - find entry in given directory. * @@ -207,161 +108,6 @@ out_ok: return ret; } -/** - * udf_find_entry - find entry in given directory. - * - * @dir: directory inode to search in - * @child: qstr of the name - * @fibh: buffer head / inode with file identifier descriptor we found - * @cfi: found file identifier descriptor with given name - * - * This function searches in the directory @dir for a file name @child. When - * found, @fibh points to the buffer head(s) (bh is NULL for in ICB - * directories) containing the file identifier descriptor (FID). In that case - * the function returns pointer to the FID in the buffer or inode - but note - * that FID may be split among two buffers (blocks) so accessing it via that - * pointer isn't easily possible. This pointer can be used only as an iterator - * for other directory manipulation functions. For inspection of the FID @cfi - * can be used - the found FID is copied there. - * - * Returns pointer to FID, NULL when nothing found, or error code. - */ -static struct fileIdentDesc *udf_find_entry(struct inode *dir, - const struct qstr *child, - struct udf_fileident_bh *fibh, - struct fileIdentDesc *cfi) -{ - struct fileIdentDesc *fi = NULL; - loff_t f_pos; - udf_pblk_t block; - int flen; - unsigned char *fname = NULL, *copy_name = NULL; - unsigned char *nameptr; - uint8_t lfi; - uint16_t liu; - loff_t size; - struct kernel_lb_addr eloc; - uint32_t elen; - sector_t offset; - struct extent_position epos = {}; - struct udf_inode_info *dinfo = UDF_I(dir); - int isdotdot = child->len == 2 && - child->name[0] == '.' && child->name[1] == '.'; - struct super_block *sb = dir->i_sb; - - size = udf_ext0_offset(dir) + dir->i_size; - f_pos = udf_ext0_offset(dir); - - fibh->sbh = fibh->ebh = NULL; - fibh->soffset = fibh->eoffset = f_pos & (sb->s_blocksize - 1); - if (dinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { - if (inode_bmap(dir, f_pos >> sb->s_blocksize_bits, &epos, - &eloc, &elen, &offset) != (EXT_RECORDED_ALLOCATED >> 30)) { - fi = ERR_PTR(-EIO); - goto out_err; - } - - block = udf_get_lb_pblock(sb, &eloc, offset); - if ((++offset << sb->s_blocksize_bits) < elen) { - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - } else - offset = 0; - - fibh->sbh = fibh->ebh = udf_tread(sb, block); - if (!fibh->sbh) { - fi = ERR_PTR(-EIO); - goto out_err; - } - } - - fname = kmalloc(UDF_NAME_LEN, GFP_NOFS); - if (!fname) { - fi = ERR_PTR(-ENOMEM); - goto out_err; - } - - while (f_pos < size) { - fi = udf_fileident_read(dir, &f_pos, fibh, cfi, &epos, &eloc, - &elen, &offset); - if (!fi) { - fi = ERR_PTR(-EIO); - goto out_err; - } - - liu = le16_to_cpu(cfi->lengthOfImpUse); - lfi = cfi->lengthFileIdent; - - if (fibh->sbh == fibh->ebh) { - nameptr = udf_get_fi_ident(fi); - } else { - int poffset; /* Unpaded ending offset */ - - poffset = fibh->soffset + sizeof(struct fileIdentDesc) + - liu + lfi; - - if (poffset >= lfi) - nameptr = (uint8_t *)(fibh->ebh->b_data + - poffset - lfi); - else { - if (!copy_name) { - copy_name = kmalloc(UDF_NAME_LEN_CS0, - GFP_NOFS); - if (!copy_name) { - fi = ERR_PTR(-ENOMEM); - goto out_err; - } - } - nameptr = copy_name; - memcpy(nameptr, udf_get_fi_ident(fi), - lfi - poffset); - memcpy(nameptr + lfi - poffset, - fibh->ebh->b_data, poffset); - } - } - - if ((cfi->fileCharacteristics & FID_FILE_CHAR_DELETED) != 0) { - if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) - continue; - } - - if ((cfi->fileCharacteristics & FID_FILE_CHAR_HIDDEN) != 0) { - if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) - continue; - } - - if ((cfi->fileCharacteristics & FID_FILE_CHAR_PARENT) && - isdotdot) - goto out_ok; - - if (!lfi) - continue; - - flen = udf_get_filename(sb, nameptr, lfi, fname, UDF_NAME_LEN); - if (flen < 0) { - fi = ERR_PTR(flen); - goto out_err; - } - - if (udf_match(flen, fname, child->len, child->name)) - goto out_ok; - } - - fi = NULL; -out_err: - if (fibh->sbh != fibh->ebh) - brelse(fibh->ebh); - brelse(fibh->sbh); -out_ok: - brelse(epos.bh); - kfree(fname); - kfree(copy_name); - - return fi; -} - static struct dentry *udf_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { @@ -582,245 +328,6 @@ store_fi: return 0; } -static struct fileIdentDesc *udf_add_entry(struct inode *dir, - struct dentry *dentry, - struct udf_fileident_bh *fibh, - struct fileIdentDesc *cfi, int *err) -{ - struct super_block *sb = dir->i_sb; - struct fileIdentDesc *fi = NULL; - unsigned char *name = NULL; - int namelen; - loff_t f_pos; - loff_t size = udf_ext0_offset(dir) + dir->i_size; - int nfidlen; - udf_pblk_t block; - struct kernel_lb_addr eloc; - uint32_t elen = 0; - sector_t offset; - struct extent_position epos = {}; - struct udf_inode_info *dinfo; - - fibh->sbh = fibh->ebh = NULL; - name = kmalloc(UDF_NAME_LEN_CS0, GFP_NOFS); - if (!name) { - *err = -ENOMEM; - goto out_err; - } - - if (dentry) { - if (!dentry->d_name.len) { - *err = -EINVAL; - goto out_err; - } - namelen = udf_put_filename(sb, dentry->d_name.name, - dentry->d_name.len, - name, UDF_NAME_LEN_CS0); - if (!namelen) { - *err = -ENAMETOOLONG; - goto out_err; - } - } else { - namelen = 0; - } - - nfidlen = ALIGN(sizeof(struct fileIdentDesc) + namelen, UDF_NAME_PAD); - - f_pos = udf_ext0_offset(dir); - - fibh->soffset = fibh->eoffset = f_pos & (dir->i_sb->s_blocksize - 1); - dinfo = UDF_I(dir); - if (dinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { - if (inode_bmap(dir, f_pos >> dir->i_sb->s_blocksize_bits, &epos, - &eloc, &elen, &offset) != (EXT_RECORDED_ALLOCATED >> 30)) { - block = udf_get_lb_pblock(dir->i_sb, - &dinfo->i_location, 0); - fibh->soffset = fibh->eoffset = sb->s_blocksize; - goto add; - } - block = udf_get_lb_pblock(dir->i_sb, &eloc, offset); - if ((++offset << dir->i_sb->s_blocksize_bits) < elen) { - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - } else - offset = 0; - - fibh->sbh = fibh->ebh = udf_tread(dir->i_sb, block); - if (!fibh->sbh) { - *err = -EIO; - goto out_err; - } - - block = dinfo->i_location.logicalBlockNum; - } - - while (f_pos < size) { - fi = udf_fileident_read(dir, &f_pos, fibh, cfi, &epos, &eloc, - &elen, &offset); - - if (!fi) { - *err = -EIO; - goto out_err; - } - - if ((cfi->fileCharacteristics & FID_FILE_CHAR_DELETED) != 0) { - if (udf_dir_entry_len(cfi) == nfidlen) { - cfi->descTag.tagSerialNum = cpu_to_le16(1); - cfi->fileVersionNum = cpu_to_le16(1); - cfi->fileCharacteristics = 0; - cfi->lengthFileIdent = namelen; - cfi->lengthOfImpUse = cpu_to_le16(0); - if (!udf_write_fi(dir, cfi, fi, fibh, NULL, - name)) - goto out_ok; - else { - *err = -EIO; - goto out_err; - } - } - } - } - -add: - f_pos += nfidlen; - - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && - sb->s_blocksize - fibh->eoffset < nfidlen) { - brelse(epos.bh); - epos.bh = NULL; - fibh->soffset -= udf_ext0_offset(dir); - fibh->eoffset -= udf_ext0_offset(dir); - f_pos -= udf_ext0_offset(dir); - if (fibh->sbh != fibh->ebh) - brelse(fibh->ebh); - brelse(fibh->sbh); - fibh->sbh = fibh->ebh = - udf_expand_dir_adinicb(dir, &block, err); - if (!fibh->sbh) - goto out_err; - epos.block = dinfo->i_location; - epos.offset = udf_file_entry_alloc_offset(dir); - /* Load extent udf_expand_dir_adinicb() has created */ - udf_current_aext(dir, &epos, &eloc, &elen, 1); - } - - /* Entry fits into current block? */ - if (sb->s_blocksize - fibh->eoffset >= nfidlen) { - fibh->soffset = fibh->eoffset; - fibh->eoffset += nfidlen; - if (fibh->sbh != fibh->ebh) { - brelse(fibh->sbh); - fibh->sbh = fibh->ebh; - } - - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - block = dinfo->i_location.logicalBlockNum; - fi = (struct fileIdentDesc *) - (dinfo->i_data + fibh->soffset - - udf_ext0_offset(dir) + - dinfo->i_lenEAttr); - } else { - block = eloc.logicalBlockNum + - ((elen - 1) >> - dir->i_sb->s_blocksize_bits); - fi = (struct fileIdentDesc *) - (fibh->sbh->b_data + fibh->soffset); - } - } else { - /* Round up last extent in the file */ - elen = (elen + sb->s_blocksize - 1) & ~(sb->s_blocksize - 1); - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - udf_write_aext(dir, &epos, &eloc, elen, 1); - dinfo->i_lenExtents = (dinfo->i_lenExtents + sb->s_blocksize - - 1) & ~(sb->s_blocksize - 1); - - fibh->soffset = fibh->eoffset - sb->s_blocksize; - fibh->eoffset += nfidlen - sb->s_blocksize; - if (fibh->sbh != fibh->ebh) { - brelse(fibh->sbh); - fibh->sbh = fibh->ebh; - } - - block = eloc.logicalBlockNum + ((elen - 1) >> - dir->i_sb->s_blocksize_bits); - fibh->ebh = udf_bread(dir, - f_pos >> dir->i_sb->s_blocksize_bits, 1, err); - if (!fibh->ebh) - goto out_err; - /* Extents could have been merged, invalidate our position */ - brelse(epos.bh); - epos.bh = NULL; - epos.block = dinfo->i_location; - epos.offset = udf_file_entry_alloc_offset(dir); - - if (!fibh->soffset) { - /* Find the freshly allocated block */ - while (udf_next_aext(dir, &epos, &eloc, &elen, 1) == - (EXT_RECORDED_ALLOCATED >> 30)) - ; - block = eloc.logicalBlockNum + ((elen - 1) >> - dir->i_sb->s_blocksize_bits); - brelse(fibh->sbh); - fibh->sbh = fibh->ebh; - fi = (struct fileIdentDesc *)(fibh->sbh->b_data); - } else { - fi = (struct fileIdentDesc *) - (fibh->sbh->b_data + sb->s_blocksize + - fibh->soffset); - } - } - - memset(cfi, 0, sizeof(struct fileIdentDesc)); - if (UDF_SB(sb)->s_udfrev >= 0x0200) - udf_new_tag((char *)cfi, TAG_IDENT_FID, 3, 1, block, - sizeof(struct tag)); - else - udf_new_tag((char *)cfi, TAG_IDENT_FID, 2, 1, block, - sizeof(struct tag)); - cfi->fileVersionNum = cpu_to_le16(1); - cfi->lengthFileIdent = namelen; - cfi->lengthOfImpUse = cpu_to_le16(0); - if (!udf_write_fi(dir, cfi, fi, fibh, NULL, name)) { - dir->i_size += nfidlen; - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - dinfo->i_lenAlloc += nfidlen; - else { - /* Find the last extent and truncate it to proper size */ - while (udf_next_aext(dir, &epos, &eloc, &elen, 1) == - (EXT_RECORDED_ALLOCATED >> 30)) - ; - elen -= dinfo->i_lenExtents - dir->i_size; - if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) - epos.offset -= sizeof(struct short_ad); - else if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) - epos.offset -= sizeof(struct long_ad); - udf_write_aext(dir, &epos, &eloc, elen, 1); - dinfo->i_lenExtents = dir->i_size; - } - - mark_inode_dirty(dir); - goto out_ok; - } else { - *err = -EIO; - goto out_err; - } - -out_err: - fi = NULL; - if (fibh->sbh != fibh->ebh) - brelse(fibh->ebh); - brelse(fibh->sbh); -out_ok: - brelse(epos.bh); - kfree(name); - return fi; -} - static void udf_fiiter_delete_entry(struct udf_fileident_iter *iter) { iter->fi.fileCharacteristics |= FID_FILE_CHAR_DELETED; @@ -831,18 +338,6 @@ static void udf_fiiter_delete_entry(struct udf_fileident_iter *iter) udf_fiiter_write_fi(iter, NULL); } -static int udf_delete_entry(struct inode *inode, struct fileIdentDesc *fi, - struct udf_fileident_bh *fibh, - struct fileIdentDesc *cfi) -{ - cfi->fileCharacteristics |= FID_FILE_CHAR_DELETED; - - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT)) - memset(&(cfi->icb), 0x00, sizeof(struct long_ad)); - - return udf_write_fi(inode, cfi, fi, fibh, NULL, NULL); -} - static int udf_add_nondir(struct dentry *dentry, struct inode *inode) { struct udf_inode_info *iinfo = UDF_I(inode); diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index e47b2f0c3e05..f764b4d15094 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -104,13 +104,6 @@ struct udf_fileident_iter { */ }; -struct udf_fileident_bh { - struct buffer_head *sbh; - struct buffer_head *ebh; - int soffset; - int eoffset; -}; - struct udf_vds_record { uint32_t block; uint32_t volDescSeqNum; @@ -139,19 +132,12 @@ struct inode *udf_find_metadata_inode_efe(struct super_block *sb, u32 meta_file_loc, u32 partition_num); /* namei.c */ -extern int udf_write_fi(struct inode *inode, struct fileIdentDesc *, - struct fileIdentDesc *, struct udf_fileident_bh *, - uint8_t *, uint8_t *); static inline unsigned int udf_dir_entry_len(struct fileIdentDesc *cfi) { return ALIGN(sizeof(struct fileIdentDesc) + le16_to_cpu(cfi->lengthOfImpUse) + cfi->lengthFileIdent, UDF_NAME_PAD); } -static inline uint8_t *udf_get_fi_ident(struct fileIdentDesc *fi) -{ - return ((uint8_t *)(fi + 1)) + le16_to_cpu(fi->lengthOfImpUse); -} /* file.c */ extern long udf_ioctl(struct file *, unsigned int, unsigned long); @@ -266,14 +252,6 @@ void udf_fiiter_release(struct udf_fileident_iter *iter); void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse); void udf_fiiter_update_elen(struct udf_fileident_iter *iter, uint32_t new_elen); int udf_fiiter_append_blk(struct udf_fileident_iter *iter); -extern struct fileIdentDesc *udf_fileident_read(struct inode *, loff_t *, - struct udf_fileident_bh *, - struct fileIdentDesc *, - struct extent_position *, - struct kernel_lb_addr *, uint32_t *, - sector_t *); -extern struct fileIdentDesc *udf_get_fileident(void *buffer, int bufsize, - int *offset); extern struct long_ad *udf_get_filelongad(uint8_t *, int, uint32_t *, int); extern struct short_ad *udf_get_fileshortad(uint8_t *, int, uint32_t *, int); From 70bfb3a8d661d4fdc742afc061b88a7f3fc9f500 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 15 Dec 2022 14:24:03 +0100 Subject: [PATCH 025/265] udf: Truncate added extents on failed expansion When a file expansion failed because we didn't have enough space for indirect extents make sure we truncate extents created so far so that we don't leave extents beyond EOF. CC: stable@vger.kernel.org Signed-off-by: Jan Kara --- fs/udf/inode.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 9655fc014c7a..e077183e2ee3 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -431,8 +431,10 @@ static int udf_do_extend_file(struct inode *inode, } if (fake) { - udf_add_aext(inode, last_pos, &last_ext->extLocation, - last_ext->extLength, 1); + err = udf_add_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); + if (err < 0) + goto out_err; count++; } else { struct kernel_lb_addr tmploc; @@ -466,7 +468,7 @@ static int udf_do_extend_file(struct inode *inode, err = udf_add_aext(inode, last_pos, &last_ext->extLocation, last_ext->extLength, 1); if (err) - return err; + goto out_err; count++; } if (new_block_bytes) { @@ -475,7 +477,7 @@ static int udf_do_extend_file(struct inode *inode, err = udf_add_aext(inode, last_pos, &last_ext->extLocation, last_ext->extLength, 1); if (err) - return err; + goto out_err; count++; } @@ -489,6 +491,11 @@ out: return -EIO; return count; +out_err: + /* Remove extents we've created so far */ + udf_clear_extent_cache(inode); + udf_truncate_extents(inode); + return err; } /* Extend the final block of the file to final_block_len bytes */ From 53cafe1d6d8ef9f93318e5bfccc0d24f27d41ced Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Fri, 16 Dec 2022 12:37:51 +0100 Subject: [PATCH 026/265] udf: Do not bother merging very long extents When merging very long extents we try to push as much length as possible to the first extent. However this is unnecessarily complicated and not really worth the trouble. Furthermore there was a bug in the logic resulting in corrupting extents in the file as syzbot reproducer shows. So just don't bother with the merging of extents that are too long together. CC: stable@vger.kernel.org Reported-by: syzbot+60f291a24acecb3c2bd5@syzkaller.appspotmail.com Signed-off-by: Jan Kara --- fs/udf/inode.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index e077183e2ee3..5498365669eb 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -997,23 +997,8 @@ static void udf_merge_extents(struct inode *inode, struct kernel_long_ad *laarr, blocksize - 1) >> blocksize_bits)))) { if (((li->extLength & UDF_EXTENT_LENGTH_MASK) + - (lip1->extLength & UDF_EXTENT_LENGTH_MASK) + - blocksize - 1) & ~UDF_EXTENT_LENGTH_MASK) { - lip1->extLength = (lip1->extLength - - (li->extLength & - UDF_EXTENT_LENGTH_MASK) + - UDF_EXTENT_LENGTH_MASK) & - ~(blocksize - 1); - li->extLength = (li->extLength & - UDF_EXTENT_FLAG_MASK) + - (UDF_EXTENT_LENGTH_MASK + 1) - - blocksize; - lip1->extLocation.logicalBlockNum = - li->extLocation.logicalBlockNum + - ((li->extLength & - UDF_EXTENT_LENGTH_MASK) >> - blocksize_bits); - } else { + (lip1->extLength & UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) <= UDF_EXTENT_LENGTH_MASK) { li->extLength = lip1->extLength + (((li->extLength & UDF_EXTENT_LENGTH_MASK) + From 33e9a53cd9f099b138578f8e1a3d60775ff8cbba Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 19 Dec 2022 19:50:14 +0100 Subject: [PATCH 027/265] udf: Handle error when expanding directory When there is an error when adding extent to the directory to expand it, make sure to propagate the error up properly. This is not expected to happen currently but let's make the code more futureproof. Signed-off-by: Jan Kara --- fs/udf/namei.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index ff9455b8c2fd..a4e8284043c1 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -192,8 +192,13 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, epos.bh = NULL; epos.block = iinfo->i_location; epos.offset = udf_file_entry_alloc_offset(inode); - udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); + ret = udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); brelse(epos.bh); + if (ret < 0) { + *err = ret; + udf_free_blocks(inode->i_sb, inode, &eloc, 0, 1); + return NULL; + } mark_inode_dirty(inode); /* Now fixup tags in moved directory entries */ From 2b10074d91e053d867148d66ba99552f576b3dd1 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 19 Dec 2022 19:54:12 +0100 Subject: [PATCH 028/265] udf: Handle error when adding extent to symlink When adding extent describing symlink data fails, make sure to handle the error properly, propagate it up and free the already allocated block. Signed-off-by: Jan Kara --- fs/udf/namei.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index a4e8284043c1..261ca813a1b8 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -613,8 +613,12 @@ static int udf_symlink(struct user_namespace *mnt_userns, struct inode *dir, iinfo->i_location.partitionReferenceNum; bsize = sb->s_blocksize; iinfo->i_lenExtents = bsize; - udf_add_aext(inode, &epos, &eloc, bsize, 0); + err = udf_add_aext(inode, &epos, &eloc, bsize, 0); brelse(epos.bh); + if (err < 0) { + udf_free_blocks(sb, inode, &eloc, 0, 1); + goto out_no_entry; + } block = udf_get_pblock(sb, block, iinfo->i_location.partitionReferenceNum, @@ -622,6 +626,7 @@ static int udf_symlink(struct user_namespace *mnt_userns, struct inode *dir, epos.bh = udf_tgetblk(sb, block); if (unlikely(!epos.bh)) { err = -ENOMEM; + udf_free_blocks(sb, inode, &eloc, 0, 1); goto out_no_entry; } lock_buffer(epos.bh); From 19fd80de0a8b5170ef34704c8984cca920dffa59 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 19 Dec 2022 20:10:35 +0100 Subject: [PATCH 029/265] udf: Handle error when adding extent to a file When adding extent to a file fails, so far we've silently squelshed the error. Make sure to propagate it up properly. Signed-off-by: Jan Kara --- fs/udf/inode.c | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 5498365669eb..6d99d1300289 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -57,15 +57,15 @@ static int udf_update_inode(struct inode *, int); static int udf_sync_inode(struct inode *inode); static int udf_alloc_i_data(struct inode *inode, size_t size); static sector_t inode_getblk(struct inode *, sector_t, int *, int *); -static int8_t udf_insert_aext(struct inode *, struct extent_position, - struct kernel_lb_addr, uint32_t); +static int udf_insert_aext(struct inode *, struct extent_position, + struct kernel_lb_addr, uint32_t); static void udf_split_extents(struct inode *, int *, int, udf_pblk_t, struct kernel_long_ad *, int *); static void udf_prealloc_extents(struct inode *, int, int, struct kernel_long_ad *, int *); static void udf_merge_extents(struct inode *, struct kernel_long_ad *, int *); -static void udf_update_extents(struct inode *, struct kernel_long_ad *, int, - int, struct extent_position *); +static int udf_update_extents(struct inode *, struct kernel_long_ad *, int, + int, struct extent_position *); static int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); static void __udf_clear_extent_cache(struct inode *inode) @@ -793,7 +793,9 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, /* write back the new extents, inserting new extents if the new number * of extents is greater than the old number, and deleting extents if * the new number of extents is less than the old number */ - udf_update_extents(inode, laarr, startnum, endnum, &prev_epos); + *err = udf_update_extents(inode, laarr, startnum, endnum, &prev_epos); + if (*err < 0) + goto out_free; newblock = udf_get_pblock(inode->i_sb, newblocknum, iinfo->i_location.partitionReferenceNum, 0); @@ -1061,21 +1063,30 @@ static void udf_merge_extents(struct inode *inode, struct kernel_long_ad *laarr, } } -static void udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr, - int startnum, int endnum, - struct extent_position *epos) +static int udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr, + int startnum, int endnum, + struct extent_position *epos) { int start = 0, i; struct kernel_lb_addr tmploc; uint32_t tmplen; + int err; if (startnum > endnum) { for (i = 0; i < (startnum - endnum); i++) udf_delete_aext(inode, *epos); } else if (startnum < endnum) { for (i = 0; i < (endnum - startnum); i++) { - udf_insert_aext(inode, *epos, laarr[i].extLocation, - laarr[i].extLength); + err = udf_insert_aext(inode, *epos, + laarr[i].extLocation, + laarr[i].extLength); + /* + * If we fail here, we are likely corrupting the extent + * list and leaking blocks. At least stop early to + * limit the damage. + */ + if (err < 0) + return err; udf_next_aext(inode, epos, &laarr[i].extLocation, &laarr[i].extLength, 1); start++; @@ -1087,6 +1098,7 @@ static void udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr udf_write_aext(inode, epos, &laarr[i].extLocation, laarr[i].extLength, 1); } + return 0; } struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, @@ -2105,12 +2117,13 @@ int8_t udf_current_aext(struct inode *inode, struct extent_position *epos, return etype; } -static int8_t udf_insert_aext(struct inode *inode, struct extent_position epos, - struct kernel_lb_addr neloc, uint32_t nelen) +static int udf_insert_aext(struct inode *inode, struct extent_position epos, + struct kernel_lb_addr neloc, uint32_t nelen) { struct kernel_lb_addr oeloc; uint32_t oelen; int8_t etype; + int err; if (epos.bh) get_bh(epos.bh); @@ -2120,10 +2133,10 @@ static int8_t udf_insert_aext(struct inode *inode, struct extent_position epos, neloc = oeloc; nelen = (etype << 30) | oelen; } - udf_add_aext(inode, &epos, &neloc, nelen, 1); + err = udf_add_aext(inode, &epos, &neloc, nelen, 1); brelse(epos.bh); - return (nelen >> 30); + return err; } int8_t udf_delete_aext(struct inode *inode, struct extent_position epos) From 0aba4860b0d0216a1a300484ff536171894d49d8 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 20 Dec 2022 12:38:45 +0100 Subject: [PATCH 030/265] udf: Allocate name buffer in directory iterator on heap Currently we allocate name buffer in directory iterators (struct udf_fileident_iter) on stack. These structures are relatively large (some 360 bytes on 64-bit architectures). For udf_rename() which needs to keep three of these structures in parallel the stack usage becomes rather heavy - 1536 bytes in total. Allocate the name buffer in the iterator from heap to avoid excessive stack usage. Link: https://lore.kernel.org/all/202212200558.lK9x1KW0-lkp@intel.com Reported-by: kernel test robot Signed-off-by: Jan Kara --- fs/udf/directory.c | 23 +++++++++++++++-------- fs/udf/udfdecl.h | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index e7e8b30876d9..a4c91905b033 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -248,9 +248,14 @@ int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, iter->elen = 0; iter->epos.bh = NULL; iter->name = NULL; + iter->namebuf = kmalloc(UDF_NAME_LEN_CS0, GFP_KERNEL); + if (!iter->namebuf) + return -ENOMEM; - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - return udf_copy_fi(iter); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + err = udf_copy_fi(iter); + goto out; + } if (inode_bmap(dir, iter->pos >> dir->i_blkbits, &iter->epos, &iter->eloc, &iter->elen, &iter->loffset) != @@ -260,17 +265,17 @@ int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, udf_err(dir->i_sb, "position %llu not allocated in directory (ino %lu)\n", (unsigned long long)pos, dir->i_ino); - return -EFSCORRUPTED; + err = -EFSCORRUPTED; + goto out; } err = udf_fiiter_load_bhs(iter); if (err < 0) - return err; + goto out; err = udf_copy_fi(iter); - if (err < 0) { +out: + if (err < 0) udf_fiiter_release(iter); - return err; - } - return 0; + return err; } int udf_fiiter_advance(struct udf_fileident_iter *iter) @@ -307,6 +312,8 @@ void udf_fiiter_release(struct udf_fileident_iter *iter) brelse(iter->bh[0]); brelse(iter->bh[1]); iter->bh[0] = iter->bh[1] = NULL; + kfree(iter->namebuf); + iter->namebuf = NULL; } static void udf_copy_to_bufs(void *buf1, int len1, void *buf2, int len2, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index f764b4d15094..d35aa42bb577 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -99,7 +99,7 @@ struct udf_fileident_iter { struct extent_position epos; /* Position after the above extent */ struct fileIdentDesc fi; /* Copied directory entry */ uint8_t *name; /* Pointer to entry name */ - uint8_t namebuf[UDF_NAME_LEN_CS0]; /* Storage for entry name in case + uint8_t *namebuf; /* Storage for entry name in case * the name is split between two blocks */ }; From e57191a8d40086537d505241a8ee49d8b62ce1d6 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 21 Dec 2022 12:18:23 +0100 Subject: [PATCH 031/265] udf: Move setting of i_lenExtents into udf_do_extend_file() When expanding file for a write into a hole, we were not updating total length of inode's extents properly. Move the update of i_lenExtents into udf_do_extend_file() so that both expanding of file by truncate and expanding of file by writing beyond EOF properly update the length of extents. As a bonus, we also correctly update the length of extents when only part of extents can be written. Signed-off-by: Jan Kara --- fs/udf/inode.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 6d99d1300289..000ce37e6e1e 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -419,6 +419,7 @@ static int udf_do_extend_file(struct inode *inode, ~(sb->s_blocksize - 1); } + add = 0; /* Can we merge with the previous extent? */ if ((last_ext->extLength & UDF_EXTENT_FLAG_MASK) == EXT_NOT_RECORDED_NOT_ALLOCATED) { @@ -451,6 +452,7 @@ static int udf_do_extend_file(struct inode *inode, if (new_block_bytes) udf_next_aext(inode, last_pos, &tmploc, &tmplen, 0); } + iinfo->i_lenExtents += add; /* Managed to do everything necessary? */ if (!new_block_bytes) @@ -469,6 +471,7 @@ static int udf_do_extend_file(struct inode *inode, last_ext->extLength, 1); if (err) goto out_err; + iinfo->i_lenExtents += add; count++; } if (new_block_bytes) { @@ -478,6 +481,7 @@ static int udf_do_extend_file(struct inode *inode, last_ext->extLength, 1); if (err) goto out_err; + iinfo->i_lenExtents += new_block_bytes; count++; } @@ -585,7 +589,6 @@ static int udf_extend_file(struct inode *inode, loff_t newsize) if (err < 0) goto out; err = 0; - iinfo->i_lenExtents = newsize; out: brelse(epos.bh); return err; From b316c443b4e85f67e34807e11ca90049d6f6a098 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 21 Dec 2022 17:53:28 +0100 Subject: [PATCH 032/265] udf: Keep i_lenExtents consistent with the total length of extents When rounding the last extent to blocksize in inode_getblk() we forgot to update also i_lenExtents to match the new extent length. This inconsistency can later confuse some assertion checks. Signed-off-by: Jan Kara --- fs/udf/inode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 000ce37e6e1e..31965c3798f2 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -677,6 +677,9 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, elen = EXT_RECORDED_ALLOCATED | ((elen + inode->i_sb->s_blocksize - 1) & ~(inode->i_sb->s_blocksize - 1)); + iinfo->i_lenExtents = + ALIGN(iinfo->i_lenExtents, + inode->i_sb->s_blocksize); udf_write_aext(inode, &cur_epos, &eloc, elen, 1); } newblock = udf_get_lb_pblock(inode->i_sb, &eloc, offset); From 96b87cbdeb50e47c25eae1e8b209857e6acf34ea Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Sat, 31 Dec 2022 18:42:05 +0100 Subject: [PATCH 033/265] fs/ext2: Replace kmap_atomic() with kmap_local_page() kmap_atomic() is deprecated in favor of kmap_local_page(). Therefore, replace kmap_atomic() with kmap_local_page(). kmap_atomic() is implemented like a kmap_local_page() which also disables page-faults and preemption (the latter only for !PREEMPT_RT kernels). However, the code within the mapping and un-mapping in ext2_make_empty() does not depend on the above-mentioned side effects. Therefore, a mere replacement of the old API with the new one is all it is required (i.e., there is no need to explicitly add any calls to pagefault_disable() and/or preempt_disable()). Suggested-by: Ira Weiny Reviewed-by: Ira Weiny Signed-off-by: Fabio M. De Francesco Signed-off-by: Jan Kara Message-Id: <20221231174205.8492-1-fmdefrancesco@gmail.com> --- fs/ext2/dir.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c index e5cbc27ba459..0f144c5c7861 100644 --- a/fs/ext2/dir.c +++ b/fs/ext2/dir.c @@ -646,7 +646,7 @@ int ext2_make_empty(struct inode *inode, struct inode *parent) unlock_page(page); goto fail; } - kaddr = kmap_atomic(page); + kaddr = kmap_local_page(page); memset(kaddr, 0, chunk_size); de = (struct ext2_dir_entry_2 *)kaddr; de->name_len = 1; @@ -661,7 +661,7 @@ int ext2_make_empty(struct inode *inode, struct inode *parent) de->inode = cpu_to_le32(parent->i_ino); memcpy (de->name, "..\0", 4); ext2_set_de_type (de, inode); - kunmap_atomic(kaddr); + kunmap_local(kaddr); ext2_commit_chunk(page, 0, chunk_size); err = ext2_handle_dirsync(inode); fail: From 02113feaf62c190b4216ea0a1a2b8d9ccdbcdf6f Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 30 Dec 2022 23:14:52 +0000 Subject: [PATCH 034/265] udf: Fix spelling mistake "lenght" -> "length" There is a spelling mistake in a udf_err message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Jan Kara Message-Id: <20221230231452.5821-1-colin.i.king@gmail.com> --- fs/udf/directory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index a4c91905b033..18a1514871fe 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -34,7 +34,7 @@ static int udf_verify_fi(struct udf_fileident_iter *iter) len = udf_dir_entry_len(&iter->fi); if (le16_to_cpu(iter->fi.lengthOfImpUse) & 3) { udf_err(iter->dir->i_sb, - "directory (ino %lu) has entry at pos %llu with unaligned lenght of impUse field\n", + "directory (ino %lu) has entry at pos %llu with unaligned length of impUse field\n", iter->dir->i_ino, (unsigned long long)iter->pos); return -EFSCORRUPTED; } From 256fe4162f8b5a1625b8603ca5f7ff79725bfb47 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 2 Jan 2023 20:14:47 +0100 Subject: [PATCH 035/265] udf: Do not update file length for failed writes to inline files When write to inline file fails (or happens only partly), we still updated length of inline data as if the whole write succeeded. Fix the update of length of inline data to happen only if the write succeeds. Reported-by: syzbot+0937935b993956ba28ab@syzkaller.appspotmail.com CC: stable@vger.kernel.org Signed-off-by: Jan Kara --- fs/udf/file.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 5c659e23e578..8be51161f3e5 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -149,26 +149,24 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; down_write(&iinfo->i_data_sem); - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - loff_t end = iocb->ki_pos + iov_iter_count(from); - - if (inode->i_sb->s_blocksize < - (udf_file_entry_alloc_offset(inode) + end)) { - err = udf_expand_file_adinicb(inode); - if (err) { - inode_unlock(inode); - udf_debug("udf_expand_adinicb: err=%d\n", err); - return err; - } - } else { - iinfo->i_lenAlloc = max(end, inode->i_size); - up_write(&iinfo->i_data_sem); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && + inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + + iocb->ki_pos + iov_iter_count(from))) { + err = udf_expand_file_adinicb(inode); + if (err) { + inode_unlock(inode); + udf_debug("udf_expand_adinicb: err=%d\n", err); + return err; } } else up_write(&iinfo->i_data_sem); retval = __generic_file_write_iter(iocb, from); out: + down_write(&iinfo->i_data_sem); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && retval > 0) + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); inode_unlock(inode); if (retval > 0) { From fc8033a34a3ca7d23353e645e6dde5d364ac5f12 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 3 Jan 2023 09:56:56 +0100 Subject: [PATCH 036/265] udf: Preserve link count of system files System files in UDF filesystem have link count 0. To not confuse VFS we fudge the link count to be 1 when reading such inodes however we forget to restore the link count of 0 when writing such inodes. Fix that. CC: stable@vger.kernel.org Signed-off-by: Jan Kara --- fs/udf/inode.c | 9 +++++++-- fs/udf/super.c | 1 + fs/udf/udf_i.h | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 31965c3798f2..9ee269d3d546 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1301,6 +1301,7 @@ reread: ret = -EIO; goto out; } + iinfo->i_hidden = hidden_inode; iinfo->i_unique = 0; iinfo->i_lenEAttr = 0; iinfo->i_lenExtents = 0; @@ -1636,8 +1637,12 @@ static int udf_update_inode(struct inode *inode, int do_sync) if (S_ISDIR(inode->i_mode) && inode->i_nlink > 0) fe->fileLinkCount = cpu_to_le16(inode->i_nlink - 1); - else - fe->fileLinkCount = cpu_to_le16(inode->i_nlink); + else { + if (iinfo->i_hidden) + fe->fileLinkCount = cpu_to_le16(0); + else + fe->fileLinkCount = cpu_to_le16(inode->i_nlink); + } fe->informationLength = cpu_to_le64(inode->i_size); diff --git a/fs/udf/super.c b/fs/udf/super.c index 06eda8177b5f..241b40e886b3 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -147,6 +147,7 @@ static struct inode *udf_alloc_inode(struct super_block *sb) ei->i_next_alloc_goal = 0; ei->i_strat4096 = 0; ei->i_streamdir = 0; + ei->i_hidden = 0; init_rwsem(&ei->i_data_sem); ei->cached_extent.lstart = -1; spin_lock_init(&ei->i_extent_cache_lock); diff --git a/fs/udf/udf_i.h b/fs/udf/udf_i.h index 06ff7006b822..312b7c9ef10e 100644 --- a/fs/udf/udf_i.h +++ b/fs/udf/udf_i.h @@ -44,7 +44,8 @@ struct udf_inode_info { unsigned i_use : 1; /* unallocSpaceEntry */ unsigned i_strat4096 : 1; unsigned i_streamdir : 1; - unsigned reserved : 25; + unsigned i_hidden : 1; /* hidden system inode */ + unsigned reserved : 24; __u8 *i_data; struct kernel_lb_addr i_locStreamdir; __u64 i_lenStreams; From 85a37983ec69cc9fcd188bc37c4de15ee326355a Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 3 Jan 2023 10:03:35 +0100 Subject: [PATCH 037/265] udf: Detect system inodes linked into directory hierarchy When UDF filesystem is corrupted, hidden system inodes can be linked into directory hierarchy which is an avenue for further serious corruption of the filesystem and kernel confusion as noticed by syzbot fuzzed images. Refuse to access system inodes linked into directory hierarchy and vice versa. CC: stable@vger.kernel.org Reported-by: syzbot+38695a20b8addcbc1084@syzkaller.appspotmail.com Signed-off-by: Jan Kara --- fs/udf/inode.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 9ee269d3d546..96873fa2f683 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1813,8 +1813,13 @@ struct inode *__udf_iget(struct super_block *sb, struct kernel_lb_addr *ino, if (!inode) return ERR_PTR(-ENOMEM); - if (!(inode->i_state & I_NEW)) + if (!(inode->i_state & I_NEW)) { + if (UDF_I(inode)->i_hidden != hidden_inode) { + iput(inode); + return ERR_PTR(-EFSCORRUPTED); + } return inode; + } memcpy(&UDF_I(inode)->i_location, ino, sizeof(struct kernel_lb_addr)); err = udf_read_inode(inode, hidden_inode); From 1fb40763a58c1f8130c0131c624060c6bbb929b8 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 5 Jan 2023 13:49:25 +0000 Subject: [PATCH 038/265] udf: remove redundant variable netype The variable netype is assigned a value that is never read, the assignment is redundant the variable can be removed. Message-Id: <20230105134925.45599-1-colin.i.king@gmail.com> Signed-off-by: Colin Ian King Signed-off-by: Jan Kara --- fs/udf/truncate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c index 036ebd892b85..775edaba82ee 100644 --- a/fs/udf/truncate.c +++ b/fs/udf/truncate.c @@ -125,7 +125,7 @@ void udf_discard_prealloc(struct inode *inode) struct kernel_lb_addr eloc; uint32_t elen; uint64_t lbcount = 0; - int8_t etype = -1, netype; + int8_t etype = -1; struct udf_inode_info *iinfo = UDF_I(inode); int bsize = 1 << inode->i_blkbits; @@ -136,7 +136,7 @@ void udf_discard_prealloc(struct inode *inode) epos.block = iinfo->i_location; /* Find the last extent in the file */ - while ((netype = udf_next_aext(inode, &epos, &eloc, &elen, 0)) != -1) { + while (udf_next_aext(inode, &epos, &eloc, &elen, 0) != -1) { brelse(prev_epos.bh); prev_epos = epos; if (prev_epos.bh) From d46aa786fa53cbc92593089374e49c94fd9063ae Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 5 Jan 2023 11:07:34 -0800 Subject: [PATCH 039/265] block: use iter_ubuf for single range This is more efficient than iter_iov. Signed-off-by: Keith Busch Reviewed-by: Christoph Hellwig [axboe: fold in iovec assumption fix] Signed-off-by: Jens Axboe --- block/blk-map.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/block/blk-map.c b/block/blk-map.c index 19940c978c73..f2135e6ee8f6 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -31,7 +31,8 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, return NULL; memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); bmd->iter = *data; - bmd->iter.iov = bmd->iov; + if (iter_is_iovec(data)) + bmd->iter.iov = bmd->iov; return bmd; } @@ -641,7 +642,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, copy = true; else if (iov_iter_is_bvec(iter)) map_bvec = true; - else if (!iter_is_iovec(iter)) + else if (!user_backed_iter(iter)) copy = true; else if (queue_virt_boundary(q)) copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); @@ -682,9 +683,8 @@ int blk_rq_map_user(struct request_queue *q, struct request *rq, struct rq_map_data *map_data, void __user *ubuf, unsigned long len, gfp_t gfp_mask) { - struct iovec iov; struct iov_iter i; - int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); + int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i); if (unlikely(ret < 0)) return ret; From 7a5fa171a2869e99fe38f61c60274e7b4bf3a5e0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 16 Jan 2023 09:52:05 +0100 Subject: [PATCH 040/265] ext2: propagate errors from ext2_prepare_chunk Propagate errors from ext2_prepare_chunk to the callers and handle them there. While touching the prototype also turn update_times into a bool from the current int used as bool. [JK: fixed up error recovery path in ext2_rename()] Signed-off-by: Christoph Hellwig Signed-off-by: Jan Kara Message-Id: <20230116085205.2342975-1-hch@lst.de> --- fs/ext2/dir.c | 13 ++++++++----- fs/ext2/ext2.h | 5 +++-- fs/ext2/namei.c | 23 +++++++++++++---------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c index 0f144c5c7861..4a6955a0a116 100644 --- a/fs/ext2/dir.c +++ b/fs/ext2/dir.c @@ -461,9 +461,9 @@ static int ext2_handle_dirsync(struct inode *dir) return err; } -void ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, - struct page *page, void *page_addr, struct inode *inode, - int update_times) +int ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, + struct page *page, void *page_addr, struct inode *inode, + bool update_times) { loff_t pos = page_offset(page) + (char *) de - (char *) page_addr; @@ -472,7 +472,10 @@ void ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, lock_page(page); err = ext2_prepare_chunk(page, pos, len); - BUG_ON(err); + if (err) { + unlock_page(page); + return err; + } de->inode = cpu_to_le32(inode->i_ino); ext2_set_de_type(de, inode); ext2_commit_chunk(page, pos, len); @@ -480,7 +483,7 @@ void ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, dir->i_mtime = dir->i_ctime = current_time(dir); EXT2_I(dir)->i_flags &= ~EXT2_BTREE_FL; mark_inode_dirty(dir); - ext2_handle_dirsync(dir); + return ext2_handle_dirsync(dir); } /* diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h index 28de11a22e5f..f7f590aac55a 100644 --- a/fs/ext2/ext2.h +++ b/fs/ext2/ext2.h @@ -734,8 +734,9 @@ extern int ext2_delete_entry(struct ext2_dir_entry_2 *dir, struct page *page, char *kaddr); extern int ext2_empty_dir (struct inode *); extern struct ext2_dir_entry_2 *ext2_dotdot(struct inode *dir, struct page **p, void **pa); -extern void ext2_set_link(struct inode *, struct ext2_dir_entry_2 *, struct page *, void *, - struct inode *, int); +int ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, + struct page *page, void *page_addr, struct inode *inode, + bool update_times); static inline void ext2_put_page(struct page *page, void *page_addr) { kunmap_local(page_addr); diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c index c056957221a2..9805bc3ee80b 100644 --- a/fs/ext2/namei.c +++ b/fs/ext2/namei.c @@ -370,8 +370,11 @@ static int ext2_rename (struct user_namespace * mnt_userns, err = PTR_ERR(new_de); goto out_dir; } - ext2_set_link(new_dir, new_de, new_page, page_addr, old_inode, 1); + err = ext2_set_link(new_dir, new_de, new_page, page_addr, + old_inode, true); ext2_put_page(new_page, page_addr); + if (err) + goto out_dir; new_inode->i_ctime = current_time(new_inode); if (dir_de) drop_nlink(new_inode); @@ -394,24 +397,24 @@ static int ext2_rename (struct user_namespace * mnt_userns, ext2_delete_entry(old_de, old_page, old_page_addr); if (dir_de) { - if (old_dir != new_dir) - ext2_set_link(old_inode, dir_de, dir_page, - dir_page_addr, new_dir, 0); + if (old_dir != new_dir) { + err = ext2_set_link(old_inode, dir_de, dir_page, + dir_page_addr, new_dir, false); + } ext2_put_page(dir_page, dir_page_addr); inode_dec_link_count(old_dir); } - ext2_put_page(old_page, old_page_addr); - return 0; - -out_dir: - if (dir_de) - ext2_put_page(dir_page, dir_page_addr); out_old: ext2_put_page(old_page, old_page_addr); out: return err; + +out_dir: + if (dir_de) + ext2_put_page(dir_page, dir_page_addr); + goto out_old; } const struct inode_operations ext2_dir_inode_operations = { From aad633dc0cf90093998b1ae0ba9f19b5f1dab644 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:31 -0500 Subject: [PATCH 041/265] fs: dlm: start midcomms before scand The scand kthread can send dlm messages out, especially dlm remove messages to free memory for unused rsb on other nodes. To send out dlm messages, midcomms must be initialized. This patch moves the midcomms start before scand is started. Cc: stable@vger.kernel.org Fixes: e7fd41792fc0 ("[DLM] The core of the DLM for GFS2/CLVM") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/lockspace.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index d0b4e2181a5f..99bc96f90779 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -381,23 +381,23 @@ static int threads_start(void) { int error; - error = dlm_scand_start(); - if (error) { - log_print("cannot start dlm_scand thread %d", error); - goto fail; - } - /* Thread for sending/receiving messages for all lockspace's */ error = dlm_midcomms_start(); if (error) { log_print("cannot start dlm midcomms %d", error); - goto scand_fail; + goto fail; + } + + error = dlm_scand_start(); + if (error) { + log_print("cannot start dlm_scand thread %d", error); + goto midcomms_fail; } return 0; - scand_fail: - dlm_scand_stop(); + midcomms_fail: + dlm_midcomms_stop(); fail: return error; } From 724b6bab0d75f1dc01fdfbf7fe8d4217a5cb90ba Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:32 -0500 Subject: [PATCH 042/265] fs: dlm: fix use after free in midcomms commit While working on processing dlm message in softirq context I experienced the following KASAN use-after-free warning: [ 151.760477] ================================================================== [ 151.761803] BUG: KASAN: use-after-free in dlm_midcomms_commit_mhandle+0x19d/0x4b0 [ 151.763414] Read of size 4 at addr ffff88811a980c60 by task lock_torture/1347 [ 151.765284] CPU: 7 PID: 1347 Comm: lock_torture Not tainted 6.1.0-rc4+ #2828 [ 151.766778] Hardware name: Red Hat KVM/RHEL-AV, BIOS 1.16.0-3.module+el8.7.0+16134+e5908aa2 04/01/2014 [ 151.768726] Call Trace: [ 151.769277] [ 151.769748] dump_stack_lvl+0x5b/0x86 [ 151.770556] print_report+0x180/0x4c8 [ 151.771378] ? kasan_complete_mode_report_info+0x7c/0x1e0 [ 151.772241] ? dlm_midcomms_commit_mhandle+0x19d/0x4b0 [ 151.773069] kasan_report+0x93/0x1a0 [ 151.773668] ? dlm_midcomms_commit_mhandle+0x19d/0x4b0 [ 151.774514] __asan_load4+0x7e/0xa0 [ 151.775089] dlm_midcomms_commit_mhandle+0x19d/0x4b0 [ 151.775890] ? create_message.isra.29.constprop.64+0x57/0xc0 [ 151.776770] send_common+0x19f/0x1b0 [ 151.777342] ? remove_from_waiters+0x60/0x60 [ 151.778017] ? lock_downgrade+0x410/0x410 [ 151.778648] ? __this_cpu_preempt_check+0x13/0x20 [ 151.779421] ? rcu_lockdep_current_cpu_online+0x88/0xc0 [ 151.780292] _convert_lock+0x46/0x150 [ 151.780893] convert_lock+0x7b/0xc0 [ 151.781459] dlm_lock+0x3ac/0x580 [ 151.781993] ? 0xffffffffc0540000 [ 151.782522] ? torture_stop+0x120/0x120 [dlm_locktorture] [ 151.783379] ? dlm_scan_rsbs+0xa70/0xa70 [ 151.784003] ? preempt_count_sub+0xd6/0x130 [ 151.784661] ? is_module_address+0x47/0x70 [ 151.785309] ? torture_stop+0x120/0x120 [dlm_locktorture] [ 151.786166] ? 0xffffffffc0540000 [ 151.786693] ? lockdep_init_map_type+0xc3/0x360 [ 151.787414] ? 0xffffffffc0540000 [ 151.787947] torture_dlm_lock_sync.isra.3+0xe9/0x150 [dlm_locktorture] [ 151.789004] ? torture_stop+0x120/0x120 [dlm_locktorture] [ 151.789858] ? 0xffffffffc0540000 [ 151.790392] ? lock_torture_cleanup+0x20/0x20 [dlm_locktorture] [ 151.791347] ? delay_tsc+0x94/0xc0 [ 151.791898] torture_ex_iter+0xc3/0xea [dlm_locktorture] [ 151.792735] ? torture_start+0x30/0x30 [dlm_locktorture] [ 151.793606] lock_torture+0x177/0x270 [dlm_locktorture] [ 151.794448] ? torture_dlm_lock_sync.isra.3+0x150/0x150 [dlm_locktorture] [ 151.795539] ? lock_torture_stats+0x80/0x80 [dlm_locktorture] [ 151.796476] ? do_raw_spin_lock+0x11e/0x1e0 [ 151.797152] ? mark_held_locks+0x34/0xb0 [ 151.797784] ? _raw_spin_unlock_irqrestore+0x30/0x70 [ 151.798581] ? __kthread_parkme+0x79/0x110 [ 151.799246] ? trace_preempt_on+0x2a/0xf0 [ 151.799902] ? __kthread_parkme+0x79/0x110 [ 151.800579] ? preempt_count_sub+0xd6/0x130 [ 151.801271] ? __kasan_check_read+0x11/0x20 [ 151.801963] ? __kthread_parkme+0xec/0x110 [ 151.802630] ? lock_torture_stats+0x80/0x80 [dlm_locktorture] [ 151.803569] kthread+0x192/0x1d0 [ 151.804104] ? kthread_complete_and_exit+0x30/0x30 [ 151.804881] ret_from_fork+0x1f/0x30 [ 151.805480] [ 151.806111] Allocated by task 1347: [ 151.806681] kasan_save_stack+0x26/0x50 [ 151.807308] kasan_set_track+0x25/0x30 [ 151.807920] kasan_save_alloc_info+0x1e/0x30 [ 151.808609] __kasan_slab_alloc+0x63/0x80 [ 151.809263] kmem_cache_alloc+0x1ad/0x830 [ 151.809916] dlm_allocate_mhandle+0x17/0x20 [ 151.810590] dlm_midcomms_get_mhandle+0x96/0x260 [ 151.811344] _create_message+0x95/0x180 [ 151.811994] create_message.isra.29.constprop.64+0x57/0xc0 [ 151.812880] send_common+0x129/0x1b0 [ 151.813467] _convert_lock+0x46/0x150 [ 151.814074] convert_lock+0x7b/0xc0 [ 151.814648] dlm_lock+0x3ac/0x580 [ 151.815199] torture_dlm_lock_sync.isra.3+0xe9/0x150 [dlm_locktorture] [ 151.816258] torture_ex_iter+0xc3/0xea [dlm_locktorture] [ 151.817129] lock_torture+0x177/0x270 [dlm_locktorture] [ 151.817986] kthread+0x192/0x1d0 [ 151.818518] ret_from_fork+0x1f/0x30 [ 151.819369] Freed by task 1336: [ 151.819890] kasan_save_stack+0x26/0x50 [ 151.820514] kasan_set_track+0x25/0x30 [ 151.821128] kasan_save_free_info+0x2e/0x50 [ 151.821812] __kasan_slab_free+0x107/0x1a0 [ 151.822483] kmem_cache_free+0x204/0x5e0 [ 151.823152] dlm_free_mhandle+0x18/0x20 [ 151.823781] dlm_mhandle_release+0x2e/0x40 [ 151.824454] rcu_core+0x583/0x1330 [ 151.825047] rcu_core_si+0xe/0x20 [ 151.825594] __do_softirq+0xf4/0x5c2 [ 151.826450] Last potentially related work creation: [ 151.827238] kasan_save_stack+0x26/0x50 [ 151.827870] __kasan_record_aux_stack+0xa2/0xc0 [ 151.828609] kasan_record_aux_stack_noalloc+0xb/0x20 [ 151.829415] call_rcu+0x4c/0x760 [ 151.829954] dlm_mhandle_delete+0x97/0xb0 [ 151.830718] dlm_process_incoming_buffer+0x2fc/0xb30 [ 151.831524] process_dlm_messages+0x16e/0x470 [ 151.832245] process_one_work+0x505/0xa10 [ 151.832905] worker_thread+0x67/0x650 [ 151.833507] kthread+0x192/0x1d0 [ 151.834046] ret_from_fork+0x1f/0x30 [ 151.834900] The buggy address belongs to the object at ffff88811a980c30 which belongs to the cache dlm_mhandle of size 88 [ 151.836894] The buggy address is located 48 bytes inside of 88-byte region [ffff88811a980c30, ffff88811a980c88) [ 151.839007] The buggy address belongs to the physical page: [ 151.839904] page:0000000076cf5d62 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11a980 [ 151.841378] flags: 0x8000000000000200(slab|zone=2) [ 151.842141] raw: 8000000000000200 0000000000000000 dead000000000122 ffff8881089b43c0 [ 151.843401] raw: 0000000000000000 0000000000220022 00000001ffffffff 0000000000000000 [ 151.844640] page dumped because: kasan: bad access detected [ 151.845822] Memory state around the buggy address: [ 151.846602] ffff88811a980b00: fb fb fb fb fc fc fc fc fa fb fb fb fb fb fb fb [ 151.847761] ffff88811a980b80: fb fb fb fc fc fc fc fa fb fb fb fb fb fb fb fb [ 151.848921] >ffff88811a980c00: fb fb fc fc fc fc fa fb fb fb fb fb fb fb fb fb [ 151.850076] ^ [ 151.851085] ffff88811a980c80: fb fc fc fc fc fa fb fb fb fb fb fb fb fb fb fb [ 151.852269] ffff88811a980d00: fc fc fc fc fa fb fb fb fb fb fb fb fb fb fb fc [ 151.853428] ================================================================== [ 151.855618] Disabling lock debugging due to kernel taint It is accessing a mhandle in dlm_midcomms_commit_mhandle() and the mhandle was freed by a call_rcu() call in dlm_process_incoming_buffer(), dlm_mhandle_delete(). It looks like it was freed because an ack of this message was received. There is a short race between committing the dlm message to be transmitted and getting an ack back. If the ack is faster than returning from dlm_midcomms_commit_msg_3_2(), then we run into a use-after free because we still need to reference the mhandle when calling srcu_read_unlock(). To avoid that, we don't allow that mhandle to be freed between dlm_midcomms_commit_msg_3_2() and srcu_read_unlock() by using rcu read lock. We can do that because mhandle is protected by rcu handling. Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index fc015a6abe17..2e60d9a2c883 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -1214,8 +1214,15 @@ void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, dlm_free_mhandle(mh); break; case DLM_VERSION_3_2: + /* held rcu read lock here, because we sending the + * dlm message out, when we do that we could receive + * an ack back which releases the mhandle and we + * get a use after free. + */ + rcu_read_lock(); dlm_midcomms_commit_msg_3_2(mh, name, namelen); srcu_read_unlock(&nodes_srcu, mh->idx); + rcu_read_unlock(); break; default: srcu_read_unlock(&nodes_srcu, mh->idx); From 7354fa4ef697191effedc2ae9a8293427708bbf5 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:33 -0500 Subject: [PATCH 043/265] fs: dlm: be sure to call dlm_send_queue_flush() If we release a midcomms node structure, there should be nothing left inside the dlm midcomms send queue. However, sometimes this is not true because I believe some DLM_FIN message was not acked... if we run into a shutdown timeout, then we should be sure there is no pending send dlm message inside this queue when releasing midcomms node structure. Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 2e60d9a2c883..a3eb19c8cec5 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -1402,6 +1402,7 @@ static void midcomms_node_release(struct rcu_head *rcu) struct midcomms_node *node = container_of(rcu, struct midcomms_node, rcu); WARN_ON_ONCE(atomic_read(&node->send_queue_cnt)); + dlm_send_queue_flush(node); kfree(node); } From 164272113b685927126c938b4a9cbd2075eb15ee Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:34 -0500 Subject: [PATCH 044/265] fs: dlm: fix race setting stop tx flag This patch sets the stop tx flag before we commit the dlm message. This flag will report about unexpected transmissions after we send the DLM_FIN message out, which should be the last message sent. When we commit the dlm fin message, it could be that we already got an ack back and the CLOSED state change already happened. We should not set this flag when we are in CLOSED state. To avoid this race we simply set the tx flag before the state change can be in progress by moving it before dlm_midcomms_commit_mhandle(). Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index a3eb19c8cec5..9d459d5bf800 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -406,6 +406,7 @@ static int dlm_send_fin(struct midcomms_node *node, if (!mh) return -ENOMEM; + set_bit(DLM_NODE_FLAG_STOP_TX, &node->flags); mh->ack_rcv = ack_rcv; m_header = (struct dlm_header *)ppc; @@ -417,7 +418,6 @@ static int dlm_send_fin(struct midcomms_node *node, pr_debug("sending fin msg to node %d\n", node->nodeid); dlm_midcomms_commit_mhandle(mh, NULL, 0); - set_bit(DLM_NODE_FLAG_STOP_TX, &node->flags); return 0; } From 15c63db8e86a72e0d5cfb9bf0cd1870e39a3e5fe Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:35 -0500 Subject: [PATCH 045/265] fs: dlm: don't set stop rx flag after node reset Similar to the stop tx flag, the rx flag should warn about a dlm message being received at DLM_FIN state change, when we are assuming no other dlm application messages. If we receive a FIN message and we are in the state DLM_FIN_WAIT2 we call midcomms_node_reset() which puts the midcomms node into DLM_CLOSED state. Afterwards we should not set the DLM_NODE_FLAG_STOP_RX flag any more. This patch changes the setting DLM_NODE_FLAG_STOP_RX in those state changes when we receive a FIN message and we assume there will be no other dlm application messages received until we hit DLM_CLOSED state. Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 9d459d5bf800..1fef99214204 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -524,6 +524,7 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, break; case DLM_FIN_WAIT1: node->state = DLM_CLOSING; + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); break; @@ -544,8 +545,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, return; } spin_unlock(&node->state_lock); - - set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); break; default: WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_RX, &node->flags)); From a58496361802070996f9bd76e941d109c4a85ebd Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:36 -0500 Subject: [PATCH 046/265] fs: dlm: move sending fin message into state change handling This patch moves the send fin handling, which should appear in a specific state change, into the state change handling while the per node state_lock is held. I experienced issues with other messages because we changed the state and a fin message was sent out in a different state. Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 1fef99214204..fd9f413cabcd 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -402,7 +402,7 @@ static int dlm_send_fin(struct midcomms_node *node, struct dlm_mhandle *mh; char *ppc; - mh = dlm_midcomms_get_mhandle(node->nodeid, mb_len, GFP_NOFS, &ppc); + mh = dlm_midcomms_get_mhandle(node->nodeid, mb_len, GFP_ATOMIC, &ppc); if (!mh) return -ENOMEM; @@ -518,8 +518,8 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, node->state = DLM_LAST_ACK; pr_debug("switch node %d to state %s case 1\n", node->nodeid, dlm_state_str(node->state)); - spin_unlock(&node->state_lock); - goto send_fin; + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); + dlm_send_fin(node, dlm_pas_fin_ack_rcv); } break; case DLM_FIN_WAIT1: @@ -563,12 +563,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, log_print_ratelimited("ignore dlm msg because seq mismatch, seq: %u, expected: %u, nodeid: %d", seq, node->seq_next, node->nodeid); } - - return; - -send_fin: - set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); - dlm_send_fin(node, dlm_pas_fin_ack_rcv); } static struct midcomms_node * @@ -1368,11 +1362,11 @@ void dlm_midcomms_remove_member(int nodeid) case DLM_CLOSE_WAIT: /* passive shutdown DLM_LAST_ACK case 2 */ node->state = DLM_LAST_ACK; - spin_unlock(&node->state_lock); - pr_debug("switch node %d to state %s case 2\n", node->nodeid, dlm_state_str(node->state)); - goto send_fin; + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); + dlm_send_fin(node, dlm_pas_fin_ack_rcv); + break; case DLM_LAST_ACK: /* probably receive fin caught it, do nothing */ break; @@ -1388,12 +1382,6 @@ void dlm_midcomms_remove_member(int nodeid) spin_unlock(&node->state_lock); srcu_read_unlock(&nodes_srcu, idx); - return; - -send_fin: - set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); - dlm_send_fin(node, dlm_pas_fin_ack_rcv); - srcu_read_unlock(&nodes_srcu, idx); } static void midcomms_node_release(struct rcu_head *rcu) @@ -1425,6 +1413,7 @@ static void midcomms_shutdown(struct midcomms_node *node) node->state = DLM_FIN_WAIT1; pr_debug("switch node %d to state %s case 2\n", node->nodeid, dlm_state_str(node->state)); + dlm_send_fin(node, dlm_act_fin_ack_rcv); break; case DLM_CLOSED: /* we have what we want */ @@ -1438,12 +1427,8 @@ static void midcomms_shutdown(struct midcomms_node *node) } spin_unlock(&node->state_lock); - if (node->state == DLM_FIN_WAIT1) { - dlm_send_fin(node, dlm_act_fin_ack_rcv); - - if (DLM_DEBUG_FENCE_TERMINATION) - msleep(5000); - } + if (DLM_DEBUG_FENCE_TERMINATION) + msleep(5000); /* wait for other side dlm + fin */ ret = wait_event_timeout(node->shutdown_wait, From 00908b3388255fc1d3782b744d07f327712f401f Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:10:37 -0500 Subject: [PATCH 047/265] fs: dlm: send FIN ack back in right cases This patch moves to send a ack back for receiving a FIN message only when we are in valid states. In other cases and there might be a sender waiting for a ack we just let it timeout at the senders time and hopefully all other cleanups will remove the FIN message on their sending queue. As an example we should never send out an ACK being in LAST_ACK state or we cannot assume a working socket communication when we are in CLOSED state. Cc: stable@vger.kernel.org Fixes: 489d8e559c65 ("fs: dlm: add reliable connection if reconnect") Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index fd9f413cabcd..ecfb3beb0bb8 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -375,7 +375,7 @@ static int dlm_send_ack(int nodeid, uint32_t seq) struct dlm_msg *msg; char *ppc; - msg = dlm_lowcomms_new_msg(nodeid, mb_len, GFP_NOFS, &ppc, + msg = dlm_lowcomms_new_msg(nodeid, mb_len, GFP_ATOMIC, &ppc, NULL, NULL); if (!msg) return -ENOMEM; @@ -498,15 +498,14 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, switch (p->header.h_cmd) { case DLM_FIN: - /* send ack before fin */ - dlm_send_ack(node->nodeid, node->seq_next); - spin_lock(&node->state_lock); pr_debug("receive fin msg from node %d with state %s\n", node->nodeid, dlm_state_str(node->state)); switch (node->state) { case DLM_ESTABLISHED: + dlm_send_ack(node->nodeid, node->seq_next); + node->state = DLM_CLOSE_WAIT; pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); @@ -523,12 +522,14 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, } break; case DLM_FIN_WAIT1: + dlm_send_ack(node->nodeid, node->seq_next); node->state = DLM_CLOSING; set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); break; case DLM_FIN_WAIT2: + dlm_send_ack(node->nodeid, node->seq_next); midcomms_node_reset(node); pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); From 54fbe0c15d9a8072e2db7f6765f50c995834e2d7 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:42 -0500 Subject: [PATCH 048/265] fs: dlm: bring back previous shutdown handling This patch mostly reverts commit 4f567acb0b86 ("fs: dlm: remove socket shutdown handling"). There can be situations where the dlm midcomms nodes hash and lowcomms connection hash are not equal, but we need to guarantee that the lowcomms are all closed on a last release of a dlm lockspace, when a shutdown is invoked. This patch guarantees that we always close all sockets managed by the lowcomms connection hash, and calls shutdown for the last message sent. This ensures we don't cut the socket, which could cause the peer to get a connection reset. In future we should try to merge the midcomms/lowcomms hashes into one hash and not handle both in separate hashes. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/lowcomms.c | 77 ++++++++++++++++++++++++++++++++++------------- fs/dlm/midcomms.c | 20 +++++------- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index 4450721ec83c..61cd6c2628fa 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -61,6 +61,7 @@ #include "memory.h" #include "config.h" +#define DLM_SHUTDOWN_WAIT_TIMEOUT msecs_to_jiffies(5000) #define NEEDED_RMEM (4*1024*1024) struct connection { @@ -99,6 +100,7 @@ struct connection { struct connection *othercon; struct work_struct rwork; /* receive worker */ struct work_struct swork; /* send worker */ + wait_queue_head_t shutdown_wait; unsigned char rx_leftover_buf[DLM_MAX_SOCKET_BUFSIZE]; int rx_leftover; int mark; @@ -282,6 +284,7 @@ static void dlm_con_init(struct connection *con, int nodeid) INIT_WORK(&con->swork, process_send_sockets); INIT_WORK(&con->rwork, process_recv_sockets); spin_lock_init(&con->addrs_lock); + init_waitqueue_head(&con->shutdown_wait); } /* @@ -790,6 +793,43 @@ static void close_connection(struct connection *con, bool and_other) up_write(&con->sock_lock); } +static void shutdown_connection(struct connection *con, bool and_other) +{ + int ret; + + if (con->othercon && and_other) + shutdown_connection(con->othercon, false); + + flush_workqueue(io_workqueue); + down_read(&con->sock_lock); + /* nothing to shutdown */ + if (!con->sock) { + up_read(&con->sock_lock); + return; + } + + ret = kernel_sock_shutdown(con->sock, SHUT_WR); + up_read(&con->sock_lock); + if (ret) { + log_print("Connection %p failed to shutdown: %d will force close", + con, ret); + goto force_close; + } else { + ret = wait_event_timeout(con->shutdown_wait, !con->sock, + DLM_SHUTDOWN_WAIT_TIMEOUT); + if (ret == 0) { + log_print("Connection %p shutdown timed out, will force close", + con); + goto force_close; + } + } + + return; + +force_close: + close_connection(con, false); +} + static struct processqueue_entry *new_processqueue_entry(int nodeid, int buflen) { @@ -1488,6 +1528,7 @@ static void process_recv_sockets(struct work_struct *work) break; case DLM_IO_EOF: close_connection(con, false); + wake_up(&con->shutdown_wait); /* CF_RECV_PENDING cleared */ break; case DLM_IO_RESCHED: @@ -1695,6 +1736,9 @@ static int work_start(void) void dlm_lowcomms_shutdown(void) { + struct connection *con; + int i, idx; + /* stop lowcomms_listen_data_ready calls */ lock_sock(listen_con.sock->sk); listen_con.sock->sk->sk_data_ready = listen_sock.sk_data_ready; @@ -1703,29 +1747,20 @@ void dlm_lowcomms_shutdown(void) cancel_work_sync(&listen_con.rwork); dlm_close_sock(&listen_con.sock); - flush_workqueue(process_workqueue); -} - -void dlm_lowcomms_shutdown_node(int nodeid, bool force) -{ - struct connection *con; - int idx; - idx = srcu_read_lock(&connections_srcu); - con = nodeid2con(nodeid, 0); - if (WARN_ON_ONCE(!con)) { - srcu_read_unlock(&connections_srcu, idx); - return; - } + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_rcu(con, &connection_hash[i], list) { + shutdown_connection(con, true); + stop_connection_io(con); + flush_workqueue(process_workqueue); + close_connection(con, true); - flush_work(&con->swork); - stop_connection_io(con); - WARN_ON_ONCE(!force && !list_empty(&con->writequeue)); - close_connection(con, true); - clean_one_writequeue(con); - if (con->othercon) - clean_one_writequeue(con->othercon); - allow_connection_io(con); + clean_one_writequeue(con); + if (con->othercon) + clean_one_writequeue(con->othercon); + allow_connection_io(con); + } + } srcu_read_unlock(&connections_srcu, idx); } diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index ecfb3beb0bb8..ecd81018d1cf 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -1418,8 +1418,7 @@ static void midcomms_shutdown(struct midcomms_node *node) break; case DLM_CLOSED: /* we have what we want */ - spin_unlock(&node->state_lock); - return; + break; default: /* busy to enter DLM_FIN_WAIT1, wait until passive * done in shutdown_wait to enter DLM_CLOSED. @@ -1436,17 +1435,12 @@ static void midcomms_shutdown(struct midcomms_node *node) node->state == DLM_CLOSED || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags), DLM_SHUTDOWN_TIMEOUT); - if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags)) { + if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags)) pr_debug("active shutdown timed out for node %d with state %s\n", node->nodeid, dlm_state_str(node->state)); - midcomms_node_reset(node); - dlm_lowcomms_shutdown_node(node->nodeid, true); - return; - } - - pr_debug("active shutdown done for node %d with state %s\n", - node->nodeid, dlm_state_str(node->state)); - dlm_lowcomms_shutdown_node(node->nodeid, false); + else + pr_debug("active shutdown done for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); } void dlm_midcomms_shutdown(void) @@ -1454,8 +1448,6 @@ void dlm_midcomms_shutdown(void) struct midcomms_node *node; int i, idx; - dlm_lowcomms_shutdown(); - mutex_lock(&close_lock); idx = srcu_read_lock(&nodes_srcu); for (i = 0; i < CONN_HASH_SIZE; i++) { @@ -1473,6 +1465,8 @@ void dlm_midcomms_shutdown(void) } srcu_read_unlock(&nodes_srcu, idx); mutex_unlock(&close_lock); + + dlm_lowcomms_shutdown(); } int dlm_midcomms_close(int nodeid) From 89835b064fe7421a60e40770c27d38024190a0c8 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:43 -0500 Subject: [PATCH 049/265] fs: dlm: ignore unexpected non dlm opts msgs This patch ignores unexpected RCOM_NAMES/RCOM_STATUS messages. To be backwards compatible, those messages are not part of the new reliable DLM OPTS encapsulation header, and have their own retransmit handling using sequence number matching When we get unexpected non dlm opts messages, we should allow them and let RCOM message handling filter them out using sequence numbers. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index ecd81018d1cf..dbc998b2748b 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -606,16 +606,8 @@ dlm_midcomms_recv_node_lookup(int nodeid, const union dlm_packet *p, case DLM_ESTABLISHED: break; default: - /* some invalid state passive shutdown - * was failed, we try to reset and - * hope it will go on. - */ - log_print("reset node %d because shutdown stuck", - node->nodeid); - - midcomms_node_reset(node); - node->state = DLM_ESTABLISHED; - break; + spin_unlock(&node->state_lock); + return NULL; } spin_unlock(&node->state_lock); } From b8b750e0c99f39223115f2672ac4cfa96ecb9edd Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:44 -0500 Subject: [PATCH 050/265] fs: dlm: wait until all midcomms nodes detect version The current dlm version detection is very complex due to backwards compatablilty with earlier dlm protocol versions. It takes some time to detect if a peer node has a specific DLM version. If it's not detected, we just cut the socket connection. There could be cases where the local node has not detected the version yet, but the peer node has. In these cases, we are trying to shutdown the dlm connection with a FIN/ACK message exchange to be sure the other peer is ready to shutdown the connection on dlm application level. However this mechanism is only available on DLM protocol version 3.2 and we need to be sure the DLM version is detected before. To make it more robust we introduce a a "best effort" wait to wait for the version detection before shutdown the dlm connection. This need to be done before the kthread recoverd for recovery handling is stopped, because recovery handling will trigger enough messages to have a version detection going on. It is a corner case which was detected by modprobe dlm_locktroture module and rmmod dlm_locktorture module directly afterwards (in a looping behaviour). In practice probably nobody would leave a lockspace immediately after joining it. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/lockspace.c | 3 +++ fs/dlm/midcomms.c | 23 +++++++++++++++++++++++ fs/dlm/midcomms.h | 1 + 3 files changed, 27 insertions(+) diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index 99bc96f90779..d9dc0b734002 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -820,6 +820,9 @@ static int release_lockspace(struct dlm_ls *ls, int force) return rv; } + if (ls_count == 1) + dlm_midcomms_version_wait(); + dlm_device_deregister(ls); if (force < 3 && dlm_user_daemon_available()) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index dbc998b2748b..cf91a5a11b4f 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -657,6 +657,7 @@ static int dlm_midcomms_version_check_3_2(struct midcomms_node *node) switch (node->version) { case DLM_VERSION_NOT_SET: node->version = DLM_VERSION_3_2; + wake_up(&node->shutdown_wait); log_print("version 0x%08x for node %d detected", DLM_VERSION_3_2, node->nodeid); break; @@ -826,6 +827,7 @@ static int dlm_midcomms_version_check_3_1(struct midcomms_node *node) switch (node->version) { case DLM_VERSION_NOT_SET: node->version = DLM_VERSION_3_1; + wake_up(&node->shutdown_wait); log_print("version 0x%08x for node %d detected", DLM_VERSION_3_1, node->nodeid); break; @@ -1386,6 +1388,27 @@ static void midcomms_node_release(struct rcu_head *rcu) kfree(node); } +void dlm_midcomms_version_wait(void) +{ + struct midcomms_node *node; + int i, idx, ret; + + idx = srcu_read_lock(&nodes_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_rcu(node, &node_hash[i], hlist) { + ret = wait_event_timeout(node->shutdown_wait, + node->version != DLM_VERSION_NOT_SET || + node->state == DLM_CLOSED || + test_bit(DLM_NODE_FLAG_CLOSE, &node->flags), + DLM_SHUTDOWN_TIMEOUT); + if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags)) + pr_debug("version wait timed out for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + } + } + srcu_read_unlock(&nodes_srcu, idx); +} + static void midcomms_shutdown(struct midcomms_node *node) { int ret; diff --git a/fs/dlm/midcomms.h b/fs/dlm/midcomms.h index bea1cee4279c..9f8c9605013d 100644 --- a/fs/dlm/midcomms.h +++ b/fs/dlm/midcomms.h @@ -20,6 +20,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, gfp_t allocation, char **ppc); void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name, int namelen); +void dlm_midcomms_version_wait(void); int dlm_midcomms_close(int nodeid); int dlm_midcomms_start(void); void dlm_midcomms_stop(void); From 317dd6ba6ccaa5e7c24039955c39df84072d5166 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:45 -0500 Subject: [PATCH 051/265] fs: dlm: make dlm sequence id more robust When joining a new lockspace, use a random number to initialize a sequence number used in messages. This makes it easier to detect sequence number mismatches in message replies during tests that repeatedly join and leave a lockspace. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/lockspace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index d9dc0b734002..9f344d76afa3 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -572,7 +572,7 @@ static int new_lockspace(const char *name, const char *cluster, spin_lock_init(&ls->ls_rcom_spin); get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t)); ls->ls_recover_status = 0; - ls->ls_recover_seq = 0; + ls->ls_recover_seq = get_random_u64(); ls->ls_recover_args = NULL; init_rwsem(&ls->ls_in_recovery); init_rwsem(&ls->ls_recv_active); From 11605353f27c23902b6fb993a57bd2f0baaff3b6 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:46 -0500 Subject: [PATCH 052/265] fs: dlm: reduce the shutdown timeout to 5 secs When a shutdown is stuck, time out after 5 seconds instead of 3 minutes. After this timeout we try a forced shutdown. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index cf91a5a11b4f..44e29f99de4e 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -146,8 +146,8 @@ /* init value for sequence numbers for testing purpose only e.g. overflows */ #define DLM_SEQ_INIT 0 -/* 3 minutes wait to sync ending of dlm */ -#define DLM_SHUTDOWN_TIMEOUT msecs_to_jiffies(3 * 60 * 1000) +/* 5 seconds wait to sync ending of dlm */ +#define DLM_SHUTDOWN_TIMEOUT msecs_to_jiffies(5000) #define DLM_VERSION_NOT_SET 0 struct midcomms_node { From 3186409711e64ed0a0fa767fad555a5e655022da Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:47 -0500 Subject: [PATCH 053/265] fs: dlm: remove newline in log_print There is an API difference between log_print() and other printk()s to put a newline or not. This one was introduced by mistake because log_print() adds a newline. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 44e29f99de4e..7827d049378c 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -467,7 +467,7 @@ static void dlm_pas_fin_ack_rcv(struct midcomms_node *node) break; default: spin_unlock(&node->state_lock); - log_print("%s: unexpected state: %d\n", + log_print("%s: unexpected state: %d", __func__, node->state); WARN_ON_ONCE(1); return; @@ -540,7 +540,7 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, break; default: spin_unlock(&node->state_lock); - log_print("%s: unexpected state: %d\n", + log_print("%s: unexpected state: %d", __func__, node->state); WARN_ON_ONCE(1); return; @@ -1269,7 +1269,7 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node) break; default: spin_unlock(&node->state_lock); - log_print("%s: unexpected state: %d\n", + log_print("%s: unexpected state: %d", __func__, node->state); WARN_ON_ONCE(1); return; @@ -1369,7 +1369,7 @@ void dlm_midcomms_remove_member(int nodeid) /* already gone, do nothing */ break; default: - log_print("%s: unexpected state: %d\n", + log_print("%s: unexpected state: %d", __func__, node->state); break; } From ef7ef015eb333c8954d7165d216ec0c774d7e4e4 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:48 -0500 Subject: [PATCH 054/265] fs: dlm: move state change into else branch Currently we can switch at first into DLM_CLOSE_WAIT state and then do another state change if a condition is true. Instead of doing two state changes we handle the other state change inside an else branch of this condition. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 7827d049378c..69e6f4705ebc 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -506,9 +506,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, case DLM_ESTABLISHED: dlm_send_ack(node->nodeid, node->seq_next); - node->state = DLM_CLOSE_WAIT; - pr_debug("switch node %d to state %s\n", - node->nodeid, dlm_state_str(node->state)); /* passive shutdown DLM_LAST_ACK case 1 * additional we check if the node is used by * cluster manager events at all. @@ -519,6 +516,10 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, node->nodeid, dlm_state_str(node->state)); set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); dlm_send_fin(node, dlm_pas_fin_ack_rcv); + } else { + node->state = DLM_CLOSE_WAIT; + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); } break; case DLM_FIN_WAIT1: From 723b197bbdf1e0adbab772b8e5e022c40db6a9fe Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Thu, 12 Jan 2023 17:18:49 -0500 Subject: [PATCH 055/265] fs: dlm: remove unnecessary waker_up() calls The wake_up() is already handled inside of midcomms_node_reset() when switching the state to CLOSED state. So there is not need to call it after midcomms_node_reset() again. Signed-off-by: Alexander Aring Signed-off-by: David Teigland --- fs/dlm/midcomms.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c index 69e6f4705ebc..c02c43e4980a 100644 --- a/fs/dlm/midcomms.c +++ b/fs/dlm/midcomms.c @@ -534,7 +534,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, midcomms_node_reset(node); pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); - wake_up(&node->shutdown_wait); break; case DLM_LAST_ACK: /* probably remove_member caught it, do nothing */ @@ -1262,7 +1261,6 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node) midcomms_node_reset(node); pr_debug("switch node %d to state %s\n", node->nodeid, dlm_state_str(node->state)); - wake_up(&node->shutdown_wait); break; case DLM_CLOSED: /* not valid but somehow we got what we want */ From 3bea4ae1c97bab48bef4b1694915b43a5543c9c7 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 16:49:00 +0100 Subject: [PATCH 056/265] udf: Zero udf name padding Padding of name in the directory entry needs to be zeroed out. Fix it. Signed-off-by: Jan Kara --- fs/udf/directory.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 18a1514871fe..e8cf13214c8c 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -367,6 +367,8 @@ static void udf_copy_fi_to_bufs(char *buf1, int len1, char *buf2, int len2, int fioff = off; int crcoff = off + sizeof(struct tag); unsigned int crclen = udf_dir_entry_len(fi) - sizeof(struct tag); + char zeros[UDF_NAME_PAD] = {}; + int endoff = off + udf_dir_entry_len(fi); udf_copy_to_bufs(buf1, len1, buf2, len2, off, fi, sizeof(struct fileIdentDesc)); @@ -375,9 +377,13 @@ static void udf_copy_fi_to_bufs(char *buf1, int len1, char *buf2, int len2, udf_copy_to_bufs(buf1, len1, buf2, len2, off, impuse, le16_to_cpu(fi->lengthOfImpUse)); off += le16_to_cpu(fi->lengthOfImpUse); - if (name) + if (name) { udf_copy_to_bufs(buf1, len1, buf2, len2, off, name, fi->lengthFileIdent); + off += fi->lengthFileIdent; + udf_copy_to_bufs(buf1, len1, buf2, len2, off, zeros, + endoff - off); + } crc = udf_crc_fi_bufs(buf1, len1, buf2, len2, crcoff, crclen); fi->descTag.descCRC = cpu_to_le16(crc); From ee454ad2fce7baa272fc0a69f93d47013ea06b07 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 11:46:19 +0100 Subject: [PATCH 057/265] udf: Propagate errors from udf_advance_blk() When we spot directory corruption when trying to load next directory extent, we didn't propagate the error up properly, leading to possibly indefinite looping on corrupted directories. Fix the problem by propagating the error properly. Signed-off-by: Jan Kara --- fs/udf/directory.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index e8cf13214c8c..b1424e2aa868 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -217,7 +217,9 @@ static int udf_fiiter_load_bhs(struct udf_fileident_iter *iter) /* Need to fetch next block to get name? */ if (off + udf_dir_entry_len(fi) > blksize) { fetch_next: - udf_fiiter_advance_blk(iter); + err = udf_fiiter_advance_blk(iter); + if (err) + goto out_brelse; iter->bh[1] = udf_fiiter_bread_blk(iter); if (!iter->bh[1]) { err = -ENOMEM; @@ -296,7 +298,9 @@ int udf_fiiter_advance(struct udf_fileident_iter *iter) iter->bh[0] = iter->bh[1]; iter->bh[1] = NULL; } else { - udf_fiiter_advance_blk(iter); + err = udf_fiiter_advance_blk(iter); + if (err < 0) + return err; } } err = udf_fiiter_load_bhs(iter); From 1ea1cd11c72d1405a6b98440a9d5ea82dfa07166 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 11:43:03 +0100 Subject: [PATCH 058/265] udf: Fix directory iteration for longer tail extents When directory's last extent has more that one block and its length is not multiple of a block side, the code wrongly decided to move to the next extent instead of processing the last partial block. This led to directory corruption. Fix the rounding issue. Signed-off-by: Jan Kara --- fs/udf/directory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index b1424e2aa868..31f1bf8ab848 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -170,7 +170,7 @@ static struct buffer_head *udf_fiiter_bread_blk(struct udf_fileident_iter *iter) static int udf_fiiter_advance_blk(struct udf_fileident_iter *iter) { iter->loffset++; - if (iter->loffset < iter->elen >> iter->dir->i_blkbits) + if (iter->loffset < DIV_ROUND_UP(iter->elen, 1<dir->i_blkbits)) return 0; iter->loffset = 0; From bd904f3c74969442b0ac42f81c0d02da54cd47eb Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 13:55:31 +0100 Subject: [PATCH 059/265] udf: Unify types in anchor block detection When detecting last recorded block and from it derived anchor block position, we were mixing unsigned long, u32, and sector_t types. Since udf supports only 32-bit block numbers this is harmless but sometimes makes things awkward. Convert everything to udf_pblk_t and also handle the situation when block device size would not fit into udf_pblk_t. Signed-off-by: Jan Kara --- fs/udf/lowlevel.c | 7 +++++-- fs/udf/super.c | 4 ++-- fs/udf/udfdecl.h | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/udf/lowlevel.c b/fs/udf/lowlevel.c index 46d697172197..c87ed942d076 100644 --- a/fs/udf/lowlevel.c +++ b/fs/udf/lowlevel.c @@ -45,7 +45,7 @@ unsigned int udf_get_last_session(struct super_block *sb) return 0; } -unsigned long udf_get_last_block(struct super_block *sb) +udf_pblk_t udf_get_last_block(struct super_block *sb) { struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); unsigned long lblock = 0; @@ -54,8 +54,11 @@ unsigned long udf_get_last_block(struct super_block *sb) * The cdrom layer call failed or returned obviously bogus value? * Try using the device size... */ - if (!cdi || cdrom_get_last_written(cdi, &lblock) || lblock == 0) + if (!cdi || cdrom_get_last_written(cdi, &lblock) || lblock == 0) { + if (sb_bdev_nr_blocks(sb) > ~(udf_pblk_t)0) + return 0; lblock = sb_bdev_nr_blocks(sb); + } if (lblock) return lblock - 1; diff --git a/fs/udf/super.c b/fs/udf/super.c index 241b40e886b3..c756d903a862 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1861,10 +1861,10 @@ static int udf_check_anchor_block(struct super_block *sb, sector_t block, * Returns < 0 on error, 0 on success. -EAGAIN is special - try next set * of anchors. */ -static int udf_scan_anchors(struct super_block *sb, sector_t *lastblock, +static int udf_scan_anchors(struct super_block *sb, udf_pblk_t *lastblock, struct kernel_lb_addr *fileset) { - sector_t last[6]; + udf_pblk_t last[6]; int i; struct udf_sb_info *sbi = UDF_SB(sb); int last_count = 0; diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index d35aa42bb577..eaf9e6fd201e 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -196,7 +196,7 @@ extern void udf_new_tag(char *, uint16_t, uint16_t, uint16_t, uint32_t, int); /* lowlevel.c */ extern unsigned int udf_get_last_session(struct super_block *); -extern unsigned long udf_get_last_block(struct super_block *); +udf_pblk_t udf_get_last_block(struct super_block *); /* partition.c */ extern uint32_t udf_get_pblock(struct super_block *, uint32_t, uint16_t, From 101ee137d32adc5b53f5c2a61fbda8f70f994845 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 13:27:07 +0100 Subject: [PATCH 060/265] udf: Drop VARCONV support UDF was supporting a strange mode where the media was containing 7 blocks of unknown data for every 32 blocks of the filesystem. I have yet to see the media that would need such conversion (maybe it comes from packet writing times) and the conversions have been inconsistent in the code. In particular any write will write to a wrong block and corrupt the media. This is an indication and no user actually needs this so let's just drop the support instead of trying to fix it. Signed-off-by: Jan Kara --- fs/udf/balloc.c | 2 +- fs/udf/directory.c | 4 ++-- fs/udf/inode.c | 11 ++++------ fs/udf/misc.c | 18 +---------------- fs/udf/namei.c | 4 ++-- fs/udf/super.c | 50 +++------------------------------------------- fs/udf/truncate.c | 2 +- fs/udf/udf_sb.h | 1 - fs/udf/udfdecl.h | 6 ------ 9 files changed, 14 insertions(+), 84 deletions(-) diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index 8e597db4d971..cdf90928b7f2 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -42,7 +42,7 @@ static int read_block_bitmap(struct super_block *sb, loc.logicalBlockNum = bitmap->s_extPosition; loc.partitionReferenceNum = UDF_SB(sb)->s_partition; - bh = udf_tread(sb, udf_get_lb_pblock(sb, &loc, block)); + bh = sb_bread(sb, udf_get_lb_pblock(sb, &loc, block)); if (!bh) retval = -EIO; diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 31f1bf8ab848..2e13c4b5fb81 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -141,7 +141,7 @@ static void udf_readahead_dir(struct udf_fileident_iter *iter) for (i = 0; i < ralen; i++) { blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, iter->loffset + i); - tmp = udf_tgetblk(iter->dir->i_sb, blk); + tmp = sb_getblk(iter->dir->i_sb, blk); if (tmp && !buffer_uptodate(tmp) && !buffer_locked(tmp)) bha[num++] = tmp; else @@ -160,7 +160,7 @@ static struct buffer_head *udf_fiiter_bread_blk(struct udf_fileident_iter *iter) udf_readahead_dir(iter); blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, iter->loffset); - return udf_tread(iter->dir->i_sb, blk); + return sb_bread(iter->dir->i_sb, blk); } /* diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 96873fa2f683..bcb56674773a 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1592,7 +1592,7 @@ static int udf_update_inode(struct inode *inode, int do_sync) unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits; struct udf_inode_info *iinfo = UDF_I(inode); - bh = udf_tgetblk(inode->i_sb, + bh = sb_getblk(inode->i_sb, udf_get_lb_pblock(inode->i_sb, &iinfo->i_location, 0)); if (!bh) { udf_debug("getblk failure\n"); @@ -1852,7 +1852,7 @@ int udf_setup_indirect_aext(struct inode *inode, udf_pblk_t block, neloc.logicalBlockNum = block; neloc.partitionReferenceNum = epos->block.partitionReferenceNum; - bh = udf_tgetblk(sb, udf_get_lb_pblock(sb, &neloc, 0)); + bh = sb_getblk(sb, udf_get_lb_pblock(sb, &neloc, 0)); if (!bh) return -EIO; lock_buffer(bh); @@ -2069,7 +2069,7 @@ int8_t udf_next_aext(struct inode *inode, struct extent_position *epos, epos->offset = sizeof(struct allocExtDesc); brelse(epos->bh); block = udf_get_lb_pblock(inode->i_sb, &epos->block, 0); - epos->bh = udf_tread(inode->i_sb, block); + epos->bh = sb_bread(inode->i_sb, block); if (!epos->bh) { udf_debug("reading block %u failed!\n", block); return -1; @@ -2290,8 +2290,5 @@ udf_pblk_t udf_block_map(struct inode *inode, sector_t block) up_read(&UDF_I(inode)->i_data_sem); brelse(epos.bh); - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_VARCONV)) - return udf_fixed_to_variable(ret); - else - return ret; + return ret; } diff --git a/fs/udf/misc.c b/fs/udf/misc.c index 1614d308d0f0..3777468d06ce 100644 --- a/fs/udf/misc.c +++ b/fs/udf/misc.c @@ -28,22 +28,6 @@ #include "udf_i.h" #include "udf_sb.h" -struct buffer_head *udf_tgetblk(struct super_block *sb, udf_pblk_t block) -{ - if (UDF_QUERY_FLAG(sb, UDF_FLAG_VARCONV)) - return sb_getblk(sb, udf_fixed_to_variable(block)); - else - return sb_getblk(sb, block); -} - -struct buffer_head *udf_tread(struct super_block *sb, udf_pblk_t block) -{ - if (UDF_QUERY_FLAG(sb, UDF_FLAG_VARCONV)) - return sb_bread(sb, udf_fixed_to_variable(block)); - else - return sb_bread(sb, block); -} - struct genericFormat *udf_add_extendedattr(struct inode *inode, uint32_t size, uint32_t type, uint8_t loc) { @@ -216,7 +200,7 @@ struct buffer_head *udf_read_tagged(struct super_block *sb, uint32_t block, if (block == 0xFFFFFFFF) return NULL; - bh = udf_tread(sb, block); + bh = sb_bread(sb, block); if (!bh) { udf_err(sb, "read failed, block=%u, location=%u\n", block, location); diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 261ca813a1b8..87257f49b5da 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -171,7 +171,7 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, 0); if (!newblock) return NULL; - dbh = udf_tgetblk(inode->i_sb, newblock); + dbh = sb_getblk(inode->i_sb, newblock); if (!dbh) return NULL; lock_buffer(dbh); @@ -623,7 +623,7 @@ static int udf_symlink(struct user_namespace *mnt_userns, struct inode *dir, block = udf_get_pblock(sb, block, iinfo->i_location.partitionReferenceNum, 0); - epos.bh = udf_tgetblk(sb, block); + epos.bh = sb_getblk(sb, block); if (unlikely(!epos.bh)) { err = -ENOMEM; udf_free_blocks(sb, inode, &eloc, 0, 1); diff --git a/fs/udf/super.c b/fs/udf/super.c index c756d903a862..58a3148173ac 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -734,7 +734,7 @@ static int udf_check_vsd(struct super_block *sb) * added */ for (; !nsr && sector < VSD_MAX_SECTOR_OFFSET; sector += sectorsize) { /* Read a block */ - bh = udf_tread(sb, sector >> sb->s_blocksize_bits); + bh = sb_bread(sb, sector >> sb->s_blocksize_bits); if (!bh) break; @@ -1839,10 +1839,6 @@ static int udf_check_anchor_block(struct super_block *sb, sector_t block, uint16_t ident; int ret; - if (UDF_QUERY_FLAG(sb, UDF_FLAG_VARCONV) && - udf_fixed_to_variable(block) >= sb_bdev_nr_blocks(sb)) - return -EAGAIN; - bh = udf_read_tagged(sb, block, block, &ident); if (!bh) return -EAGAIN; @@ -1924,46 +1920,6 @@ static int udf_scan_anchors(struct super_block *sb, udf_pblk_t *lastblock, return udf_check_anchor_block(sb, sbi->s_session + 512, fileset); } -/* - * Find an anchor volume descriptor and load Volume Descriptor Sequence from - * area specified by it. The function expects sbi->s_lastblock to be the last - * block on the media. - * - * Return <0 on error, 0 if anchor found. -EAGAIN is special meaning anchor - * was not found. - */ -static int udf_find_anchor(struct super_block *sb, - struct kernel_lb_addr *fileset) -{ - struct udf_sb_info *sbi = UDF_SB(sb); - sector_t lastblock = sbi->s_last_block; - int ret; - - ret = udf_scan_anchors(sb, &lastblock, fileset); - if (ret != -EAGAIN) - goto out; - - /* No anchor found? Try VARCONV conversion of block numbers */ - UDF_SET_FLAG(sb, UDF_FLAG_VARCONV); - lastblock = udf_variable_to_fixed(sbi->s_last_block); - /* Firstly, we try to not convert number of the last block */ - ret = udf_scan_anchors(sb, &lastblock, fileset); - if (ret != -EAGAIN) - goto out; - - lastblock = sbi->s_last_block; - /* Secondly, we try with converted number of the last block */ - ret = udf_scan_anchors(sb, &lastblock, fileset); - if (ret < 0) { - /* VARCONV didn't help. Clear it. */ - UDF_CLEAR_FLAG(sb, UDF_FLAG_VARCONV); - } -out: - if (ret == 0) - sbi->s_last_block = lastblock; - return ret; -} - /* * Check Volume Structure Descriptor, find Anchor block and load Volume * Descriptor Sequence. @@ -2004,7 +1960,7 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, /* Look for anchor block and load Volume Descriptor Sequence */ sbi->s_anchor = uopt->anchor; - ret = udf_find_anchor(sb, fileset); + ret = udf_scan_anchors(sb, &sbi->s_last_block, fileset); if (ret < 0) { if (!silent && ret == -EAGAIN) udf_warn(sb, "No anchor found\n"); @@ -2455,7 +2411,7 @@ static unsigned int udf_count_free_bitmap(struct super_block *sb, if (bytes) { brelse(bh); newblock = udf_get_lb_pblock(sb, &loc, ++block); - bh = udf_tread(sb, newblock); + bh = sb_bread(sb, newblock); if (!bh) { udf_debug("read failed\n"); goto out; diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c index 775edaba82ee..871856c69df5 100644 --- a/fs/udf/truncate.c +++ b/fs/udf/truncate.c @@ -240,7 +240,7 @@ int udf_truncate_extents(struct inode *inode) brelse(epos.bh); epos.offset = sizeof(struct allocExtDesc); epos.block = eloc; - epos.bh = udf_tread(sb, + epos.bh = sb_bread(sb, udf_get_lb_pblock(sb, &eloc, 0)); /* Error reading indirect block? */ if (!epos.bh) diff --git a/fs/udf/udf_sb.h b/fs/udf/udf_sb.h index 6bccff3c70f5..9af6ff7f9747 100644 --- a/fs/udf/udf_sb.h +++ b/fs/udf/udf_sb.h @@ -23,7 +23,6 @@ #define UDF_FLAG_STRICT 5 #define UDF_FLAG_UNDELETE 6 #define UDF_FLAG_UNHIDE 7 -#define UDF_FLAG_VARCONV 8 #define UDF_FLAG_UID_FORGET 11 /* save -1 for uid to disk */ #define UDF_FLAG_GID_FORGET 12 #define UDF_FLAG_UID_SET 13 diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index eaf9e6fd201e..88667a80795d 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -34,9 +34,6 @@ extern __printf(3, 4) void _udf_warn(struct super_block *sb, #define udf_debug(fmt, ...) \ pr_debug("%s:%d:%s: " fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__) -#define udf_fixed_to_variable(x) ( ( ( (x) >> 5 ) * 39 ) + ( (x) & 0x0000001F ) ) -#define udf_variable_to_fixed(x) ( ( ( (x) / 39 ) << 5 ) + ( (x) % 39 ) ) - #define UDF_EXTENT_LENGTH_MASK 0x3FFFFFFF #define UDF_EXTENT_FLAG_MASK 0xC0000000 @@ -179,9 +176,6 @@ extern int8_t udf_current_aext(struct inode *, struct extent_position *, extern void udf_update_extra_perms(struct inode *inode, umode_t mode); /* misc.c */ -extern struct buffer_head *udf_tgetblk(struct super_block *sb, - udf_pblk_t block); -extern struct buffer_head *udf_tread(struct super_block *sb, udf_pblk_t block); extern struct genericFormat *udf_add_extendedattr(struct inode *, uint32_t, uint32_t, uint8_t); extern struct genericFormat *udf_get_extendedattr(struct inode *, uint32_t, From de80dae03c436b43017dd5e479486f1f73fa6769 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 12:05:58 +0100 Subject: [PATCH 061/265] udf: Move incrementing of goal block directly into inode_getblk() inode_getblk() sets goal block for the next allocation to the currently allocated block. This is obviously one less than what the goal block should be which we fixup in udf_get_block(). Just set the right goal block directly in inode_getblk(). Signed-off-by: Jan Kara --- fs/udf/inode.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index bcb56674773a..6f190d0d590c 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -339,11 +339,6 @@ static int udf_get_block(struct inode *inode, sector_t block, iinfo = UDF_I(inode); down_write(&iinfo->i_data_sem); - if (block == iinfo->i_next_alloc_block + 1) { - iinfo->i_next_alloc_block++; - iinfo->i_next_alloc_goal++; - } - /* * Block beyond EOF and prealloc extents? Just discard preallocation * as it is not useful and complicates things. @@ -810,8 +805,8 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, goto out_free; } *new = 1; - iinfo->i_next_alloc_block = block; - iinfo->i_next_alloc_goal = newblocknum; + iinfo->i_next_alloc_block = block + 1; + iinfo->i_next_alloc_goal = newblocknum + 1; inode->i_ctime = current_time(inode); if (IS_SYNC(inode)) From f3a30be777508c371067ca2b47a55c4cb142d5f5 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 12:29:22 +0100 Subject: [PATCH 062/265] udf: Factor out block mapping into udf_map_block() Create new block mapping function udf_map_block() that takes new udf_map_rq structure describing mapping request. We will convert other places to use this function for block mapping. Signed-off-by: Jan Kara --- fs/udf/inode.c | 72 ++++++++++++++++++++++++++++++++---------------- fs/udf/udfdecl.h | 1 + 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 6f190d0d590c..3e0adfee29c4 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -52,6 +52,8 @@ #define FE_DELETE_PERMS (FE_PERM_U_DELETE | FE_PERM_G_DELETE | \ FE_PERM_O_DELETE) +struct udf_map_rq; + static umode_t udf_convert_permissions(struct fileEntry *); static int udf_update_inode(struct inode *, int); static int udf_sync_inode(struct inode *inode); @@ -320,43 +322,67 @@ int udf_expand_file_adinicb(struct inode *inode) return err; } -static int udf_get_block(struct inode *inode, sector_t block, - struct buffer_head *bh_result, int create) +#define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ + +#define UDF_BLK_MAPPED 0x01 /* Block was successfully mapped */ +#define UDF_BLK_NEW 0x02 /* Block was freshly allocated */ + +struct udf_map_rq { + sector_t lblk; + udf_pblk_t pblk; + int iflags; /* UDF_MAP_ flags determining behavior */ + int oflags; /* UDF_BLK_ flags reporting results */ +}; + +static int udf_map_block(struct inode *inode, struct udf_map_rq *map) { int err, new; - sector_t phys = 0; - struct udf_inode_info *iinfo; + struct udf_inode_info *iinfo = UDF_I(inode); - if (!create) { - phys = udf_block_map(inode, block); - if (phys) - map_bh(bh_result, inode->i_sb, phys); + map->oflags = 0; + if (!(map->iflags & UDF_MAP_CREATE)) { + map->pblk = udf_block_map(inode, map->lblk); + if (map->pblk != 0) + map->oflags |= UDF_BLK_MAPPED; return 0; } - err = -EIO; - new = 0; - iinfo = UDF_I(inode); - down_write(&iinfo->i_data_sem); /* * Block beyond EOF and prealloc extents? Just discard preallocation * as it is not useful and complicates things. */ - if (((loff_t)block) << inode->i_blkbits > iinfo->i_lenExtents) + if (((loff_t)map->lblk) << inode->i_blkbits > iinfo->i_lenExtents) udf_discard_prealloc(inode); udf_clear_extent_cache(inode); - phys = inode_getblk(inode, block, &err, &new); - if (!phys) - goto abort; - - if (new) - set_buffer_new(bh_result); - map_bh(bh_result, inode->i_sb, phys); - -abort: + map->pblk = inode_getblk(inode, map->lblk, &err, &new); up_write(&iinfo->i_data_sem); - return err; + if (err) + return err; + map->oflags |= UDF_BLK_MAPPED; + if (new) + map->oflags |= UDF_BLK_NEW; + return 0; +} + +static int udf_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + int err; + struct udf_map_rq map = { + .lblk = block, + .iflags = create ? UDF_MAP_CREATE : 0, + }; + + err = udf_map_block(inode, &map); + if (err < 0) + return err; + if (map.oflags & UDF_BLK_MAPPED) { + map_bh(bh_result, inode->i_sb, map.pblk); + if (map.oflags & UDF_BLK_NEW) + set_buffer_new(bh_result); + } + return 0; } static struct buffer_head *udf_getblk(struct inode *inode, udf_pblk_t block, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 88667a80795d..d791458fe52a 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -138,6 +138,7 @@ static inline unsigned int udf_dir_entry_len(struct fileIdentDesc *cfi) /* file.c */ extern long udf_ioctl(struct file *, unsigned int, unsigned long); + /* inode.c */ extern struct inode *__udf_iget(struct super_block *, struct kernel_lb_addr *, bool hidden_inode); From 4215db46d5389066bba103f87d6fb4aa392ec849 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 13:41:12 +0100 Subject: [PATCH 063/265] udf: Use udf_bread() in udf_get_pblock_virt15() Use udf_bread() instead of mapping and reading buffer head manually in udf_get_pblock_virt15(). Signed-off-by: Jan Kara --- fs/udf/partition.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/udf/partition.c b/fs/udf/partition.c index 4cbf40575965..92765d2f6958 100644 --- a/fs/udf/partition.c +++ b/fs/udf/partition.c @@ -54,6 +54,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, struct udf_part_map *map; struct udf_virtual_data *vdata; struct udf_inode_info *iinfo = UDF_I(sbi->s_vat_inode); + int err; map = &sbi->s_partmaps[partition]; vdata = &map->s_type_specific.s_virtual; @@ -79,9 +80,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, index = vdata->s_start_offset / sizeof(uint32_t) + block; } - loc = udf_block_map(sbi->s_vat_inode, newblock); - - bh = sb_bread(sb, loc); + bh = udf_bread(sbi->s_vat_inode, newblock, 0, &err); if (!bh) { udf_debug("get_pblock(UDF_VIRTUAL_MAP:%p,%u,%u) VAT: %u[%u]\n", sb, block, partition, loc, index); From 08931b78932d56185753af77b93b8dcaace31e04 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 14:08:10 +0100 Subject: [PATCH 064/265] udf: Use udf_bread() in udf_load_vat() Use udf_bread() instead of mapping and loadign buffer head manually in udf_load_vat(). Signed-off-by: Jan Kara --- fs/udf/super.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/udf/super.c b/fs/udf/super.c index 58a3148173ac..df5287c5d659 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1176,7 +1176,6 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) struct udf_part_map *map = &sbi->s_partmaps[p_index]; struct buffer_head *bh = NULL; struct udf_inode_info *vati; - uint32_t pos; struct virtualAllocationTable20 *vat20; sector_t blocks = sb_bdev_nr_blocks(sb); @@ -1198,10 +1197,14 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) } else if (map->s_partition_type == UDF_VIRTUAL_MAP20) { vati = UDF_I(sbi->s_vat_inode); if (vati->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { - pos = udf_block_map(sbi->s_vat_inode, 0); - bh = sb_bread(sb, pos); - if (!bh) - return -EIO; + int err = 0; + + bh = udf_bread(sbi->s_vat_inode, 0, 0, &err); + if (!bh) { + if (!err) + err = -EFSCORRUPTED; + return err; + } vat20 = (struct virtualAllocationTable20 *)bh->b_data; } else { vat20 = (struct virtualAllocationTable20 *) From f33321b29b672d0d1e565e3c1a6886bfd14d4673 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 14:26:04 +0100 Subject: [PATCH 065/265] udf: Do not call udf_block_map() on ICB files Currently udf_symlink_filler() called udf_block_map() even on files which have data stored inside the ICB. This is invalid as we cannot map blocks for such files (although so far the error got silently ignored). The call happened because we could not call block mapping function once we've acquired i_data_sem and determined whether the file has data stored in the ICB. For symlinks the situation is luckily simple as they get never modified so file type never changes once it is set. Hence we can check the file type even without i_data_sem. Just drop the i_data_sem locking and move block mapping to where it is needed. Signed-off-by: Jan Kara --- fs/udf/symlink.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index f3642f9c23f8..451d6d6c701e 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -109,27 +109,24 @@ static int udf_symlink_filler(struct file *file, struct folio *folio) unsigned char *symlink; int err; unsigned char *p = page_address(page); - struct udf_inode_info *iinfo; + struct udf_inode_info *iinfo = UDF_I(inode); uint32_t pos; /* We don't support symlinks longer than one block */ if (inode->i_size > inode->i_sb->s_blocksize) { err = -ENAMETOOLONG; - goto out_unmap; + goto out_unlock; } - iinfo = UDF_I(inode); - pos = udf_block_map(inode, 0); - - down_read(&iinfo->i_data_sem); if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { symlink = iinfo->i_data + iinfo->i_lenEAttr; } else { + pos = udf_block_map(inode, 0); bh = sb_bread(inode->i_sb, pos); if (!bh) { err = -EIO; - goto out_unlock_inode; + goto out_err; } symlink = bh->b_data; @@ -138,17 +135,15 @@ static int udf_symlink_filler(struct file *file, struct folio *folio) err = udf_pc_to_char(inode->i_sb, symlink, inode->i_size, p, PAGE_SIZE); brelse(bh); if (err) - goto out_unlock_inode; + goto out_err; - up_read(&iinfo->i_data_sem); SetPageUptodate(page); unlock_page(page); return 0; -out_unlock_inode: - up_read(&iinfo->i_data_sem); +out_err: SetPageError(page); -out_unmap: +out_unlock: unlock_page(page); return err; } From 15a08f51624a252ddadf7ce95ad5256d634873fc Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 14:32:45 +0100 Subject: [PATCH 066/265] udf: Convert udf_symlink_filler() to use udf_bread() Convert udf_symlink_filler() to use udf_bread() instead of mapping and reading buffer head manually. Signed-off-by: Jan Kara --- fs/udf/symlink.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index 451d6d6c701e..a6cabaa5f1c2 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -107,10 +107,9 @@ static int udf_symlink_filler(struct file *file, struct folio *folio) struct inode *inode = page->mapping->host; struct buffer_head *bh = NULL; unsigned char *symlink; - int err; + int err = 0; unsigned char *p = page_address(page); struct udf_inode_info *iinfo = UDF_I(inode); - uint32_t pos; /* We don't support symlinks longer than one block */ if (inode->i_size > inode->i_sb->s_blocksize) { @@ -121,14 +120,12 @@ static int udf_symlink_filler(struct file *file, struct folio *folio) if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { symlink = iinfo->i_data + iinfo->i_lenEAttr; } else { - pos = udf_block_map(inode, 0); - bh = sb_bread(inode->i_sb, pos); - + bh = udf_bread(inode, 0, 0, &err); if (!bh) { - err = -EIO; + if (!err) + err = -EFSCORRUPTED; goto out_err; } - symlink = bh->b_data; } From 364a6665d573128790bf52481ed2898579fbc472 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 14:38:35 +0100 Subject: [PATCH 067/265] udf: Fold udf_block_map() into udf_map_block() udf_block_map() has now only a single caller. Fold it there. Signed-off-by: Jan Kara --- fs/udf/inode.c | 38 ++++++++++++++------------------------ fs/udf/udfdecl.h | 1 - 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 3e0adfee29c4..08808683c7fd 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -341,9 +341,21 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) map->oflags = 0; if (!(map->iflags & UDF_MAP_CREATE)) { - map->pblk = udf_block_map(inode, map->lblk); - if (map->pblk != 0) + struct kernel_lb_addr eloc; + uint32_t elen; + sector_t offset; + struct extent_position epos = {}; + + down_read(&iinfo->i_data_sem); + if (inode_bmap(inode, map->lblk, &epos, &eloc, &elen, &offset) + == (EXT_RECORDED_ALLOCATED >> 30)) { + map->pblk = udf_get_lb_pblock(inode->i_sb, &eloc, + offset); map->oflags |= UDF_BLK_MAPPED; + } + up_read(&iinfo->i_data_sem); + brelse(epos.bh); + return 0; } @@ -2291,25 +2303,3 @@ int8_t inode_bmap(struct inode *inode, sector_t block, return etype; } - -udf_pblk_t udf_block_map(struct inode *inode, sector_t block) -{ - struct kernel_lb_addr eloc; - uint32_t elen; - sector_t offset; - struct extent_position epos = {}; - udf_pblk_t ret; - - down_read(&UDF_I(inode)->i_data_sem); - - if (inode_bmap(inode, block, &epos, &eloc, &elen, &offset) == - (EXT_RECORDED_ALLOCATED >> 30)) - ret = udf_get_lb_pblock(inode->i_sb, &eloc, offset); - else - ret = 0; - - up_read(&UDF_I(inode)->i_data_sem); - brelse(epos.bh); - - return ret; -} diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index d791458fe52a..98b4d89b4368 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -158,7 +158,6 @@ extern struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, extern int udf_setsize(struct inode *, loff_t); extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); -extern udf_pblk_t udf_block_map(struct inode *inode, sector_t block); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); extern int udf_setup_indirect_aext(struct inode *inode, udf_pblk_t block, From b3c03fcef2712eb957a7633806a98d2c84d55468 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 14:57:34 +0100 Subject: [PATCH 068/265] udf: Pass mapping request into inode_getblk() Pass struct udf_map_rq into inode_getblk() instead of unfolding it and the putting the results back. Signed-off-by: Jan Kara --- fs/udf/inode.c | 56 +++++++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 08808683c7fd..31088e905150 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -58,7 +58,7 @@ static umode_t udf_convert_permissions(struct fileEntry *); static int udf_update_inode(struct inode *, int); static int udf_sync_inode(struct inode *inode); static int udf_alloc_i_data(struct inode *inode, size_t size); -static sector_t inode_getblk(struct inode *, sector_t, int *, int *); +static int inode_getblk(struct inode *inode, struct udf_map_rq *map); static int udf_insert_aext(struct inode *, struct extent_position, struct kernel_lb_addr, uint32_t); static void udf_split_extents(struct inode *, int *, int, udf_pblk_t, @@ -336,7 +336,7 @@ struct udf_map_rq { static int udf_map_block(struct inode *inode, struct udf_map_rq *map) { - int err, new; + int err; struct udf_inode_info *iinfo = UDF_I(inode); map->oflags = 0; @@ -367,14 +367,9 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) if (((loff_t)map->lblk) << inode->i_blkbits > iinfo->i_lenExtents) udf_discard_prealloc(inode); udf_clear_extent_cache(inode); - map->pblk = inode_getblk(inode, map->lblk, &err, &new); + err = inode_getblk(inode, map); up_write(&iinfo->i_data_sem); - if (err) - return err; - map->oflags |= UDF_BLK_MAPPED; - if (new) - map->oflags |= UDF_BLK_NEW; - return 0; + return err; } static int udf_get_block(struct inode *inode, sector_t block, @@ -627,8 +622,7 @@ out: return err; } -static sector_t inode_getblk(struct inode *inode, sector_t block, - int *err, int *new) +static int inode_getblk(struct inode *inode, struct udf_map_rq *map) { struct kernel_long_ad laarr[EXTENT_MERGE_SIZE]; struct extent_position prev_epos, cur_epos, next_epos; @@ -637,21 +631,20 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, struct kernel_lb_addr eloc, tmpeloc; int c = 1; loff_t lbcount = 0, b_off = 0; - udf_pblk_t newblocknum, newblock = 0; + udf_pblk_t newblocknum; sector_t offset = 0; int8_t etype; struct udf_inode_info *iinfo = UDF_I(inode); udf_pblk_t goal = 0, pgoal = iinfo->i_location.logicalBlockNum; int lastblock = 0; bool isBeyondEOF; + int ret = 0; - *err = 0; - *new = 0; prev_epos.offset = udf_file_entry_alloc_offset(inode); prev_epos.block = iinfo->i_location; prev_epos.bh = NULL; cur_epos = next_epos = prev_epos; - b_off = (loff_t)block << inode->i_sb->s_blocksize_bits; + b_off = (loff_t)map->lblk << inode->i_sb->s_blocksize_bits; /* find the extent which contains the block we are looking for. alternate between laarr[0] and laarr[1] for locations of the @@ -715,13 +708,13 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, inode->i_sb->s_blocksize); udf_write_aext(inode, &cur_epos, &eloc, elen, 1); } - newblock = udf_get_lb_pblock(inode->i_sb, &eloc, offset); + map->oflags = UDF_BLK_MAPPED; + map->pblk = udf_get_lb_pblock(inode->i_sb, &eloc, offset); goto out_free; } /* Are we beyond EOF and preallocated extent? */ if (etype == -1) { - int ret; loff_t hole_len; isBeyondEOF = true; @@ -741,10 +734,8 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, /* Create extents for the hole between EOF and offset */ hole_len = (loff_t)offset << inode->i_blkbits; ret = udf_do_extend_file(inode, &prev_epos, laarr, hole_len); - if (ret < 0) { - *err = ret; + if (ret < 0) goto out_free; - } c = 0; offset = 0; count += ret; @@ -794,7 +785,7 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, if ((laarr[c].extLength >> 30) == (EXT_NOT_RECORDED_ALLOCATED >> 30)) newblocknum = laarr[c].extLocation.logicalBlockNum + offset; else { /* otherwise, allocate a new block */ - if (iinfo->i_next_alloc_block == block) + if (iinfo->i_next_alloc_block == map->lblk) goal = iinfo->i_next_alloc_goal; if (!goal) { @@ -804,11 +795,9 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, newblocknum = udf_new_block(inode->i_sb, inode, iinfo->i_location.partitionReferenceNum, - goal, err); - if (!newblocknum) { - *err = -ENOSPC; + goal, &ret); + if (!newblocknum) goto out_free; - } if (isBeyondEOF) iinfo->i_lenExtents += inode->i_sb->s_blocksize; } @@ -832,18 +821,18 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, /* write back the new extents, inserting new extents if the new number * of extents is greater than the old number, and deleting extents if * the new number of extents is less than the old number */ - *err = udf_update_extents(inode, laarr, startnum, endnum, &prev_epos); - if (*err < 0) + ret = udf_update_extents(inode, laarr, startnum, endnum, &prev_epos); + if (ret < 0) goto out_free; - newblock = udf_get_pblock(inode->i_sb, newblocknum, + map->pblk = udf_get_pblock(inode->i_sb, newblocknum, iinfo->i_location.partitionReferenceNum, 0); - if (!newblock) { - *err = -EIO; + if (!map->pblk) { + ret = -EFSCORRUPTED; goto out_free; } - *new = 1; - iinfo->i_next_alloc_block = block + 1; + map->oflags = UDF_BLK_NEW | UDF_BLK_MAPPED; + iinfo->i_next_alloc_block = map->lblk + 1; iinfo->i_next_alloc_goal = newblocknum + 1; inode->i_ctime = current_time(inode); @@ -851,11 +840,12 @@ static sector_t inode_getblk(struct inode *inode, sector_t block, udf_sync_inode(inode); else mark_inode_dirty(inode); + ret = 0; out_free: brelse(prev_epos.bh); brelse(cur_epos.bh); brelse(next_epos.bh); - return newblock; + return ret; } static void udf_split_extents(struct inode *inode, int *c, int offset, From 809425217805c2866a7a3019eeff92312edbd7e8 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 15:18:41 +0100 Subject: [PATCH 069/265] udf: Add flag to disable block preallocation In some cases we don't want to create block preallocation when allocating blocks. Add a flag to udf_map_rq controlling the behavior. Signed-off-by: Jan Kara --- fs/udf/inode.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 31088e905150..1a6627c91bd8 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -322,7 +322,8 @@ int udf_expand_file_adinicb(struct inode *inode) return err; } -#define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ +#define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ +#define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ #define UDF_BLK_MAPPED 0x01 /* Block was successfully mapped */ #define UDF_BLK_NEW 0x02 /* Block was freshly allocated */ @@ -381,6 +382,14 @@ static int udf_get_block(struct inode *inode, sector_t block, .iflags = create ? UDF_MAP_CREATE : 0, }; + /* + * We preallocate blocks only for regular files. It also makes sense + * for directories but there's a problem when to drop the + * preallocation. We might use some delayed work for that but I feel + * it's overengineering for a filesystem like UDF. + */ + if (!S_ISREG(inode->i_mode)) + map.iflags |= UDF_MAP_NOPREALLOC; err = udf_map_block(inode, &map); if (err < 0) return err; @@ -808,11 +817,7 @@ static int inode_getblk(struct inode *inode, struct udf_map_rq *map) * block */ udf_split_extents(inode, &c, offset, newblocknum, laarr, &endnum); - /* We preallocate blocks only for regular files. It also makes sense - * for directories but there's a problem when to drop the - * preallocation. We might use some delayed work for that but I feel - * it's overengineering for a filesystem like UDF. */ - if (S_ISREG(inode->i_mode)) + if (!(map->iflags & UDF_MAP_NOPREALLOC)) udf_prealloc_extents(inode, c, lastblock, laarr, &endnum); /* merge any continuous blocks in laarr */ From 541e047b14c8f07de05262fce948ee7667117986 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 15:30:06 +0100 Subject: [PATCH 070/265] udf: Use udf_map_block() in udf_getblk() Use the new function udf_map_block() in udf_getblk(). Signed-off-by: Jan Kara --- fs/udf/inode.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 1a6627c91bd8..d3877a356715 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -405,14 +405,15 @@ static struct buffer_head *udf_getblk(struct inode *inode, udf_pblk_t block, int create, int *err) { struct buffer_head *bh; - struct buffer_head dummy; + struct udf_map_rq map = { + .lblk = block, + .iflags = UDF_MAP_NOPREALLOC | (create ? UDF_MAP_CREATE : 0), + }; - dummy.b_state = 0; - dummy.b_blocknr = -1000; - *err = udf_get_block(inode, block, &dummy, create); - if (!*err && buffer_mapped(&dummy)) { - bh = sb_getblk(inode->i_sb, dummy.b_blocknr); - if (buffer_new(&dummy)) { + *err = udf_map_block(inode, &map); + if (!*err && map.oflags & UDF_BLK_MAPPED) { + bh = sb_getblk(inode->i_sb, map.pblk); + if (map.oflags & UDF_BLK_NEW) { lock_buffer(bh); memset(bh->b_data, 0x00, inode->i_sb->s_blocksize); set_buffer_uptodate(bh); From 32f123a3f34283f9c6446de87861696f0502b02e Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 15:33:41 +0100 Subject: [PATCH 071/265] udf: Fold udf_getblk() into udf_bread() udf_getblk() has a single call site. Fold it there. Signed-off-by: Jan Kara --- fs/udf/inode.c | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index d3877a356715..adef328c4278 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -401,31 +401,6 @@ static int udf_get_block(struct inode *inode, sector_t block, return 0; } -static struct buffer_head *udf_getblk(struct inode *inode, udf_pblk_t block, - int create, int *err) -{ - struct buffer_head *bh; - struct udf_map_rq map = { - .lblk = block, - .iflags = UDF_MAP_NOPREALLOC | (create ? UDF_MAP_CREATE : 0), - }; - - *err = udf_map_block(inode, &map); - if (!*err && map.oflags & UDF_BLK_MAPPED) { - bh = sb_getblk(inode->i_sb, map.pblk); - if (map.oflags & UDF_BLK_NEW) { - lock_buffer(bh); - memset(bh->b_data, 0x00, inode->i_sb->s_blocksize); - set_buffer_uptodate(bh); - unlock_buffer(bh); - mark_buffer_dirty_inode(bh, inode); - } - return bh; - } - - return NULL; -} - /* Extend the file with new blocks totaling 'new_block_bytes', * return the number of extents added */ @@ -1140,11 +1115,29 @@ struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, int create, int *err) { struct buffer_head *bh = NULL; + struct udf_map_rq map = { + .lblk = block, + .iflags = UDF_MAP_NOPREALLOC | (create ? UDF_MAP_CREATE : 0), + }; - bh = udf_getblk(inode, block, create, err); - if (!bh) + *err = udf_map_block(inode, &map); + if (*err || !(map.oflags & UDF_BLK_MAPPED)) return NULL; + bh = sb_getblk(inode->i_sb, map.pblk); + if (!bh) { + *err = -ENOMEM; + return NULL; + } + if (map.oflags & UDF_BLK_NEW) { + lock_buffer(bh); + memset(bh->b_data, 0x00, inode->i_sb->s_blocksize); + set_buffer_uptodate(bh); + unlock_buffer(bh); + mark_buffer_dirty_inode(bh, inode); + return bh; + } + if (bh_read(bh, 0) >= 0) return bh; From f950fd0529130a617b3da526da9fb6a896ce87c2 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 19:42:35 +0100 Subject: [PATCH 072/265] udf: Protect rename against modification of moved directory When we are renaming a directory to a different directory, we need to update '..' entry in the moved directory. However nothing prevents moved directory from being modified and even converted from the in-ICB format to the normal format which results in a crash. Fix the problem by locking the moved directory. Reported-by: syzbot+aebf90eea2671c43112a@syzkaller.appspotmail.com Signed-off-by: Jan Kara --- fs/udf/namei.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 87257f49b5da..663b66014c98 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -786,6 +786,11 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, if (!empty_dir(new_inode)) goto out_oiter; } + /* + * We need to protect against old_inode getting converted from + * ICB to normal directory. + */ + inode_lock_nested(old_inode, I_MUTEX_NONDIR2); retval = udf_fiiter_find_entry(old_inode, &dotdot_name, &diriter); if (retval == -ENOENT) { @@ -794,8 +799,10 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, old_inode->i_ino); retval = -EFSCORRUPTED; } - if (retval) + if (retval) { + inode_unlock(old_inode); goto out_oiter; + } has_diriter = true; tloc = lelb_to_cpu(diriter.fi.icb.extLocation); if (udf_get_lb_pblock(old_inode->i_sb, &tloc, 0) != @@ -873,6 +880,7 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, udf_dir_entry_len(&diriter.fi)); udf_fiiter_write_fi(&diriter, NULL); udf_fiiter_release(&diriter); + inode_unlock(old_inode); inode_dec_link_count(old_dir); if (new_inode) @@ -884,8 +892,10 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, } return 0; out_oiter: - if (has_diriter) + if (has_diriter) { udf_fiiter_release(&diriter); + inode_unlock(old_inode); + } udf_fiiter_release(&oiter); return retval; From 6a3b37e0ea20bcb50e519c75fae3387199f35400 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 19 Jan 2023 12:28:37 +0100 Subject: [PATCH 073/265] udf: Push i_data_sem locking into udf_expand_file_adinicb() The checks we do in udf_setsize() and udf_file_write_iter() are safe to do only with i_rwsem locked as it stabilizes both file type and file size. Hence we don't need to lock i_data_sem before we enter udf_expand_file_adinicb() which simplifies the locking somewhat. Signed-off-by: Jan Kara --- fs/udf/file.c | 11 +++++------ fs/udf/inode.c | 18 ++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 8be51161f3e5..60524814c594 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -148,7 +148,6 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (retval <= 0) goto out; - down_write(&iinfo->i_data_sem); if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + iocb->ki_pos + iov_iter_count(from))) { @@ -158,15 +157,15 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) udf_debug("udf_expand_adinicb: err=%d\n", err); return err; } - } else - up_write(&iinfo->i_data_sem); + } retval = __generic_file_write_iter(iocb, from); out: - down_write(&iinfo->i_data_sem); - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && retval > 0) + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && retval > 0) { + down_write(&iinfo->i_data_sem); iinfo->i_lenAlloc = inode->i_size; - up_write(&iinfo->i_data_sem); + up_write(&iinfo->i_data_sem); + } inode_unlock(inode); if (retval > 0) { diff --git a/fs/udf/inode.c b/fs/udf/inode.c index adef328c4278..627cdcb2f456 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -247,7 +247,6 @@ const struct address_space_operations udf_aops = { /* * Expand file stored in ICB to a normal one-block-file * - * This function requires i_data_sem for writing and releases it. * This function requires i_mutex held */ int udf_expand_file_adinicb(struct inode *inode) @@ -259,6 +258,7 @@ int udf_expand_file_adinicb(struct inode *inode) WARN_ON_ONCE(!inode_is_locked(inode)); if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; else @@ -269,11 +269,6 @@ int udf_expand_file_adinicb(struct inode *inode) mark_inode_dirty(inode); return 0; } - /* - * Release i_data_sem so that we can lock a page - page lock ranks - * above i_data_sem. i_mutex still protects us against file changes. - */ - up_write(&iinfo->i_data_sem); page = find_or_create_page(inode->i_mapping, 0, GFP_NOFS); if (!page) @@ -1160,19 +1155,18 @@ int udf_setsize(struct inode *inode, loff_t newsize) iinfo = UDF_I(inode); if (newsize > inode->i_size) { - down_write(&iinfo->i_data_sem); if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { - if (bsize < + if (bsize >= (udf_file_entry_alloc_offset(inode) + newsize)) { - err = udf_expand_file_adinicb(inode); - if (err) - return err; down_write(&iinfo->i_data_sem); - } else { iinfo->i_lenAlloc = newsize; goto set_size; } + err = udf_expand_file_adinicb(inode); + if (err) + return err; } + down_write(&iinfo->i_data_sem); err = udf_extend_file(inode, newsize); if (err) { up_write(&iinfo->i_data_sem); From 2d532616c768c7473b7ebcf39aa16c824c73b7fc Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 19 Jan 2023 12:37:04 +0100 Subject: [PATCH 074/265] udf: Push i_data_sem locking into udf_extend_file() Push i_data_sem locking into udf_extend_file(). It somewhat simplifies the code around it. Signed-off-by: Jan Kara --- fs/udf/inode.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 627cdcb2f456..b4e4aacdaabc 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -555,6 +555,7 @@ static int udf_extend_file(struct inode *inode, loff_t newsize) else BUG(); + down_write(&iinfo->i_data_sem); /* * When creating hole in file, just don't bother with preserving * preallocation. It likely won't be very useful anyway. @@ -599,6 +600,7 @@ static int udf_extend_file(struct inode *inode, loff_t newsize) err = 0; out: brelse(epos.bh); + up_write(&iinfo->i_data_sem); return err; } @@ -1160,20 +1162,17 @@ int udf_setsize(struct inode *inode, loff_t newsize) (udf_file_entry_alloc_offset(inode) + newsize)) { down_write(&iinfo->i_data_sem); iinfo->i_lenAlloc = newsize; + up_write(&iinfo->i_data_sem); goto set_size; } err = udf_expand_file_adinicb(inode); if (err) return err; } - down_write(&iinfo->i_data_sem); err = udf_extend_file(inode, newsize); - if (err) { - up_write(&iinfo->i_data_sem); + if (err) return err; - } set_size: - up_write(&iinfo->i_data_sem); truncate_setsize(inode, newsize); } else { if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { From 96eeaaaea592079fcbf0b18a2b6f99165b32c942 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 19 Jan 2023 12:44:34 +0100 Subject: [PATCH 075/265] udf: Simplify error handling in udf_file_write_iter() When udf_expand_file_adinicb() fails, we can now use the standard exit path instead of implementing our own. Signed-off-by: Jan Kara --- fs/udf/file.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 60524814c594..596d703fb6c8 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -140,7 +140,6 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); struct udf_inode_info *iinfo = UDF_I(inode); - int err; inode_lock(inode); @@ -151,12 +150,9 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + iocb->ki_pos + iov_iter_count(from))) { - err = udf_expand_file_adinicb(inode); - if (err) { - inode_unlock(inode); - udf_debug("udf_expand_adinicb: err=%d\n", err); - return err; - } + retval = udf_expand_file_adinicb(inode); + if (retval) + goto out; } retval = __generic_file_write_iter(iocb, from); From b9a861fd527ab123e76effb492b4eb7e8115d4ca Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 19 Jan 2023 12:46:09 +0100 Subject: [PATCH 076/265] udf: Protect truncate and file type conversion with invalidate_lock Protect truncate and file type conversion in udf_file_write_iter() with invalidate lock. That will allow us to serialize these paths with page faults so that the page fault can determine the file type in a racefree way. Signed-off-by: Jan Kara --- fs/udf/file.c | 2 ++ fs/udf/inode.c | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 596d703fb6c8..cf050bdffd9e 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -150,7 +150,9 @@ static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + iocb->ki_pos + iov_iter_count(from))) { + filemap_invalidate_lock(inode->i_mapping); retval = udf_expand_file_adinicb(inode); + filemap_invalidate_unlock(inode->i_mapping); if (retval) goto out; } diff --git a/fs/udf/inode.c b/fs/udf/inode.c index b4e4aacdaabc..f57ef7d0a207 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1145,7 +1145,7 @@ struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, int udf_setsize(struct inode *inode, loff_t newsize) { - int err; + int err = 0; struct udf_inode_info *iinfo; unsigned int bsize = i_blocksize(inode); @@ -1155,6 +1155,7 @@ int udf_setsize(struct inode *inode, loff_t newsize) if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) return -EPERM; + filemap_invalidate_lock(inode->i_mapping); iinfo = UDF_I(inode); if (newsize > inode->i_size) { if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { @@ -1167,11 +1168,11 @@ int udf_setsize(struct inode *inode, loff_t newsize) } err = udf_expand_file_adinicb(inode); if (err) - return err; + goto out_unlock; } err = udf_extend_file(inode, newsize); if (err) - return err; + goto out_unlock; set_size: truncate_setsize(inode, newsize); } else { @@ -1189,14 +1190,14 @@ set_size: err = block_truncate_page(inode->i_mapping, newsize, udf_get_block); if (err) - return err; + goto out_unlock; truncate_setsize(inode, newsize); down_write(&iinfo->i_data_sem); udf_clear_extent_cache(inode); err = udf_truncate_extents(inode); up_write(&iinfo->i_data_sem); if (err) - return err; + goto out_unlock; } update_time: inode->i_mtime = inode->i_ctime = current_time(inode); @@ -1204,7 +1205,9 @@ update_time: udf_sync_inode(inode); else mark_inode_dirty(inode); - return 0; +out_unlock: + filemap_invalidate_unlock(inode->i_mapping); + return err; } /* From 3c21204818ae45504b5d8ce8902748ef2306f0f3 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 19 Jan 2023 10:51:21 +0100 Subject: [PATCH 077/265] udf: Allocate blocks on write page fault Currently if file with holes is mapped, udf allocates blocks for dirtied pages during page writeback. This however creates problems when to truncate final extent to proper size and currently we leave the last extent untruncated which violates UDF standard. So allocate blocks on write page fault instead. In that case the last extent gets truncated the file is closed and everything is happy. Signed-off-by: Jan Kara --- fs/udf/file.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++- fs/udf/inode.c | 1 - fs/udf/udfdecl.h | 1 + 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index cf050bdffd9e..322115c8369d 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -134,6 +134,57 @@ const struct address_space_operations udf_adinicb_aops = { .direct_IO = udf_adinicb_direct_IO, }; +static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct inode *inode = file_inode(vma->vm_file); + struct address_space *mapping = inode->i_mapping; + struct page *page = vmf->page; + loff_t size; + unsigned int end; + vm_fault_t ret = VM_FAULT_LOCKED; + int err; + + sb_start_pagefault(inode->i_sb); + file_update_time(vma->vm_file); + filemap_invalidate_lock_shared(mapping); + lock_page(page); + size = i_size_read(inode); + if (page->mapping != inode->i_mapping || page_offset(page) >= size) { + unlock_page(page); + ret = VM_FAULT_NOPAGE; + goto out_unlock; + } + /* Space is already allocated for in-ICB file */ + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + goto out_dirty; + if (page->index == size >> PAGE_SHIFT) + end = size & ~PAGE_MASK; + else + end = PAGE_SIZE; + err = __block_write_begin(page, 0, end, udf_get_block); + if (!err) + err = block_commit_write(page, 0, end); + if (err < 0) { + unlock_page(page); + ret = block_page_mkwrite_return(err); + goto out_unlock; + } +out_dirty: + set_page_dirty(page); + wait_for_stable_page(page); +out_unlock: + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct udf_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = udf_page_mkwrite, +}; + static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) { ssize_t retval; @@ -238,11 +289,19 @@ static int udf_release_file(struct inode *inode, struct file *filp) return 0; } +static int udf_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + file_accessed(file); + vma->vm_ops = &udf_file_vm_ops; + + return 0; +} + const struct file_operations udf_file_operations = { .read_iter = generic_file_read_iter, .unlocked_ioctl = udf_ioctl, .open = generic_file_open, - .mmap = generic_file_mmap, + .mmap = udf_file_mmap, .write_iter = udf_file_write_iter, .release = udf_release_file, .fsync = generic_file_fsync, diff --git a/fs/udf/inode.c b/fs/udf/inode.c index f57ef7d0a207..bac4f905bbd1 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -68,7 +68,6 @@ static void udf_prealloc_extents(struct inode *, int, int, static void udf_merge_extents(struct inode *, struct kernel_long_ad *, int *); static int udf_update_extents(struct inode *, struct kernel_long_ad *, int, int, struct extent_position *); -static int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); static void __udf_clear_extent_cache(struct inode *inode) { diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 98b4d89b4368..5ba59ab90d48 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -160,6 +160,7 @@ extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); +int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); extern int udf_setup_indirect_aext(struct inode *inode, udf_pblk_t block, struct extent_position *epos); extern int __udf_add_aext(struct inode *inode, struct extent_position *epos, From 7010839ccfd4f875c23c1758f6321ee6312cc548 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 14:59:14 +0100 Subject: [PATCH 078/265] fs: gracefully handle ->get_block not mapping bh in __mpage_writepage When filesystem's ->get_block function does not map the buffer head when called from __mpage_writepage(), the function will happily go and pass bogus bdev and block number to bio allocation routines which leads to crashes sooner or later. E.g. UDF can do this because it doesn't want to allocate blocks from ->writepages callbacks. It allocates blocks on write or page fault but writeback can still spot dirty buffers without underlying blocks allocated e.g. if blocksize < pagesize, the tail page is dirtied (which means all its buffers are dirtied), and truncate extends the file so that some buffer starts to be within i_size. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/mpage.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/mpage.c b/fs/mpage.c index 0f8ae954a579..ce53179428db 100644 --- a/fs/mpage.c +++ b/fs/mpage.c @@ -532,6 +532,8 @@ static int __mpage_writepage(struct page *page, struct writeback_control *wbc, map_bh.b_size = 1 << blkbits; if (mpd->get_block(inode, block_in_file, &map_bh, 1)) goto confused; + if (!buffer_mapped(&map_bh)) + goto confused; if (buffer_new(&map_bh)) clean_bdev_bh_alias(&map_bh); if (buffer_boundary(&map_bh)) { From 36580ed08776224f25f9a34ede6eed7a4e11aed8 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 18 Jan 2023 15:09:49 +0100 Subject: [PATCH 079/265] udf: Do not allocate blocks on page writeback Now when we allocate blocks on write page fault there should be no block allocation happening on page writeback. So just ignore the 'create' flag passed to udf_get_block(). Note that we can spot dirty buffers without underlying blocks allocated in writeback when we race with expanding truncate. However in that case these buffers do not contain valid data so we can safely ignore them and we would just create ourselves problem when to trim the tail extent. Signed-off-by: Jan Kara --- fs/udf/inode.c | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index bac4f905bbd1..356e15daef18 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -68,6 +68,8 @@ static void udf_prealloc_extents(struct inode *, int, int, static void udf_merge_extents(struct inode *, struct kernel_long_ad *, int *); static int udf_update_extents(struct inode *, struct kernel_long_ad *, int, int, struct extent_position *); +static int udf_get_block_wb(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create); static void __udf_clear_extent_cache(struct inode *inode) { @@ -186,7 +188,7 @@ static void udf_write_failed(struct address_space *mapping, loff_t to) static int udf_writepages(struct address_space *mapping, struct writeback_control *wbc) { - return mpage_writepages(mapping, wbc, udf_get_block); + return mpage_writepages(mapping, wbc, udf_get_block_wb); } static int udf_read_folio(struct file *file, struct folio *folio) @@ -367,23 +369,15 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) return err; } -static int udf_get_block(struct inode *inode, sector_t block, - struct buffer_head *bh_result, int create) +static int __udf_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int flags) { int err; struct udf_map_rq map = { .lblk = block, - .iflags = create ? UDF_MAP_CREATE : 0, + .iflags = flags, }; - /* - * We preallocate blocks only for regular files. It also makes sense - * for directories but there's a problem when to drop the - * preallocation. We might use some delayed work for that but I feel - * it's overengineering for a filesystem like UDF. - */ - if (!S_ISREG(inode->i_mode)) - map.iflags |= UDF_MAP_NOPREALLOC; err = udf_map_block(inode, &map); if (err < 0) return err; @@ -395,6 +389,34 @@ static int udf_get_block(struct inode *inode, sector_t block, return 0; } +int udf_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + int flags = create ? UDF_MAP_CREATE : 0; + + /* + * We preallocate blocks only for regular files. It also makes sense + * for directories but there's a problem when to drop the + * preallocation. We might use some delayed work for that but I feel + * it's overengineering for a filesystem like UDF. + */ + if (!S_ISREG(inode->i_mode)) + flags |= UDF_MAP_NOPREALLOC; + return __udf_get_block(inode, block, bh_result, flags); +} + +/* + * We shouldn't be allocating blocks on page writeback since we allocate them + * on page fault. We can spot dirty buffers without allocated blocks though + * when truncate expands file. These however don't have valid data so we can + * safely ignore them. So never allocate blocks from page writeback. + */ +static int udf_get_block_wb(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + return __udf_get_block(inode, block, bh_result, 0); +} + /* Extend the file with new blocks totaling 'new_block_bytes', * return the number of extents added */ From 36ec52ea038b18a53e198116ef7d7e70c87db046 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 23 Jan 2023 14:18:47 +0100 Subject: [PATCH 080/265] udf: Fix file corruption when appending just after end of preallocated extent When we append new block just after the end of preallocated extent, the code in inode_getblk() wrongly determined we're going to use the preallocated extent which resulted in adding block into a wrong logical offset in the file. Sequence like this manifests it: xfs_io -f -c "pwrite 0x2cacf 0xd122" -c "truncate 0x2dd6f" \ -c "pwrite 0x27fd9 0x69a9" -c "pwrite 0x32981 0x7244" The code that determined the use of preallocated extent is actually stale because udf_do_extend_file() does not create preallocation anymore so after calling that function we are sure there's no usable preallocation. Just remove the faulty condition. CC: stable@vger.kernel.org Fixes: 16d055656814 ("udf: Discard preallocation before extending file with a hole") Signed-off-by: Jan Kara --- fs/udf/inode.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 356e15daef18..51deada8b928 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -742,19 +742,17 @@ static int inode_getblk(struct inode *inode, struct udf_map_rq *map) c = 0; offset = 0; count += ret; - /* We are not covered by a preallocated extent? */ - if ((laarr[0].extLength & UDF_EXTENT_FLAG_MASK) != - EXT_NOT_RECORDED_ALLOCATED) { - /* Is there any real extent? - otherwise we overwrite - * the fake one... */ - if (count) - c = !c; - laarr[c].extLength = EXT_NOT_RECORDED_NOT_ALLOCATED | - inode->i_sb->s_blocksize; - memset(&laarr[c].extLocation, 0x00, - sizeof(struct kernel_lb_addr)); - count++; - } + /* + * Is there any real extent? - otherwise we overwrite the fake + * one... + */ + if (count) + c = !c; + laarr[c].extLength = EXT_NOT_RECORDED_NOT_ALLOCATED | + inode->i_sb->s_blocksize; + memset(&laarr[c].extLocation, 0x00, + sizeof(struct kernel_lb_addr)); + count++; endnum = c + 1; lastblock = 1; } else { From f54aa97fb7e5329a373f9df4e5e213ced4fc8759 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 23 Jan 2023 14:29:15 +0100 Subject: [PATCH 081/265] udf: Fix off-by-one error when discarding preallocation The condition determining whether the preallocation can be used had an off-by-one error so we didn't discard preallocation when new allocation was just following it. This can then confuse code in inode_getblk(). CC: stable@vger.kernel.org Fixes: 16d055656814 ("udf: Discard preallocation before extending file with a hole") Signed-off-by: Jan Kara --- fs/udf/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 51deada8b928..ee440d16411e 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -361,7 +361,7 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) * Block beyond EOF and prealloc extents? Just discard preallocation * as it is not useful and complicates things. */ - if (((loff_t)map->lblk) << inode->i_blkbits > iinfo->i_lenExtents) + if (((loff_t)map->lblk) << inode->i_blkbits >= iinfo->i_lenExtents) udf_discard_prealloc(inode); udf_clear_extent_cache(inode); err = inode_getblk(inode, map); From b7c31e6f14779324a39927cdaf493dc193576da1 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 11:40:28 +0100 Subject: [PATCH 082/265] udf: Unify .read_folio for normal and in-ICB files Switching address_space_operations while a file is used is difficult to do in a race-free way. To be able to use single address_space_operations in UDF, make udf_read_folio() handle both normal and in-ICB files. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 15 +++------------ fs/udf/inode.c | 9 ++++++++- fs/udf/udfdecl.h | 2 ++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 322115c8369d..2666234a5204 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -38,7 +38,7 @@ #include "udf_i.h" #include "udf_sb.h" -static void __udf_adinicb_readpage(struct page *page) +void udf_adinicb_readpage(struct page *page) { struct inode *inode = page->mapping->host; char *kaddr; @@ -57,15 +57,6 @@ static void __udf_adinicb_readpage(struct page *page) kunmap_atomic(kaddr); } -static int udf_adinicb_read_folio(struct file *file, struct folio *folio) -{ - BUG_ON(!folio_test_locked(folio)); - __udf_adinicb_readpage(&folio->page); - folio_unlock(folio); - - return 0; -} - static int udf_adinicb_writepage(struct page *page, struct writeback_control *wbc) { @@ -100,7 +91,7 @@ static int udf_adinicb_write_begin(struct file *file, *pagep = page; if (!PageUptodate(page)) - __udf_adinicb_readpage(page); + udf_adinicb_readpage(page); return 0; } @@ -127,7 +118,7 @@ static int udf_adinicb_write_end(struct file *file, struct address_space *mappin const struct address_space_operations udf_adinicb_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, - .read_folio = udf_adinicb_read_folio, + .read_folio = udf_read_folio, .writepage = udf_adinicb_writepage, .write_begin = udf_adinicb_write_begin, .write_end = udf_adinicb_write_end, diff --git a/fs/udf/inode.c b/fs/udf/inode.c index ee440d16411e..a1816f067c14 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -191,8 +191,15 @@ static int udf_writepages(struct address_space *mapping, return mpage_writepages(mapping, wbc, udf_get_block_wb); } -static int udf_read_folio(struct file *file, struct folio *folio) +int udf_read_folio(struct file *file, struct folio *folio) { + struct udf_inode_info *iinfo = UDF_I(file_inode(file)); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + udf_adinicb_readpage(&folio->page); + folio_unlock(folio); + return 0; + } return mpage_read_folio(folio, udf_get_block); } diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 5ba59ab90d48..6b93b393cb46 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -138,6 +138,7 @@ static inline unsigned int udf_dir_entry_len(struct fileIdentDesc *cfi) /* file.c */ extern long udf_ioctl(struct file *, unsigned int, unsigned long); +void udf_adinicb_readpage(struct page *page); /* inode.c */ extern struct inode *__udf_iget(struct super_block *, struct kernel_lb_addr *, @@ -158,6 +159,7 @@ extern struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, extern int udf_setsize(struct inode *, loff_t); extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); +int udf_read_folio(struct file *file, struct folio *folio); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); From 79d3c6dbada4a20193467f72c531ab99c173bf4f Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 11:54:08 +0100 Subject: [PATCH 083/265] udf: Convert in-ICB files to use udf_writepages() Switching address_space_operations while a file is used is difficult to do in a race-free way. To be able to use single address_space_operations in UDF, make in-ICB files use udf_writepages(). Reported-by: syzbot+c27475eb921c46bbdc62@syzkaller.appspotmail.com Reported-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 21 +-------------------- fs/udf/inode.c | 29 ++++++++++++++++++++++++++--- fs/udf/udfdecl.h | 1 + 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 2666234a5204..7a8dbad86e41 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -57,25 +57,6 @@ void udf_adinicb_readpage(struct page *page) kunmap_atomic(kaddr); } -static int udf_adinicb_writepage(struct page *page, - struct writeback_control *wbc) -{ - struct inode *inode = page->mapping->host; - char *kaddr; - struct udf_inode_info *iinfo = UDF_I(inode); - - BUG_ON(!PageLocked(page)); - - kaddr = kmap_atomic(page); - memcpy(iinfo->i_data + iinfo->i_lenEAttr, kaddr, i_size_read(inode)); - SetPageUptodate(page); - kunmap_atomic(kaddr); - mark_inode_dirty(inode); - unlock_page(page); - - return 0; -} - static int udf_adinicb_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, struct page **pagep, @@ -119,7 +100,7 @@ const struct address_space_operations udf_adinicb_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, .read_folio = udf_read_folio, - .writepage = udf_adinicb_writepage, + .writepages = udf_writepages, .write_begin = udf_adinicb_write_begin, .write_end = udf_adinicb_write_end, .direct_IO = udf_adinicb_direct_IO, diff --git a/fs/udf/inode.c b/fs/udf/inode.c index a1816f067c14..f0ab4dd0d8ce 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -185,10 +185,33 @@ static void udf_write_failed(struct address_space *mapping, loff_t to) } } -static int udf_writepages(struct address_space *mapping, - struct writeback_control *wbc) +static int udf_adinicb_writepage(struct page *page, + struct writeback_control *wbc, void *data) { - return mpage_writepages(mapping, wbc, udf_get_block_wb); + struct inode *inode = page->mapping->host; + char *kaddr; + struct udf_inode_info *iinfo = UDF_I(inode); + + BUG_ON(!PageLocked(page)); + + kaddr = kmap_atomic(page); + memcpy(iinfo->i_data + iinfo->i_lenEAttr, kaddr, i_size_read(inode)); + SetPageUptodate(page); + kunmap_atomic(kaddr); + unlock_page(page); + mark_inode_dirty(inode); + + return 0; +} + +int udf_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct udf_inode_info *iinfo = UDF_I(inode); + + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) + return mpage_writepages(mapping, wbc, udf_get_block_wb); + return write_cache_pages(mapping, wbc, udf_adinicb_writepage, NULL); } int udf_read_folio(struct file *file, struct folio *folio) diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 6b93b393cb46..48647eab26a6 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -160,6 +160,7 @@ extern int udf_setsize(struct inode *, loff_t); extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); int udf_read_folio(struct file *file, struct folio *folio); +int udf_writepages(struct address_space *mapping, struct writeback_control *wbc); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); From d5abfb1b7b26086db19ee430dea7282f01d4ef44 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 11:59:32 +0100 Subject: [PATCH 084/265] udf: Convert in-ICB files to use udf_direct_IO() Switching address_space_operations while a file is used is difficult to do in a race-free way. To be able to use single address_space_operations in UDF, make in-ICB files use udf_direct_IO(). Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 8 +------- fs/udf/inode.c | 5 ++++- fs/udf/udfdecl.h | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 7a8dbad86e41..6bab6aa7770a 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -76,12 +76,6 @@ static int udf_adinicb_write_begin(struct file *file, return 0; } -static ssize_t udf_adinicb_direct_IO(struct kiocb *iocb, struct iov_iter *iter) -{ - /* Fallback to buffered I/O. */ - return 0; -} - static int udf_adinicb_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) @@ -103,7 +97,7 @@ const struct address_space_operations udf_adinicb_aops = { .writepages = udf_writepages, .write_begin = udf_adinicb_write_begin, .write_end = udf_adinicb_write_end, - .direct_IO = udf_adinicb_direct_IO, + .direct_IO = udf_direct_IO, }; static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index f0ab4dd0d8ce..d01b97e9e4f4 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -243,7 +243,7 @@ static int udf_write_begin(struct file *file, struct address_space *mapping, return ret; } -static ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; @@ -251,6 +251,9 @@ static ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) size_t count = iov_iter_count(iter); ssize_t ret; + /* Fallback to buffered IO for in-ICB files */ + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return 0; ret = blockdev_direct_IO(iocb, inode, iter, udf_get_block); if (unlikely(ret < 0 && iov_iter_rw(iter) == WRITE)) udf_write_failed(mapping, iocb->ki_pos + count); diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 48647eab26a6..a851613465c6 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -161,6 +161,7 @@ extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); int udf_read_folio(struct file *file, struct folio *folio); int udf_writepages(struct address_space *mapping, struct writeback_control *wbc); +ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); From 60b99a1b9fa731453e1b69a3e0b3e4dcab7a6ea5 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:07:37 +0100 Subject: [PATCH 085/265] udf: Convert in-ICB files to use udf_write_begin() Switching address_space_operations while a file is used is difficult to do in a race-free way. To be able to use single address_space_operations in UDF, make in-ICB files use udf_write_begin(). Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 21 +-------------------- fs/udf/inode.c | 24 +++++++++++++++++++----- fs/udf/udfdecl.h | 3 +++ 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 6bab6aa7770a..16aecf4b2387 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -57,25 +57,6 @@ void udf_adinicb_readpage(struct page *page) kunmap_atomic(kaddr); } -static int udf_adinicb_write_begin(struct file *file, - struct address_space *mapping, loff_t pos, - unsigned len, struct page **pagep, - void **fsdata) -{ - struct page *page; - - if (WARN_ON_ONCE(pos >= PAGE_SIZE)) - return -EIO; - page = grab_cache_page_write_begin(mapping, 0); - if (!page) - return -ENOMEM; - *pagep = page; - - if (!PageUptodate(page)) - udf_adinicb_readpage(page); - return 0; -} - static int udf_adinicb_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) @@ -95,7 +76,7 @@ const struct address_space_operations udf_adinicb_aops = { .invalidate_folio = block_invalidate_folio, .read_folio = udf_read_folio, .writepages = udf_writepages, - .write_begin = udf_adinicb_write_begin, + .write_begin = udf_write_begin, .write_end = udf_adinicb_write_end, .direct_IO = udf_direct_IO, }; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index d01b97e9e4f4..afe061d2020e 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -231,16 +231,30 @@ static void udf_readahead(struct readahead_control *rac) mpage_readahead(rac, udf_get_block); } -static int udf_write_begin(struct file *file, struct address_space *mapping, +int udf_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, struct page **pagep, void **fsdata) { + struct udf_inode_info *iinfo = UDF_I(file_inode(file)); + struct page *page; int ret; - ret = block_write_begin(mapping, pos, len, pagep, udf_get_block); - if (unlikely(ret)) - udf_write_failed(mapping, pos + len); - return ret; + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + ret = block_write_begin(mapping, pos, len, pagep, + udf_get_block); + if (unlikely(ret)) + udf_write_failed(mapping, pos + len); + return ret; + } + if (WARN_ON_ONCE(pos >= PAGE_SIZE)) + return -EIO; + page = grab_cache_page_write_begin(mapping, 0); + if (!page) + return -ENOMEM; + *pagep = page; + if (!PageUptodate(page)) + udf_adinicb_readpage(page); + return 0; } ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index a851613465c6..32decf6b6a21 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -161,6 +161,9 @@ extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); int udf_read_folio(struct file *file, struct folio *folio); int udf_writepages(struct address_space *mapping, struct writeback_control *wbc); +int udf_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, + struct page **pagep, void **fsdata); ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); From c694e40ba231e854fcddd25cce46db3918fca291 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:13:57 +0100 Subject: [PATCH 086/265] udf: Convert all file types to use udf_write_end() Switching address_space_operations while a file is used is difficult to do in a race-free way. To be able to use single address_space_operations in UDF, create udf_write_end() function that is able to handle both normal and in-ICB files. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 16 +--------------- fs/udf/inode.c | 22 +++++++++++++++++++++- fs/udf/udfdecl.h | 3 +++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 16aecf4b2387..8a37cd593883 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -57,27 +57,13 @@ void udf_adinicb_readpage(struct page *page) kunmap_atomic(kaddr); } -static int udf_adinicb_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, - struct page *page, void *fsdata) -{ - struct inode *inode = page->mapping->host; - loff_t last_pos = pos + copied; - if (last_pos > inode->i_size) - i_size_write(inode, last_pos); - set_page_dirty(page); - unlock_page(page); - put_page(page); - return copied; -} - const struct address_space_operations udf_adinicb_aops = { .dirty_folio = block_dirty_folio, .invalidate_folio = block_invalidate_folio, .read_folio = udf_read_folio, .writepages = udf_writepages, .write_begin = udf_write_begin, - .write_end = udf_adinicb_write_end, + .write_end = udf_write_end, .direct_IO = udf_direct_IO, }; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index afe061d2020e..094ac6f22c72 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -257,6 +257,26 @@ int udf_write_begin(struct file *file, struct address_space *mapping, return 0; } +int udf_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ + struct inode *inode = file_inode(file); + loff_t last_pos; + + if (UDF_I(inode)->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) + return generic_write_end(file, mapping, pos, len, copied, page, + fsdata); + last_pos = pos + copied; + if (last_pos > inode->i_size) + i_size_write(inode, last_pos); + set_page_dirty(page); + unlock_page(page); + put_page(page); + + return copied; +} + ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { struct file *file = iocb->ki_filp; @@ -286,7 +306,7 @@ const struct address_space_operations udf_aops = { .readahead = udf_readahead, .writepages = udf_writepages, .write_begin = udf_write_begin, - .write_end = generic_write_end, + .write_end = udf_write_end, .direct_IO = udf_direct_IO, .bmap = udf_bmap, .migrate_folio = buffer_migrate_folio, diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 32decf6b6a21..304c2ec81589 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -164,6 +164,9 @@ int udf_writepages(struct address_space *mapping, struct writeback_control *wbc) int udf_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, struct page **pagep, void **fsdata); +int udf_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata); ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); From 907c6c2ffa6a6d28f59db22ac0686ca3103c61d8 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:16:38 +0100 Subject: [PATCH 087/265] udf: Add handling of in-ICB files to udf_bmap() Add detection of in-ICB files to udf_bmap() and return error in that case. This will allow us o use single address_space_operations in UDF. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/inode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 094ac6f22c72..c65a5d76d15a 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -296,6 +296,10 @@ ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) static sector_t udf_bmap(struct address_space *mapping, sector_t block) { + struct udf_inode_info *iinfo = UDF_I(mapping->host); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return -EINVAL; return generic_block_bmap(mapping, block, udf_get_block); } From 37a8a39f7ad34d4ad31c7f5cf6b2237113963c48 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:23:04 +0100 Subject: [PATCH 088/265] udf: Switch to single address_space_operations Now that udf_aops and udf_adiniicb_aops are functionally identical, just drop udf_adiniicb_aops. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 10 ---------- fs/udf/inode.c | 8 +------- fs/udf/namei.c | 10 ++-------- fs/udf/udfdecl.h | 1 - 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 8a37cd593883..84e0b241940d 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -57,16 +57,6 @@ void udf_adinicb_readpage(struct page *page) kunmap_atomic(kaddr); } -const struct address_space_operations udf_adinicb_aops = { - .dirty_folio = block_dirty_folio, - .invalidate_folio = block_invalidate_folio, - .read_folio = udf_read_folio, - .writepages = udf_writepages, - .write_begin = udf_write_begin, - .write_end = udf_write_end, - .direct_IO = udf_direct_IO, -}; - static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index c65a5d76d15a..97662898704c 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -364,8 +364,6 @@ int udf_expand_file_adinicb(struct inode *inode) iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; else iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - /* from now on we have normal address_space methods */ - inode->i_data.a_ops = &udf_aops; set_page_dirty(page); unlock_page(page); up_write(&iinfo->i_data_sem); @@ -379,7 +377,6 @@ int udf_expand_file_adinicb(struct inode *inode) kunmap_atomic(kaddr); unlock_page(page); iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - inode->i_data.a_ops = &udf_adinicb_aops; iinfo->i_lenAlloc = inode->i_size; up_write(&iinfo->i_data_sem); } @@ -1566,10 +1563,7 @@ reread: case ICBTAG_FILE_TYPE_REGULAR: case ICBTAG_FILE_TYPE_UNDEF: case ICBTAG_FILE_TYPE_VAT20: - if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - inode->i_data.a_ops = &udf_adinicb_aops; - else - inode->i_data.a_ops = &udf_aops; + inode->i_data.a_ops = &udf_aops; inode->i_op = &udf_file_inode_operations; inode->i_fop = &udf_file_operations; inode->i_mode |= S_IFREG; diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 663b66014c98..51a518cb6fc2 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -377,10 +377,7 @@ static int udf_create(struct user_namespace *mnt_userns, struct inode *dir, if (IS_ERR(inode)) return PTR_ERR(inode); - if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - inode->i_data.a_ops = &udf_adinicb_aops; - else - inode->i_data.a_ops = &udf_aops; + inode->i_data.a_ops = &udf_aops; inode->i_op = &udf_file_inode_operations; inode->i_fop = &udf_file_operations; mark_inode_dirty(inode); @@ -396,10 +393,7 @@ static int udf_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, if (IS_ERR(inode)) return PTR_ERR(inode); - if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) - inode->i_data.a_ops = &udf_adinicb_aops; - else - inode->i_data.a_ops = &udf_aops; + inode->i_data.a_ops = &udf_aops; inode->i_op = &udf_file_inode_operations; inode->i_fop = &udf_file_operations; mark_inode_dirty(inode); diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 304c2ec81589..d8c0de3b224e 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -80,7 +80,6 @@ extern const struct inode_operations udf_file_inode_operations; extern const struct file_operations udf_file_operations; extern const struct inode_operations udf_symlink_inode_operations; extern const struct address_space_operations udf_aops; -extern const struct address_space_operations udf_adinicb_aops; extern const struct address_space_operations udf_symlink_aops; struct udf_fileident_iter { From 759e4d74c072aadba7cf7c026434ef7e267e4520 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:25:46 +0100 Subject: [PATCH 089/265] udf: Mark aops implementation static Mark functions implementing aops static since they are not needed outside of inode.c anymore. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/inode.c | 19 ++++++++++--------- fs/udf/udfdecl.h | 9 --------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 97662898704c..084b1f319713 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -204,7 +204,8 @@ static int udf_adinicb_writepage(struct page *page, return 0; } -int udf_writepages(struct address_space *mapping, struct writeback_control *wbc) +static int udf_writepages(struct address_space *mapping, + struct writeback_control *wbc) { struct inode *inode = mapping->host; struct udf_inode_info *iinfo = UDF_I(inode); @@ -214,7 +215,7 @@ int udf_writepages(struct address_space *mapping, struct writeback_control *wbc) return write_cache_pages(mapping, wbc, udf_adinicb_writepage, NULL); } -int udf_read_folio(struct file *file, struct folio *folio) +static int udf_read_folio(struct file *file, struct folio *folio) { struct udf_inode_info *iinfo = UDF_I(file_inode(file)); @@ -231,9 +232,9 @@ static void udf_readahead(struct readahead_control *rac) mpage_readahead(rac, udf_get_block); } -int udf_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, - struct page **pagep, void **fsdata) +static int udf_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, + struct page **pagep, void **fsdata) { struct udf_inode_info *iinfo = UDF_I(file_inode(file)); struct page *page; @@ -257,9 +258,9 @@ int udf_write_begin(struct file *file, struct address_space *mapping, return 0; } -int udf_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, - struct page *page, void *fsdata) +static int udf_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) { struct inode *inode = file_inode(file); loff_t last_pos; @@ -277,7 +278,7 @@ int udf_write_end(struct file *file, struct address_space *mapping, return copied; } -ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +static ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index d8c0de3b224e..337daf97d5b4 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -158,15 +158,6 @@ extern struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, extern int udf_setsize(struct inode *, loff_t); extern void udf_evict_inode(struct inode *); extern int udf_write_inode(struct inode *, struct writeback_control *wbc); -int udf_read_folio(struct file *file, struct folio *folio); -int udf_writepages(struct address_space *mapping, struct writeback_control *wbc); -int udf_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, - struct page **pagep, void **fsdata); -int udf_write_end(struct file *file, struct address_space *mapping, - loff_t pos, unsigned len, unsigned copied, - struct page *page, void *fsdata); -ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter); extern int8_t inode_bmap(struct inode *, sector_t, struct extent_position *, struct kernel_lb_addr *, uint32_t *, sector_t *); int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); From 174cb748d88819d8a6420aa5040f9f065fcbdbab Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:29:57 +0100 Subject: [PATCH 090/265] udf: Move udf_adinicb_readpage() to inode.c udf_adinicb_readpage() is only called from aops functions, move it to the same file as its callers and also drop the stale comment - invalidate_lock is protecting us against races with truncate. Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/file.c | 19 ------------------- fs/udf/inode.c | 15 +++++++++++++++ fs/udf/udfdecl.h | 1 - 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/fs/udf/file.c b/fs/udf/file.c index 84e0b241940d..26592577b7da 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -38,25 +38,6 @@ #include "udf_i.h" #include "udf_sb.h" -void udf_adinicb_readpage(struct page *page) -{ - struct inode *inode = page->mapping->host; - char *kaddr; - struct udf_inode_info *iinfo = UDF_I(inode); - loff_t isize = i_size_read(inode); - - /* - * We have to be careful here as truncate can change i_size under us. - * So just sample it once and use the same value everywhere. - */ - kaddr = kmap_atomic(page); - memcpy(kaddr, iinfo->i_data + iinfo->i_lenEAttr, isize); - memset(kaddr + isize, 0, PAGE_SIZE - isize); - flush_dcache_page(page); - SetPageUptodate(page); - kunmap_atomic(kaddr); -} - static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 084b1f319713..b2bad81a253e 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -215,6 +215,21 @@ static int udf_writepages(struct address_space *mapping, return write_cache_pages(mapping, wbc, udf_adinicb_writepage, NULL); } +static void udf_adinicb_readpage(struct page *page) +{ + struct inode *inode = page->mapping->host; + char *kaddr; + struct udf_inode_info *iinfo = UDF_I(inode); + loff_t isize = i_size_read(inode); + + kaddr = kmap_atomic(page); + memcpy(kaddr, iinfo->i_data + iinfo->i_lenEAttr, isize); + memset(kaddr + isize, 0, PAGE_SIZE - isize); + flush_dcache_page(page); + SetPageUptodate(page); + kunmap_atomic(kaddr); +} + static int udf_read_folio(struct file *file, struct folio *folio) { struct udf_inode_info *iinfo = UDF_I(file_inode(file)); diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 337daf97d5b4..88692512a466 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -137,7 +137,6 @@ static inline unsigned int udf_dir_entry_len(struct fileIdentDesc *cfi) /* file.c */ extern long udf_ioctl(struct file *, unsigned int, unsigned long); -void udf_adinicb_readpage(struct page *page); /* inode.c */ extern struct inode *__udf_iget(struct super_block *, struct kernel_lb_addr *, From 7b7f68655fca72de0d342412034cbdb2102adb7f Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 12:35:45 +0100 Subject: [PATCH 091/265] udf: Switch udf_adinicb_readpage() to kmap_local_page() Instead of using kmap_atomic() use kmap_local_page() in udf_adinicb_readpage(). Reviewed-by: Christoph Hellwig Signed-off-by: Jan Kara --- fs/udf/inode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index b2bad81a253e..72bca08d1fca 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -222,12 +222,12 @@ static void udf_adinicb_readpage(struct page *page) struct udf_inode_info *iinfo = UDF_I(inode); loff_t isize = i_size_read(inode); - kaddr = kmap_atomic(page); + kaddr = kmap_local_page(page); memcpy(kaddr, iinfo->i_data + iinfo->i_lenEAttr, isize); memset(kaddr + isize, 0, PAGE_SIZE - isize); flush_dcache_page(page); SetPageUptodate(page); - kunmap_atomic(kaddr); + kunmap_local(kaddr); } static int udf_read_folio(struct file *file, struct folio *folio) From 5cfc45321a6d81c2587b38f26187dbe0018c6b04 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 10:23:33 +0100 Subject: [PATCH 092/265] udf: Convert udf_adinicb_writepage() to memcpy_to_page() Instead of mapping the page manually with kmap() atomic, use helper memcpy_to_page(). Also delete the pointless SetPageUptodate() call. Signed-off-by: Jan Kara --- fs/udf/inode.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 72bca08d1fca..1d661faffde0 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -189,15 +189,11 @@ static int udf_adinicb_writepage(struct page *page, struct writeback_control *wbc, void *data) { struct inode *inode = page->mapping->host; - char *kaddr; struct udf_inode_info *iinfo = UDF_I(inode); BUG_ON(!PageLocked(page)); - - kaddr = kmap_atomic(page); - memcpy(iinfo->i_data + iinfo->i_lenEAttr, kaddr, i_size_read(inode)); - SetPageUptodate(page); - kunmap_atomic(kaddr); + memcpy_to_page(page, 0, iinfo->i_data + iinfo->i_lenEAttr, + i_size_read(inode)); unlock_page(page); mark_inode_dirty(inode); From 1eeceaec794ef937d161379d89504990c82133ad Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 10:33:37 +0100 Subject: [PATCH 093/265] udf: Convert udf_expand_file_adinicb() to avoid kmap_atomic() Remove the last two remaining kmap_atomic() uses in UDF in udf_expand_file_adinicb(). The first use can be actually conveniently replaced with udf_adinicb_readpage(), the second with memcpy_to_page(). Signed-off-by: Jan Kara --- fs/udf/inode.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 1d661faffde0..3b2adf4cbc57 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -336,7 +336,6 @@ const struct address_space_operations udf_aops = { int udf_expand_file_adinicb(struct inode *inode) { struct page *page; - char *kaddr; struct udf_inode_info *iinfo = UDF_I(inode); int err; @@ -358,16 +357,8 @@ int udf_expand_file_adinicb(struct inode *inode) if (!page) return -ENOMEM; - if (!PageUptodate(page)) { - kaddr = kmap_atomic(page); - memset(kaddr + iinfo->i_lenAlloc, 0x00, - PAGE_SIZE - iinfo->i_lenAlloc); - memcpy(kaddr, iinfo->i_data + iinfo->i_lenEAttr, - iinfo->i_lenAlloc); - flush_dcache_page(page); - SetPageUptodate(page); - kunmap_atomic(kaddr); - } + if (!PageUptodate(page)) + udf_adinicb_readpage(page); down_write(&iinfo->i_data_sem); memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, iinfo->i_lenAlloc); @@ -384,9 +375,8 @@ int udf_expand_file_adinicb(struct inode *inode) /* Restore everything back so that we don't lose data... */ lock_page(page); down_write(&iinfo->i_data_sem); - kaddr = kmap_atomic(page); - memcpy(iinfo->i_data + iinfo->i_lenEAttr, kaddr, inode->i_size); - kunmap_atomic(kaddr); + memcpy_to_page(page, 0, iinfo->i_data + iinfo->i_lenEAttr, + inode->i_size); unlock_page(page); iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; iinfo->i_lenAlloc = inode->i_size; From f386c802a6fda8f9fe4a5cf418c49aa84dfc52e4 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 24 Jan 2023 14:49:52 +0100 Subject: [PATCH 094/265] udf: Don't return bh from udf_expand_dir_adinicb() Nobody uses the bh returned from udf_expand_dir_adinicb(). Don't return it. Signed-off-by: Jan Kara --- fs/udf/namei.c | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index 51a518cb6fc2..d68a0c2b58b7 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -136,8 +136,7 @@ static struct dentry *udf_lookup(struct inode *dir, struct dentry *dentry, return d_splice_alias(inode, dentry); } -static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, - udf_pblk_t *block, int *err) +static int udf_expand_dir_adinicb(struct inode *inode, udf_pblk_t *block) { udf_pblk_t newblock; struct buffer_head *dbh = NULL; @@ -157,23 +156,23 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, if (!inode->i_size) { iinfo->i_alloc_type = alloctype; mark_inode_dirty(inode); - return NULL; + return 0; } /* alloc block, and copy data to it */ *block = udf_new_block(inode->i_sb, inode, iinfo->i_location.partitionReferenceNum, - iinfo->i_location.logicalBlockNum, err); + iinfo->i_location.logicalBlockNum, &ret); if (!(*block)) - return NULL; + return ret; newblock = udf_get_pblock(inode->i_sb, *block, iinfo->i_location.partitionReferenceNum, 0); - if (!newblock) - return NULL; + if (newblock == 0xffffffff) + return -EFSCORRUPTED; dbh = sb_getblk(inode->i_sb, newblock); if (!dbh) - return NULL; + return -ENOMEM; lock_buffer(dbh); memcpy(dbh->b_data, iinfo->i_data, inode->i_size); memset(dbh->b_data + inode->i_size, 0, @@ -195,9 +194,9 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, ret = udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); brelse(epos.bh); if (ret < 0) { - *err = ret; + brelse(dbh); udf_free_blocks(inode->i_sb, inode, &eloc, 0, 1); - return NULL; + return ret; } mark_inode_dirty(inode); @@ -213,6 +212,7 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, impuse = NULL; udf_fiiter_write_fi(&iter, impuse); } + brelse(dbh); /* * We don't expect the iteration to fail as the directory has been * already verified to be correct @@ -220,7 +220,7 @@ static struct buffer_head *udf_expand_dir_adinicb(struct inode *inode, WARN_ON_ONCE(ret); udf_fiiter_release(&iter); - return dbh; + return 0; } static int udf_fiiter_add_entry(struct inode *dir, struct dentry *dentry, @@ -266,17 +266,10 @@ static int udf_fiiter_add_entry(struct inode *dir, struct dentry *dentry, } if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && blksize - udf_ext0_offset(dir) - iter->pos < nfidlen) { - struct buffer_head *retbh; - udf_fiiter_release(iter); - /* - * FIXME: udf_expand_dir_adinicb does not need to return bh - * once other users are gone - */ - retbh = udf_expand_dir_adinicb(dir, &block, &ret); - if (!retbh) + ret = udf_expand_dir_adinicb(dir, &block); + if (ret) return ret; - brelse(retbh); ret = udf_fiiter_init(iter, dir, dir->i_size); if (ret < 0) return ret; From c2efd13a2ed4f29bf9ef14ac2fbb7474084655f8 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 17:56:06 +0100 Subject: [PATCH 095/265] udf: Limit file size to 4TB UDF disk format supports in principle file sizes up to 1<<64-1. However the file space (including holes) is described by a linked list of extents, each of which can have at most 1GB. Thus the creation and handling of extents gets unusably slow beyond certain point. Limit the file size to 4TB to avoid locking up the kernel too easily. Signed-off-by: Jan Kara --- fs/udf/super.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fs/udf/super.c b/fs/udf/super.c index df5287c5d659..6304e3c5c3d9 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -86,6 +86,13 @@ enum { #define UDF_MAX_LVID_NESTING 1000 enum { UDF_MAX_LINKS = 0xffff }; +/* + * We limit filesize to 4TB. This is arbitrary as the on-disk format supports + * more but because the file space is described by a linked list of extents, + * each of which can have at most 1GB, the creation and handling of extents + * gets unusably slow beyond certain point... + */ +#define UDF_MAX_FILESIZE (1ULL << 42) /* These are the "meat" - everything else is stuffing */ static int udf_fill_super(struct super_block *, void *, int); @@ -2257,7 +2264,7 @@ static int udf_fill_super(struct super_block *sb, void *options, int silent) ret = -ENOMEM; goto error_out; } - sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_maxbytes = UDF_MAX_FILESIZE; sb->s_max_links = UDF_MAX_LINKS; return 0; From 085cf7b7e2ef7bdd88c8f2f56a17d756bd18f1a4 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Wed, 25 Jan 2023 19:31:38 +0100 Subject: [PATCH 096/265] udf: Fix file counting in LVID numFiles entry in LVID should actually contain number for non-dir file entries, not the number of non-dir inodes. Move the counting from inode allocation / deallocation into directory entry handling functions. Signed-off-by: Jan Kara --- fs/udf/ialloc.c | 31 +++---------------------------- fs/udf/namei.c | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/fs/udf/ialloc.c b/fs/udf/ialloc.c index b5d611cee749..ffd9ccbce0d0 100644 --- a/fs/udf/ialloc.c +++ b/fs/udf/ialloc.c @@ -28,21 +28,7 @@ void udf_free_inode(struct inode *inode) { - struct super_block *sb = inode->i_sb; - struct udf_sb_info *sbi = UDF_SB(sb); - struct logicalVolIntegrityDescImpUse *lvidiu = udf_sb_lvidiu(sb); - - if (lvidiu) { - mutex_lock(&sbi->s_alloc_mutex); - if (S_ISDIR(inode->i_mode)) - le32_add_cpu(&lvidiu->numDirs, -1); - else - le32_add_cpu(&lvidiu->numFiles, -1); - udf_updated_lvid(sb); - mutex_unlock(&sbi->s_alloc_mutex); - } - - udf_free_blocks(sb, NULL, &UDF_I(inode)->i_location, 0, 1); + udf_free_blocks(inode->i_sb, NULL, &UDF_I(inode)->i_location, 0, 1); } struct inode *udf_new_inode(struct inode *dir, umode_t mode) @@ -54,7 +40,6 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode) uint32_t start = UDF_I(dir)->i_location.logicalBlockNum; struct udf_inode_info *iinfo; struct udf_inode_info *dinfo = UDF_I(dir); - struct logicalVolIntegrityDescImpUse *lvidiu; int err; inode = new_inode(sb); @@ -92,18 +77,8 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode) return ERR_PTR(err); } - lvidiu = udf_sb_lvidiu(sb); - if (lvidiu) { - iinfo->i_unique = lvid_get_unique_id(sb); - inode->i_generation = iinfo->i_unique; - mutex_lock(&sbi->s_alloc_mutex); - if (S_ISDIR(mode)) - le32_add_cpu(&lvidiu->numDirs, 1); - else - le32_add_cpu(&lvidiu->numFiles, 1); - udf_updated_lvid(sb); - mutex_unlock(&sbi->s_alloc_mutex); - } + iinfo->i_unique = lvid_get_unique_id(sb); + inode->i_generation = iinfo->i_unique; inode_init_owner(&init_user_ns, inode, dir, mode); if (UDF_QUERY_FLAG(sb, UDF_FLAG_UID_SET)) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index d68a0c2b58b7..55b0d78a581f 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -336,6 +336,21 @@ static void udf_fiiter_delete_entry(struct udf_fileident_iter *iter) udf_fiiter_write_fi(iter, NULL); } +static void udf_add_fid_counter(struct super_block *sb, bool dir, int val) +{ + struct logicalVolIntegrityDescImpUse *lvidiu = udf_sb_lvidiu(sb); + + if (!lvidiu) + return; + mutex_lock(&UDF_SB(sb)->s_alloc_mutex); + if (dir) + le32_add_cpu(&lvidiu->numDirs, val); + else + le32_add_cpu(&lvidiu->numFiles, val); + udf_updated_lvid(sb); + mutex_unlock(&UDF_SB(sb)->s_alloc_mutex); +} + static int udf_add_nondir(struct dentry *dentry, struct inode *inode) { struct udf_inode_info *iinfo = UDF_I(inode); @@ -357,6 +372,7 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode) dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); udf_fiiter_release(&iter); + udf_add_fid_counter(dir->i_sb, false, 1); d_instantiate_new(dentry, inode); return 0; @@ -457,6 +473,7 @@ static int udf_mkdir(struct user_namespace *mnt_userns, struct inode *dir, iter.fi.fileCharacteristics |= FID_FILE_CHAR_DIRECTORY; udf_fiiter_write_fi(&iter, NULL); udf_fiiter_release(&iter); + udf_add_fid_counter(dir->i_sb, true, 1); inc_nlink(dir); dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); @@ -509,6 +526,7 @@ static int udf_rmdir(struct inode *dir, struct dentry *dentry) clear_nlink(inode); inode->i_size = 0; inode_dec_link_count(dir); + udf_add_fid_counter(dir->i_sb, true, -1); inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); mark_inode_dirty(dir); @@ -544,6 +562,7 @@ static int udf_unlink(struct inode *dir, struct dentry *dentry) dir->i_ctime = dir->i_mtime = current_time(dir); mark_inode_dirty(dir); inode_dec_link_count(inode); + udf_add_fid_counter(dir->i_sb, false, -1); inode->i_ctime = dir->i_ctime; ret = 0; end_unlink: @@ -730,6 +749,7 @@ static int udf_link(struct dentry *old_dentry, struct inode *dir, udf_fiiter_release(&iter); inc_nlink(inode); + udf_add_fid_counter(dir->i_sb, false, 1); inode->i_ctime = current_time(inode); mark_inode_dirty(inode); dir->i_ctime = dir->i_mtime = current_time(dir); @@ -854,6 +874,8 @@ static int udf_rename(struct user_namespace *mnt_userns, struct inode *old_dir, if (new_inode) { new_inode->i_ctime = current_time(new_inode); inode_dec_link_count(new_inode); + udf_add_fid_counter(old_dir->i_sb, S_ISDIR(new_inode->i_mode), + -1); } old_dir->i_ctime = old_dir->i_mtime = current_time(old_dir); new_dir->i_ctime = new_dir->i_mtime = current_time(new_dir); From 439bc39b3cf0014b1b75075812f7ef0f8baa9674 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 25 Jan 2023 07:58:38 +0100 Subject: [PATCH 097/265] fs: move sb_init_dio_done_wq out of direct-io.c sb_init_dio_done_wq is also used by the iomap code, so move it to super.c in preparation for building direct-io.c conditionally. Signed-off-by: Christoph Hellwig Reviewed-by: Eric Biggers Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230125065839.191256-2-hch@lst.de Signed-off-by: Jens Axboe --- fs/direct-io.c | 24 ------------------------ fs/internal.h | 4 +--- fs/super.c | 24 ++++++++++++++++++++++++ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/fs/direct-io.c b/fs/direct-io.c index 03d381377ae1..ab0d7ea89813 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -558,30 +558,6 @@ static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) return ret; } -/* - * Create workqueue for deferred direct IO completions. We allocate the - * workqueue when it's first needed. This avoids creating workqueue for - * filesystems that don't need it and also allows us to create the workqueue - * late enough so the we can include s_id in the name of the workqueue. - */ -int sb_init_dio_done_wq(struct super_block *sb) -{ - struct workqueue_struct *old; - struct workqueue_struct *wq = alloc_workqueue("dio/%s", - WQ_MEM_RECLAIM, 0, - sb->s_id); - if (!wq) - return -ENOMEM; - /* - * This has to be atomic as more DIOs can race to create the workqueue - */ - old = cmpxchg(&sb->s_dio_done_wq, NULL, wq); - /* Someone created workqueue before us? Free ours... */ - if (old) - destroy_workqueue(wq); - return 0; -} - static int dio_set_defer_completion(struct dio *dio) { struct super_block *sb = dio->inode->i_sb; diff --git a/fs/internal.h b/fs/internal.h index a803cc3cf716..cb0c0749661a 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -120,6 +120,7 @@ extern bool trylock_super(struct super_block *sb); struct super_block *user_get_super(dev_t, bool excl); void put_super(struct super_block *sb); extern bool mount_capable(struct fs_context *); +int sb_init_dio_done_wq(struct super_block *sb); /* * open.c @@ -187,9 +188,6 @@ extern void mnt_pin_kill(struct mount *m); */ extern const struct dentry_operations ns_dentry_operations; -/* direct-io.c: */ -int sb_init_dio_done_wq(struct super_block *sb); - /* * fs/stat.c: */ diff --git a/fs/super.c b/fs/super.c index 12c08cb20405..904adfbacdcf 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1764,3 +1764,27 @@ int thaw_super(struct super_block *sb) return thaw_super_locked(sb); } EXPORT_SYMBOL(thaw_super); + +/* + * Create workqueue for deferred direct IO completions. We allocate the + * workqueue when it's first needed. This avoids creating workqueue for + * filesystems that don't need it and also allows us to create the workqueue + * late enough so the we can include s_id in the name of the workqueue. + */ +int sb_init_dio_done_wq(struct super_block *sb) +{ + struct workqueue_struct *old; + struct workqueue_struct *wq = alloc_workqueue("dio/%s", + WQ_MEM_RECLAIM, 0, + sb->s_id); + if (!wq) + return -ENOMEM; + /* + * This has to be atomic as more DIOs can race to create the workqueue + */ + old = cmpxchg(&sb->s_dio_done_wq, NULL, wq); + /* Someone created workqueue before us? Free ours... */ + if (old) + destroy_workqueue(wq); + return 0; +} From 9636e650e16f6b01f0044f7662074958c23e4707 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 25 Jan 2023 07:58:39 +0100 Subject: [PATCH 098/265] fs: build the legacy direct I/O code conditionally Add a new LEGACY_DIRECT_IO config symbol that is only selected by the file systems that still use the legacy blockdev_direct_IO code, so that kernels without support for those file systems don't need to build the code. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Reviewed-by: Eric Biggers Link: https://lore.kernel.org/r/20230125065839.191256-3-hch@lst.de Signed-off-by: Jens Axboe --- fs/Kconfig | 4 ++++ fs/Makefile | 3 ++- fs/affs/Kconfig | 1 + fs/exfat/Kconfig | 1 + fs/ext2/Kconfig | 1 + fs/fat/Kconfig | 1 + fs/hfs/Kconfig | 1 + fs/hfsplus/Kconfig | 1 + fs/jfs/Kconfig | 1 + fs/nilfs2/Kconfig | 1 + fs/ntfs3/Kconfig | 1 + fs/ocfs2/Kconfig | 1 + fs/reiserfs/Kconfig | 1 + fs/udf/Kconfig | 1 + 14 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fs/Kconfig b/fs/Kconfig index 2685a4d0d353..e99830c65033 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -18,6 +18,10 @@ config VALIDATE_FS_PARSER config FS_IOMAP bool +# old blockdev_direct_IO implementation. Use iomap for new code instead +config LEGACY_DIRECT_IO + bool + if BLOCK source "fs/ext2/Kconfig" diff --git a/fs/Makefile b/fs/Makefile index 4dea17840761..606c029e1c9b 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -19,13 +19,14 @@ obj-y := open.o read_write.o file_table.o super.o \ kernel_read_file.o remap_range.o ifeq ($(CONFIG_BLOCK),y) -obj-y += buffer.o direct-io.o mpage.o +obj-y += buffer.o mpage.o else obj-y += no-block.o endif obj-$(CONFIG_PROC_FS) += proc_namespace.o +obj-$(CONFIG_LEGACY_DIRECT_IO) += direct-io.o obj-y += notify/ obj-$(CONFIG_EPOLL) += eventpoll.o obj-y += anon_inodes.o diff --git a/fs/affs/Kconfig b/fs/affs/Kconfig index eb9d0ab850cb..962b86374e1c 100644 --- a/fs/affs/Kconfig +++ b/fs/affs/Kconfig @@ -2,6 +2,7 @@ config AFFS_FS tristate "Amiga FFS file system support" depends on BLOCK + select LEGACY_DIRECT_IO help The Fast File System (FFS) is the common file system used on hard disks by Amiga(tm) systems since AmigaOS Version 1.3 (34.20). Say Y diff --git a/fs/exfat/Kconfig b/fs/exfat/Kconfig index 5a65071b5ecf..147edeb04469 100644 --- a/fs/exfat/Kconfig +++ b/fs/exfat/Kconfig @@ -3,6 +3,7 @@ config EXFAT_FS tristate "exFAT filesystem support" select NLS + select LEGACY_DIRECT_IO help This allows you to mount devices formatted with the exFAT file system. exFAT is typically used on SD-Cards or USB sticks. diff --git a/fs/ext2/Kconfig b/fs/ext2/Kconfig index 1248ff4ef562..77393fda99af 100644 --- a/fs/ext2/Kconfig +++ b/fs/ext2/Kconfig @@ -2,6 +2,7 @@ config EXT2_FS tristate "Second extended fs support" select FS_IOMAP + select LEGACY_DIRECT_IO help Ext2 is a standard Linux file system for hard disks. diff --git a/fs/fat/Kconfig b/fs/fat/Kconfig index 238cc55f84c4..afe83b4e7172 100644 --- a/fs/fat/Kconfig +++ b/fs/fat/Kconfig @@ -2,6 +2,7 @@ config FAT_FS tristate select NLS + select LEGACY_DIRECT_IO help If you want to use one of the FAT-based file systems (the MS-DOS and VFAT (Windows 95) file systems), then you must say Y or M here diff --git a/fs/hfs/Kconfig b/fs/hfs/Kconfig index 129926b5142d..d985066006d5 100644 --- a/fs/hfs/Kconfig +++ b/fs/hfs/Kconfig @@ -3,6 +3,7 @@ config HFS_FS tristate "Apple Macintosh file system support" depends on BLOCK select NLS + select LEGACY_DIRECT_IO help If you say Y here, you will be able to mount Macintosh-formatted floppy disks and hard drive partitions with full read-write access. diff --git a/fs/hfsplus/Kconfig b/fs/hfsplus/Kconfig index 7d4229aecec0..8034e7827a69 100644 --- a/fs/hfsplus/Kconfig +++ b/fs/hfsplus/Kconfig @@ -4,6 +4,7 @@ config HFSPLUS_FS depends on BLOCK select NLS select NLS_UTF8 + select LEGACY_DIRECT_IO help If you say Y here, you will be able to mount extended format Macintosh-formatted hard drive partitions with full read-write access. diff --git a/fs/jfs/Kconfig b/fs/jfs/Kconfig index 05cb0e8e4382..51e856f0e4b8 100644 --- a/fs/jfs/Kconfig +++ b/fs/jfs/Kconfig @@ -3,6 +3,7 @@ config JFS_FS tristate "JFS filesystem support" select NLS select CRC32 + select LEGACY_DIRECT_IO help This is a port of IBM's Journaled Filesystem . More information is available in the file . diff --git a/fs/nilfs2/Kconfig b/fs/nilfs2/Kconfig index 254d102e79c9..7d59567465e1 100644 --- a/fs/nilfs2/Kconfig +++ b/fs/nilfs2/Kconfig @@ -2,6 +2,7 @@ config NILFS2_FS tristate "NILFS2 file system support" select CRC32 + select LEGACY_DIRECT_IO help NILFS2 is a log-structured file system (LFS) supporting continuous snapshotting. In addition to versioning capability of the entire diff --git a/fs/ntfs3/Kconfig b/fs/ntfs3/Kconfig index 6e4cbc48ab8e..96cc236f7f7b 100644 --- a/fs/ntfs3/Kconfig +++ b/fs/ntfs3/Kconfig @@ -2,6 +2,7 @@ config NTFS3_FS tristate "NTFS Read-Write file system support" select NLS + select LEGACY_DIRECT_IO help Windows OS native file system (NTFS) support up to NTFS version 3.1. diff --git a/fs/ocfs2/Kconfig b/fs/ocfs2/Kconfig index 5d11380d8724..304d12186ccd 100644 --- a/fs/ocfs2/Kconfig +++ b/fs/ocfs2/Kconfig @@ -7,6 +7,7 @@ config OCFS2_FS select QUOTA select QUOTA_TREE select FS_POSIX_ACL + select LEGACY_DIRECT_IO help OCFS2 is a general purpose extent based shared disk cluster file system with many similarities to ext3. It supports 64 bit inode diff --git a/fs/reiserfs/Kconfig b/fs/reiserfs/Kconfig index 33c8b0dd07a2..4d22ecfe0fab 100644 --- a/fs/reiserfs/Kconfig +++ b/fs/reiserfs/Kconfig @@ -2,6 +2,7 @@ config REISERFS_FS tristate "Reiserfs support (deprecated)" select CRC32 + select LEGACY_DIRECT_IO help Reiserfs is deprecated and scheduled to be removed from the kernel in 2025. If you are still using it, please migrate to another diff --git a/fs/udf/Kconfig b/fs/udf/Kconfig index 26e1a49f3ba7..82e8bfa2dfd9 100644 --- a/fs/udf/Kconfig +++ b/fs/udf/Kconfig @@ -3,6 +3,7 @@ config UDF_FS tristate "UDF file system support" select CRC_ITU_T select NLS + select LEGACY_DIRECT_IO help This is a file system used on some CD-ROMs and DVDs. Since the file system is supported by multiple operating systems and is more From 0c4fe008c9cb2215b3f838769886857ae986014b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:20 +0000 Subject: [PATCH 099/265] io_uring: rearrange defer list checks There should be nothing in the ->work_llist for non DEFER_TASKRUN rings, so we can skip flag checks and test the list emptiness directly. Also move it out of io_run_local_work() for inlining. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/331d63fd15ca79b35b95c82a82d9246110686392.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 3 --- io_uring/io_uring.h | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index db623b3185c8..da9b0397f9a1 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1343,9 +1343,6 @@ int io_run_local_work(struct io_ring_ctx *ctx) bool locked; int ret; - if (llist_empty(&ctx->work_llist)) - return 0; - __set_current_state(TASK_RUNNING); locked = mutex_trylock(&ctx->uring_lock); ret = __io_run_local_work(ctx, &locked); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index ab4b2a1c3b7e..7607a4992a94 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -289,7 +289,7 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) int ret = 0; int ret2; - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + if (!llist_empty(&ctx->work_llist)) ret = io_run_local_work(ctx); /* want to run this after in case more is added */ From f36ba6cf1ab6b05a538aae9cca896917db14ba27 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:21 +0000 Subject: [PATCH 100/265] io_uring: don't iterate cq wait fast path Task work runners keep running until all queues tw items are exhausted. It's also rare for defer tw to queue normal tw and vise versa. Taking it into account, there is only a dim chance that further iterating the io_cqring_wait() fast path will get us anything and so we can remove the loop there. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1f9565726661266abaa5d921e97433c831759ecf.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index da9b0397f9a1..dfc94654e082 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2510,18 +2510,16 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, if (!io_allowed_run_tw(ctx)) return -EEXIST; - - do { - /* always run at least 1 task work to process local work */ - ret = io_run_task_work_ctx(ctx); + if (!llist_empty(&ctx->work_llist)) { + ret = io_run_local_work(ctx); if (ret < 0) return ret; - io_cqring_overflow_flush(ctx); - - /* if user messes with these they will just get an early return */ - if (__io_cqring_events_user(ctx) >= min_events) - return 0; - } while (ret > 0); + } + io_run_task_work(); + io_cqring_overflow_flush(ctx); + /* if user messes with these they will just get an early return */ + if (__io_cqring_events_user(ctx) >= min_events) + return 0; if (sig) { #ifdef CONFIG_COMPAT From 1414d62985848d095af5a400a4ca074a4888b77f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:22 +0000 Subject: [PATCH 101/265] io_uring: kill io_run_task_work_ctx There is only one user of io_run_task_work_ctx(), inline it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/40953c65f7c88fb00cdc4d870ca5d5319fb3d7ea.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 6 +++++- io_uring/io_uring.h | 20 -------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index dfc94654e082..5326e2d94055 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2455,7 +2455,11 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, int io_run_task_work_sig(struct io_ring_ctx *ctx) { - if (io_run_task_work_ctx(ctx) > 0) + if (!llist_empty(&ctx->work_llist)) { + if (io_run_local_work(ctx) > 0) + return 1; + } + if (io_run_task_work() > 0) return 1; if (task_sigpending(current)) return -EINTR; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 7607a4992a94..9d36271eb6d3 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -284,26 +284,6 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx) return task_work_pending(current) || !wq_list_empty(&ctx->work_llist); } -static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) -{ - int ret = 0; - int ret2; - - if (!llist_empty(&ctx->work_llist)) - ret = io_run_local_work(ctx); - - /* want to run this after in case more is added */ - ret2 = io_run_task_work(); - - /* Try propagate error in favour of if tasks were run, - * but still make sure to run them if requested - */ - if (ret >= 0) - ret += ret2; - - return ret; -} - static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) { bool locked; From 140102ae9a9f2f83f0592b98b3c5c6119d9a9b32 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:23 +0000 Subject: [PATCH 102/265] io_uring: move defer tw task checks Most places that want to run local tw explicitly and in advance check if they are allowed to do so. Don't rely on a similar check in __io_run_local_work(), leave it as a just-in-case warning and make sure callers checks capabilities themselves. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/990fe0e8e70fd4d57e43625e5ce8fba584821d1a.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 15 ++++++--------- io_uring/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 5326e2d94055..97b749203ba8 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1301,14 +1301,13 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked) struct llist_node *node; struct llist_node fake; struct llist_node *current_final = NULL; - int ret; + int ret = 0; unsigned int loops = 1; - if (unlikely(ctx->submitter_task != current)) + if (WARN_ON_ONCE(ctx->submitter_task != current)) return -EEXIST; node = io_llist_xchg(&ctx->work_llist, &fake); - ret = 0; again: while (node != current_final) { struct llist_node *next = node->next; @@ -2514,11 +2513,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, if (!io_allowed_run_tw(ctx)) return -EEXIST; - if (!llist_empty(&ctx->work_llist)) { - ret = io_run_local_work(ctx); - if (ret < 0) - return ret; - } + if (!llist_empty(&ctx->work_llist)) + io_run_local_work(ctx); io_run_task_work(); io_cqring_overflow_flush(ctx); /* if user messes with these they will just get an early return */ @@ -3055,7 +3051,8 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && + io_allowed_defer_tw_run(ctx)) ret |= io_run_local_work(ctx) > 0; ret |= io_cancel_defer_files(ctx, task, cancel_all); mutex_lock(&ctx->uring_lock); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 9d36271eb6d3..011932935c36 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -367,6 +367,11 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return container_of(node, struct io_kiocb, comp_list); } +static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) +{ + return likely(ctx->submitter_task == current); +} + static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) { return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) || From 3fcf19d592d5cb63eb209400b22055651e3c27d0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:24 +0000 Subject: [PATCH 103/265] io_uring: parse check_cq out of wq waiting We already avoid flushing overflows in io_cqring_wait_schedule() but only return an error for the outer loop to handle it. Minimise it even further by moving all ->check_cq parsing there. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9dfcec3121013f98208dbf79368d636d74e1231a.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 97b749203ba8..524ef5a2bb9c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2471,21 +2471,13 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, ktime_t *timeout) { int ret; - unsigned long check_cq; + if (unlikely(READ_ONCE(ctx->check_cq))) + return 1; /* make sure we run task_work before checking for signals */ ret = io_run_task_work_sig(ctx); if (ret || io_should_wake(iowq)) return ret; - - check_cq = READ_ONCE(ctx->check_cq); - if (unlikely(check_cq)) { - /* let the caller flush overflows, retry */ - if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) - return 1; - if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) - return -EBADR; - } if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) return -ETIME; @@ -2551,13 +2543,25 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, trace_io_uring_cqring_wait(ctx, min_events); do { - if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) { - finish_wait(&ctx->cq_wait, &iowq.wq); - io_cqring_do_overflow_flush(ctx); - } + unsigned long check_cq; + prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + + check_cq = READ_ONCE(ctx->check_cq); + if (unlikely(check_cq)) { + /* let the caller flush overflows, retry */ + if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) { + finish_wait(&ctx->cq_wait, &iowq.wq); + io_cqring_do_overflow_flush(ctx); + } + if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) { + ret = -EBADR; + break; + } + } + if (__io_cqring_events_user(ctx) >= min_events) break; cond_resched(); From 846072f16eed3b3fb4e59b677f3ed8afb8509b89 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:25 +0000 Subject: [PATCH 104/265] io_uring: mimimise io_cqring_wait_schedule io_cqring_wait_schedule() is called after we started waiting on the cq wq and set the state to TASK_INTERRUPTIBLE, for that reason we have to constantly worry whether we has returned the state back to running or not. Leave only quick checks in io_cqring_wait_schedule() and move the rest including running task work to the callers. Note, we run tw in the loop after the sched checks because of the fast path in the beginning of the function. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2814fabe75e2e019e7ca43ea07daa94564349805.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 524ef5a2bb9c..2cbf33996c3b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2470,24 +2470,19 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, struct io_wait_queue *iowq, ktime_t *timeout) { - int ret; - if (unlikely(READ_ONCE(ctx->check_cq))) return 1; - /* make sure we run task_work before checking for signals */ - ret = io_run_task_work_sig(ctx); - if (ret || io_should_wake(iowq)) - return ret; + if (unlikely(!llist_empty(&ctx->work_llist))) + return 1; + if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL))) + return 1; + if (unlikely(task_sigpending(current))) + return -EINTR; + if (unlikely(io_should_wake(iowq))) + return 0; if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) return -ETIME; - - /* - * Run task_work after scheduling. If we got woken because of - * task_work being processed, run it now rather than let the caller - * do another wait loop. - */ - ret = io_run_task_work_sig(ctx); - return ret < 0 ? ret : 1; + return 0; } /* @@ -2548,6 +2543,16 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + if (ret < 0) + break; + /* + * Run task_work after scheduling and before io_should_wake(). + * If we got woken because of task_work being processed, run it + * now rather than let the caller do another wait loop. + */ + io_run_task_work(); + if (!llist_empty(&ctx->work_llist)) + io_run_local_work(ctx); check_cq = READ_ONCE(ctx->check_cq); if (unlikely(check_cq)) { @@ -2562,10 +2567,12 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, } } - if (__io_cqring_events_user(ctx) >= min_events) + if (io_should_wake(&iowq)) { + ret = 0; break; + } cond_resched(); - } while (ret > 0); + } while (1); finish_wait(&ctx->cq_wait, &iowq.wq); restore_saved_sigmask_unless(ret == -EINTR); From 490c00eb4fa5e5e25e0127240f6d6c1b499da95b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:26 +0000 Subject: [PATCH 105/265] io_uring: simplify io_has_work ->work_llist should never be non-empty for a non DEFER_TASKRUN ring, so we can safely skip checking the flag. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/26af9f73c09a56c9a035f94db56127358688f3aa.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2cbf33996c3b..4cb9cce23c90 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2419,8 +2419,7 @@ struct io_wait_queue { static inline bool io_has_work(struct io_ring_ctx *ctx) { return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || - ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && - !llist_empty(&ctx->work_llist)); + !llist_empty(&ctx->work_llist); } static inline bool io_should_wake(struct io_wait_queue *iowq) From 326a9e482e2134d7a44b7f8f9a721b38c6bbb146 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:27 +0000 Subject: [PATCH 106/265] io_uring: set TASK_RUNNING right after schedule Instead of constantly watching that the state of the task is running before executing tw or taking locks in io_cqring_wait(), switch it back to TASK_RUNNING immediately. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/246dddee247d89fd52023f785ed17cc34962a008.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4cb9cce23c90..2ec011f0ba7d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2544,6 +2544,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); if (ret < 0) break; + __set_current_state(TASK_RUNNING); /* * Run task_work after scheduling and before io_should_wake(). * If we got woken because of task_work being processed, run it @@ -2556,10 +2557,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, check_cq = READ_ONCE(ctx->check_cq); if (unlikely(check_cq)) { /* let the caller flush overflows, retry */ - if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) { - finish_wait(&ctx->cq_wait, &iowq.wq); + if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) io_cqring_do_overflow_flush(ctx); - } if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) { ret = -EBADR; break; From 46ae7eef44f6dfd825a3bcfa43392d3ad9836ada Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:28 +0000 Subject: [PATCH 107/265] io_uring: optimise non-timeout waiting Unlike the jiffy scheduling version, schedule_hrtimeout() jumps a few functions before getting into schedule() even if there is no actual timeout needed. Some tests showed that it takes up to 1% of CPU cycles. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/89f880574eceee6f4899783377ead234df7b3d04.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2ec011f0ba7d..6229a49c0c33 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2479,7 +2479,9 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, return -EINTR; if (unlikely(io_should_wake(iowq))) return 0; - if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) + if (*timeout == KTIME_MAX) + schedule(); + else if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) return -ETIME; return 0; } From d33a39e577687e12d4468e9dd999375b9973d700 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 5 Jan 2023 11:22:29 +0000 Subject: [PATCH 108/265] io_uring: keep timeout in io_wait_queue Move waiting timeout into io_wait_queue Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e4b48a9e26a3b1cf97c80121e62d4b5ab873d28d.1672916894.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6229a49c0c33..fdea6fbc3fad 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2414,6 +2414,7 @@ struct io_wait_queue { struct io_ring_ctx *ctx; unsigned cq_tail; unsigned nr_timeouts; + ktime_t timeout; }; static inline bool io_has_work(struct io_ring_ctx *ctx) @@ -2466,8 +2467,7 @@ int io_run_task_work_sig(struct io_ring_ctx *ctx) /* when returns >0, the caller should retry */ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, - struct io_wait_queue *iowq, - ktime_t *timeout) + struct io_wait_queue *iowq) { if (unlikely(READ_ONCE(ctx->check_cq))) return 1; @@ -2479,9 +2479,9 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, return -EINTR; if (unlikely(io_should_wake(iowq))) return 0; - if (*timeout == KTIME_MAX) + if (iowq->timeout == KTIME_MAX) schedule(); - else if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS)) + else if (!schedule_hrtimeout(&iowq->timeout, HRTIMER_MODE_ABS)) return -ETIME; return 0; } @@ -2496,7 +2496,6 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, { struct io_wait_queue iowq; struct io_rings *rings = ctx->rings; - ktime_t timeout = KTIME_MAX; int ret; if (!io_allowed_run_tw(ctx)) @@ -2522,20 +2521,21 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, return ret; } - if (uts) { - struct timespec64 ts; - - if (get_timespec64(&ts, uts)) - return -EFAULT; - timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); - } - init_waitqueue_func_entry(&iowq.wq, io_wake_function); iowq.wq.private = current; INIT_LIST_HEAD(&iowq.wq.entry); iowq.ctx = ctx; iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts); iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; + iowq.timeout = KTIME_MAX; + + if (uts) { + struct timespec64 ts; + + if (get_timespec64(&ts, uts)) + return -EFAULT; + iowq.timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns()); + } trace_io_uring_cqring_wait(ctx, min_events); do { @@ -2543,7 +2543,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); - ret = io_cqring_wait_schedule(ctx, &iowq, &timeout); + ret = io_cqring_wait_schedule(ctx, &iowq); if (ret < 0) break; __set_current_state(TASK_RUNNING); From cbeb47a7b5f003429ded32b1fb3a7108ce5c1b54 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Tue, 3 Jan 2023 08:05:07 -0800 Subject: [PATCH 109/265] io_uring/msg_ring: Pass custom flags to the cqe This patch adds a new flag (IORING_MSG_RING_FLAGS_PASS) in the message ring operations (IORING_OP_MSG_RING). This new flag enables the sender to specify custom flags, which will be copied over to cqe->flags in the receiving ring. These custom flags should be specified using the sqe->file_index field. This mechanism provides additional flexibility when sending messages between rings. Signed-off-by: Breno Leitao Link: https://lore.kernel.org/r/20230103160507.617416-1-leitao@debian.org Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 2 ++ io_uring/msg_ring.c | 24 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 2780bce62faf..636a4c2c1294 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -347,6 +347,8 @@ enum { * applicable for IORING_MSG_DATA, obviously. */ #define IORING_MSG_RING_CQE_SKIP (1U << 0) +/* Pass through the flags from sqe->file_index to cqe->flags */ +#define IORING_MSG_RING_FLAGS_PASS (1U << 1) /* * IO completion data structure (Completion Queue Entry) diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 15602a136821..27992bda1ffa 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -13,6 +13,11 @@ #include "filetable.h" #include "msg_ring.h" + +/* All valid masks for MSG_RING */ +#define IORING_MSG_RING_MASK (IORING_MSG_RING_CQE_SKIP | \ + IORING_MSG_RING_FLAGS_PASS) + struct io_msg { struct file *file; struct file *src_file; @@ -21,7 +26,10 @@ struct io_msg { u32 len; u32 cmd; u32 src_fd; - u32 dst_fd; + union { + u32 dst_fd; + u32 cqe_flags; + }; u32 flags; }; @@ -114,9 +122,12 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *target_ctx = req->file->private_data; struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); + u32 flags = 0; int ret; - if (msg->src_fd || msg->dst_fd || msg->flags) + if (msg->src_fd || msg->flags & ~IORING_MSG_RING_FLAGS_PASS) + return -EINVAL; + if (!(msg->flags & IORING_MSG_RING_FLAGS_PASS) && msg->dst_fd) return -EINVAL; if (target_ctx->flags & IORING_SETUP_R_DISABLED) return -EBADFD; @@ -124,15 +135,18 @@ static int io_msg_ring_data(struct io_kiocb *req, unsigned int issue_flags) if (io_msg_need_remote(target_ctx)) return io_msg_exec_remote(req, io_msg_tw_complete); + if (msg->flags & IORING_MSG_RING_FLAGS_PASS) + flags = msg->cqe_flags; + ret = -EOVERFLOW; if (target_ctx->flags & IORING_SETUP_IOPOLL) { if (unlikely(io_double_lock_ctx(target_ctx, issue_flags))) return -EAGAIN; - if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0)) + if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, flags)) ret = 0; io_double_unlock_ctx(target_ctx); } else { - if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0)) + if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, flags)) ret = 0; } return ret; @@ -241,7 +255,7 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) msg->src_fd = READ_ONCE(sqe->addr3); msg->dst_fd = READ_ONCE(sqe->file_index); msg->flags = READ_ONCE(sqe->msg_ring_flags); - if (msg->flags & ~IORING_MSG_RING_CQE_SKIP) + if (msg->flags & ~IORING_MSG_RING_MASK) return -EINVAL; return 0; From 81594e7e7a146888f0bac4fa782b0b5d3c37fdff Mon Sep 17 00:00:00 2001 From: Dmitrii Bundin Date: Mon, 9 Jan 2023 21:58:54 +0300 Subject: [PATCH 110/265] io_uring: remove excessive unlikely on IS_ERR The IS_ERR function uses the IS_ERR_VALUE macro under the hood which already wraps the condition into unlikely. Signed-off-by: Dmitrii Bundin Link: https://lore.kernel.org/r/20230109185854.25698-1-dmitrii.bundin.a@gmail.com Signed-off-by: Jens Axboe --- io_uring/rw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index 9c3ddd46a1ad..3a3aef7e7c33 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -410,7 +410,7 @@ static inline int io_import_iovec(int rw, struct io_kiocb *req, unsigned int issue_flags) { *iovec = __io_import_iovec(rw, req, s, issue_flags); - if (unlikely(IS_ERR(*iovec))) + if (IS_ERR(*iovec)) return PTR_ERR(*iovec); iov_iter_save_state(&s->iter, &s->iter_state); From dde40322ae20f2d6b0bcb781a9eedcfc7ca3aa73 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:03 +0000 Subject: [PATCH 111/265] io_uring: move submitter_task out of cold cacheline ->submitter_task is used somewhat more frequent now than before, i.e. for local tw enqueue and run, let's move it from the end of ctx, which is full of cold data, to the first cacheline with mostly constants. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/415ca91dc5ad1dec612b892e489cda98e1069542.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 128a67a40065..8dfb6c4a35d9 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -195,11 +195,7 @@ struct io_alloc_cache { struct io_ring_ctx { /* const or read-mostly hot data */ struct { - struct percpu_ref refs; - - struct io_rings *rings; unsigned int flags; - enum task_work_notify_mode notify_method; unsigned int compat: 1; unsigned int drain_next: 1; unsigned int restricted: 1; @@ -210,6 +206,11 @@ struct io_ring_ctx { unsigned int syscall_iopoll: 1; /* all CQEs should be posted only by the submitter task */ unsigned int task_complete: 1; + + enum task_work_notify_mode notify_method; + struct io_rings *rings; + struct task_struct *submitter_task; + struct percpu_ref refs; } ____cacheline_aligned_in_smp; /* submission data */ @@ -320,7 +321,6 @@ struct io_ring_ctx { /* Keep this last, we don't need it for the fast path */ struct io_restriction restrictions; - struct task_struct *submitter_task; /* slow path rsrc auxilary data, used by update/register */ struct io_rsrc_node *rsrc_backup_node; From bd550173acc2dc782d9c57852f6b6e71f5d9a159 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:04 +0000 Subject: [PATCH 112/265] io_uring: refactor io_wake_function Remove a local variable ctx in io_wake_function(), we don't need it if io_should_wake() triggers it to wake up. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e60eb1008aebe286aab7d34c772ed01c447bddb1.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index fdea6fbc3fad..5e1a0845e4b6 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2439,15 +2439,13 @@ static inline bool io_should_wake(struct io_wait_queue *iowq) static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, int wake_flags, void *key) { - struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, - wq); - struct io_ring_ctx *ctx = iowq->ctx; + struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, wq); /* * Cannot safely flush overflowed CQEs from here, ensure we wake up * the task, and the next invocation will do it. */ - if (io_should_wake(iowq) || io_has_work(ctx)) + if (io_should_wake(iowq) || io_has_work(iowq->ctx)) return autoremove_wake_function(curr, mode, wake_flags, key); return -1; } From 2f413956cc8a72fc11c2779228112559bbca8279 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:05 +0000 Subject: [PATCH 113/265] io_uring: don't set TASK_RUNNING in local tw runner The CQ waiting loop sets TASK_RUNNING before trying to execute task_work, no need to repeat it in io_run_local_work(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9d9422c429ef3f9457b4f4b8288bf4789564f33b.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 5e1a0845e4b6..e61e2a0f23b1 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1339,11 +1339,9 @@ again: int io_run_local_work(struct io_ring_ctx *ctx) { - bool locked; + bool locked = mutex_trylock(&ctx->uring_lock); int ret; - __set_current_state(TASK_RUNNING); - locked = mutex_trylock(&ctx->uring_lock); ret = __io_run_local_work(ctx, &locked); if (locked) mutex_unlock(&ctx->uring_lock); @@ -2453,6 +2451,7 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, int io_run_task_work_sig(struct io_ring_ctx *ctx) { if (!llist_empty(&ctx->work_llist)) { + __set_current_state(TASK_RUNNING); if (io_run_local_work(ctx) > 0) return 1; } From 3e5655552a8299492d54117a15b6ddf5a2e7512c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:06 +0000 Subject: [PATCH 114/265] io_uring: mark io_run_local_work static io_run_local_work is enclosed in io_uring.c, we don't need to export it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b477fb81f5e77044f724a06fe245d5c078659364.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- io_uring/io_uring.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index e61e2a0f23b1..79d94381d967 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1337,7 +1337,7 @@ again: } -int io_run_local_work(struct io_ring_ctx *ctx) +static int io_run_local_work(struct io_ring_ctx *ctx) { bool locked = mutex_trylock(&ctx->uring_lock); int ret; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 011932935c36..78896d1f7916 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -29,7 +29,6 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); bool io_req_cqe_overflow(struct io_kiocb *req); int io_run_task_work_sig(struct io_ring_ctx *ctx); int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked); -int io_run_local_work(struct io_ring_ctx *ctx); void io_req_defer_failed(struct io_kiocb *req, s32 res); void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); From 360173ab9e1a8a50bc9092ae8c741f0a05d499b7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:07 +0000 Subject: [PATCH 115/265] io_uring: move io_run_local_work_locked io_run_local_work_locked() is only used in io_uring.c, move it there. With that we can also make __io_run_local_work() static. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/91757bcb33e5774e49fed6f2b6e058630608119b.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 18 +++++++++++++++++- io_uring/io_uring.h | 17 ----------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 79d94381d967..1e79f37b071b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1296,7 +1296,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) } } -int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked) +static int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked) { struct llist_node *node; struct llist_node fake; @@ -1337,6 +1337,22 @@ again: } +static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) +{ + bool locked; + int ret; + + if (llist_empty(&ctx->work_llist)) + return 0; + + locked = true; + ret = __io_run_local_work(ctx, &locked); + /* shouldn't happen! */ + if (WARN_ON_ONCE(!locked)) + mutex_lock(&ctx->uring_lock); + return ret; +} + static int io_run_local_work(struct io_ring_ctx *ctx) { bool locked = mutex_trylock(&ctx->uring_lock); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 78896d1f7916..b5975e353aa1 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -28,7 +28,6 @@ enum { struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); bool io_req_cqe_overflow(struct io_kiocb *req); int io_run_task_work_sig(struct io_ring_ctx *ctx); -int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked); void io_req_defer_failed(struct io_kiocb *req, s32 res); void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); @@ -283,22 +282,6 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx) return task_work_pending(current) || !wq_list_empty(&ctx->work_llist); } -static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) -{ - bool locked; - int ret; - - if (llist_empty(&ctx->work_llist)) - return 0; - - locked = true; - ret = __io_run_local_work(ctx, &locked); - /* shouldn't happen! */ - if (WARN_ON_ONCE(!locked)) - mutex_lock(&ctx->uring_lock); - return ret; -} - static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) { if (!*locked) { From 7b235dd82ad32c1626e51303d94ec5ef4d7bc994 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:08 +0000 Subject: [PATCH 116/265] io_uring: separate wq for ring polling Don't use ->cq_wait for ring polling but add a separate wait queue for it. We need it for following patches. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/dea0be0bf990503443c5c6c337fc66824af7d590.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 +- io_uring/io_uring.c | 3 ++- io_uring/io_uring.h | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 8dfb6c4a35d9..0d94ee191c15 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -319,7 +319,7 @@ struct io_ring_ctx { } ____cacheline_aligned_in_smp; /* Keep this last, we don't need it for the fast path */ - + struct wait_queue_head poll_wq; struct io_restriction restrictions; /* slow path rsrc auxilary data, used by update/register */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1e79f37b071b..2ee2bcaeadfd 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -316,6 +316,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1); mutex_init(&ctx->uring_lock); init_waitqueue_head(&ctx->cq_wait); + init_waitqueue_head(&ctx->poll_wq); spin_lock_init(&ctx->completion_lock); spin_lock_init(&ctx->timeout_lock); INIT_WQ_LIST(&ctx->iopoll_list); @@ -2786,7 +2787,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) struct io_ring_ctx *ctx = file->private_data; __poll_t mask = 0; - poll_wait(file, &ctx->cq_wait, wait); + poll_wait(file, &ctx->poll_wq, wait); /* * synchronizes with barrier from wq_has_sleeper call in * io_commit_cqring diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index b5975e353aa1..c75bbb94703c 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -220,9 +220,18 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx) smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); } +static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) +{ + if (waitqueue_active(&ctx->poll_wq)) + __wake_up(&ctx->poll_wq, TASK_NORMAL, 0, + poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); +} + /* requires smb_mb() prior, see wq_has_sleeper() */ static inline void __io_cqring_wake(struct io_ring_ctx *ctx) { + io_poll_wq_wake(ctx); + /* * Trigger waitqueue handler on all waiters on our waitqueue. This * won't necessarily wake up all the tasks, io_should_wake() will make From bca39f39058567643487cd654970717705784ba3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:09 +0000 Subject: [PATCH 117/265] io_uring: add lazy poll_wq activation Even though io_poll_wq_wake()'s waitqueue_active reuses a barrier we do for another waitqueue, it's not going to be the case in the future and so we want to have a fast path for it when the ring has never been polled. Move poll_wq wake ups into __io_commit_cqring_flush() using a new flag called ->poll_activated. The idea behind the flag is to set it when the ring was polled for the first time. This requires additional sync to not miss events, which is done here by using task_work for ->task_complete rings, and by default enabling the flag for all other types of rings. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/060785e8e9137a920b232c0c7f575b131af19cac.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 ++ io_uring/io_uring.c | 60 +++++++++++++++++++++++++++++++++- io_uring/io_uring.h | 7 ++-- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 0d94ee191c15..7b5e90520278 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -206,6 +206,7 @@ struct io_ring_ctx { unsigned int syscall_iopoll: 1; /* all CQEs should be posted only by the submitter task */ unsigned int task_complete: 1; + unsigned int poll_activated: 1; enum task_work_notify_mode notify_method; struct io_rings *rings; @@ -357,6 +358,7 @@ struct io_ring_ctx { u32 iowq_limits[2]; bool iowq_limits_set; + struct callback_head poll_wq_task_work; struct list_head defer_list; unsigned sq_thread_idle; /* protected by ->completion_lock */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2ee2bcaeadfd..de71730d9051 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -573,6 +573,8 @@ static void io_eventfd_flush_signal(struct io_ring_ctx *ctx) void __io_commit_cqring_flush(struct io_ring_ctx *ctx) { + if (ctx->poll_activated) + io_poll_wq_wake(ctx); if (ctx->off_timeout_used) io_flush_timeouts(ctx); if (ctx->drain_active) { @@ -2782,11 +2784,53 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) kfree(ctx); } +static __cold void io_activate_pollwq_cb(struct callback_head *cb) +{ + struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx, + poll_wq_task_work); + + mutex_lock(&ctx->uring_lock); + ctx->poll_activated = true; + mutex_unlock(&ctx->uring_lock); + + /* + * Wake ups for some events between start of polling and activation + * might've been lost due to loose synchronisation. + */ + wake_up_all(&ctx->poll_wq); + percpu_ref_put(&ctx->refs); +} + +static __cold void io_activate_pollwq(struct io_ring_ctx *ctx) +{ + spin_lock(&ctx->completion_lock); + /* already activated or in progress */ + if (ctx->poll_activated || ctx->poll_wq_task_work.func) + goto out; + if (WARN_ON_ONCE(!ctx->task_complete)) + goto out; + if (!ctx->submitter_task) + goto out; + /* + * with ->submitter_task only the submitter task completes requests, we + * only need to sync with it, which is done by injecting a tw + */ + init_task_work(&ctx->poll_wq_task_work, io_activate_pollwq_cb); + percpu_ref_get(&ctx->refs); + if (task_work_add(ctx->submitter_task, &ctx->poll_wq_task_work, TWA_SIGNAL)) + percpu_ref_put(&ctx->refs); +out: + spin_unlock(&ctx->completion_lock); +} + static __poll_t io_uring_poll(struct file *file, poll_table *wait) { struct io_ring_ctx *ctx = file->private_data; __poll_t mask = 0; + if (unlikely(!ctx->poll_activated)) + io_activate_pollwq(ctx); + poll_wait(file, &ctx->poll_wq, wait); /* * synchronizes with barrier from wq_has_sleeper call in @@ -3593,6 +3637,13 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, !(ctx->flags & IORING_SETUP_SQPOLL)) ctx->task_complete = true; + /* + * lazy poll_wq activation relies on ->task_complete for synchronisation + * purposes, see io_activate_pollwq() + */ + if (!ctx->task_complete) + ctx->poll_activated = true; + /* * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user * space applications don't need to do io completion events @@ -3886,8 +3937,15 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; - if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) + if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) { WRITE_ONCE(ctx->submitter_task, get_task_struct(current)); + /* + * Lazy activation attempts would fail if it was polled before + * submitter_task is set. + */ + if (wq_has_sleeper(&ctx->poll_wq)) + io_activate_pollwq(ctx); + } if (ctx->restrictions.registered) ctx->restricted = 1; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index c75bbb94703c..5113e0ddb01d 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -222,7 +222,7 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx) static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) { - if (waitqueue_active(&ctx->poll_wq)) + if (wq_has_sleeper(&ctx->poll_wq)) __wake_up(&ctx->poll_wq, TASK_NORMAL, 0, poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); } @@ -230,8 +230,6 @@ static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) /* requires smb_mb() prior, see wq_has_sleeper() */ static inline void __io_cqring_wake(struct io_ring_ctx *ctx) { - io_poll_wq_wake(ctx); - /* * Trigger waitqueue handler on all waiters on our waitqueue. This * won't necessarily wake up all the tasks, io_should_wake() will make @@ -316,7 +314,8 @@ static inline void io_req_complete_defer(struct io_kiocb *req) static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) { - if (unlikely(ctx->off_timeout_used || ctx->drain_active || ctx->has_evfd)) + if (unlikely(ctx->off_timeout_used || ctx->drain_active || + ctx->has_evfd || ctx->poll_activated)) __io_commit_cqring_flush(ctx); } From 3181e22fb79910c7071e84a43af93ac89e8a7106 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:10 +0000 Subject: [PATCH 118/265] io_uring: wake up optimisations Flush completions is done either from the submit syscall or by the task_work, both are in the context of the submitter task, and when it goes for a single threaded rings like implied by ->task_complete, there won't be any waiters on ->cq_wait but the master task. That means that there can be no tasks sleeping on cq_wait while we run __io_submit_flush_completions() and so waking up can be skipped. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/60ad9768ec74435a0ddaa6eec0ffa7729474f69f.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index de71730d9051..f2e1dd076d98 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -621,6 +621,25 @@ static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx) io_cqring_wake(ctx); } +static inline void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx) + __releases(ctx->completion_lock) +{ + io_commit_cqring(ctx); + __io_cq_unlock(ctx); + io_commit_cqring_flush(ctx); + + /* + * As ->task_complete implies that the ring is single tasked, cq_wait + * may only be waited on by the current in io_cqring_wait(), but since + * it will re-check the wakeup conditions once we return we can safely + * skip waking it up. + */ + if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) { + smp_mb(); + __io_cqring_wake(ctx); + } +} + void io_cq_unlock_post(struct io_ring_ctx *ctx) __releases(ctx->completion_lock) { @@ -1480,7 +1499,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) } } } - __io_cq_unlock_post(ctx); + __io_cq_unlock_post_flush(ctx); if (!wq_list_empty(&ctx->submit_state.compl_reqs)) { io_free_batch_list(ctx, state->compl_reqs.first); From 130bd686d9be918e4cc8c03abf5794ba2d860502 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:11 +0000 Subject: [PATCH 119/265] io_uring: waitqueue-less cq waiting With DEFER_TASKRUN only ctx->submitter_task might be waiting for CQEs, we can use this to optimise io_cqring_wait(). Replace ->cq_wait waitqueue with waking the task directly. It works but misses an important optimisation covered by the following patch, so this patch without follow ups might hurt performance. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/103d174d35d919d4cb0922d8a9c93a8f0c35f74a.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index f2e1dd076d98..be26829f7d20 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1263,7 +1263,7 @@ static void io_req_local_work_add(struct io_kiocb *req) percpu_ref_put(&ctx->refs); return; } - /* need it for the following io_cqring_wake() */ + /* needed for the following wake up */ smp_mb__after_atomic(); if (unlikely(atomic_read(&req->task->io_uring->in_idle))) { @@ -1274,10 +1274,9 @@ static void io_req_local_work_add(struct io_kiocb *req) if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); - if (ctx->has_evfd) io_eventfd_signal(ctx); - __io_cqring_wake(ctx); + wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); percpu_ref_put(&ctx->refs); } @@ -2576,12 +2575,17 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, do { unsigned long check_cq; - prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, - TASK_INTERRUPTIBLE); + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { + set_current_state(TASK_INTERRUPTIBLE); + } else { + prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, + TASK_INTERRUPTIBLE); + } + ret = io_cqring_wait_schedule(ctx, &iowq); + __set_current_state(TASK_RUNNING); if (ret < 0) break; - __set_current_state(TASK_RUNNING); /* * Run task_work after scheduling and before io_should_wake(). * If we got woken because of task_work being processed, run it @@ -2609,7 +2613,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, cond_resched(); } while (1); - finish_wait(&ctx->cq_wait, &iowq.wq); + if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) + finish_wait(&ctx->cq_wait, &iowq.wq); restore_saved_sigmask_unless(ret == -EINTR); return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; From d80c0f00d04766972c95e72b7535a842d6f4680d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:12 +0000 Subject: [PATCH 120/265] io_uring: add io_req_local_work_add wake fast path Don't wake the master task after queueing a deferred tw unless it's currently waiting in io_cqring_wait. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/717702d772825a6647e6c315b4690277ba84c3fc.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 1 + io_uring/io_uring.c | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 7b5e90520278..cc0cf0705b8f 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -295,6 +295,7 @@ struct io_ring_ctx { spinlock_t completion_lock; bool poll_multi_queue; + bool cq_waiting; /* * ->iopoll_list is protected by the ctx->uring_lock for diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index be26829f7d20..3f92028a7ee8 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1276,7 +1276,9 @@ static void io_req_local_work_add(struct io_kiocb *req) atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); if (ctx->has_evfd) io_eventfd_signal(ctx); - wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); + + if (READ_ONCE(ctx->cq_waiting)) + wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE); percpu_ref_put(&ctx->refs); } @@ -2576,6 +2578,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, unsigned long check_cq; if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { + WRITE_ONCE(ctx->cq_waiting, 1); set_current_state(TASK_INTERRUPTIBLE); } else { prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, @@ -2584,6 +2587,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ret = io_cqring_wait_schedule(ctx, &iowq); __set_current_state(TASK_RUNNING); + WRITE_ONCE(ctx->cq_waiting, 0); + if (ret < 0) break; /* From c3f4d39ee4bc06f975ff45012398342ff276eb69 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 9 Jan 2023 14:46:13 +0000 Subject: [PATCH 121/265] io_uring: optimise deferred tw execution We needed fake nodes in __io_run_local_work() and to avoid unecessary wake ups while the task already running task_works, but we don't need them anymore since wake ups are protected by cq_waiting, which is always cleared by the time we're executing deferred task_work items. Note that because of loose sync around cq_waiting clearing io_req_local_work_add() may wake the task more than once, but that's fine and should be rare to not hurt perf. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8839534891f0a2f1076e78554a31ea7e099f7de5.1673274244.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 3f92028a7ee8..653361746203 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1322,17 +1322,16 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) static int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked) { struct llist_node *node; - struct llist_node fake; - struct llist_node *current_final = NULL; + unsigned int loops = 0; int ret = 0; - unsigned int loops = 1; if (WARN_ON_ONCE(ctx->submitter_task != current)) return -EEXIST; - - node = io_llist_xchg(&ctx->work_llist, &fake); + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) + atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); again: - while (node != current_final) { + node = io_llist_xchg(&ctx->work_llist, NULL); + while (node) { struct llist_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); @@ -1341,23 +1340,14 @@ again: ret++; node = next; } + loops++; - if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) - atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); - - node = io_llist_cmpxchg(&ctx->work_llist, &fake, NULL); - if (node != &fake) { - loops++; - current_final = &fake; - node = io_llist_xchg(&ctx->work_llist, &fake); + if (!llist_empty(&ctx->work_llist)) goto again; - } - if (*locked) io_submit_flush_completions(ctx); trace_io_uring_local_work_run(ctx, ret, loops); return ret; - } static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) From 88b80534f60f5ddf5f42218b280e0370f95eae78 Mon Sep 17 00:00:00 2001 From: Quanfa Fu Date: Sun, 15 Jan 2023 15:15:19 +0800 Subject: [PATCH 122/265] io_uring: make io_sqpoll_wait_sq return void Change the return type to void since it always return 0, and no need to do the checking in syscall io_uring_enter. Signed-off-by: Quanfa Fu Link: https://lore.kernel.org/r/20230115071519.554282-1-quanfafu@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 8 +++----- io_uring/sqpoll.c | 3 +-- io_uring/sqpoll.h | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 653361746203..8fddc010ffe2 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3412,11 +3412,9 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, } if (flags & IORING_ENTER_SQ_WAKEUP) wake_up(&ctx->sq_data->wait); - if (flags & IORING_ENTER_SQ_WAIT) { - ret = io_sqpoll_wait_sq(ctx); - if (ret) - goto out; - } + if (flags & IORING_ENTER_SQ_WAIT) + io_sqpoll_wait_sq(ctx); + ret = to_submit; } else if (to_submit) { ret = io_uring_add_tctx_node(ctx); diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 559652380672..0119d3f1a556 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -312,7 +312,7 @@ static int io_sq_thread(void *data) do_exit(0); } -int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) +void io_sqpoll_wait_sq(struct io_ring_ctx *ctx) { DEFINE_WAIT(wait); @@ -327,7 +327,6 @@ int io_sqpoll_wait_sq(struct io_ring_ctx *ctx) } while (!signal_pending(current)); finish_wait(&ctx->sqo_sq_wait, &wait); - return 0; } __cold int io_sq_offload_create(struct io_ring_ctx *ctx, diff --git a/io_uring/sqpoll.h b/io_uring/sqpoll.h index 0c3fbcd1f583..e1b8d508d22d 100644 --- a/io_uring/sqpoll.h +++ b/io_uring/sqpoll.h @@ -26,4 +26,4 @@ void io_sq_thread_stop(struct io_sq_data *sqd); void io_sq_thread_park(struct io_sq_data *sqd); void io_sq_thread_unpark(struct io_sq_data *sqd); void io_put_sq_data(struct io_sq_data *sqd); -int io_sqpoll_wait_sq(struct io_ring_ctx *ctx); +void io_sqpoll_wait_sq(struct io_ring_ctx *ctx); From b0b7a7d24b66109a940d09d8d2dcf513e4eaf3a1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 16 Jan 2023 16:48:57 +0000 Subject: [PATCH 123/265] io_uring: return back links tw run optimisation io_submit_flush_completions() may queue new requests for tw execution, especially true for linked requests. Recheck the tw list for emptiness after flushing completions. Note that this doesn't really fix the commit referenced below, but it does reinstate an optimization that existed before that got merged. Fixes: f88262e60bb9 ("io_uring: lockless task list") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6328acdbb5e60efc762b18003382de077e6e1367.1673887636.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 8fddc010ffe2..c47af3a3dacf 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1344,8 +1344,11 @@ again: if (!llist_empty(&ctx->work_llist)) goto again; - if (*locked) + if (*locked) { io_submit_flush_completions(ctx); + if (!llist_empty(&ctx->work_llist)) + goto again; + } trace_io_uring_local_work_run(ctx, ret, loops); return ret; } From 89800a2dd570919bfe01ced90c80e3b472d1c723 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 16 Jan 2023 16:48:58 +0000 Subject: [PATCH 124/265] io_uring: don't export io_put_task() io_put_task() is only used in uring.c so enclose it there together with __io_put_task(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/43c7f9227e2ab215f1a6069dadbc5382bed346fe.1673887636.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 11 ++++++++++- io_uring/io_uring.h | 10 ---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c47af3a3dacf..f49d0036657f 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -715,7 +715,7 @@ static void io_cqring_overflow_flush(struct io_ring_ctx *ctx) io_cqring_do_overflow_flush(ctx); } -void __io_put_task(struct task_struct *task, int nr) +static void __io_put_task(struct task_struct *task, int nr) { struct io_uring_task *tctx = task->io_uring; @@ -725,6 +725,15 @@ void __io_put_task(struct task_struct *task, int nr) put_task_struct_many(task, nr); } +/* must to be called somewhat shortly after putting a request */ +static inline void io_put_task(struct task_struct *task, int nr) +{ + if (likely(task == current)) + task->io_uring->cached_refs += nr; + else + __io_put_task(task, nr); +} + void io_task_refs_refill(struct io_uring_task *tctx) { unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 5113e0ddb01d..c68edf9872a5 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -70,7 +70,6 @@ void io_wq_submit_work(struct io_wq_work *work); void io_free_req(struct io_kiocb *req); void io_queue_next(struct io_kiocb *req); -void __io_put_task(struct task_struct *task, int nr); void io_task_refs_refill(struct io_uring_task *tctx); bool __io_alloc_req_refill(struct io_ring_ctx *ctx); @@ -319,15 +318,6 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) __io_commit_cqring_flush(ctx); } -/* must to be called somewhat shortly after putting a request */ -static inline void io_put_task(struct task_struct *task, int nr) -{ - if (likely(task == current)) - task->io_uring->cached_refs += nr; - else - __io_put_task(task, nr); -} - static inline void io_get_task_refs(int nr) { struct io_uring_task *tctx = current->io_uring; From 31f084b7b0288fd51740b1e1efdb0ff61fb81e48 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 16 Jan 2023 16:48:59 +0000 Subject: [PATCH 125/265] io_uring: simplify fallback execution Lock the ring with uring_lock in io_fallback_req_func(), which should make it a bit safer and easier. With that we also don't need refs pinning as io_ring_exit_work() will wait until uring_lock is freed. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/56170e6a0cbfc8edee2794c6613e8f6f1d76d276.1673887636.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index f49d0036657f..c314dc111d5d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -245,17 +245,15 @@ static __cold void io_fallback_req_func(struct work_struct *work) fallback_work.work); struct llist_node *node = llist_del_all(&ctx->fallback_llist); struct io_kiocb *req, *tmp; - bool locked = false; + bool locked = true; - percpu_ref_get(&ctx->refs); + mutex_lock(&ctx->uring_lock); llist_for_each_entry_safe(req, tmp, node, io_task_work.node) req->io_task_work.func(req, &locked); - - if (locked) { - io_submit_flush_completions(ctx); - mutex_unlock(&ctx->uring_lock); - } - percpu_ref_put(&ctx->refs); + if (WARN_ON_ONCE(!locked)) + return; + io_submit_flush_completions(ctx); + mutex_unlock(&ctx->uring_lock); } static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits) From 632ffe0956740ba56ae3f83778f3bf97edd57c69 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 16 Jan 2023 16:49:00 +0000 Subject: [PATCH 126/265] io_uring: optimise ctx flags layout There may be different cost for reeading just one byte or more, so it's benificial to keep ctx flag bits that we access together in a single byte. That affected code generation of __io_cq_unlock_post_flush() and removed one memory load. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/bbe8ca4705704690319d65e45845f9fc9d35f420.1673887636.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index cc0cf0705b8f..0efe4d784358 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -196,17 +196,17 @@ struct io_ring_ctx { /* const or read-mostly hot data */ struct { unsigned int flags; - unsigned int compat: 1; unsigned int drain_next: 1; unsigned int restricted: 1; unsigned int off_timeout_used: 1; unsigned int drain_active: 1; - unsigned int drain_disabled: 1; unsigned int has_evfd: 1; - unsigned int syscall_iopoll: 1; /* all CQEs should be posted only by the submitter task */ unsigned int task_complete: 1; + unsigned int syscall_iopoll: 1; unsigned int poll_activated: 1; + unsigned int drain_disabled: 1; + unsigned int compat: 1; enum task_work_notify_mode notify_method; struct io_rings *rings; From 68a2cc1bba98144340f6ed66f3aed57e914766d0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 16 Jan 2023 16:49:01 +0000 Subject: [PATCH 127/265] io_uring: refactor __io_req_complete_post Keep parts of __io_req_complete_post() relying on req->flags together so the value can be cached. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2b4fbb42f404a0e75c4d9f0a5b16f314a839d0a9.1673887636.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c314dc111d5d..597b757e791e 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -974,14 +974,14 @@ static void __io_req_complete_post(struct io_kiocb *req) req->link = NULL; } } + io_put_kbuf_comp(req); + io_dismantle_req(req); io_req_put_rsrc(req); /* * Selected buffer deallocation in io_clean_op() assumes that * we don't hold ->completion_lock. Clean them here to avoid * deadlocks. */ - io_put_kbuf_comp(req); - io_dismantle_req(req); io_put_task(req->task, 1); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; From a7dd27828b00be8c0c7520c53baf0b360f4d8bea Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 12 Jan 2023 06:44:10 -0800 Subject: [PATCH 128/265] io_uring: Rename struct io_op_def The current io_op_def struct is becoming huge and the name is a bit generic. The goal of this patch is to rename this struct to `io_issue_def`. This struct will contain the hot functions associated with the issue code path. For now, this patch only renames the structure, and an upcoming patch will break up the structure in two, moving the non-issue fields to a secondary struct. Signed-off-by: Breno Leitao Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20230112144411.2624698-1-leitao@debian.org Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 26 +++++++++++++------------- io_uring/opdef.c | 16 ++++++++-------- io_uring/opdef.h | 4 ++-- io_uring/poll.c | 2 +- io_uring/rw.c | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 597b757e791e..e5b51abe0060 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -406,7 +406,7 @@ static inline void io_arm_ltimeout(struct io_kiocb *req) static void io_prep_async_work(struct io_kiocb *req) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct io_ring_ctx *ctx = req->ctx; if (!(req->flags & REQ_F_CREDS)) { @@ -1009,7 +1009,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) void io_req_defer_failed(struct io_kiocb *req, s32 res) __must_hold(&ctx->uring_lock) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; lockdep_assert_held(&req->ctx->uring_lock); @@ -1741,8 +1741,8 @@ unsigned int io_file_get_flags(struct file *file) bool io_alloc_async_data(struct io_kiocb *req) { - WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); - req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); + WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size); + req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL); if (req->async_data) { req->flags |= REQ_F_ASYNC_DATA; return false; @@ -1752,7 +1752,7 @@ bool io_alloc_async_data(struct io_kiocb *req) int io_req_prep_async(struct io_kiocb *req) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; /* assign early for deferred execution for non-fixed file */ if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE)) @@ -1761,7 +1761,7 @@ int io_req_prep_async(struct io_kiocb *req) return 0; if (WARN_ON_ONCE(req_has_async_data(req))) return -EFAULT; - if (!io_op_defs[req->opcode].manual_alloc) { + if (!io_issue_defs[req->opcode].manual_alloc) { if (io_alloc_async_data(req)) return -EAGAIN; } @@ -1829,7 +1829,7 @@ static void io_clean_op(struct io_kiocb *req) } if (req->flags & REQ_F_NEED_CLEANUP) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; if (def->cleanup) def->cleanup(req); @@ -1855,7 +1855,7 @@ static void io_clean_op(struct io_kiocb *req) static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) { - if (req->file || !io_op_defs[req->opcode].needs_file) + if (req->file || !io_issue_defs[req->opcode].needs_file) return true; if (req->flags & REQ_F_FIXED_FILE) @@ -1868,7 +1868,7 @@ static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; const struct cred *creds = NULL; int ret; @@ -1922,7 +1922,7 @@ struct io_wq_work *io_wq_free_work(struct io_wq_work *work) void io_wq_submit_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ; bool needs_poll = false; int ret = 0, err = -ECANCELED; @@ -2137,7 +2137,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, const struct io_uring_sqe *sqe) __must_hold(&ctx->uring_lock) { - const struct io_op_def *def; + const struct io_issue_def *def; unsigned int sqe_flags; int personality; u8 opcode; @@ -2155,7 +2155,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->opcode = 0; return -EINVAL; } - def = &io_op_defs[opcode]; + def = &io_issue_defs[opcode]; if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) { /* enforce forwards compatibility on users */ if (sqe_flags & ~SQE_VALID_FLAGS) @@ -3859,7 +3859,7 @@ static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, for (i = 0; i < nr_args; i++) { p->ops[i].op = i; - if (!io_op_defs[i].not_supported) + if (!io_issue_defs[i].not_supported) p->ops[i].flags = IO_URING_OP_SUPPORTED; } p->ops_len = i; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 3aa0d65c50e3..3c95e70a625e 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -46,7 +46,7 @@ static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, return -EOPNOTSUPP; } -const struct io_op_def io_op_defs[] = { +const struct io_issue_def io_issue_defs[] = { [IORING_OP_NOP] = { .audit_skip = 1, .iopoll = 1, @@ -536,7 +536,7 @@ const struct io_op_def io_op_defs[] = { const char *io_uring_get_opcode(u8 opcode) { if (opcode < IORING_OP_LAST) - return io_op_defs[opcode].name; + return io_issue_defs[opcode].name; return "INVALID"; } @@ -544,12 +544,12 @@ void __init io_uring_optable_init(void) { int i; - BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); + BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); - for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) { - BUG_ON(!io_op_defs[i].prep); - if (io_op_defs[i].prep != io_eopnotsupp_prep) - BUG_ON(!io_op_defs[i].issue); - WARN_ON_ONCE(!io_op_defs[i].name); + for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { + BUG_ON(!io_issue_defs[i].prep); + if (io_issue_defs[i].prep != io_eopnotsupp_prep) + BUG_ON(!io_issue_defs[i].issue); + WARN_ON_ONCE(!io_issue_defs[i].name); } } diff --git a/io_uring/opdef.h b/io_uring/opdef.h index df7e13d9bfba..d718e2ab1ff7 100644 --- a/io_uring/opdef.h +++ b/io_uring/opdef.h @@ -2,7 +2,7 @@ #ifndef IOU_OP_DEF_H #define IOU_OP_DEF_H -struct io_op_def { +struct io_issue_def { /* needs req->file assigned */ unsigned needs_file : 1; /* should block plug */ @@ -41,7 +41,7 @@ struct io_op_def { void (*fail)(struct io_kiocb *); }; -extern const struct io_op_def io_op_defs[]; +extern const struct io_issue_def io_issue_defs[]; void io_uring_optable_init(void); #endif diff --git a/io_uring/poll.c b/io_uring/poll.c index 2ac1366adbd7..8339a92b4510 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -678,7 +678,7 @@ alloc_apoll: int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags) { - const struct io_op_def *def = &io_op_defs[req->opcode]; + const struct io_issue_def *def = &io_issue_defs[req->opcode]; struct async_poll *apoll; struct io_poll_table ipt; __poll_t mask = POLLPRI | POLLERR | EPOLLET; diff --git a/io_uring/rw.c b/io_uring/rw.c index 3a3aef7e7c33..02e710e77145 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, struct io_rw_state *s, bool force) { - if (!force && !io_op_defs[req->opcode].prep_async) + if (!force && !io_issue_defs[req->opcode].prep_async) return 0; if (!req_has_async_data(req)) { struct io_async_rw *iorw; From f30bd4d03824fb437bf080c2b2f926cfee3f09d0 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Thu, 12 Jan 2023 06:44:11 -0800 Subject: [PATCH 129/265] io_uring: Split io_issue_def struct This patch removes some "cold" fields from `struct io_issue_def`. The plan is to keep only highly used fields into `struct io_issue_def`, so, it may be hot in the cache. The hot fields are basically all the bitfields and the callback functions for .issue and .prep. The other less frequently used fields are now located in a secondary and cold struct, called `io_cold_def`. This is the size for the structs: Before: io_issue_def = 56 bytes After: io_issue_def = 24 bytes; io_cold_def = 40 bytes Signed-off-by: Breno Leitao Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20230112144411.2624698-2-leitao@debian.org Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 15 +- io_uring/opdef.c | 327 ++++++++++++++++++++++++++++++-------------- io_uring/opdef.h | 9 +- io_uring/rw.c | 2 +- 4 files changed, 238 insertions(+), 115 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index e5b51abe0060..c84dce6944db 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1009,7 +1009,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) void io_req_defer_failed(struct io_kiocb *req, s32 res) __must_hold(&ctx->uring_lock) { - const struct io_issue_def *def = &io_issue_defs[req->opcode]; + const struct io_cold_def *def = &io_cold_defs[req->opcode]; lockdep_assert_held(&req->ctx->uring_lock); @@ -1741,8 +1741,8 @@ unsigned int io_file_get_flags(struct file *file) bool io_alloc_async_data(struct io_kiocb *req) { - WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size); - req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL); + WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size); + req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL); if (req->async_data) { req->flags |= REQ_F_ASYNC_DATA; return false; @@ -1752,20 +1752,21 @@ bool io_alloc_async_data(struct io_kiocb *req) int io_req_prep_async(struct io_kiocb *req) { + const struct io_cold_def *cdef = &io_cold_defs[req->opcode]; const struct io_issue_def *def = &io_issue_defs[req->opcode]; /* assign early for deferred execution for non-fixed file */ if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE)) req->file = io_file_get_normal(req, req->cqe.fd); - if (!def->prep_async) + if (!cdef->prep_async) return 0; if (WARN_ON_ONCE(req_has_async_data(req))) return -EFAULT; - if (!io_issue_defs[req->opcode].manual_alloc) { + if (!def->manual_alloc) { if (io_alloc_async_data(req)) return -EAGAIN; } - return def->prep_async(req); + return cdef->prep_async(req); } static u32 io_get_sequence(struct io_kiocb *req) @@ -1829,7 +1830,7 @@ static void io_clean_op(struct io_kiocb *req) } if (req->flags & REQ_F_NEED_CLEANUP) { - const struct io_issue_def *def = &io_issue_defs[req->opcode]; + const struct io_cold_def *def = &io_cold_defs[req->opcode]; if (def->cleanup) def->cleanup(req); diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 3c95e70a625e..5238ecd7af6a 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -50,7 +50,6 @@ const struct io_issue_def io_issue_defs[] = { [IORING_OP_NOP] = { .audit_skip = 1, .iopoll = 1, - .name = "NOP", .prep = io_nop_prep, .issue = io_nop, }, @@ -64,13 +63,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READV", .prep = io_prep_rw, .issue = io_read, - .prep_async = io_readv_prep_async, - .cleanup = io_readv_writev_cleanup, - .fail = io_rw_fail, }, [IORING_OP_WRITEV] = { .needs_file = 1, @@ -82,18 +76,12 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITEV", .prep = io_prep_rw, .issue = io_write, - .prep_async = io_writev_prep_async, - .cleanup = io_readv_writev_cleanup, - .fail = io_rw_fail, }, [IORING_OP_FSYNC] = { .needs_file = 1, .audit_skip = 1, - .name = "FSYNC", .prep = io_fsync_prep, .issue = io_fsync, }, @@ -106,11 +94,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READ_FIXED", .prep = io_prep_rw, .issue = io_read, - .fail = io_rw_fail, }, [IORING_OP_WRITE_FIXED] = { .needs_file = 1, @@ -122,30 +107,24 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITE_FIXED", .prep = io_prep_rw, .issue = io_write, - .fail = io_rw_fail, }, [IORING_OP_POLL_ADD] = { .needs_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "POLL_ADD", .prep = io_poll_add_prep, .issue = io_poll_add, }, [IORING_OP_POLL_REMOVE] = { .audit_skip = 1, - .name = "POLL_REMOVE", .prep = io_poll_remove_prep, .issue = io_poll_remove, }, [IORING_OP_SYNC_FILE_RANGE] = { .needs_file = 1, .audit_skip = 1, - .name = "SYNC_FILE_RANGE", .prep = io_sfr_prep, .issue = io_sync_file_range, }, @@ -155,14 +134,9 @@ const struct io_issue_def io_issue_defs[] = { .pollout = 1, .ioprio = 1, .manual_alloc = 1, - .name = "SENDMSG", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_sendmsg, - .prep_async = io_sendmsg_prep_async, - .cleanup = io_sendmsg_recvmsg_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif @@ -174,29 +148,21 @@ const struct io_issue_def io_issue_defs[] = { .buffer_select = 1, .ioprio = 1, .manual_alloc = 1, - .name = "RECVMSG", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_recvmsg_prep, .issue = io_recvmsg, - .prep_async = io_recvmsg_prep_async, - .cleanup = io_sendmsg_recvmsg_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_TIMEOUT] = { .audit_skip = 1, - .async_size = sizeof(struct io_timeout_data), - .name = "TIMEOUT", .prep = io_timeout_prep, .issue = io_timeout, }, [IORING_OP_TIMEOUT_REMOVE] = { /* used by timeout updates' prep() */ .audit_skip = 1, - .name = "TIMEOUT_REMOVE", .prep = io_timeout_remove_prep, .issue = io_timeout_remove, }, @@ -206,7 +172,6 @@ const struct io_issue_def io_issue_defs[] = { .pollin = 1, .poll_exclusive = 1, .ioprio = 1, /* used for flags */ - .name = "ACCEPT", #if defined(CONFIG_NET) .prep = io_accept_prep, .issue = io_accept, @@ -216,14 +181,11 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_ASYNC_CANCEL] = { .audit_skip = 1, - .name = "ASYNC_CANCEL", .prep = io_async_cancel_prep, .issue = io_async_cancel, }, [IORING_OP_LINK_TIMEOUT] = { .audit_skip = 1, - .async_size = sizeof(struct io_timeout_data), - .name = "LINK_TIMEOUT", .prep = io_link_timeout_prep, .issue = io_no_issue, }, @@ -231,46 +193,36 @@ const struct io_issue_def io_issue_defs[] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, - .name = "CONNECT", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_connect), .prep = io_connect_prep, .issue = io_connect, - .prep_async = io_connect_prep_async, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_FALLOCATE] = { .needs_file = 1, - .name = "FALLOCATE", .prep = io_fallocate_prep, .issue = io_fallocate, }, [IORING_OP_OPENAT] = { - .name = "OPENAT", .prep = io_openat_prep, .issue = io_openat, - .cleanup = io_open_cleanup, }, [IORING_OP_CLOSE] = { - .name = "CLOSE", .prep = io_close_prep, .issue = io_close, }, [IORING_OP_FILES_UPDATE] = { .audit_skip = 1, .iopoll = 1, - .name = "FILES_UPDATE", .prep = io_files_update_prep, .issue = io_files_update, }, [IORING_OP_STATX] = { .audit_skip = 1, - .name = "STATX", .prep = io_statx_prep, .issue = io_statx, - .cleanup = io_statx_cleanup, }, [IORING_OP_READ] = { .needs_file = 1, @@ -282,11 +234,8 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "READ", .prep = io_prep_rw, .issue = io_read, - .fail = io_rw_fail, }, [IORING_OP_WRITE] = { .needs_file = 1, @@ -298,21 +247,16 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .iopoll = 1, .iopoll_queue = 1, - .async_size = sizeof(struct io_async_rw), - .name = "WRITE", .prep = io_prep_rw, .issue = io_write, - .fail = io_rw_fail, }, [IORING_OP_FADVISE] = { .needs_file = 1, .audit_skip = 1, - .name = "FADVISE", .prep = io_fadvise_prep, .issue = io_fadvise, }, [IORING_OP_MADVISE] = { - .name = "MADVISE", .prep = io_madvise_prep, .issue = io_madvise, }, @@ -323,13 +267,9 @@ const struct io_issue_def io_issue_defs[] = { .audit_skip = 1, .ioprio = 1, .manual_alloc = 1, - .name = "SEND", #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_send, - .fail = io_sendrecv_fail, - .prep_async = io_send_prep_async, #else .prep = io_eopnotsupp_prep, #endif @@ -341,25 +281,20 @@ const struct io_issue_def io_issue_defs[] = { .buffer_select = 1, .audit_skip = 1, .ioprio = 1, - .name = "RECV", #if defined(CONFIG_NET) .prep = io_recvmsg_prep, .issue = io_recv, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_OPENAT2] = { - .name = "OPENAT2", .prep = io_openat2_prep, .issue = io_openat2, - .cleanup = io_open_cleanup, }, [IORING_OP_EPOLL_CTL] = { .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "EPOLL", #if defined(CONFIG_EPOLL) .prep = io_epoll_ctl_prep, .issue = io_epoll_ctl, @@ -372,21 +307,18 @@ const struct io_issue_def io_issue_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "SPLICE", .prep = io_splice_prep, .issue = io_splice, }, [IORING_OP_PROVIDE_BUFFERS] = { .audit_skip = 1, .iopoll = 1, - .name = "PROVIDE_BUFFERS", .prep = io_provide_buffers_prep, .issue = io_provide_buffers, }, [IORING_OP_REMOVE_BUFFERS] = { .audit_skip = 1, .iopoll = 1, - .name = "REMOVE_BUFFERS", .prep = io_remove_buffers_prep, .issue = io_remove_buffers, }, @@ -395,13 +327,11 @@ const struct io_issue_def io_issue_defs[] = { .hash_reg_file = 1, .unbound_nonreg_file = 1, .audit_skip = 1, - .name = "TEE", .prep = io_tee_prep, .issue = io_tee, }, [IORING_OP_SHUTDOWN] = { .needs_file = 1, - .name = "SHUTDOWN", #if defined(CONFIG_NET) .prep = io_shutdown_prep, .issue = io_shutdown, @@ -410,72 +340,51 @@ const struct io_issue_def io_issue_defs[] = { #endif }, [IORING_OP_RENAMEAT] = { - .name = "RENAMEAT", .prep = io_renameat_prep, .issue = io_renameat, - .cleanup = io_renameat_cleanup, }, [IORING_OP_UNLINKAT] = { - .name = "UNLINKAT", .prep = io_unlinkat_prep, .issue = io_unlinkat, - .cleanup = io_unlinkat_cleanup, }, [IORING_OP_MKDIRAT] = { - .name = "MKDIRAT", .prep = io_mkdirat_prep, .issue = io_mkdirat, - .cleanup = io_mkdirat_cleanup, }, [IORING_OP_SYMLINKAT] = { - .name = "SYMLINKAT", .prep = io_symlinkat_prep, .issue = io_symlinkat, - .cleanup = io_link_cleanup, }, [IORING_OP_LINKAT] = { - .name = "LINKAT", .prep = io_linkat_prep, .issue = io_linkat, - .cleanup = io_link_cleanup, }, [IORING_OP_MSG_RING] = { .needs_file = 1, .iopoll = 1, - .name = "MSG_RING", .prep = io_msg_ring_prep, .issue = io_msg_ring, - .cleanup = io_msg_ring_cleanup, }, [IORING_OP_FSETXATTR] = { .needs_file = 1, - .name = "FSETXATTR", .prep = io_fsetxattr_prep, .issue = io_fsetxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_SETXATTR] = { - .name = "SETXATTR", .prep = io_setxattr_prep, .issue = io_setxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_FGETXATTR] = { .needs_file = 1, - .name = "FGETXATTR", .prep = io_fgetxattr_prep, .issue = io_fgetxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_GETXATTR] = { - .name = "GETXATTR", .prep = io_getxattr_prep, .issue = io_getxattr, - .cleanup = io_xattr_cleanup, }, [IORING_OP_SOCKET] = { .audit_skip = 1, - .name = "SOCKET", #if defined(CONFIG_NET) .prep = io_socket_prep, .issue = io_socket, @@ -486,16 +395,12 @@ const struct io_issue_def io_issue_defs[] = { [IORING_OP_URING_CMD] = { .needs_file = 1, .plug = 1, - .name = "URING_CMD", .iopoll = 1, .iopoll_queue = 1, - .async_size = uring_cmd_pdu_size(1), .prep = io_uring_cmd_prep, .issue = io_uring_cmd, - .prep_async = io_uring_cmd_prep_async, }, [IORING_OP_SEND_ZC] = { - .name = "SEND_ZC", .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, @@ -503,32 +408,243 @@ const struct io_issue_def io_issue_defs[] = { .ioprio = 1, .manual_alloc = 1, #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_send_zc_prep, .issue = io_send_zc, - .prep_async = io_send_prep_async, - .cleanup = io_send_zc_cleanup, - .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif }, [IORING_OP_SENDMSG_ZC] = { - .name = "SENDMSG_ZC", .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, .ioprio = 1, .manual_alloc = 1, #if defined(CONFIG_NET) - .async_size = sizeof(struct io_async_msghdr), .prep = io_send_zc_prep, .issue = io_sendmsg_zc, +#else + .prep = io_eopnotsupp_prep, +#endif + }, +}; + + +const struct io_cold_def io_cold_defs[] = { + [IORING_OP_NOP] = { + .name = "NOP", + }, + [IORING_OP_READV] = { + .async_size = sizeof(struct io_async_rw), + .name = "READV", + .prep_async = io_readv_prep_async, + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_WRITEV] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITEV", + .prep_async = io_writev_prep_async, + .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, + }, + [IORING_OP_FSYNC] = { + .name = "FSYNC", + }, + [IORING_OP_READ_FIXED] = { + .async_size = sizeof(struct io_async_rw), + .name = "READ_FIXED", + .fail = io_rw_fail, + }, + [IORING_OP_WRITE_FIXED] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITE_FIXED", + .fail = io_rw_fail, + }, + [IORING_OP_POLL_ADD] = { + .name = "POLL_ADD", + }, + [IORING_OP_POLL_REMOVE] = { + .name = "POLL_REMOVE", + }, + [IORING_OP_SYNC_FILE_RANGE] = { + .name = "SYNC_FILE_RANGE", + }, + [IORING_OP_SENDMSG] = { + .name = "SENDMSG", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_sendmsg_prep_async, + .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_RECVMSG] = { + .name = "RECVMSG", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_recvmsg_prep_async, + .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_TIMEOUT] = { + .async_size = sizeof(struct io_timeout_data), + .name = "TIMEOUT", + }, + [IORING_OP_TIMEOUT_REMOVE] = { + .name = "TIMEOUT_REMOVE", + }, + [IORING_OP_ACCEPT] = { + .name = "ACCEPT", + }, + [IORING_OP_ASYNC_CANCEL] = { + .name = "ASYNC_CANCEL", + }, + [IORING_OP_LINK_TIMEOUT] = { + .async_size = sizeof(struct io_timeout_data), + .name = "LINK_TIMEOUT", + }, + [IORING_OP_CONNECT] = { + .name = "CONNECT", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_connect), + .prep_async = io_connect_prep_async, +#endif + }, + [IORING_OP_FALLOCATE] = { + .name = "FALLOCATE", + }, + [IORING_OP_OPENAT] = { + .name = "OPENAT", + .cleanup = io_open_cleanup, + }, + [IORING_OP_CLOSE] = { + .name = "CLOSE", + }, + [IORING_OP_FILES_UPDATE] = { + .name = "FILES_UPDATE", + }, + [IORING_OP_STATX] = { + .name = "STATX", + .cleanup = io_statx_cleanup, + }, + [IORING_OP_READ] = { + .async_size = sizeof(struct io_async_rw), + .name = "READ", + .fail = io_rw_fail, + }, + [IORING_OP_WRITE] = { + .async_size = sizeof(struct io_async_rw), + .name = "WRITE", + .fail = io_rw_fail, + }, + [IORING_OP_FADVISE] = { + .name = "FADVISE", + }, + [IORING_OP_MADVISE] = { + .name = "MADVISE", + }, + [IORING_OP_SEND] = { + .name = "SEND", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .fail = io_sendrecv_fail, + .prep_async = io_send_prep_async, +#endif + }, + [IORING_OP_RECV] = { + .name = "RECV", +#if defined(CONFIG_NET) + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_OPENAT2] = { + .name = "OPENAT2", + .cleanup = io_open_cleanup, + }, + [IORING_OP_EPOLL_CTL] = { + .name = "EPOLL", + }, + [IORING_OP_SPLICE] = { + .name = "SPLICE", + }, + [IORING_OP_PROVIDE_BUFFERS] = { + .name = "PROVIDE_BUFFERS", + }, + [IORING_OP_REMOVE_BUFFERS] = { + .name = "REMOVE_BUFFERS", + }, + [IORING_OP_TEE] = { + .name = "TEE", + }, + [IORING_OP_SHUTDOWN] = { + .name = "SHUTDOWN", + }, + [IORING_OP_RENAMEAT] = { + .name = "RENAMEAT", + .cleanup = io_renameat_cleanup, + }, + [IORING_OP_UNLINKAT] = { + .name = "UNLINKAT", + .cleanup = io_unlinkat_cleanup, + }, + [IORING_OP_MKDIRAT] = { + .name = "MKDIRAT", + .cleanup = io_mkdirat_cleanup, + }, + [IORING_OP_SYMLINKAT] = { + .name = "SYMLINKAT", + .cleanup = io_link_cleanup, + }, + [IORING_OP_LINKAT] = { + .name = "LINKAT", + .cleanup = io_link_cleanup, + }, + [IORING_OP_MSG_RING] = { + .name = "MSG_RING", + .cleanup = io_msg_ring_cleanup, + }, + [IORING_OP_FSETXATTR] = { + .name = "FSETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_SETXATTR] = { + .name = "SETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_FGETXATTR] = { + .name = "FGETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_GETXATTR] = { + .name = "GETXATTR", + .cleanup = io_xattr_cleanup, + }, + [IORING_OP_SOCKET] = { + .name = "SOCKET", + }, + [IORING_OP_URING_CMD] = { + .name = "URING_CMD", + .async_size = uring_cmd_pdu_size(1), + .prep_async = io_uring_cmd_prep_async, + }, + [IORING_OP_SEND_ZC] = { + .name = "SEND_ZC", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep_async = io_send_prep_async, + .cleanup = io_send_zc_cleanup, + .fail = io_sendrecv_fail, +#endif + }, + [IORING_OP_SENDMSG_ZC] = { + .name = "SENDMSG_ZC", +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), .prep_async = io_sendmsg_prep_async, .cleanup = io_send_zc_cleanup, .fail = io_sendrecv_fail, -#else - .prep = io_eopnotsupp_prep, #endif }, }; @@ -536,7 +652,7 @@ const struct io_issue_def io_issue_defs[] = { const char *io_uring_get_opcode(u8 opcode) { if (opcode < IORING_OP_LAST) - return io_issue_defs[opcode].name; + return io_cold_defs[opcode].name; return "INVALID"; } @@ -544,12 +660,13 @@ void __init io_uring_optable_init(void) { int i; + BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { BUG_ON(!io_issue_defs[i].prep); if (io_issue_defs[i].prep != io_eopnotsupp_prep) BUG_ON(!io_issue_defs[i].issue); - WARN_ON_ONCE(!io_issue_defs[i].name); + WARN_ON_ONCE(!io_cold_defs[i].name); } } diff --git a/io_uring/opdef.h b/io_uring/opdef.h index d718e2ab1ff7..c22c8696e749 100644 --- a/io_uring/opdef.h +++ b/io_uring/opdef.h @@ -29,19 +29,24 @@ struct io_issue_def { unsigned iopoll_queue : 1; /* opcode specific path will handle ->async_data allocation if needed */ unsigned manual_alloc : 1; + + int (*issue)(struct io_kiocb *, unsigned int); + int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); +}; + +struct io_cold_def { /* size of async data needed, if any */ unsigned short async_size; const char *name; - int (*prep)(struct io_kiocb *, const struct io_uring_sqe *); - int (*issue)(struct io_kiocb *, unsigned int); int (*prep_async)(struct io_kiocb *); void (*cleanup)(struct io_kiocb *); void (*fail)(struct io_kiocb *); }; extern const struct io_issue_def io_issue_defs[]; +extern const struct io_cold_def io_cold_defs[]; void io_uring_optable_init(void); #endif diff --git a/io_uring/rw.c b/io_uring/rw.c index 02e710e77145..efe6bfda9ca9 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, struct io_rw_state *s, bool force) { - if (!force && !io_issue_defs[req->opcode].prep_async) + if (!force && !io_cold_defs[req->opcode].prep_async) return 0; if (!req_has_async_data(req)) { struct io_async_rw *iorw; From 8572df941cbef2b295282535b013828e7df39471 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 21 Jan 2023 19:53:41 -0700 Subject: [PATCH 130/265] io_uring/msg-ring: ensure flags passing works for task_work completions If the target ring is using IORING_SETUP_SINGLE_ISSUER and we're posting a message from a different thread, then we need to ensure that the fallback task_work that posts the CQE knwos about the flags passing as well. If not we'll always be posting 0 as the flags. Fixes: 3563d7ed58a5 ("io_uring/msg_ring: Pass custom flags to the cqe") Signed-off-by: Jens Axboe --- io_uring/msg_ring.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 27992bda1ffa..8803c0979e2a 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -99,6 +99,11 @@ static void io_msg_tw_complete(struct callback_head *head) if (current->flags & PF_EXITING) { ret = -EOWNERDEAD; } else { + u32 flags = 0; + + if (msg->flags & IORING_MSG_RING_FLAGS_PASS) + flags = msg->cqe_flags; + /* * If the target ring is using IOPOLL mode, then we need to be * holding the uring_lock for posting completions. Other ring @@ -107,7 +112,7 @@ static void io_msg_tw_complete(struct callback_head *head) */ if (target_ctx->flags & IORING_SETUP_IOPOLL) mutex_lock(&target_ctx->uring_lock); - if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0)) + if (!io_post_aux_cqe(target_ctx, msg->user_data, msg->len, flags)) ret = -EOVERFLOW; if (target_ctx->flags & IORING_SETUP_IOPOLL) mutex_unlock(&target_ctx->uring_lock); From b5d3ae202fbfe055aa2a8ae8524531ee1dcab717 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 24 Jan 2023 08:24:25 -0700 Subject: [PATCH 131/265] io_uring: handle TIF_NOTIFY_RESUME when checking for task_work If TIF_NOTIFY_RESUME is set, then we need to call resume_user_mode_work() for PF_IO_WORKER threads. They never return to usermode, hence never get a chance to process any items that are marked by this flag. Most notably this includes the final put of files, but also any throttling markers set by block cgroups. Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Jens Axboe --- io_uring/io_uring.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index c68edf9872a5..d58cfe062da9 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include "io-wq.h" @@ -274,6 +275,13 @@ static inline int io_run_task_work(void) */ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) clear_notify_signal(); + /* + * PF_IO_WORKER never returns to userspace, so check here if we have + * notify work that needs processing. + */ + if (current->flags & PF_IO_WORKER && + test_thread_flag(TIF_NOTIFY_RESUME)) + resume_user_mode_work(NULL); if (task_work_pending(current)) { __set_current_state(TASK_RUNNING); task_work_run(); From c1755c25a7190494b45861284b4a30bd9cd813ff Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Wed, 18 Jan 2023 07:56:30 -0800 Subject: [PATCH 132/265] io_uring: Enable KASAN for request cache Every io_uring request is represented by struct io_kiocb, which is cached locally by io_uring (not SLAB/SLUB) in the list called submit_state.freelist. This patch simply enabled KASAN for this free list. This list is initially created by KMEM_CACHE, but later, managed by io_uring. This patch basically poisons the objects that are not used (i.e., they are the free list), and unpoisons it when the object is allocated/removed from the list. Touching these poisoned objects while in the freelist will cause a KASAN warning. Suggested-by: Jens Axboe Signed-off-by: Breno Leitao Reviewed-by: Pavel Begunkov Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 3 ++- io_uring/io_uring.h | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c84dce6944db..407834255a59 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -151,7 +151,7 @@ static void io_move_task_work_from_local(struct io_ring_ctx *ctx); static void __io_submit_flush_completions(struct io_ring_ctx *ctx); static __cold void io_fallback_tw(struct io_uring_task *tctx); -static struct kmem_cache *req_cachep; +struct kmem_cache *req_cachep; struct sock *io_uring_get_socket(struct file *file) { @@ -230,6 +230,7 @@ static inline void req_fail_link_node(struct io_kiocb *req, int res) static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx) { wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); + kasan_poison_object_data(req_cachep, req); } static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref) diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index d58cfe062da9..9270156288aa 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include "io-wq.h" @@ -347,12 +348,16 @@ static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx) return true; } +extern struct kmem_cache *req_cachep; + static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { - struct io_wq_work_node *node; + struct io_kiocb *req; - node = wq_stack_extract(&ctx->submit_state.free_list); - return container_of(node, struct io_kiocb, comp_list); + req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list); + kasan_unpoison_object_data(req_cachep, req); + wq_stack_extract(&ctx->submit_state.free_list); + return req; } static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) From f499254474a83bf60191c86de82c1fec1d8eb9f9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 20 Jan 2023 09:10:30 -0700 Subject: [PATCH 133/265] io_uring: pass in io_issue_def to io_assign_file() This generates better code for me, avoiding an extra load on arm64, and both call sites already have this variable available for easy passing. Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 407834255a59..4958725e0b97 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1855,9 +1855,10 @@ static void io_clean_op(struct io_kiocb *req) req->flags &= ~IO_REQ_CLEAN_FLAGS; } -static bool io_assign_file(struct io_kiocb *req, unsigned int issue_flags) +static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def, + unsigned int issue_flags) { - if (req->file || !io_issue_defs[req->opcode].needs_file) + if (req->file || !def->needs_file) return true; if (req->flags & REQ_F_FIXED_FILE) @@ -1874,7 +1875,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) const struct cred *creds = NULL; int ret; - if (unlikely(!io_assign_file(req, issue_flags))) + if (unlikely(!io_assign_file(req, def, issue_flags))) return -EBADF; if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred())) @@ -1943,7 +1944,7 @@ fail: io_req_task_queue_fail(req, err); return; } - if (!io_assign_file(req, issue_flags)) { + if (!io_assign_file(req, def, issue_flags)) { err = -EBADF; work->flags |= IO_WQ_WORK_CANCEL; goto fail; From c10bb64684813a326174c3eebcafb3ee5af52ca3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:13 +0000 Subject: [PATCH 134/265] io_uring: use user visible tail in io_uring_poll() We return POLLIN from io_uring_poll() depending on whether there are CQEs for the userspace, and so we should use the user visible tail pointer instead of a transient cached value. Cc: stable@vger.kernel.org Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/228ffcbf30ba98856f66ffdb9a6a60ead1dd96c0.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4958725e0b97..1681be25cf58 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2886,7 +2886,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) * pushes them to do the flush. */ - if (io_cqring_events(ctx) || io_has_work(ctx)) + if (__io_cqring_events_user(ctx) || io_has_work(ctx)) mask |= EPOLLIN | EPOLLRDNORM; return mask; From b2aa66aff60c841b2c93242752c25abf4c82a28c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:14 +0000 Subject: [PATCH 135/265] io_uring: kill outdated comment about overflow flush __io_cqring_overflow_flush() doesn't return anything anymore, remove outdate comment. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4ce2bcbb17eac80cdf883fd1459d5ee6586e238c.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 1 - 1 file changed, 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1681be25cf58..b865275898b1 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -666,7 +666,6 @@ static void io_cqring_overflow_kill(struct io_ring_ctx *ctx) } } -/* Returns true if there are no backlogged entries after the flush */ static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx) { size_t cqe_size = sizeof(struct io_uring_cqe); From b5083dfa36676e7b5d72bf3d70f429a0d08c5075 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:15 +0000 Subject: [PATCH 136/265] io_uring: improve io_get_sqe Return an SQE from io_get_sqe() as a parameter and use the return value to determine if it failed or not. This enables the compiler to compile out the sqe NULL check when we know that the return SQE is valid. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9cceb11329240ea097dffef6bf0a675bca14cf42.1674484266.git.asml.silence@gmail.com [axboe: remove bogus const modifier on return value] Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b865275898b1..c1f27626c205 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2368,7 +2368,7 @@ static void io_commit_sqring(struct io_ring_ctx *ctx) * used, it's important that those reads are done through READ_ONCE() to * prevent a re-load down the line. */ -static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) +static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe) { unsigned head, mask = ctx->sq_entries - 1; unsigned sq_idx = ctx->cached_sq_head++ & mask; @@ -2386,14 +2386,15 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) /* double index for 128-byte SQEs, twice as long */ if (ctx->flags & IORING_SETUP_SQE128) head <<= 1; - return &ctx->sq_sqes[head]; + *sqe = &ctx->sq_sqes[head]; + return true; } /* drop invalid entries */ ctx->cq_extra--; WRITE_ONCE(ctx->rings->sq_dropped, READ_ONCE(ctx->rings->sq_dropped) + 1); - return NULL; + return false; } int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) @@ -2417,8 +2418,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) if (unlikely(!io_alloc_req_refill(ctx))) break; req = io_alloc_req(ctx); - sqe = io_get_sqe(ctx); - if (unlikely(!sqe)) { + if (unlikely(!io_get_sqe(ctx, &sqe))) { io_req_add_to_cache(req, ctx); break; } From c8576f3e612d3c5b01d434ae296b9b76d7907708 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:16 +0000 Subject: [PATCH 137/265] io_uring: refactor req allocation Follow the io_get_sqe pattern returning the result via a pointer and hide request cache refill inside io_alloc_req(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8c37c2e8a3cb5e4cd6a8ae3b91371227a92708a6.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 7 +++---- io_uring/io_uring.h | 19 +++++++++++-------- io_uring/notif.c | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c1f27626c205..2b046a80f75b 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2415,9 +2415,8 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) const struct io_uring_sqe *sqe; struct io_kiocb *req; - if (unlikely(!io_alloc_req_refill(ctx))) + if (unlikely(!io_alloc_req(ctx, &req))) break; - req = io_alloc_req(ctx); if (unlikely(!io_get_sqe(ctx, &sqe))) { io_req_add_to_cache(req, ctx); break; @@ -2736,14 +2735,14 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) static void io_req_caches_free(struct io_ring_ctx *ctx) { + struct io_kiocb *req; int nr = 0; mutex_lock(&ctx->uring_lock); io_flush_cached_locked_reqs(ctx, &ctx->submit_state); while (!io_req_cache_empty(ctx)) { - struct io_kiocb *req = io_alloc_req(ctx); - + req = io_extract_req(ctx); kmem_cache_free(req_cachep, req); nr++; } diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 9270156288aa..f90816aac896 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -341,16 +341,9 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) return !ctx->submit_state.free_list.next; } -static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx) -{ - if (unlikely(io_req_cache_empty(ctx))) - return __io_alloc_req_refill(ctx); - return true; -} - extern struct kmem_cache *req_cachep; -static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) { struct io_kiocb *req; @@ -360,6 +353,16 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return req; } +static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) +{ + if (unlikely(io_req_cache_empty(ctx))) { + if (!__io_alloc_req_refill(ctx)) + return false; + } + *req = io_extract_req(ctx); + return true; +} + static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) { return likely(ctx->submitter_task == current); diff --git a/io_uring/notif.c b/io_uring/notif.c index c4bb793ebf0e..09dfd0832d19 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -68,9 +68,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) struct io_kiocb *notif; struct io_notif_data *nd; - if (unlikely(!io_alloc_req_refill(ctx))) + if (unlikely(!io_alloc_req(ctx, ¬if))) return NULL; - notif = io_alloc_req(ctx); notif->opcode = IORING_OP_NOP; notif->flags = 0; notif->file = NULL; From 5afa4650713918e60865ed42d9439e82f6d24773 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:17 +0000 Subject: [PATCH 138/265] io_uring: refactor io_put_task helpers Add a helper for putting refs from the target task context, rename __io_put_task() and add a couple of comments around. Use the remote version for __io_req_complete_post(), the local is only needed for __io_submit_flush_completions(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3bf92ebd594769d8a5d648472a8e335f2031d542.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2b046a80f75b..106cba919023 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -713,7 +713,8 @@ static void io_cqring_overflow_flush(struct io_ring_ctx *ctx) io_cqring_do_overflow_flush(ctx); } -static void __io_put_task(struct task_struct *task, int nr) +/* can be called by any task */ +static void io_put_task_remote(struct task_struct *task, int nr) { struct io_uring_task *tctx = task->io_uring; @@ -723,13 +724,19 @@ static void __io_put_task(struct task_struct *task, int nr) put_task_struct_many(task, nr); } +/* used by a task to put its own references */ +static void io_put_task_local(struct task_struct *task, int nr) +{ + task->io_uring->cached_refs += nr; +} + /* must to be called somewhat shortly after putting a request */ static inline void io_put_task(struct task_struct *task, int nr) { if (likely(task == current)) - task->io_uring->cached_refs += nr; + io_put_task_local(task, nr); else - __io_put_task(task, nr); + io_put_task_remote(task, nr); } void io_task_refs_refill(struct io_uring_task *tctx) @@ -982,7 +989,7 @@ static void __io_req_complete_post(struct io_kiocb *req) * we don't hold ->completion_lock. Clean them here to avoid * deadlocks. */ - io_put_task(req->task, 1); + io_put_task_remote(req->task, 1); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; } @@ -1105,7 +1112,7 @@ __cold void io_free_req(struct io_kiocb *req) io_req_put_rsrc(req); io_dismantle_req(req); - io_put_task(req->task, 1); + io_put_task_remote(req->task, 1); spin_lock(&ctx->completion_lock); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); From cb6bf7f285c270d7808807128b5bce414e3f254a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:18 +0000 Subject: [PATCH 139/265] io_uring: refactor tctx_task_work Merge almost identical sections of tctx_task_work(), this will make code modifications later easier and also inlines handle_tw_list(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d06592d91e3e7559e7a4dbb8907d110863008dc7.1674484266.git.asml.silence@gmail.com [axboe: fold in setting count to zero patch from Tom Rix] Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 106cba919023..55101013f3ee 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1166,7 +1166,7 @@ static unsigned int handle_tw_list(struct llist_node *node, { unsigned int count = 0; - while (node != last) { + while (node && node != last) { struct llist_node *next = node->next; struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); @@ -1226,23 +1226,20 @@ void tctx_task_work(struct callback_head *cb) task_work); struct llist_node fake = {}; struct llist_node *node; - unsigned int loops = 1; - unsigned int count; + unsigned int loops = 0; + unsigned int count = 0; if (unlikely(current->flags & PF_EXITING)) { io_fallback_tw(tctx); return; } - node = io_llist_xchg(&tctx->task_list, &fake); - count = handle_tw_list(node, &ctx, &uring_locked, NULL); - node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL); - while (node != &fake) { + do { loops++; node = io_llist_xchg(&tctx->task_list, &fake); count += handle_tw_list(node, &ctx, &uring_locked, &fake); node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL); - } + } while (node != &fake); ctx_flush_and_put(ctx, &uring_locked); From 50470fc5723ae0adb2f429a8b27ff6bf1a41913e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 23 Jan 2023 14:37:19 +0000 Subject: [PATCH 140/265] io_uring: return normal tw run linking optimisation io_submit_flush_completions() may produce new task_work items, so it's a good idea to recheck the task_work list after flushing completions. The optimisation is not new and was accidentially removed by f88262e60bb9 ("io_uring: lockless task list") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a7ed5ede84de190832cc33ebbcdd6e91cd90f5b6.1674484266.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 55101013f3ee..9c92ca081c11 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1238,6 +1238,15 @@ void tctx_task_work(struct callback_head *cb) loops++; node = io_llist_xchg(&tctx->task_list, &fake); count += handle_tw_list(node, &ctx, &uring_locked, &fake); + + /* skip expensive cmpxchg if there are items in the list */ + if (READ_ONCE(tctx->task_list.first) != &fake) + continue; + if (uring_locked && !wq_list_empty(&ctx->submit_state.compl_reqs)) { + io_submit_flush_completions(ctx); + if (READ_ONCE(tctx->task_list.first) != &fake) + continue; + } node = io_llist_cmpxchg(&tctx->task_list, &fake, NULL); } while (node != &fake); From fcc926bb857949dbfa51a7d95f3f5ebc657f198c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 27 Jan 2023 09:28:13 -0700 Subject: [PATCH 141/265] io_uring: add a conditional reschedule to the IOPOLL cancelation loop If the kernel is configured with CONFIG_PREEMPT_NONE, we could be sitting in a tight loop reaping events but not giving them a chance to finish. This results in a trace ala: rcu: INFO: rcu_sched self-detected stall on CPU rcu: 2-...!: (5249 ticks this GP) idle=935c/1/0x4000000000000000 softirq=4265/4274 fqs=1 (t=5251 jiffies g=465 q=4135 ncpus=4) rcu: rcu_sched kthread starved for 5249 jiffies! g465 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 rcu: Unless rcu_sched kthread gets sufficient CPU time, OOM is now expected behavior. rcu: RCU grace-period kthread stack dump: task:rcu_sched state:R running task stack:0 pid:12 ppid:2 flags:0x00000008 Call trace: __switch_to+0xb0/0xc8 __schedule+0x43c/0x520 schedule+0x4c/0x98 schedule_timeout+0xbc/0xdc rcu_gp_fqs_loop+0x308/0x344 rcu_gp_kthread+0xd8/0xf0 kthread+0xb8/0xc8 ret_from_fork+0x10/0x20 rcu: Stack dump where RCU GP kthread last ran: Task dump for CPU 0: task:kworker/u8:10 state:R running task stack:0 pid:89 ppid:2 flags:0x0000000a Workqueue: events_unbound io_ring_exit_work Call trace: __switch_to+0xb0/0xc8 0xffff0000c8fefd28 CPU: 2 PID: 95 Comm: kworker/u8:13 Not tainted 6.2.0-rc5-00042-g40316e337c80-dirty #2759 Hardware name: linux,dummy-virt (DT) Workqueue: events_unbound io_ring_exit_work pstate: 61400005 (nZCv daif +PAN -UAO -TCO +DIT -SSBS BTYPE=--) pc : io_do_iopoll+0x344/0x360 lr : io_do_iopoll+0xb8/0x360 sp : ffff800009bebc60 x29: ffff800009bebc60 x28: 0000000000000000 x27: 0000000000000000 x26: ffff0000c0f67d48 x25: ffff0000c0f67840 x24: ffff800008950024 x23: 0000000000000001 x22: 0000000000000000 x21: ffff0000c27d3200 x20: ffff0000c0f67840 x19: ffff0000c0f67800 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000001 x13: 0000000000000001 x12: 0000000000000000 x11: 0000000000000179 x10: 0000000000000870 x9 : ffff800009bebd60 x8 : ffff0000c27d3ad0 x7 : fefefefefefefeff x6 : 0000646e756f626e x5 : ffff0000c0f67840 x4 : 0000000000000000 x3 : ffff0000c2398000 x2 : 0000000000000000 x1 : 0000000000000000 x0 : 0000000000000000 Call trace: io_do_iopoll+0x344/0x360 io_uring_try_cancel_requests+0x21c/0x334 io_ring_exit_work+0x90/0x40c process_one_work+0x1a4/0x254 worker_thread+0x1ec/0x258 kthread+0xb8/0xc8 ret_from_fork+0x10/0x20 Add a cond_resched() in the cancelation IOPOLL loop to fix this. Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 1 + 1 file changed, 1 insertion(+) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 9c92ca081c11..fab581a31dc1 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3160,6 +3160,7 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, while (!wq_list_empty(&ctx->iopoll_list)) { io_iopoll_try_reap_events(ctx); ret = true; + cond_resched(); } } From f58680085478dd292435727210122960d38e8014 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 27 Jan 2023 09:50:31 -0700 Subject: [PATCH 142/265] io_uring: add reschedule point to handle_tw_list() If CONFIG_PREEMPT_NONE is set and the task_work chains are long, we could be running into issues blocking others for too long. Add a reschedule check in handle_tw_list(), and flush the ctx if we need to reschedule. Cc: stable@vger.kernel.org # 5.10+ Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index fab581a31dc1..acf6d9680d76 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1179,10 +1179,16 @@ static unsigned int handle_tw_list(struct llist_node *node, /* if not contended, grab and improve batching */ *locked = mutex_trylock(&(*ctx)->uring_lock); percpu_ref_get(&(*ctx)->refs); - } + } else if (!*locked) + *locked = mutex_trylock(&(*ctx)->uring_lock); req->io_task_work.func(req, locked); node = next; count++; + if (unlikely(need_resched())) { + ctx_flush_and_put(*ctx, locked); + *ctx = NULL; + cond_resched(); + } } return count; From 6bb30855560e6343e7b88595d7c3159d0f848a04 Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Fri, 27 Jan 2023 05:52:24 -0800 Subject: [PATCH 143/265] io_uring: if a linked request has REQ_F_FORCE_ASYNC then run it async REQ_F_FORCE_ASYNC was being ignored for re-queueing linked requests. Instead obey that flag. Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20230127135227.3646353-2-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index acf6d9680d76..0e42160a3051 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1417,10 +1417,12 @@ void io_req_task_submit(struct io_kiocb *req, bool *locked) { io_tw_lock(req->ctx, locked); /* req->task == current here, checking PF_EXITING is safe */ - if (likely(!(req->task->flags & PF_EXITING))) - io_queue_sqe(req); - else + if (unlikely(req->task->flags & PF_EXITING)) io_req_defer_failed(req, -EFAULT); + else if (req->flags & REQ_F_FORCE_ASYNC) + io_queue_iowq(req, locked); + else + io_queue_sqe(req); } void io_req_task_queue_fail(struct io_kiocb *req, int ret) From aebb224fd4fc7352cd839ad90414c548387142fd Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Fri, 27 Jan 2023 05:52:25 -0800 Subject: [PATCH 144/265] io_uring: for requests that require async, force it Some requests require being run async as they do not support non-blocking. Instead of trying to issue these requests, getting -EAGAIN and then queueing them for async issue, rather just force async upfront. Add WARN_ON_ONCE to make sure surprising code paths do not come up, however in those cases the bug would end up being a blocking io_uring_enter(2) which should not be critical. Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20230127135227.3646353-3-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/advise.c | 4 ++-- io_uring/fs.c | 20 ++++++++++---------- io_uring/net.c | 4 ++-- io_uring/splice.c | 7 +++---- io_uring/statx.c | 4 ++-- io_uring/sync.c | 14 ++++++++------ io_uring/xattr.c | 14 ++++++-------- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/io_uring/advise.c b/io_uring/advise.c index 449c6f14649f..cf600579bffe 100644 --- a/io_uring/advise.c +++ b/io_uring/advise.c @@ -39,6 +39,7 @@ int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ma->addr = READ_ONCE(sqe->addr); ma->len = READ_ONCE(sqe->len); ma->advice = READ_ONCE(sqe->fadvise_advice); + req->flags |= REQ_F_FORCE_ASYNC; return 0; #else return -EOPNOTSUPP; @@ -51,8 +52,7 @@ int io_madvise(struct io_kiocb *req, unsigned int issue_flags) struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice); io_req_set_res(req, ret, 0); diff --git a/io_uring/fs.c b/io_uring/fs.c index 7100c293c13a..f6a69a549fd4 100644 --- a/io_uring/fs.c +++ b/io_uring/fs.c @@ -74,6 +74,7 @@ int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -82,8 +83,7 @@ int io_renameat(struct io_kiocb *req, unsigned int issue_flags) struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd, ren->newpath, ren->flags); @@ -123,6 +123,7 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return PTR_ERR(un->filename); req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -131,8 +132,7 @@ int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); if (un->flags & AT_REMOVEDIR) ret = do_rmdir(un->dfd, un->filename); @@ -170,6 +170,7 @@ int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return PTR_ERR(mkd->filename); req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -178,8 +179,7 @@ int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_mkdirat(mkd->dfd, mkd->filename, mkd->mode); @@ -220,6 +220,7 @@ int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -228,8 +229,7 @@ int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_symlinkat(sl->oldpath, sl->new_dfd, sl->newpath); @@ -265,6 +265,7 @@ int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -273,8 +274,7 @@ int io_linkat(struct io_kiocb *req, unsigned int issue_flags) struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_linkat(lnk->old_dfd, lnk->oldpath, lnk->new_dfd, lnk->newpath, lnk->flags); diff --git a/io_uring/net.c b/io_uring/net.c index 90326b279965..aeb1d016e2e9 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -90,6 +90,7 @@ int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EINVAL; shutdown->how = READ_ONCE(sqe->len); + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -99,8 +100,7 @@ int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) struct socket *sock; int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); sock = sock_from_file(req->file); if (unlikely(!sock)) diff --git a/io_uring/splice.c b/io_uring/splice.c index 53e4232d0866..2a4bbb719531 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -34,6 +34,7 @@ static int __io_splice_prep(struct io_kiocb *req, if (unlikely(sp->flags & ~valid_flags)) return -EINVAL; sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in); + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -52,8 +53,7 @@ int io_tee(struct io_kiocb *req, unsigned int issue_flags) struct file *in; long ret = 0; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); if (sp->flags & SPLICE_F_FD_IN_FIXED) in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); @@ -94,8 +94,7 @@ int io_splice(struct io_kiocb *req, unsigned int issue_flags) struct file *in; long ret = 0; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); if (sp->flags & SPLICE_F_FD_IN_FIXED) in = io_file_get_fixed(req, sp->splice_fd_in, issue_flags); diff --git a/io_uring/statx.c b/io_uring/statx.c index d8fc933d3f59..abb874209caa 100644 --- a/io_uring/statx.c +++ b/io_uring/statx.c @@ -48,6 +48,7 @@ int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -56,8 +57,7 @@ int io_statx(struct io_kiocb *req, unsigned int issue_flags) struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_statx(sx->dfd, sx->filename, sx->flags, sx->mask, sx->buffer); io_req_set_res(req, ret, 0); diff --git a/io_uring/sync.c b/io_uring/sync.c index 64e87ea2b8fb..255f68c37e55 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -32,6 +32,8 @@ int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sync->off = READ_ONCE(sqe->off); sync->len = READ_ONCE(sqe->len); sync->flags = READ_ONCE(sqe->sync_range_flags); + req->flags |= REQ_F_FORCE_ASYNC; + return 0; } @@ -41,8 +43,7 @@ int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) int ret; /* sync_file_range always requires a blocking context */ - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = sync_file_range(req->file, sync->off, sync->len, sync->flags); io_req_set_res(req, ret, 0); @@ -62,6 +63,7 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sync->off = READ_ONCE(sqe->off); sync->len = READ_ONCE(sqe->len); + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -72,8 +74,7 @@ int io_fsync(struct io_kiocb *req, unsigned int issue_flags) int ret; /* fsync always requires a blocking context */ - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = vfs_fsync_range(req->file, sync->off, end > 0 ? end : LLONG_MAX, sync->flags & IORING_FSYNC_DATASYNC); @@ -91,6 +92,7 @@ int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) sync->off = READ_ONCE(sqe->off); sync->len = READ_ONCE(sqe->addr); sync->mode = READ_ONCE(sqe->len); + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -100,8 +102,8 @@ int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) int ret; /* fallocate always requiring blocking context */ - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); + ret = vfs_fallocate(req->file, sync->mode, sync->off, sync->len); if (ret >= 0) fsnotify_modify(req->file); diff --git a/io_uring/xattr.c b/io_uring/xattr.c index 6201a9f442c6..e1c810e0b85a 100644 --- a/io_uring/xattr.c +++ b/io_uring/xattr.c @@ -75,6 +75,7 @@ static int __io_getxattr_prep(struct io_kiocb *req, } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -109,8 +110,7 @@ int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = do_getxattr(mnt_idmap(req->file->f_path.mnt), req->file->f_path.dentry, @@ -127,8 +127,7 @@ int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) struct path path; int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); retry: ret = filename_lookup(AT_FDCWD, ix->filename, lookup_flags, &path, NULL); @@ -174,6 +173,7 @@ static int __io_setxattr_prep(struct io_kiocb *req, } req->flags |= REQ_F_NEED_CLEANUP; + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -222,8 +222,7 @@ int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags) { int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); ret = __io_setxattr(req, issue_flags, &req->file->f_path); io_xattr_finish(req, ret); @@ -237,8 +236,7 @@ int io_setxattr(struct io_kiocb *req, unsigned int issue_flags) struct path path; int ret; - if (issue_flags & IO_URING_F_NONBLOCK) - return -EAGAIN; + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); retry: ret = filename_lookup(AT_FDCWD, ix->filename, lookup_flags, &path, NULL); From c31cc60fddd11134031e7f9eb76812353cfaac84 Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Fri, 27 Jan 2023 05:52:26 -0800 Subject: [PATCH 145/265] io_uring: always go async for unsupported fadvise flags No point in issuing -> return -EAGAIN -> go async, when it can be done upfront. Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20230127135227.3646353-4-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/advise.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/io_uring/advise.c b/io_uring/advise.c index cf600579bffe..7085804c513c 100644 --- a/io_uring/advise.c +++ b/io_uring/advise.c @@ -62,6 +62,18 @@ int io_madvise(struct io_kiocb *req, unsigned int issue_flags) #endif } +static bool io_fadvise_force_async(struct io_fadvise *fa) +{ + switch (fa->advice) { + case POSIX_FADV_NORMAL: + case POSIX_FADV_RANDOM: + case POSIX_FADV_SEQUENTIAL: + return false; + default: + return true; + } +} + int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); @@ -72,6 +84,8 @@ int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) fa->offset = READ_ONCE(sqe->off); fa->len = READ_ONCE(sqe->len); fa->advice = READ_ONCE(sqe->fadvise_advice); + if (io_fadvise_force_async(fa)) + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -80,16 +94,7 @@ int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); int ret; - if (issue_flags & IO_URING_F_NONBLOCK) { - switch (fa->advice) { - case POSIX_FADV_NORMAL: - case POSIX_FADV_RANDOM: - case POSIX_FADV_SEQUENTIAL: - break; - default: - return -EAGAIN; - } - } + WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK && io_fadvise_force_async(fa)); ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice); if (ret < 0) From 0ffae640ad83de46865c6b8dc3fda370823e4f1d Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Fri, 27 Jan 2023 05:52:27 -0800 Subject: [PATCH 146/265] io_uring: always go async for unsupported open flags No point in issuing -> return -EAGAIN -> go async, when it can be done upfront. Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20230127135227.3646353-5-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/openclose.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/io_uring/openclose.c b/io_uring/openclose.c index 67178e4bb282..a1b98c81a52d 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -31,6 +31,15 @@ struct io_close { u32 file_slot; }; +static bool io_openat_force_async(struct io_open *open) +{ + /* + * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, + * it'll always -EAGAIN + */ + return open->how.flags & (O_TRUNC | O_CREAT | O_TMPFILE); +} + static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_open *open = io_kiocb_to_cmd(req, struct io_open); @@ -61,6 +70,8 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe open->nofile = rlimit(RLIMIT_NOFILE); req->flags |= REQ_F_NEED_CLEANUP; + if (io_openat_force_async(open)) + req->flags |= REQ_F_FORCE_ASYNC; return 0; } @@ -108,12 +119,7 @@ int io_openat2(struct io_kiocb *req, unsigned int issue_flags) nonblock_set = op.open_flag & O_NONBLOCK; resolve_nonblock = open->how.resolve & RESOLVE_CACHED; if (issue_flags & IO_URING_F_NONBLOCK) { - /* - * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, - * it'll always -EAGAIN - */ - if (open->how.flags & (O_TRUNC | O_CREAT | O_TMPFILE)) - return -EAGAIN; + WARN_ON_ONCE(io_openat_force_async(open)); op.lookup_flags |= LOOKUP_CACHED; op.open_flag |= O_NONBLOCK; } From 2f2bb1ffc9983e227424d0787289da5483b0c74f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 6 Feb 2023 08:20:46 -0700 Subject: [PATCH 147/265] io_uring: mark task TASK_RUNNING before handling resume/task work Just like for task_work, set the task mode to TASK_RUNNING before doing any potential resume work. We're not holding any locks at this point, but we may have already set the task state to TASK_INTERRUPTIBLE in preparation for going to sleep waiting for events. Ensure that we set it back to TASK_RUNNING if we have work to process, to avoid warnings on calling blocking operations with !TASK_RUNNING. Fixes: b5d3ae202fbf ("io_uring: handle TIF_NOTIFY_RESUME when checking for task_work") Reported-by: kernel test robot Link: https://lore.kernel.org/oe-lkp/202302062208.24d3e563-oliver.sang@intel.com Signed-off-by: Jens Axboe --- io_uring/io_uring.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index f90816aac896..2711865f1e19 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -281,8 +281,10 @@ static inline int io_run_task_work(void) * notify work that needs processing. */ if (current->flags & PF_IO_WORKER && - test_thread_flag(TIF_NOTIFY_RESUME)) + test_thread_flag(TIF_NOTIFY_RESUME)) { + __set_current_state(TASK_RUNNING); resume_user_mode_work(NULL); + } if (task_work_pending(current)) { __set_current_state(TASK_RUNNING); task_work_run(); From 1e0d4adf17e7ef03281d7b16555e7c1508c8ed2d Mon Sep 17 00:00:00 2001 From: Vladislav Efanov Date: Thu, 2 Feb 2023 17:04:56 +0300 Subject: [PATCH 148/265] udf: Check consistency of Space Bitmap Descriptor Bits, which are related to Bitmap Descriptor logical blocks, are not reset when buffer headers are allocated for them. As the result, these logical blocks can be treated as free and be used for other blocks.This can cause usage of one buffer header for several types of data. UDF issues WARNING in this situation: WARNING: CPU: 0 PID: 2703 at fs/udf/inode.c:2014 __udf_add_aext+0x685/0x7d0 fs/udf/inode.c:2014 RIP: 0010:__udf_add_aext+0x685/0x7d0 fs/udf/inode.c:2014 Call Trace: udf_setup_indirect_aext+0x573/0x880 fs/udf/inode.c:1980 udf_add_aext+0x208/0x2e0 fs/udf/inode.c:2067 udf_insert_aext fs/udf/inode.c:2233 [inline] udf_update_extents fs/udf/inode.c:1181 [inline] inode_getblk+0x1981/0x3b70 fs/udf/inode.c:885 Found by Linux Verification Center (linuxtesting.org) with syzkaller. [JK: Somewhat cleaned up the boundary checks] Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Vladislav Efanov Signed-off-by: Jan Kara --- fs/udf/balloc.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c index cdf90928b7f2..14b9db4c80f0 100644 --- a/fs/udf/balloc.c +++ b/fs/udf/balloc.c @@ -36,18 +36,41 @@ static int read_block_bitmap(struct super_block *sb, unsigned long bitmap_nr) { struct buffer_head *bh = NULL; - int retval = 0; + int i; + int max_bits, off, count; struct kernel_lb_addr loc; loc.logicalBlockNum = bitmap->s_extPosition; loc.partitionReferenceNum = UDF_SB(sb)->s_partition; bh = sb_bread(sb, udf_get_lb_pblock(sb, &loc, block)); - if (!bh) - retval = -EIO; - bitmap->s_block_bitmap[bitmap_nr] = bh; - return retval; + if (!bh) + return -EIO; + + /* Check consistency of Space Bitmap buffer. */ + max_bits = sb->s_blocksize * 8; + if (!bitmap_nr) { + off = sizeof(struct spaceBitmapDesc) << 3; + count = min(max_bits - off, bitmap->s_nr_groups); + } else { + /* + * Rough check if bitmap number is too big to have any bitmap + * blocks reserved. + */ + if (bitmap_nr > + (bitmap->s_nr_groups >> (sb->s_blocksize_bits + 3)) + 2) + return 0; + off = 0; + count = bitmap->s_nr_groups - bitmap_nr * max_bits + + (sizeof(struct spaceBitmapDesc) << 3); + count = min(count, max_bits); + } + + for (i = 0; i < count; i++) + if (udf_test_bit(i + off, bh->b_data)) + return -EFSCORRUPTED; + return 0; } static int __load_block_bitmap(struct super_block *sb, From f8d0dd0bc302b237dfa2ef5836e6ee375c0a1773 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Fri, 27 Jan 2023 08:29:06 -0800 Subject: [PATCH 149/265] udf: remove reporting loc in debug output clang build fails with fs/udf/partition.c:86:28: error: variable 'loc' is uninitialized when used here [-Werror,-Wuninitialized] sb, block, partition, loc, index); ^~~ loc is now only known when bh is valid. So remove reporting loc in debug output. Fixes: 4215db46d538 ("udf: Use udf_bread() in udf_get_pblock_virt15()") Reported-by: kernel test robot Reported-by: "kernelci.org bot" Signed-off-by: Tom Rix Reviewed-by: Nathan Chancellor Reviewed-by: Nick Desaulniers Signed-off-by: Jan Kara --- fs/udf/partition.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/udf/partition.c b/fs/udf/partition.c index 92765d2f6958..5bcfe78d5cab 100644 --- a/fs/udf/partition.c +++ b/fs/udf/partition.c @@ -82,8 +82,8 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, bh = udf_bread(sbi->s_vat_inode, newblock, 0, &err); if (!bh) { - udf_debug("get_pblock(UDF_VIRTUAL_MAP:%p,%u,%u) VAT: %u[%u]\n", - sb, block, partition, loc, index); + udf_debug("get_pblock(UDF_VIRTUAL_MAP:%p,%u,%u)\n", + sb, block, partition); return 0xFFFFFFFF; } From 2e0a547164b1384a87fd3500a01297222b0971b0 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Fri, 3 Feb 2023 16:35:14 -0500 Subject: [PATCH 150/265] fanotify: Ensure consistent variable type for response The user space API for the response variable is __u32. This patch makes sure that the whole path through the kernel uses u32 so that there is no sign extension or truncation of the user space response. Suggested-by: Steve Grubb Link: https://lore.kernel.org/r/12617626.uLZWGnKmhe@x2 Signed-off-by: Richard Guy Briggs Acked-by: Paul Moore Tested-by: Steve Grubb Acked-by: Steve Grubb Signed-off-by: Jan Kara Message-Id: <3778cb0b3501bc4e686ba7770b20eb9ab0506cf4.1675373475.git.rgb@redhat.com> --- fs/notify/fanotify/fanotify.h | 2 +- fs/notify/fanotify/fanotify_user.c | 6 +++--- include/linux/audit.h | 6 +++--- kernel/auditsc.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index 57f51a9a3015..f899d610bc08 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -425,7 +425,7 @@ FANOTIFY_PE(struct fanotify_event *event) struct fanotify_perm_event { struct fanotify_event fae; struct path path; - unsigned short response; /* userspace answer to the event */ + u32 response; /* userspace answer to the event */ unsigned short state; /* state of the event */ int fd; /* fd we passed to userspace for this event */ }; diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 4546da4a54f9..caa1211bac8c 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -289,7 +289,7 @@ static int create_fd(struct fsnotify_group *group, const struct path *path, */ static void finish_permission_event(struct fsnotify_group *group, struct fanotify_perm_event *event, - unsigned int response) + u32 response) __releases(&group->notification_lock) { bool destroy = false; @@ -310,9 +310,9 @@ static int process_access_response(struct fsnotify_group *group, { struct fanotify_perm_event *event; int fd = response_struct->fd; - int response = response_struct->response; + u32 response = response_struct->response; - pr_debug("%s: group=%p fd=%d response=%d\n", __func__, group, + pr_debug("%s: group=%p fd=%d response=%u\n", __func__, group, fd, response); /* * make sure the response is valid, if invalid we do nothing and either diff --git a/include/linux/audit.h b/include/linux/audit.h index 3608992848d3..d6b7d0c7ce43 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -416,7 +416,7 @@ extern void __audit_log_capset(const struct cred *new, const struct cred *old); extern void __audit_mmap_fd(int fd, int flags); extern void __audit_openat2_how(struct open_how *how); extern void __audit_log_kern_module(char *name); -extern void __audit_fanotify(unsigned int response); +extern void __audit_fanotify(u32 response); extern void __audit_tk_injoffset(struct timespec64 offset); extern void __audit_ntp_log(const struct audit_ntp_data *ad); extern void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, @@ -523,7 +523,7 @@ static inline void audit_log_kern_module(char *name) __audit_log_kern_module(name); } -static inline void audit_fanotify(unsigned int response) +static inline void audit_fanotify(u32 response) { if (!audit_dummy_context()) __audit_fanotify(response); @@ -679,7 +679,7 @@ static inline void audit_log_kern_module(char *name) { } -static inline void audit_fanotify(unsigned int response) +static inline void audit_fanotify(u32 response) { } static inline void audit_tk_injoffset(struct timespec64 offset) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 547c88be8a28..d1fb821de104 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -2877,7 +2877,7 @@ void __audit_log_kern_module(char *name) context->type = AUDIT_KERN_MODULE; } -void __audit_fanotify(unsigned int response) +void __audit_fanotify(u32 response) { audit_log(audit_context(), GFP_KERNEL, AUDIT_FANOTIFY, "resp=%u", response); From 70529a199574c15a40f46b14256633b02ba10ca2 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Fri, 3 Feb 2023 16:35:15 -0500 Subject: [PATCH 151/265] fanotify: define struct members to hold response decision context This patch adds a flag, FAN_INFO and an extensible buffer to provide additional information about response decisions. The buffer contains one or more headers defining the information type and the length of the following information. The patch defines one additional information type, FAN_RESPONSE_INFO_AUDIT_RULE, to audit a rule number. This will allow for the creation of other information types in the future if other users of the API identify different needs. The kernel can be tested if it supports a given info type by supplying the complete info extension but setting fd to FAN_NOFD. It will return the expected size but not issue an audit record. Suggested-by: Steve Grubb Link: https://lore.kernel.org/r/2745105.e9J7NaK4W3@x2 Suggested-by: Jan Kara Link: https://lore.kernel.org/r/20201001101219.GE17860@quack2.suse.cz Tested-by: Steve Grubb Acked-by: Steve Grubb Signed-off-by: Richard Guy Briggs Signed-off-by: Jan Kara Message-Id: <10177cfcae5480926b7176321a28d9da6835b667.1675373475.git.rgb@redhat.com> --- fs/notify/fanotify/fanotify.c | 5 +- fs/notify/fanotify/fanotify.h | 4 ++ fs/notify/fanotify/fanotify_user.c | 86 ++++++++++++++++++++++-------- include/linux/fanotify.h | 5 ++ include/uapi/linux/fanotify.h | 30 ++++++++++- 5 files changed, 107 insertions(+), 23 deletions(-) diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index a2a15bc4df28..24ec1d66d5a8 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -262,7 +262,7 @@ static int fanotify_get_response(struct fsnotify_group *group, } /* userspace responded, convert to something usable */ - switch (event->response & ~FAN_AUDIT) { + switch (event->response & FANOTIFY_RESPONSE_ACCESS) { case FAN_ALLOW: ret = 0; break; @@ -563,6 +563,9 @@ static struct fanotify_event *fanotify_alloc_perm_event(const struct path *path, pevent->fae.type = FANOTIFY_EVENT_TYPE_PATH_PERM; pevent->response = 0; + pevent->hdr.type = FAN_RESPONSE_INFO_NONE; + pevent->hdr.pad = 0; + pevent->hdr.len = 0; pevent->state = FAN_EVENT_INIT; pevent->path = *path; path_get(path); diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index f899d610bc08..e8a3c28c5d12 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -428,6 +428,10 @@ struct fanotify_perm_event { u32 response; /* userspace answer to the event */ unsigned short state; /* state of the event */ int fd; /* fd we passed to userspace for this event */ + union { + struct fanotify_response_info_header hdr; + struct fanotify_response_info_audit_rule audit_rule; + }; }; static inline struct fanotify_perm_event * diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index caa1211bac8c..8f430bfad487 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -283,19 +283,42 @@ static int create_fd(struct fsnotify_group *group, const struct path *path, return client_fd; } +static int process_access_response_info(const char __user *info, + size_t info_len, + struct fanotify_response_info_audit_rule *friar) +{ + if (info_len != sizeof(*friar)) + return -EINVAL; + + if (copy_from_user(friar, info, sizeof(*friar))) + return -EFAULT; + + if (friar->hdr.type != FAN_RESPONSE_INFO_AUDIT_RULE) + return -EINVAL; + if (friar->hdr.pad != 0) + return -EINVAL; + if (friar->hdr.len != sizeof(*friar)) + return -EINVAL; + + return info_len; +} + /* * Finish processing of permission event by setting it to ANSWERED state and * drop group->notification_lock. */ static void finish_permission_event(struct fsnotify_group *group, - struct fanotify_perm_event *event, - u32 response) + struct fanotify_perm_event *event, u32 response, + struct fanotify_response_info_audit_rule *friar) __releases(&group->notification_lock) { bool destroy = false; assert_spin_locked(&group->notification_lock); - event->response = response; + event->response = response & ~FAN_INFO; + if (response & FAN_INFO) + memcpy(&event->audit_rule, friar, sizeof(*friar)); + if (event->state == FAN_EVENT_CANCELED) destroy = true; else @@ -306,20 +329,27 @@ static void finish_permission_event(struct fsnotify_group *group, } static int process_access_response(struct fsnotify_group *group, - struct fanotify_response *response_struct) + struct fanotify_response *response_struct, + const char __user *info, + size_t info_len) { struct fanotify_perm_event *event; int fd = response_struct->fd; u32 response = response_struct->response; + int ret = info_len; + struct fanotify_response_info_audit_rule friar; - pr_debug("%s: group=%p fd=%d response=%u\n", __func__, group, - fd, response); + pr_debug("%s: group=%p fd=%d response=%u buf=%p size=%zu\n", __func__, + group, fd, response, info, info_len); /* * make sure the response is valid, if invalid we do nothing and either * userspace can send a valid response or we will clean it up after the * timeout */ - switch (response & ~FAN_AUDIT) { + if (response & ~FANOTIFY_RESPONSE_VALID_MASK) + return -EINVAL; + + switch (response & FANOTIFY_RESPONSE_ACCESS) { case FAN_ALLOW: case FAN_DENY: break; @@ -327,10 +357,20 @@ static int process_access_response(struct fsnotify_group *group, return -EINVAL; } - if (fd < 0) + if ((response & FAN_AUDIT) && !FAN_GROUP_FLAG(group, FAN_ENABLE_AUDIT)) return -EINVAL; - if ((response & FAN_AUDIT) && !FAN_GROUP_FLAG(group, FAN_ENABLE_AUDIT)) + if (response & FAN_INFO) { + ret = process_access_response_info(info, info_len, &friar); + if (ret < 0) + return ret; + if (fd == FAN_NOFD) + return ret; + } else { + ret = 0; + } + + if (fd < 0) return -EINVAL; spin_lock(&group->notification_lock); @@ -340,9 +380,9 @@ static int process_access_response(struct fsnotify_group *group, continue; list_del_init(&event->fae.fse.list); - finish_permission_event(group, event, response); + finish_permission_event(group, event, response, &friar); wake_up(&group->fanotify_data.access_waitq); - return 0; + return ret; } spin_unlock(&group->notification_lock); @@ -804,7 +844,7 @@ static ssize_t fanotify_read(struct file *file, char __user *buf, if (ret <= 0) { spin_lock(&group->notification_lock); finish_permission_event(group, - FANOTIFY_PERM(event), FAN_DENY); + FANOTIFY_PERM(event), FAN_DENY, NULL); wake_up(&group->fanotify_data.access_waitq); } else { spin_lock(&group->notification_lock); @@ -827,28 +867,32 @@ static ssize_t fanotify_read(struct file *file, char __user *buf, static ssize_t fanotify_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) { - struct fanotify_response response = { .fd = -1, .response = -1 }; + struct fanotify_response response; struct fsnotify_group *group; int ret; + const char __user *info_buf = buf + sizeof(struct fanotify_response); + size_t info_len; if (!IS_ENABLED(CONFIG_FANOTIFY_ACCESS_PERMISSIONS)) return -EINVAL; group = file->private_data; + pr_debug("%s: group=%p count=%zu\n", __func__, group, count); + if (count < sizeof(response)) return -EINVAL; - count = sizeof(response); - - pr_debug("%s: group=%p count=%zu\n", __func__, group, count); - - if (copy_from_user(&response, buf, count)) + if (copy_from_user(&response, buf, sizeof(response))) return -EFAULT; - ret = process_access_response(group, &response); + info_len = count - sizeof(response); + + ret = process_access_response(group, &response, info_buf, info_len); if (ret < 0) count = ret; + else + count = sizeof(response) + ret; return count; } @@ -876,7 +920,7 @@ static int fanotify_release(struct inode *ignored, struct file *file) event = list_first_entry(&group->fanotify_data.access_list, struct fanotify_perm_event, fae.fse.list); list_del_init(&event->fae.fse.list); - finish_permission_event(group, event, FAN_ALLOW); + finish_permission_event(group, event, FAN_ALLOW, NULL); spin_lock(&group->notification_lock); } @@ -893,7 +937,7 @@ static int fanotify_release(struct inode *ignored, struct file *file) fsnotify_destroy_event(group, fsn_event); } else { finish_permission_event(group, FANOTIFY_PERM(event), - FAN_ALLOW); + FAN_ALLOW, NULL); } spin_lock(&group->notification_lock); } diff --git a/include/linux/fanotify.h b/include/linux/fanotify.h index 8ad743def6f3..4f1c4f603118 100644 --- a/include/linux/fanotify.h +++ b/include/linux/fanotify.h @@ -122,6 +122,11 @@ #define ALL_FANOTIFY_EVENT_BITS (FANOTIFY_OUTGOING_EVENTS | \ FANOTIFY_EVENT_FLAGS) +/* These masks check for invalid bits in permission responses. */ +#define FANOTIFY_RESPONSE_ACCESS (FAN_ALLOW | FAN_DENY) +#define FANOTIFY_RESPONSE_FLAGS (FAN_AUDIT | FAN_INFO) +#define FANOTIFY_RESPONSE_VALID_MASK (FANOTIFY_RESPONSE_ACCESS | FANOTIFY_RESPONSE_FLAGS) + /* Do not use these old uapi constants internally */ #undef FAN_ALL_CLASS_BITS #undef FAN_ALL_INIT_FLAGS diff --git a/include/uapi/linux/fanotify.h b/include/uapi/linux/fanotify.h index 436258214bb0..cd14c94e9a1e 100644 --- a/include/uapi/linux/fanotify.h +++ b/include/uapi/linux/fanotify.h @@ -188,15 +188,43 @@ struct fanotify_event_info_error { __u32 error_count; }; +/* + * User space may need to record additional information about its decision. + * The extra information type records what kind of information is included. + * The default is none. We also define an extra information buffer whose + * size is determined by the extra information type. + * + * If the information type is Audit Rule, then the information following + * is the rule number that triggered the user space decision that + * requires auditing. + */ + +#define FAN_RESPONSE_INFO_NONE 0 +#define FAN_RESPONSE_INFO_AUDIT_RULE 1 + struct fanotify_response { __s32 fd; __u32 response; }; +struct fanotify_response_info_header { + __u8 type; + __u8 pad; + __u16 len; +}; + +struct fanotify_response_info_audit_rule { + struct fanotify_response_info_header hdr; + __u32 rule_number; + __u32 subj_trust; + __u32 obj_trust; +}; + /* Legit userspace responses to a _PERM event */ #define FAN_ALLOW 0x01 #define FAN_DENY 0x02 -#define FAN_AUDIT 0x10 /* Bit mask to create audit record for result */ +#define FAN_AUDIT 0x10 /* Bitmask to create audit record for result */ +#define FAN_INFO 0x20 /* Bitmask to indicate additional information */ /* No fd set in event */ #define FAN_NOFD -1 From 032bffd494e3924cc8b854b696ef9b5b7396b883 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Fri, 3 Feb 2023 16:35:16 -0500 Subject: [PATCH 152/265] fanotify,audit: Allow audit to use the full permission event response This patch passes the full response so that the audit function can use all of it. The audit function was updated to log the additional information in the AUDIT_FANOTIFY record. Currently the only type of fanotify info that is defined is an audit rule number, but convert it to hex encoding to future-proof the field. Hex encoding suggested by Paul Moore . The {subj,obj}_trust values are {0,1,2}, corresponding to no, yes, unknown. Sample records: type=FANOTIFY msg=audit(1600385147.372:590): resp=2 fan_type=1 fan_info=3137 subj_trust=3 obj_trust=5 type=FANOTIFY msg=audit(1659730979.839:284): resp=1 fan_type=0 fan_info=0 subj_trust=2 obj_trust=2 Suggested-by: Steve Grubb Link: https://lore.kernel.org/r/3075502.aeNJFYEL58@x2 Tested-by: Steve Grubb Acked-by: Steve Grubb Signed-off-by: Richard Guy Briggs Signed-off-by: Jan Kara Message-Id: --- fs/notify/fanotify/fanotify.c | 3 ++- include/linux/audit.h | 9 +++++---- kernel/auditsc.c | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index 24ec1d66d5a8..29bdd99b29fa 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -273,7 +273,8 @@ static int fanotify_get_response(struct fsnotify_group *group, /* Check if the response should be audited */ if (event->response & FAN_AUDIT) - audit_fanotify(event->response & ~FAN_AUDIT); + audit_fanotify(event->response & ~FAN_AUDIT, + &event->audit_rule); pr_debug("%s: group=%p event=%p about to return ret=%d\n", __func__, group, event, ret); diff --git a/include/linux/audit.h b/include/linux/audit.h index d6b7d0c7ce43..31086a72e32a 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -14,6 +14,7 @@ #include #include #include +#include #define AUDIT_INO_UNSET ((unsigned long)-1) #define AUDIT_DEV_UNSET ((dev_t)-1) @@ -416,7 +417,7 @@ extern void __audit_log_capset(const struct cred *new, const struct cred *old); extern void __audit_mmap_fd(int fd, int flags); extern void __audit_openat2_how(struct open_how *how); extern void __audit_log_kern_module(char *name); -extern void __audit_fanotify(u32 response); +extern void __audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar); extern void __audit_tk_injoffset(struct timespec64 offset); extern void __audit_ntp_log(const struct audit_ntp_data *ad); extern void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, @@ -523,10 +524,10 @@ static inline void audit_log_kern_module(char *name) __audit_log_kern_module(name); } -static inline void audit_fanotify(u32 response) +static inline void audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar) { if (!audit_dummy_context()) - __audit_fanotify(response); + __audit_fanotify(response, friar); } static inline void audit_tk_injoffset(struct timespec64 offset) @@ -679,7 +680,7 @@ static inline void audit_log_kern_module(char *name) { } -static inline void audit_fanotify(u32 response) +static inline void audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar) { } static inline void audit_tk_injoffset(struct timespec64 offset) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index d1fb821de104..5a5994659b44 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -64,6 +64,7 @@ #include #include #include // struct open_how +#include #include "audit.h" @@ -2877,10 +2878,21 @@ void __audit_log_kern_module(char *name) context->type = AUDIT_KERN_MODULE; } -void __audit_fanotify(u32 response) +void __audit_fanotify(u32 response, struct fanotify_response_info_audit_rule *friar) { - audit_log(audit_context(), GFP_KERNEL, - AUDIT_FANOTIFY, "resp=%u", response); + /* {subj,obj}_trust values are {0,1,2}: no,yes,unknown */ + switch (friar->hdr.type) { + case FAN_RESPONSE_INFO_NONE: + audit_log(audit_context(), GFP_KERNEL, AUDIT_FANOTIFY, + "resp=%u fan_type=%u fan_info=0 subj_trust=2 obj_trust=2", + response, FAN_RESPONSE_INFO_NONE); + break; + case FAN_RESPONSE_INFO_AUDIT_RULE: + audit_log(audit_context(), GFP_KERNEL, AUDIT_FANOTIFY, + "resp=%u fan_type=%u fan_info=%X subj_trust=%u obj_trust=%u", + response, friar->hdr.type, friar->rule_number, + friar->subj_trust, friar->obj_trust); + } } void __audit_tk_injoffset(struct timespec64 offset) From 51e38c92bed26f648ec187c4400fa7512fcd8067 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sat, 4 Feb 2023 10:34:27 -0800 Subject: [PATCH 153/265] udf: Use unsigned variables for size calculations To avoid confusing the compiler about possible negative sizes, switch various size variables that can never be negative from int to u32. Seen with GCC 13: ../fs/udf/directory.c: In function 'udf_copy_fi': ../include/linux/fortify-string.h:57:33: warning: '__builtin_memcpy' pointer overflow between offset 80 and size [-2147483648, -1] [-Warray-bounds=] 57 | #define __underlying_memcpy __builtin_memcpy | ^ ... ../fs/udf/directory.c:102:9: note: in expansion of macro 'memcpy' 102 | memcpy(&iter->fi, iter->bh[0]->b_data + off, len); | ^~~~~~ Cc: Jan Kara Signed-off-by: Kees Cook Signed-off-by: Jan Kara Message-Id: <20230204183427.never.856-kees@kernel.org> --- fs/udf/directory.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index 2e13c4b5fb81..e0bb73d414dd 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -71,8 +71,9 @@ static int udf_verify_fi(struct udf_fileident_iter *iter) static int udf_copy_fi(struct udf_fileident_iter *iter) { struct udf_inode_info *iinfo = UDF_I(iter->dir); - int blksize = 1 << iter->dir->i_blkbits; - int err, off, len, nameoff; + u32 blksize = 1 << iter->dir->i_blkbits; + u32 off, len, nameoff; + int err; /* Skip copying when we are at EOF */ if (iter->pos >= iter->dir->i_size) { From df97f64dfa317a5485daf247b6c043a584ef95f9 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 9 Feb 2023 10:33:09 +0100 Subject: [PATCH 154/265] udf: Avoid directory type conversion failure due to ENOMEM When converting directory from in-ICB to normal format, the last iteration through the directory fixing up directory enteries can fail due to ENOMEM. We do not expect this iteration to fail since the directory is already verified to be correct and it is difficult to undo the conversion at this point. So just use GFP_NOFAIL to make sure the small allocation cannot fail. Reported-by: syzbot+111eaa994ff74f8d440f@syzkaller.appspotmail.com Fixes: 0aba4860b0d0 ("udf: Allocate name buffer in directory iterator on heap") Signed-off-by: Jan Kara --- fs/udf/directory.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/udf/directory.c b/fs/udf/directory.c index e0bb73d414dd..654536d2b609 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -251,9 +251,12 @@ int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, iter->elen = 0; iter->epos.bh = NULL; iter->name = NULL; - iter->namebuf = kmalloc(UDF_NAME_LEN_CS0, GFP_KERNEL); - if (!iter->namebuf) - return -ENOMEM; + /* + * When directory is verified, we don't expect directory iteration to + * fail and it can be difficult to undo without corrupting filesystem. + * So just do not allow memory allocation failures here. + */ + iter->namebuf = kmalloc(UDF_NAME_LEN_CS0, GFP_KERNEL | __GFP_NOFAIL); if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { err = udf_copy_fi(iter); From fbe870a72fd1ddc5e08c23764e23e5766f54aa87 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 1 Feb 2023 15:33:33 -0500 Subject: [PATCH 155/265] io_uring,audit: don't log IORING_OP_MADVISE fadvise and madvise both provide hints for caching or access pattern for file and memory respectively. Skip them. Fixes: 5bd2182d58e9 ("audit,io_uring,io-wq: add some basic audit support to io_uring") Signed-off-by: Richard Guy Briggs Link: https://lore.kernel.org/r/b5dfdcd541115c86dbc774aa9dd502c964849c5f.1675282642.git.rgb@redhat.com Acked-by: Paul Moore Signed-off-by: Jens Axboe --- io_uring/opdef.c | 1 + 1 file changed, 1 insertion(+) diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 5238ecd7af6a..cca7c5b55208 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -257,6 +257,7 @@ const struct io_issue_def io_issue_defs[] = { .issue = io_fadvise, }, [IORING_OP_MADVISE] = { + .audit_skip = 1, .prep = io_madvise_prep, .issue = io_madvise, }, From 4c89493f35c4a8143b7bdc23ff446ddbadfb39d9 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Mon, 12 Dec 2022 01:02:46 -0800 Subject: [PATCH 156/265] btrfs: drop unused trans parameter of drop_delayed_ref drop_delayed_ref() doesn't use the btrfs_trans_handle it gets passed in, so remove it. Reviewed-by: Qu Wenruo Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 573ebab886e2..663e7493926f 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -437,8 +437,7 @@ int btrfs_delayed_ref_lock(struct btrfs_delayed_ref_root *delayed_refs, return 0; } -static inline void drop_delayed_ref(struct btrfs_trans_handle *trans, - struct btrfs_delayed_ref_root *delayed_refs, +static inline void drop_delayed_ref(struct btrfs_delayed_ref_root *delayed_refs, struct btrfs_delayed_ref_head *head, struct btrfs_delayed_ref_node *ref) { @@ -482,10 +481,10 @@ static bool merge_ref(struct btrfs_trans_handle *trans, mod = -next->ref_mod; } - drop_delayed_ref(trans, delayed_refs, head, next); + drop_delayed_ref(delayed_refs, head, next); ref->ref_mod += mod; if (ref->ref_mod == 0) { - drop_delayed_ref(trans, delayed_refs, head, ref); + drop_delayed_ref(delayed_refs, head, ref); done = true; } else { /* @@ -641,7 +640,7 @@ static int insert_delayed_ref(struct btrfs_trans_handle *trans, /* remove existing tail if its ref_mod is zero */ if (exist->ref_mod == 0) - drop_delayed_ref(trans, root, href, exist); + drop_delayed_ref(root, href, exist); spin_unlock(&href->lock); return ret; inserted: From f09f7851b78b2fdb9d4c3bd5b1a086ab2ab955dd Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Mon, 12 Dec 2022 01:02:47 -0800 Subject: [PATCH 157/265] btrfs: remove trans parameter of merge_ref Now that drop_delayed_ref() doesn't get the btrfs_trans_handle passed in anymore, we can get rid of it in merge_ref() as well. Reviewed-by: Qu Wenruo Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 663e7493926f..046ba49b8f94 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -451,8 +451,7 @@ static inline void drop_delayed_ref(struct btrfs_delayed_ref_root *delayed_refs, atomic_dec(&delayed_refs->num_entries); } -static bool merge_ref(struct btrfs_trans_handle *trans, - struct btrfs_delayed_ref_root *delayed_refs, +static bool merge_ref(struct btrfs_delayed_ref_root *delayed_refs, struct btrfs_delayed_ref_head *head, struct btrfs_delayed_ref_node *ref, u64 seq) @@ -523,7 +522,7 @@ again: ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node); if (seq && ref->seq >= seq) continue; - if (merge_ref(trans, delayed_refs, head, ref, seq)) + if (merge_ref(delayed_refs, head, ref, seq)) goto again; } } From afe2d748b08ad15ae945ad3b0c2408f8b7ecea22 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Mon, 12 Dec 2022 01:02:48 -0800 Subject: [PATCH 158/265] btrfs: drop trans parameter of insert_delayed_ref Now that drop_delayed_ref() doesn't need a btrfs_trans_handle, drop it from insert_delayed_ref() as well. Reviewed-by: Qu Wenruo Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 046ba49b8f94..678ce95c04ac 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -599,8 +599,7 @@ void btrfs_delete_ref_head(struct btrfs_delayed_ref_root *delayed_refs, * Return 0 for insert. * Return >0 for merge. */ -static int insert_delayed_ref(struct btrfs_trans_handle *trans, - struct btrfs_delayed_ref_root *root, +static int insert_delayed_ref(struct btrfs_delayed_ref_root *root, struct btrfs_delayed_ref_head *href, struct btrfs_delayed_ref_node *ref) { @@ -976,7 +975,7 @@ int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, head_ref = add_delayed_ref_head(trans, head_ref, record, action, &qrecord_inserted); - ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node); + ret = insert_delayed_ref(delayed_refs, head_ref, &ref->node); spin_unlock(&delayed_refs->lock); /* @@ -1068,7 +1067,7 @@ int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, head_ref = add_delayed_ref_head(trans, head_ref, record, action, &qrecord_inserted); - ret = insert_delayed_ref(trans, delayed_refs, head_ref, &ref->node); + ret = insert_delayed_ref(delayed_refs, head_ref, &ref->node); spin_unlock(&delayed_refs->lock); /* From 0c555c97ef5a96b57c8955bb3cc1be9d3fbf166e Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Mon, 12 Dec 2022 01:02:49 -0800 Subject: [PATCH 159/265] btrfs: directly pass in fs_info to btrfs_merge_delayed_refs Now that none of the functions called by btrfs_merge_delayed_refs() needs a btrfs_trans_handle, directly pass in a btrfs_fs_info to btrfs_merge_delayed_refs(). Reviewed-by: Qu Wenruo Reviewed-by: Josef Bacik Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/delayed-ref.c | 3 +-- fs/btrfs/delayed-ref.h | 2 +- fs/btrfs/extent-tree.c | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index 678ce95c04ac..886ffb232eac 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -497,11 +497,10 @@ static bool merge_ref(struct btrfs_delayed_ref_root *delayed_refs, return done; } -void btrfs_merge_delayed_refs(struct btrfs_trans_handle *trans, +void btrfs_merge_delayed_refs(struct btrfs_fs_info *fs_info, struct btrfs_delayed_ref_root *delayed_refs, struct btrfs_delayed_ref_head *head) { - struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_delayed_ref_node *ref; struct rb_node *node; u64 seq = 0; diff --git a/fs/btrfs/delayed-ref.h b/fs/btrfs/delayed-ref.h index d6304b690ec4..2eb34abf700f 100644 --- a/fs/btrfs/delayed-ref.h +++ b/fs/btrfs/delayed-ref.h @@ -357,7 +357,7 @@ int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, struct btrfs_delayed_extent_op *extent_op); -void btrfs_merge_delayed_refs(struct btrfs_trans_handle *trans, +void btrfs_merge_delayed_refs(struct btrfs_fs_info *fs_info, struct btrfs_delayed_ref_root *delayed_refs, struct btrfs_delayed_ref_head *head); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 72ba13b027a9..d1a4e51f8fbc 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1966,7 +1966,7 @@ static int btrfs_run_delayed_refs_for_head(struct btrfs_trans_handle *trans, cond_resched(); spin_lock(&locked_ref->lock); - btrfs_merge_delayed_refs(trans, delayed_refs, locked_ref); + btrfs_merge_delayed_refs(fs_info, delayed_refs, locked_ref); } return 0; @@ -2013,7 +2013,7 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, * insert_inline_extent_backref()). */ spin_lock(&locked_ref->lock); - btrfs_merge_delayed_refs(trans, delayed_refs, locked_ref); + btrfs_merge_delayed_refs(fs_info, delayed_refs, locked_ref); ret = btrfs_run_delayed_refs_for_head(trans, locked_ref, &actual_count); From fccf0c842ed4857cff8067b72a79fcf64d972c2f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Wed, 7 Dec 2022 10:18:04 -0500 Subject: [PATCH 160/265] btrfs: move btrfs_abort_transaction to transaction.c While trying to sync messages.[ch] I ended up with this dependency on messages.h in the rest of btrfs-progs code base because it's where btrfs_abort_transaction() was now held. We want to keep messages.[ch] limited to the kernel code, and the btrfs_abort_transaction() code better fits in the transaction code and not in messages. Reviewed-by: Johannes Thumshirn Signed-off-by: Josef Bacik Reviewed-by: David Sterba [ move the __cold attributes ] Signed-off-by: David Sterba --- fs/btrfs/messages.c | 30 ------------------------------ fs/btrfs/messages.h | 34 ---------------------------------- fs/btrfs/transaction.c | 29 +++++++++++++++++++++++++++++ fs/btrfs/transaction.h | 31 +++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 64 deletions(-) diff --git a/fs/btrfs/messages.c b/fs/btrfs/messages.c index 625bbbbb2608..fde5aaa6e7c9 100644 --- a/fs/btrfs/messages.c +++ b/fs/btrfs/messages.c @@ -292,36 +292,6 @@ void __cold btrfs_err_32bit_limit(struct btrfs_fs_info *fs_info) } #endif -/* - * We only mark the transaction aborted and then set the file system read-only. - * This will prevent new transactions from starting or trying to join this - * one. - * - * This means that error recovery at the call site is limited to freeing - * any local memory allocations and passing the error code up without - * further cleanup. The transaction should complete as it normally would - * in the call path but will return -EIO. - * - * We'll complete the cleanup in btrfs_end_transaction and - * btrfs_commit_transaction. - */ -__cold -void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, - const char *function, - unsigned int line, int errno, bool first_hit) -{ - struct btrfs_fs_info *fs_info = trans->fs_info; - - WRITE_ONCE(trans->aborted, errno); - WRITE_ONCE(trans->transaction->aborted, errno); - if (first_hit && errno == -ENOSPC) - btrfs_dump_space_info_for_trans_abort(fs_info); - /* Wake up anybody who may be waiting on this transaction */ - wake_up(&fs_info->transaction_wait); - wake_up(&fs_info->transaction_blocked_wait); - __btrfs_handle_fs_error(fs_info, function, line, errno, NULL); -} - /* * __btrfs_panic decodes unexpected, fatal errors from the caller, issues an * alert, and either panics or BUGs, depending on mount options. diff --git a/fs/btrfs/messages.h b/fs/btrfs/messages.h index 190af1f698d9..8c516ee58ff9 100644 --- a/fs/btrfs/messages.h +++ b/fs/btrfs/messages.h @@ -6,7 +6,6 @@ #include struct btrfs_fs_info; -struct btrfs_trans_handle; static inline __printf(2, 3) __cold void btrfs_no_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...) @@ -178,39 +177,6 @@ void __btrfs_handle_fs_error(struct btrfs_fs_info *fs_info, const char *function const char * __attribute_const__ btrfs_decode_error(int errno); -__cold -void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, - const char *function, - unsigned int line, int errno, bool first_hit); - -bool __cold abort_should_print_stack(int errno); - -/* - * Call btrfs_abort_transaction as early as possible when an error condition is - * detected, that way the exact stack trace is reported for some errors. - */ -#define btrfs_abort_transaction(trans, errno) \ -do { \ - bool first = false; \ - /* Report first abort since mount */ \ - if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ - &((trans)->fs_info->fs_state))) { \ - first = true; \ - if (WARN(abort_should_print_stack(errno), \ - KERN_ERR \ - "BTRFS: Transaction aborted (error %d)\n", \ - (errno))) { \ - /* Stack trace printed. */ \ - } else { \ - btrfs_err((trans)->fs_info, \ - "Transaction aborted (error %d)", \ - (errno)); \ - } \ - } \ - __btrfs_abort_transaction((trans), __func__, \ - __LINE__, (errno), first); \ -} while (0) - #define btrfs_handle_fs_error(fs_info, errno, fmt, args...) \ __btrfs_handle_fs_error((fs_info), __func__, __LINE__, \ (errno), fmt, ##args) diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index b8c52e89688c..528efe559866 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -2604,6 +2604,35 @@ int btrfs_clean_one_deleted_snapshot(struct btrfs_fs_info *fs_info) return (ret < 0) ? 0 : 1; } +/* + * We only mark the transaction aborted and then set the file system read-only. + * This will prevent new transactions from starting or trying to join this + * one. + * + * This means that error recovery at the call site is limited to freeing + * any local memory allocations and passing the error code up without + * further cleanup. The transaction should complete as it normally would + * in the call path but will return -EIO. + * + * We'll complete the cleanup in btrfs_end_transaction and + * btrfs_commit_transaction. + */ +void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, + const char *function, + unsigned int line, int errno, bool first_hit) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + + WRITE_ONCE(trans->aborted, errno); + WRITE_ONCE(trans->transaction->aborted, errno); + if (first_hit && errno == -ENOSPC) + btrfs_dump_space_info_for_trans_abort(fs_info); + /* Wake up anybody who may be waiting on this transaction */ + wake_up(&fs_info->transaction_wait); + wake_up(&fs_info->transaction_blocked_wait); + __btrfs_handle_fs_error(fs_info, function, line, errno, NULL); +} + int __init btrfs_transaction_init(void) { btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle", diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 97f6c39f59c8..fa728ab80826 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -202,6 +202,34 @@ static inline void btrfs_clear_skip_qgroup(struct btrfs_trans_handle *trans) delayed_refs->qgroup_to_skip = 0; } +bool __cold abort_should_print_stack(int errno); + +/* + * Call btrfs_abort_transaction as early as possible when an error condition is + * detected, that way the exact stack trace is reported for some errors. + */ +#define btrfs_abort_transaction(trans, errno) \ +do { \ + bool first = false; \ + /* Report first abort since mount */ \ + if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ + &((trans)->fs_info->fs_state))) { \ + first = true; \ + if (WARN(abort_should_print_stack(errno), \ + KERN_ERR \ + "BTRFS: Transaction aborted (error %d)\n", \ + (errno))) { \ + /* Stack trace printed. */ \ + } else { \ + btrfs_debug((trans)->fs_info, \ + "Transaction aborted (error %d)", \ + (errno)); \ + } \ + } \ + __btrfs_abort_transaction((trans), __func__, \ + __LINE__, (errno), first); \ +} while (0) + int btrfs_end_transaction(struct btrfs_trans_handle *trans); struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, unsigned int num_items); @@ -236,6 +264,9 @@ void btrfs_put_transaction(struct btrfs_transaction *transaction); void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, struct btrfs_root *root); void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); +void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, + const char *function, + unsigned int line, int errno, bool first_hit); int __init btrfs_transaction_init(void); void __cold btrfs_transaction_exit(void); From 0e47b25cafb29338722f68e8c5a260aaf18ce92c Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:52 -0500 Subject: [PATCH 161/265] btrfs: fix uninitialized variable warning in btrfs_cleanup_ordered_extents We can conditionally pass in a locked page, and then we'll use that page range to skip marking errors as that will happen in another layer. However this causes the compiler to complain because it doesn't understand we only use these values when we have the page. Make the compiler stop complaining by setting these values to 0. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 98a800b8bd43..77c2acc06891 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -228,7 +228,7 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, { unsigned long index = offset >> PAGE_SHIFT; unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT; - u64 page_start, page_end; + u64 page_start = 0, page_end = 0; struct page *page; if (locked_page) { From ab199013592abb3499b8316a800a39ab61e6719f Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:53 -0500 Subject: [PATCH 162/265] btrfs: fix uninitialized variable warning in get_inode_gen Anybody that calls get_inode_gen() can have an uninitialized gen if there's an error. This isn't a big deal because all the users just exit if they get an error, however it makes -Wmaybe-uninitialized complain, so fix this up to always initialize the passed in gen, this quiets all of the uninitialized warnings in send.c. Reviewed-by: Qu Wenruo Reviewed-by: Qu Wenruo Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index d50182b6deec..7817bf8d0c0b 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -956,14 +956,12 @@ out: static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen) { int ret; - struct btrfs_inode_info info; + struct btrfs_inode_info info = { 0 }; - if (!gen) - return -EPERM; + ASSERT(gen); ret = get_inode_info(root, ino, &info); - if (!ret) - *gen = info.gen; + *gen = info.gen; return ret; } From efbf35a102b20246cfe4409c6ae92e72ecb67ab8 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:54 -0500 Subject: [PATCH 163/265] btrfs: fix uninitialized variable warning in btrfs_update_block_group reclaim isn't set in the alloc case, however we only care about reclaim in the !alloc case. This isn't an actual problem, however -Wmaybe-uninitialized will complain, so initialize reclaim to quiet the compiler. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 708d843daa72..e90800388a41 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -3330,7 +3330,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, spin_unlock(&info->delalloc_root_lock); while (total) { - bool reclaim; + bool reclaim = false; cache = btrfs_lookup_block_group(info, bytenr); if (!cache) { From 598643250c47c2a2e2339f25e2be36165eb229a9 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:55 -0500 Subject: [PATCH 164/265] btrfs: fix uninitialized variable warnings in __set_extent_bit and convert_extent_bit We will pass in the parent and p pointer into our tree_search function to avoid doing a second search when inserting a new extent state into the tree. However because this is conditional upon passing in these pointers the compiler seems to think these values can be uninitialized if we're using -Wmaybe-uninitialized. Fix this by initializing these values. Reviewed-by: Qu Wenruo Reviewed-by: Qu Wenruo Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-io-tree.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index 3c7766dfaa69..b2bab7672340 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -972,8 +972,8 @@ static int __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, { struct extent_state *state; struct extent_state *prealloc = NULL; - struct rb_node **p; - struct rb_node *parent; + struct rb_node **p = NULL; + struct rb_node *parent = NULL; int err = 0; u64 last_start; u64 last_end; @@ -1218,8 +1218,8 @@ int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, { struct extent_state *state; struct extent_state *prealloc = NULL; - struct rb_node **p; - struct rb_node *parent; + struct rb_node **p = NULL; + struct rb_node *parent = NULL; int err = 0; u64 last_start; u64 last_end; From 12adffe6cfb8e279e8ab49504501d894ff79822a Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:57 -0500 Subject: [PATCH 165/265] btrfs: fix uninitialized variable warning in btrfs_sb_log_location We only have 3 possible mirrors, and we have ASSERT()'s to make sure we're not passing in an invalid super mirror into this function, so technically this value isn't uninitialized. However -Wmaybe-uninitialized will complain, so set it to U64_MAX so if we don't have ASSERT()'s turned on it'll error out later on when it see's the zone is beyond our maximum zones. Reviewed-by: Naohiro Aota Reviewed-by: Johannes Thumshirn Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 1f503e8e42d4..379d678e3894 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -160,7 +160,7 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, */ static inline u32 sb_zone_number(int shift, int mirror) { - u64 zone; + u64 zone = U64_MAX; ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX); switch (mirror) { From cd30d3bc78d9acbd505d0d6a4cff3b87e40a8187 Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Wed, 21 Dec 2022 16:47:45 +0000 Subject: [PATCH 166/265] btrfs: zoned: fix uninitialized variable warning in btrfs_get_dev_zones Fix an uninitialized warning we get with -Wmaybe-uninitialized where it thought zno may have been uninitialized, in both cases it depends on zinfo->zone_cache but we know the value won't change between checks. Reported-by: Josef Bacik Link: https://lore.kernel.org/linux-btrfs/af6c527cbd8bdc782e50bd33996ee83acc3a16fb.1671221596.git.josef@toxicpanda.com/ Signed-off-by: Naohiro Aota Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 379d678e3894..d46701a77b17 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -220,7 +220,6 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, struct blk_zone *zones, unsigned int *nr_zones) { struct btrfs_zoned_device_info *zinfo = device->zone_info; - u32 zno; int ret; if (!*nr_zones) @@ -235,6 +234,7 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, /* Check cache */ if (zinfo->zone_cache) { unsigned int i; + u32 zno; ASSERT(IS_ALIGNED(pos, zinfo->zone_size)); zno = pos >> zinfo->zone_size_shift; @@ -274,9 +274,12 @@ static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, return -EIO; /* Populate cache */ - if (zinfo->zone_cache) + if (zinfo->zone_cache) { + u32 zno = pos >> zinfo->zone_size_shift; + memcpy(zinfo->zone_cache + zno, zones, sizeof(*zinfo->zone_cache) * *nr_zones); + } return 0; } From a6ca692ec22bdac5ae700e82ff575a1b5141fa40 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:51 -0500 Subject: [PATCH 167/265] btrfs: fix uninitialized variable warning in run_one_async_start With -Wmaybe-uninitialized compiler complains about ret being possibly uninitialized, which isn't possible as the WQ_ constants are set only from our code, however we can handle the default case and get rid of the warning. The value is set to BLK_STS_IOERR so it does not issue any IO and could be potentially detected, but this is basically a "cannot happen" error. To catch any problems during development use the assert. Signed-off-by: Josef Bacik Reviewed-by: David Sterba [ set the error in default: ] Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 3aa04224315e..7586a8e9b718 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -717,6 +717,10 @@ static void run_one_async_start(struct btrfs_work *work) ret = btrfs_submit_bio_start_direct_io(async->inode, async->bio, async->dio_file_offset); break; + default: + /* Can't happen so return something that would prevent the IO. */ + ret = BLK_STS_IOERR; + ASSERT(0); } if (ret) async->status = ret; From 1ec49744ba83f0429c5c706708610f7821a7b6f4 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 16 Dec 2022 15:15:58 -0500 Subject: [PATCH 168/265] btrfs: turn on -Wmaybe-uninitialized We had a recent bug that would have been caught by a newer compiler with -Wmaybe-uninitialized and would have saved us a month of failing tests that I didn't have time to investigate. Reviewed-by: Qu Wenruo Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index 555c962fdad6..460eced3f5bd 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -11,7 +11,8 @@ condflags := \ $(call cc-option, -Wunused-but-set-variable) \ $(call cc-option, -Wunused-const-variable) \ $(call cc-option, -Wpacked-not-aligned) \ - $(call cc-option, -Wstringop-truncation) + $(call cc-option, -Wstringop-truncation) \ + $(call cc-option, -Wmaybe-uninitialized) subdir-ccflags-y += $(condflags) # The following turn off the warnings enabled by -Wextra subdir-ccflags-y += -Wno-missing-field-initializers From d31de3785047a24959eda835b0bafb1f8629f8a9 Mon Sep 17 00:00:00 2001 From: Peng Hao Date: Mon, 9 Jan 2023 21:08:31 +0100 Subject: [PATCH 169/265] btrfs: go to matching label when cleaning em in btrfs_submit_direct When btrfs_get_chunk_map fails to allocate a new em the cleanup does not need to be done so the goto target is out_err, which is consistent with current coding style. Signed-off-by: Peng Hao Reviewed-by: David Sterba [ update changelog ] Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 77c2acc06891..7fa1db6a474a 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8080,7 +8080,7 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, if (IS_ERR(em)) { status = errno_to_blk_status(PTR_ERR(em)); em = NULL; - goto out_err_em; + goto out_err; } ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(dio_bio), logical, &geom); From ce394a7f39032bc2a85b070af608e3ae8b2cefda Mon Sep 17 00:00:00 2001 From: Yushan Zhou Date: Tue, 3 Jan 2023 13:11:37 +0800 Subject: [PATCH 170/265] btrfs: use PAGE_{ALIGN, ALIGNED, ALIGN_DOWN} macro The header file linux/mm.h provides PAGE_ALIGN, PAGE_ALIGNED, PAGE_ALIGN_DOWN macros. Use these macros to make code more concise. Signed-off-by: Yushan Zhou Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 2 +- fs/btrfs/defrag.c | 2 +- fs/btrfs/inode.c | 5 ++--- fs/btrfs/lzo.c | 2 +- fs/btrfs/relocation.c | 2 +- fs/btrfs/send.c | 4 ++-- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 5122ca79f7ea..4a5aeb8dd479 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -1609,7 +1609,7 @@ static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end, index_end = end >> PAGE_SHIFT; /* Don't miss unaligned end */ - if (!IS_ALIGNED(end, PAGE_SIZE)) + if (!PAGE_ALIGNED(end)) index_end++; curr_sample_pos = 0; diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index d81b764a7644..2737af7e2780 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -999,7 +999,7 @@ next: } #define CLUSTER_SIZE (SZ_256K) -static_assert(IS_ALIGNED(CLUSTER_SIZE, PAGE_SIZE)); +static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); /* * Defrag one contiguous target range. diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 7fa1db6a474a..49a2e118f561 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -10995,9 +10995,8 @@ static int btrfs_add_swap_extent(struct swap_info_struct *sis, return 0; max_pages = sis->max - bsi->nr_pages; - first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT; - next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len, - PAGE_SIZE) >> PAGE_SHIFT; + first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT; + next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT; if (first_ppage >= next_ppage) return 0; diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c index d5e78cbc8fbc..71f6d8302d50 100644 --- a/fs/btrfs/lzo.c +++ b/fs/btrfs/lzo.c @@ -280,7 +280,7 @@ int lzo_compress_pages(struct list_head *ws, struct address_space *mapping, } /* Check if we have reached page boundary */ - if (IS_ALIGNED(cur_in, PAGE_SIZE)) { + if (PAGE_ALIGNED(cur_in)) { put_page(page_in); page_in = NULL; } diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 31ec4a7658ce..ef13a9d4e370 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2825,7 +2825,7 @@ static noinline_for_stack int prealloc_file_extent_cluster( * * Here we have to manually invalidate the range (i_size, PAGE_END + 1). */ - if (!IS_ALIGNED(i_size, PAGE_SIZE)) { + if (!PAGE_ALIGNED(i_size)) { struct address_space *mapping = inode->vfs_inode.i_mapping; struct btrfs_fs_info *fs_info = inode->root->fs_info; const u32 sectorsize = fs_info->sectorsize; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 7817bf8d0c0b..1c2dc8e0323d 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -5633,7 +5633,7 @@ static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, * boundary in the send buffer. This means that there may be a gap * between the beginning of the command and the file data. */ - data_offset = ALIGN(sctx->send_size, PAGE_SIZE); + data_offset = PAGE_ALIGN(sctx->send_size); if (data_offset > sctx->send_max_size || sctx->send_max_size - data_offset < disk_num_bytes) { ret = -EOVERFLOW; @@ -5757,7 +5757,7 @@ static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path, sent += size; } - if (sctx->clean_page_cache && IS_ALIGNED(end, PAGE_SIZE)) { + if (sctx->clean_page_cache && PAGE_ALIGNED(end)) { /* * Always operate only on ranges that are a multiple of the page * size. This is not only to prevent zeroing parts of a page in From 5cce1780dc47906fa06c7be850532c4d1a43822c Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 10 Jan 2023 14:56:40 +0000 Subject: [PATCH 171/265] btrfs: use a negative value for BTRFS_LOG_FORCE_COMMIT Currently we use the value 1 for BTRFS_LOG_FORCE_COMMIT, but that value has a few inconveniences: 1) If it's ever used by btrfs_log_inode(), or any function down the call chain, we have to remember to btrfs_set_log_full_commit(), which is repetitive and has a chance to be forgotten in future use cases. btrfs_log_inode_parent() only calls btrfs_set_log_full_commit() when it gets a negative value from btrfs_log_inode(); 2) Down the call chain of btrfs_log_inode(), we may have functions that need to force a log commit, but can return either an error (negative value), false (0) or true (1). So they are forced to return some random negative to force a log commit - using BTRFS_LOG_FORCE_COMMIT would make the intention more clear. Currently the only example is flush_dir_items_batch(). So turn BTRFS_LOG_FORCE_COMMIT into a negative value. The chosen value is -(MAX_ERRNO + 1), so that it does not overlap any errno value and makes it easier to debug. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 10 +++------- fs/btrfs/tree-log.h | 9 +++++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 58599189bd18..94fc8b08254c 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3652,11 +3652,10 @@ static int flush_dir_items_batch(struct btrfs_trans_handle *trans, /* * If for some unexpected reason the last item's index is not greater - * than the last index we logged, warn and return an error to fallback - * to a transaction commit. + * than the last index we logged, warn and force a transaction commit. */ if (WARN_ON(last_index <= inode->last_dir_index_offset)) - ret = -EUCLEAN; + ret = BTRFS_LOG_FORCE_COMMIT; else inode->last_dir_index_offset = last_index; out: @@ -5604,10 +5603,8 @@ static int add_conflicting_inode(struct btrfs_trans_handle *trans, * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction * commits. */ - if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) { - btrfs_set_log_full_commit(trans); + if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) return BTRFS_LOG_FORCE_COMMIT; - } inode = btrfs_iget(root->fs_info->sb, ino, root); /* @@ -6466,7 +6463,6 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * result in losing the file after a log replay. */ if (full_dir_logging && inode->last_unlink_trans >= trans->transid) { - btrfs_set_log_full_commit(trans); ret = BTRFS_LOG_FORCE_COMMIT; goto out_unlock; } diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index 85cd24cb0540..bdeb5216718f 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -13,8 +13,13 @@ /* return value for btrfs_log_dentry_safe that means we don't need to log it at all */ #define BTRFS_NO_LOG_SYNC 256 -/* We can't use the tree log for whatever reason, force a transaction commit */ -#define BTRFS_LOG_FORCE_COMMIT (1) +/* + * We can't use the tree log for whatever reason, force a transaction commit. + * We use a negative value because there are functions through the logging code + * that need to return an error (< 0 value), false (0) or true (1). Any negative + * value will do, as it will cause the log to be marked for a full sync. + */ +#define BTRFS_LOG_FORCE_COMMIT (-(MAX_ERRNO + 1)) struct btrfs_log_ctx { int log_ret; From 235e1c7b872f9cf16e8a3e6050a05774b8763c58 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 10 Jan 2023 14:56:41 +0000 Subject: [PATCH 172/265] btrfs: use a single variable to track return value for log_dir_items() We currently use 'ret' and 'err' to track the return value for log_dir_items(), which is confusing and likely the cause for previous bugs where log_dir_items() did not return an error when it should, fixed in previous patches. So change this and use only a single variable, 'ret', to track the return value. This is simpler and makes it similar to most of the existing code. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 94fc8b08254c..997ba92481cb 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3793,7 +3793,6 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, struct btrfs_key min_key; struct btrfs_root *root = inode->root; struct btrfs_root *log = root->log_root; - int err = 0; int ret; u64 last_old_dentry_offset = min_offset - 1; u64 last_offset = (u64)-1; @@ -3834,8 +3833,8 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, path->slots[0]); if (tmp.type == BTRFS_DIR_INDEX_KEY) last_old_dentry_offset = tmp.offset; - } else if (ret < 0) { - err = ret; + } else if (ret > 0) { + ret = 0; } goto done; @@ -3858,7 +3857,6 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, if (tmp.type == BTRFS_DIR_INDEX_KEY) last_old_dentry_offset = tmp.offset; } else if (ret < 0) { - err = ret; goto done; } @@ -3880,12 +3878,15 @@ static noinline int log_dir_items(struct btrfs_trans_handle *trans, */ search: ret = btrfs_search_slot(NULL, root, &min_key, path, 0, 0); - if (ret > 0) + if (ret > 0) { ret = btrfs_next_item(root, path); + if (ret > 0) { + /* There are no more keys in the inode's root. */ + ret = 0; + goto done; + } + } if (ret < 0) - err = ret; - /* If ret is 1, there are no more keys in the inode's root. */ - if (ret != 0) goto done; /* @@ -3896,8 +3897,8 @@ search: ret = process_dir_items_leaf(trans, inode, path, dst_path, ctx, &last_old_dentry_offset); if (ret != 0) { - if (ret < 0) - err = ret; + if (ret > 0) + ret = 0; goto done; } path->slots[0] = btrfs_header_nritems(path->nodes[0]); @@ -3908,10 +3909,10 @@ search: */ ret = btrfs_next_leaf(root, path); if (ret) { - if (ret == 1) + if (ret == 1) { last_offset = (u64)-1; - else - err = ret; + ret = 0; + } goto done; } btrfs_item_key_to_cpu(path->nodes[0], &min_key, path->slots[0]); @@ -3942,7 +3943,7 @@ done: btrfs_release_path(path); btrfs_release_path(dst_path); - if (err == 0) { + if (ret == 0) { *last_offset_ret = last_offset; /* * In case the leaf was changed in the current transaction but @@ -3953,15 +3954,13 @@ done: * a range, last_old_dentry_offset is == to last_offset. */ ASSERT(last_old_dentry_offset <= last_offset); - if (last_old_dentry_offset < last_offset) { + if (last_old_dentry_offset < last_offset) ret = insert_dir_log_key(trans, log, path, ino, last_old_dentry_offset + 1, last_offset); - if (ret) - err = ret; - } } - return err; + + return ret; } /* From 36d4556745fe60e0e3c8d9933c2610b1c641b2f0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:12:43 +0100 Subject: [PATCH 173/265] btrfs: remove the wait argument to btrfs_start_ordered_extent Given that wait is always set to 1, so remove the argument. Last use of wait with 0 was in 0c304304feab ("Btrfs: remove csum_bytes_left"). Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/defrag.c | 2 +- fs/btrfs/file.c | 2 +- fs/btrfs/inode.c | 8 ++++---- fs/btrfs/ordered-data.c | 25 +++++++++++-------------- fs/btrfs/ordered-data.h | 2 +- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index 2737af7e2780..8065341d831a 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -765,7 +765,7 @@ again: break; unlock_page(page); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); lock_page(page); /* diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index af046d22300e..5cc5a1faaef5 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1017,7 +1017,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, unlock_page(pages[i]); put_page(pages[i]); } - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); return -EAGAIN; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 49a2e118f561..3c49742f0d45 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2969,7 +2969,7 @@ again: unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); unlock_page(page); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); goto again; } @@ -4987,7 +4987,7 @@ again: unlock_extent(io_tree, block_start, block_end, &cached_state); unlock_page(page); put_page(page); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); goto again; } @@ -7392,7 +7392,7 @@ static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, */ if (writing || test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); else ret = nowait ? -EAGAIN : -ENOTBLK; btrfs_put_ordered_extent(ordered); @@ -8552,7 +8552,7 @@ again: unlock_extent(io_tree, page_start, page_end, &cached_state); unlock_page(page); up_read(&BTRFS_I(inode)->i_mmap_lock); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); goto again; } diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 57d8c72737e1..6c24b69e2d0a 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -616,7 +616,7 @@ static void btrfs_run_ordered_extent_work(struct btrfs_work *work) struct btrfs_ordered_extent *ordered; ordered = container_of(work, struct btrfs_ordered_extent, flush_work); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); complete(&ordered->completion); } @@ -716,13 +716,12 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, } /* - * Used to start IO or wait for a given ordered extent to finish. + * Start IO and wait for a given ordered extent to finish. * - * If wait is one, this effectively waits on page writeback for all the pages - * in the extent, and it waits on the io completion code to insert - * metadata into the btree corresponding to the extent + * Wait on page writeback for all the pages in the extent and the IO completion + * code to insert metadata into the btree corresponding to the extent. */ -void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) +void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry) { u64 start = entry->file_offset; u64 end = start + entry->num_bytes - 1; @@ -744,12 +743,10 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) */ if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags)) filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, end); - if (wait) { - if (!freespace_inode) - btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); - wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, - &entry->flags)); - } + + if (!freespace_inode) + btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); + wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)); } /* @@ -800,7 +797,7 @@ int btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len) btrfs_put_ordered_extent(ordered); break; } - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); end = ordered->file_offset; /* * If the ordered extent had an error save the error but don't @@ -1061,7 +1058,7 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, break; } unlock_extent(&inode->io_tree, start, end, cachedp); - btrfs_start_ordered_extent(ordered, 1); + btrfs_start_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); } } diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 89f82b78f590..ae3ed748acaf 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -187,7 +187,7 @@ void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, struct btrfs_ordered_sum *sum); struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *inode, u64 file_offset); -void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait); +void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry); int btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len); struct btrfs_ordered_extent * btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset); From cfc2de0fce015d4249c674ef9f5e0b4817ba5c53 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 15 Dec 2022 16:06:31 -0800 Subject: [PATCH 174/265] btrfs: pass find_free_extent_ctl to allocator tracepoints The allocator tracepoints currently have a pile of values from ffe_ctl. In modifying the allocator and adding more tracepoints, I found myself adding to the already long argument list of the tracepoints. It makes it a lot simpler to just send in the ffe_ctl itself. Reviewed-by: Johannes Thumshirn Reviewed-by: Anand Jain Signed-off-by: Boris Burkov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 92 +++--------------------------------- fs/btrfs/extent-tree.h | 75 +++++++++++++++++++++++++++++ fs/btrfs/super.c | 1 + include/trace/events/btrfs.h | 41 ++++++++-------- 4 files changed, 103 insertions(+), 106 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index d1a4e51f8fbc..b26cbf6cae8b 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -16,7 +16,8 @@ #include #include #include -#include "misc.h" +#include "ctree.h" +#include "extent-tree.h" #include "tree-log.h" #include "disk-io.h" #include "print-tree.h" @@ -31,7 +32,6 @@ #include "space-info.h" #include "block-rsv.h" #include "delalloc-space.h" -#include "block-group.h" #include "discard.h" #include "rcu-string.h" #include "zoned.h" @@ -3453,81 +3453,6 @@ btrfs_release_block_group(struct btrfs_block_group *cache, btrfs_put_block_group(cache); } -enum btrfs_extent_allocation_policy { - BTRFS_EXTENT_ALLOC_CLUSTERED, - BTRFS_EXTENT_ALLOC_ZONED, -}; - -/* - * Structure used internally for find_free_extent() function. Wraps needed - * parameters. - */ -struct find_free_extent_ctl { - /* Basic allocation info */ - u64 ram_bytes; - u64 num_bytes; - u64 min_alloc_size; - u64 empty_size; - u64 flags; - int delalloc; - - /* Where to start the search inside the bg */ - u64 search_start; - - /* For clustered allocation */ - u64 empty_cluster; - struct btrfs_free_cluster *last_ptr; - bool use_cluster; - - bool have_caching_bg; - bool orig_have_caching_bg; - - /* Allocation is called for tree-log */ - bool for_treelog; - - /* Allocation is called for data relocation */ - bool for_data_reloc; - - /* RAID index, converted from flags */ - int index; - - /* - * Current loop number, check find_free_extent_update_loop() for details - */ - int loop; - - /* - * Whether we're refilling a cluster, if true we need to re-search - * current block group but don't try to refill the cluster again. - */ - bool retry_clustered; - - /* - * Whether we're updating free space cache, if true we need to re-search - * current block group but don't try updating free space cache again. - */ - bool retry_unclustered; - - /* If current block group is cached */ - int cached; - - /* Max contiguous hole found */ - u64 max_extent_size; - - /* Total free space from free space cache, not always contiguous */ - u64 total_free_space; - - /* Found result */ - u64 found_offset; - - /* Hint where to start looking for an empty space */ - u64 hint_byte; - - /* Allocation policy */ - enum btrfs_extent_allocation_policy policy; -}; - - /* * Helper function for find_free_extent(). * @@ -3559,8 +3484,7 @@ static int find_free_extent_clustered(struct btrfs_block_group *bg, if (offset) { /* We have a block, we're done */ spin_unlock(&last_ptr->refill_lock); - trace_btrfs_reserve_extent_cluster(cluster_bg, - ffe_ctl->search_start, ffe_ctl->num_bytes); + trace_btrfs_reserve_extent_cluster(cluster_bg, ffe_ctl); *cluster_bg_ret = cluster_bg; ffe_ctl->found_offset = offset; return 0; @@ -3610,10 +3534,8 @@ refill_cluster: if (offset) { /* We found one, proceed */ spin_unlock(&last_ptr->refill_lock); - trace_btrfs_reserve_extent_cluster(bg, - ffe_ctl->search_start, - ffe_ctl->num_bytes); ffe_ctl->found_offset = offset; + trace_btrfs_reserve_extent_cluster(bg, ffe_ctl); return 0; } } else if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT && @@ -4296,8 +4218,7 @@ static noinline int find_free_extent(struct btrfs_root *root, ins->objectid = 0; ins->offset = 0; - trace_find_free_extent(root, ffe_ctl->num_bytes, ffe_ctl->empty_size, - ffe_ctl->flags); + trace_find_free_extent(root, ffe_ctl); space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags); if (!space_info) { @@ -4468,8 +4389,7 @@ have_block_group: ins->objectid = ffe_ctl->search_start; ins->offset = ffe_ctl->num_bytes; - trace_btrfs_reserve_extent(block_group, ffe_ctl->search_start, - ffe_ctl->num_bytes); + trace_btrfs_reserve_extent(block_group, ffe_ctl); btrfs_release_block_group(block_group, ffe_ctl->delalloc); break; loop: diff --git a/fs/btrfs/extent-tree.h b/fs/btrfs/extent-tree.h index ae5425253603..64fa8ad7914a 100644 --- a/fs/btrfs/extent-tree.h +++ b/fs/btrfs/extent-tree.h @@ -3,6 +3,81 @@ #ifndef BTRFS_EXTENT_TREE_H #define BTRFS_EXTENT_TREE_H +#include "misc.h" +#include "block-group.h" + +struct btrfs_free_cluster; + +enum btrfs_extent_allocation_policy { + BTRFS_EXTENT_ALLOC_CLUSTERED, + BTRFS_EXTENT_ALLOC_ZONED, +}; + +struct find_free_extent_ctl { + /* Basic allocation info */ + u64 ram_bytes; + u64 num_bytes; + u64 min_alloc_size; + u64 empty_size; + u64 flags; + int delalloc; + + /* Where to start the search inside the bg */ + u64 search_start; + + /* For clustered allocation */ + u64 empty_cluster; + struct btrfs_free_cluster *last_ptr; + bool use_cluster; + + bool have_caching_bg; + bool orig_have_caching_bg; + + /* Allocation is called for tree-log */ + bool for_treelog; + + /* Allocation is called for data relocation */ + bool for_data_reloc; + + /* RAID index, converted from flags */ + int index; + + /* + * Current loop number, check find_free_extent_update_loop() for details + */ + int loop; + + /* + * Whether we're refilling a cluster, if true we need to re-search + * current block group but don't try to refill the cluster again. + */ + bool retry_clustered; + + /* + * Whether we're updating free space cache, if true we need to re-search + * current block group but don't try updating free space cache again. + */ + bool retry_unclustered; + + /* If current block group is cached */ + int cached; + + /* Max contiguous hole found */ + u64 max_extent_size; + + /* Total free space from free space cache, not always contiguous */ + u64 total_free_space; + + /* Found result */ + u64 found_offset; + + /* Hint where to start looking for an empty space */ + u64 hint_byte; + + /* Allocation policy */ + enum btrfs_extent_allocation_policy policy; +}; + enum btrfs_inline_ref_type { BTRFS_REF_TYPE_INVALID, BTRFS_REF_TYPE_BLOCK, diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 433ce221dc5c..e5136baef9af 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -58,6 +58,7 @@ #include "scrub.h" #include "verity.h" #include "super.h" +#include "extent-tree.h" #define CREATE_TRACE_POINTS #include diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 6548b5b5aa60..8a422e29d064 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -32,6 +32,7 @@ struct prelim_ref; struct btrfs_space_info; struct btrfs_raid_bio; struct raid56_bio_trace_info; +struct find_free_extent_ctl; #define show_ref_type(type) \ __print_symbolic(type, \ @@ -1241,38 +1242,38 @@ DEFINE_EVENT(btrfs__reserved_extent, btrfs_reserved_extent_free, TRACE_EVENT(find_free_extent, - TP_PROTO(const struct btrfs_root *root, u64 num_bytes, - u64 empty_size, u64 data), + TP_PROTO(const struct btrfs_root *root, + const struct find_free_extent_ctl *ffe_ctl), - TP_ARGS(root, num_bytes, empty_size, data), + TP_ARGS(root, ffe_ctl), TP_STRUCT__entry_btrfs( __field( u64, root_objectid ) __field( u64, num_bytes ) __field( u64, empty_size ) - __field( u64, data ) + __field( u64, flags ) ), TP_fast_assign_btrfs(root->fs_info, __entry->root_objectid = root->root_key.objectid; - __entry->num_bytes = num_bytes; - __entry->empty_size = empty_size; - __entry->data = data; + __entry->num_bytes = ffe_ctl->num_bytes; + __entry->empty_size = ffe_ctl->empty_size; + __entry->flags = ffe_ctl->flags; ), TP_printk_btrfs("root=%llu(%s) len=%llu empty_size=%llu flags=%llu(%s)", show_root_type(__entry->root_objectid), - __entry->num_bytes, __entry->empty_size, __entry->data, - __print_flags((unsigned long)__entry->data, "|", + __entry->num_bytes, __entry->empty_size, __entry->flags, + __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS)) ); DECLARE_EVENT_CLASS(btrfs__reserve_extent, - TP_PROTO(const struct btrfs_block_group *block_group, u64 start, - u64 len), + TP_PROTO(const struct btrfs_block_group *block_group, + const struct find_free_extent_ctl *ffe_ctl), - TP_ARGS(block_group, start, len), + TP_ARGS(block_group, ffe_ctl), TP_STRUCT__entry_btrfs( __field( u64, bg_objectid ) @@ -1284,8 +1285,8 @@ DECLARE_EVENT_CLASS(btrfs__reserve_extent, TP_fast_assign_btrfs(block_group->fs_info, __entry->bg_objectid = block_group->start; __entry->flags = block_group->flags; - __entry->start = start; - __entry->len = len; + __entry->start = ffe_ctl->search_start; + __entry->len = ffe_ctl->num_bytes; ), TP_printk_btrfs("root=%llu(%s) block_group=%llu flags=%llu(%s) " @@ -1299,18 +1300,18 @@ DECLARE_EVENT_CLASS(btrfs__reserve_extent, DEFINE_EVENT(btrfs__reserve_extent, btrfs_reserve_extent, - TP_PROTO(const struct btrfs_block_group *block_group, u64 start, - u64 len), + TP_PROTO(const struct btrfs_block_group *block_group, + const struct find_free_extent_ctl *ffe_ctl), - TP_ARGS(block_group, start, len) + TP_ARGS(block_group, ffe_ctl) ); DEFINE_EVENT(btrfs__reserve_extent, btrfs_reserve_extent_cluster, - TP_PROTO(const struct btrfs_block_group *block_group, u64 start, - u64 len), + TP_PROTO(const struct btrfs_block_group *block_group, + const struct find_free_extent_ctl *ffe_ctl), - TP_ARGS(block_group, start, len) + TP_ARGS(block_group, ffe_ctl) ); TRACE_EVENT(btrfs_find_cluster, From 854c2f365d7e0b5b1250953e03860f09a7847c39 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 15 Dec 2022 16:06:32 -0800 Subject: [PATCH 175/265] btrfs: add more find_free_extent tracepoints find_free_extent is a complicated function. It consists (at least) of: - a hint that jumps into the middle of a for loop macro - a middle loop trying every raid level - an outer loop ascending through ffe loop levels - complicated logic for skipping some of those ffe loop levels - multiple underlying in-bg allocators (zoned, cluster, no cluster) Which is all to say that more tracing is helpful for debugging its behavior. Add two new tracepoints: at the entrance to the block_groups loop (hit for every raid level and every ffe_ctl loop) and at the point we seriously consider a block_group for allocation. This way we can see the whole path through the algorithm, including hints, multiple loops, etc. Reviewed-by: Johannes Thumshirn Signed-off-by: Boris Burkov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 4 ++ fs/btrfs/extent-tree.h | 3 ++ include/trace/events/btrfs.h | 81 ++++++++++++++++++++++++++++++++++-- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index b26cbf6cae8b..203c8cbb9983 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4261,6 +4261,7 @@ static noinline int find_free_extent(struct btrfs_root *root, block_group->flags); btrfs_lock_block_group(block_group, ffe_ctl->delalloc); + ffe_ctl->hinted = true; goto have_block_group; } } else if (block_group) { @@ -4268,6 +4269,7 @@ static noinline int find_free_extent(struct btrfs_root *root, } } search: + trace_find_free_extent_search_loop(root, ffe_ctl); ffe_ctl->have_caching_bg = false; if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) || ffe_ctl->index == 0) @@ -4277,6 +4279,7 @@ search: &space_info->block_groups[ffe_ctl->index], list) { struct btrfs_block_group *bg_ret; + ffe_ctl->hinted = false; /* If the block group is read-only, we can skip it entirely. */ if (unlikely(block_group->ro)) { if (ffe_ctl->for_treelog) @@ -4318,6 +4321,7 @@ search: } have_block_group: + trace_find_free_extent_have_block_group(root, ffe_ctl, block_group); ffe_ctl->cached = btrfs_block_group_done(block_group); if (unlikely(!ffe_ctl->cached)) { ffe_ctl->have_caching_bg = true; diff --git a/fs/btrfs/extent-tree.h b/fs/btrfs/extent-tree.h index 64fa8ad7914a..daa5e3505886 100644 --- a/fs/btrfs/extent-tree.h +++ b/fs/btrfs/extent-tree.h @@ -76,6 +76,9 @@ struct find_free_extent_ctl { /* Allocation policy */ enum btrfs_extent_allocation_policy policy; + + /* Whether or not the allocator is currently following a hint */ + bool hinted; }; enum btrfs_inline_ref_type { diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 8a422e29d064..cc2eaab28d58 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1268,6 +1268,77 @@ TRACE_EVENT(find_free_extent, BTRFS_GROUP_FLAGS)) ); +TRACE_EVENT(find_free_extent_search_loop, + + TP_PROTO(const struct btrfs_root *root, + const struct find_free_extent_ctl *ffe_ctl), + + TP_ARGS(root, ffe_ctl), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, num_bytes ) + __field( u64, empty_size ) + __field( u64, flags ) + __field( u64, loop ) + ), + + TP_fast_assign_btrfs(root->fs_info, + __entry->root_objectid = root->root_key.objectid; + __entry->num_bytes = ffe_ctl->num_bytes; + __entry->empty_size = ffe_ctl->empty_size; + __entry->flags = ffe_ctl->flags; + __entry->loop = ffe_ctl->loop; + ), + + TP_printk_btrfs("root=%llu(%s) len=%llu empty_size=%llu flags=%llu(%s) loop=%llu", + show_root_type(__entry->root_objectid), + __entry->num_bytes, __entry->empty_size, __entry->flags, + __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS), + __entry->loop) +); + +TRACE_EVENT(find_free_extent_have_block_group, + + TP_PROTO(const struct btrfs_root *root, + const struct find_free_extent_ctl *ffe_ctl, + const struct btrfs_block_group *block_group), + + TP_ARGS(root, ffe_ctl, block_group), + + TP_STRUCT__entry_btrfs( + __field( u64, root_objectid ) + __field( u64, num_bytes ) + __field( u64, empty_size ) + __field( u64, flags ) + __field( u64, loop ) + __field( bool, hinted ) + __field( u64, bg_start ) + __field( u64, bg_flags ) + ), + + TP_fast_assign_btrfs(root->fs_info, + __entry->root_objectid = root->root_key.objectid; + __entry->num_bytes = ffe_ctl->num_bytes; + __entry->empty_size = ffe_ctl->empty_size; + __entry->flags = ffe_ctl->flags; + __entry->loop = ffe_ctl->loop; + __entry->hinted = ffe_ctl->hinted; + __entry->bg_start = block_group->start; + __entry->bg_flags = block_group->flags; + ), + + TP_printk_btrfs( +"root=%llu(%s) len=%llu empty_size=%llu flags=%llu(%s) loop=%llu hinted=%d block_group=%llu bg_flags=%llu(%s)", + show_root_type(__entry->root_objectid), + __entry->num_bytes, __entry->empty_size, __entry->flags, + __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS), + __entry->loop, __entry->hinted, + __entry->bg_start, __entry->bg_flags, + __print_flags((unsigned long)__entry->bg_flags, "|", + BTRFS_GROUP_FLAGS)) +); + DECLARE_EVENT_CLASS(btrfs__reserve_extent, TP_PROTO(const struct btrfs_block_group *block_group, @@ -1280,6 +1351,8 @@ DECLARE_EVENT_CLASS(btrfs__reserve_extent, __field( u64, flags ) __field( u64, start ) __field( u64, len ) + __field( u64, loop ) + __field( bool, hinted ) ), TP_fast_assign_btrfs(block_group->fs_info, @@ -1287,15 +1360,17 @@ DECLARE_EVENT_CLASS(btrfs__reserve_extent, __entry->flags = block_group->flags; __entry->start = ffe_ctl->search_start; __entry->len = ffe_ctl->num_bytes; + __entry->loop = ffe_ctl->loop; + __entry->hinted = ffe_ctl->hinted; ), - TP_printk_btrfs("root=%llu(%s) block_group=%llu flags=%llu(%s) " - "start=%llu len=%llu", + TP_printk_btrfs( +"root=%llu(%s) block_group=%llu flags=%llu(%s) start=%llu len=%llu loop=%llu hinted=%d", show_root_type(BTRFS_EXTENT_TREE_OBJECTID), __entry->bg_objectid, __entry->flags, __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS), - __entry->start, __entry->len) + __entry->start, __entry->len, __entry->loop, __entry->hinted) ); DEFINE_EVENT(btrfs__reserve_extent, btrfs_reserve_extent, From 52bb7a2166af490317ce2cca1865b6630e86aca8 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 15 Dec 2022 16:06:33 -0800 Subject: [PATCH 176/265] btrfs: introduce size class to block group allocator The aim of this patch is to reduce the fragmentation of block groups under certain unhappy workloads. It is particularly effective when the size of extents correlates with their lifetime, which is something we have observed causing fragmentation in the fleet at Meta. This patch categorizes extents into size classes: - x < 128KiB: "small" - 128KiB < x < 8MiB: "medium" - x > 8MiB: "large" and as much as possible reduces allocations of extents into block groups that don't match the size class. This takes advantage of any (possible) correlation between size and lifetime and also leaves behind predictable re-usable gaps when extents are freed; small writes don't gum up bigger holes. Size classes are implemented in the following way: - Mark each new block group with a size class of the first allocation that goes into it. - Add two new passes to ffe: "unset size class" and "wrong size class". First, try only matching block groups, then try unset ones, then allow allocation of new ones, and finally allow mismatched block groups. - Filtering is done just by skipping inappropriate ones, there is no special size class indexing. Other solutions I considered were: - A best fit allocator with an rb-tree. This worked well, as small writes didn't leak big holes from large freed extents, but led to regressions in ffe and write performance due to lock contention on the rb-tree with every allocation possibly updating it in parallel. Perhaps something clever could be done to do the updates in the background while being "right enough". - A fixed size "working set". This prevents freeing an extent drastically changing where writes currently land, and seems like a good option too. Doesn't take advantage of size in any way. - The same size class idea, but implemented with xarray marks. This turned out to be slower than looping the linked list and skipping wrong block groups, and is also less flexible since we must have only 3 size classes (max #marks). With the current approach we can have as many as we like. Performance testing was done via: https://github.com/josefbacik/fsperf Of particular relevance are the new fragmentation specific tests. A brief summary of the testing results: - Neutral results on existing tests. There are some minor regressions and improvements here and there, but nothing that truly stands out as notable. - Improvement on new tests where size class and extent lifetime are correlated. Fragmentation in these cases is completely eliminated and write performance is generally a little better. There is also significant improvement where extent sizes are just a bit larger than the size class boundaries. - Regression on one new tests: where the allocations are sized intentionally a hair under the borders of the size classes. Results are neutral on the test that intentionally attacks this new scheme by mixing extent size and lifetime. The full dump of the performance results can be found here: https://bur.io/fsperf/size-class-2022-11-15.txt (there are ANSI escape codes, so best to curl and view in terminal) Here is a snippet from the full results for a new test which mixes buffered writes appending to a long lived set of files and large short lived fallocates: bufferedappendvsfallocate results metric baseline current stdev diff ====================================================================================== avg_commit_ms 31.13 29.20 2.67 -6.22% bg_count 14 15.60 0 11.43% commits 11.10 12.20 0.32 9.91% elapsed 27.30 26.40 2.98 -3.30% end_state_mount_ns 11122551.90 10635118.90 851143.04 -4.38% end_state_umount_ns 1.36e+09 1.35e+09 12248056.65 -1.07% find_free_extent_calls 116244.30 114354.30 964.56 -1.63% find_free_extent_ns_max 599507.20 1047168.20 103337.08 74.67% find_free_extent_ns_mean 3607.19 3672.11 101.20 1.80% find_free_extent_ns_min 500 512 6.67 2.40% find_free_extent_ns_p50 2848 2876 37.65 0.98% find_free_extent_ns_p95 4916 5000 75.45 1.71% find_free_extent_ns_p99 20734.49 20920.48 1670.93 0.90% frag_pct_max 61.67 0 8.05 -100.00% frag_pct_mean 43.59 0 6.10 -100.00% frag_pct_min 25.91 0 16.60 -100.00% frag_pct_p50 42.53 0 7.25 -100.00% frag_pct_p95 61.67 0 8.05 -100.00% frag_pct_p99 61.67 0 8.05 -100.00% fragmented_bg_count 6.10 0 1.45 -100.00% max_commit_ms 49.80 46 5.37 -7.63% sys_cpu 2.59 2.62 0.29 1.39% write_bw_bytes 1.62e+08 1.68e+08 17975843.50 3.23% write_clat_ns_mean 57426.39 54475.95 2292.72 -5.14% write_clat_ns_p50 46950.40 42905.60 2101.35 -8.62% write_clat_ns_p99 148070.40 143769.60 2115.17 -2.90% write_io_kbytes 4194304 4194304 0 0.00% write_iops 2476.15 2556.10 274.29 3.23% write_lat_ns_max 2101667.60 2251129.50 370556.59 7.11% write_lat_ns_mean 59374.91 55682.00 2523.09 -6.22% write_lat_ns_min 17353.10 16250 1646.08 -6.36% There are some mixed improvements/regressions in most metrics along with an elimination of fragmentation in this workload. On the balance, the drastic 1->0 improvement in the happy cases seems worth the mix of regressions and improvements we do observe. Some considerations for future work: - Experimenting with more size classes - More hinting/search ordering work to approximate a best-fit allocator Signed-off-by: Boris Burkov Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 107 +++++++++++++++++++++++++++++------ fs/btrfs/block-group.h | 20 ++++++- fs/btrfs/extent-tree.c | 71 ++++++++++++----------- fs/btrfs/extent-tree.h | 3 + include/trace/events/btrfs.h | 9 ++- 5 files changed, 156 insertions(+), 54 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index e90800388a41..6557b1b7f89a 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include "misc.h" #include "ctree.h" @@ -3379,6 +3380,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, cache->space_info->disk_used -= num_bytes * factor; reclaim = should_reclaim_block_group(cache, num_bytes); + spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); @@ -3433,32 +3435,42 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, * reservation and return -EAGAIN, otherwise this function always succeeds. */ int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, - u64 ram_bytes, u64 num_bytes, int delalloc) + u64 ram_bytes, u64 num_bytes, int delalloc, + bool force_wrong_size_class) { struct btrfs_space_info *space_info = cache->space_info; + enum btrfs_block_group_size_class size_class; int ret = 0; spin_lock(&space_info->lock); spin_lock(&cache->lock); if (cache->ro) { ret = -EAGAIN; - } else { - cache->reserved += num_bytes; - space_info->bytes_reserved += num_bytes; - trace_btrfs_space_reservation(cache->fs_info, "space_info", - space_info->flags, num_bytes, 1); - btrfs_space_info_update_bytes_may_use(cache->fs_info, - space_info, -ram_bytes); - if (delalloc) - cache->delalloc_bytes += num_bytes; - - /* - * Compression can use less space than we reserved, so wake - * tickets if that happens - */ - if (num_bytes < ram_bytes) - btrfs_try_granting_tickets(cache->fs_info, space_info); + goto out; } + + if (btrfs_is_block_group_data_only(cache)) { + size_class = btrfs_calc_block_group_size_class(num_bytes); + ret = btrfs_use_block_group_size_class(cache, size_class, force_wrong_size_class); + if (ret) + goto out; + } + cache->reserved += num_bytes; + space_info->bytes_reserved += num_bytes; + trace_btrfs_space_reservation(cache->fs_info, "space_info", + space_info->flags, num_bytes, 1); + btrfs_space_info_update_bytes_may_use(cache->fs_info, + space_info, -ram_bytes); + if (delalloc) + cache->delalloc_bytes += num_bytes; + + /* + * Compression can use less space than we reserved, so wake tickets if + * that happens. + */ + if (num_bytes < ram_bytes) + btrfs_try_granting_tickets(cache->fs_info, space_info); +out: spin_unlock(&cache->lock); spin_unlock(&space_info->lock); return ret; @@ -4218,3 +4230,64 @@ void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount bg->swap_extents -= amount; spin_unlock(&bg->lock); } + +enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size) +{ + if (size <= SZ_128K) + return BTRFS_BG_SZ_SMALL; + if (size <= SZ_8M) + return BTRFS_BG_SZ_MEDIUM; + return BTRFS_BG_SZ_LARGE; +} + +/* + * Handle a block group allocating an extent in a size class + * + * @bg: The block group we allocated in. + * @size_class: The size class of the allocation. + * @force_wrong_size_class: Whether we are desperate enough to allow + * mismatched size classes. + * + * Returns: 0 if the size class was valid for this block_group, -EAGAIN in the + * case of a race that leads to the wrong size class without + * force_wrong_size_class set. + * + * find_free_extent will skip block groups with a mismatched size class until + * it really needs to avoid ENOSPC. In that case it will set + * force_wrong_size_class. However, if a block group is newly allocated and + * doesn't yet have a size class, then it is possible for two allocations of + * different sizes to race and both try to use it. The loser is caught here and + * has to retry. + */ +int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, + enum btrfs_block_group_size_class size_class, + bool force_wrong_size_class) +{ + ASSERT(size_class != BTRFS_BG_SZ_NONE); + + /* The new allocation is in the right size class, do nothing */ + if (bg->size_class == size_class) + return 0; + /* + * The new allocation is in a mismatched size class. + * This means one of two things: + * + * 1. Two tasks in find_free_extent for different size_classes raced + * and hit the same empty block_group. Make the loser try again. + * 2. A call to find_free_extent got desperate enough to set + * 'force_wrong_slab'. Don't change the size_class, but allow the + * allocation. + */ + if (bg->size_class != BTRFS_BG_SZ_NONE) { + if (force_wrong_size_class) + return 0; + return -EAGAIN; + } + /* + * The happy new block group case: the new allocation is the first + * one in the block_group so we set size_class. + */ + bg->size_class = size_class; + + return 0; +} diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index a02ea76fd6cf..2b9d328b589e 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -12,6 +12,17 @@ enum btrfs_disk_cache_state { BTRFS_DC_SETUP, }; +enum btrfs_block_group_size_class { + /* Unset */ + BTRFS_BG_SZ_NONE, + /* 0 < size <= 128K */ + BTRFS_BG_SZ_SMALL, + /* 128K < size <= 8M */ + BTRFS_BG_SZ_MEDIUM, + /* 8M < size < BG_LENGTH */ + BTRFS_BG_SZ_LARGE, +}; + /* * This describes the state of the block_group for async discard. This is due * to the two pass nature of it where extent discarding is prioritized over @@ -233,6 +244,7 @@ struct btrfs_block_group { struct list_head active_bg_list; struct work_struct zone_finish_work; struct extent_buffer *last_eb; + enum btrfs_block_group_size_class size_class; }; static inline u64 btrfs_block_group_end(struct btrfs_block_group *block_group) @@ -302,7 +314,8 @@ int btrfs_setup_space_cache(struct btrfs_trans_handle *trans); int btrfs_update_block_group(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes, bool alloc); int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, - u64 ram_bytes, u64 num_bytes, int delalloc); + u64 ram_bytes, u64 num_bytes, int delalloc, + bool force_wrong_size_class); void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, int delalloc); int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags, @@ -346,4 +359,9 @@ void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); bool btrfs_inc_block_group_swap_extents(struct btrfs_block_group *bg); void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount); +enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size); +int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, + enum btrfs_block_group_size_class size_class, + bool force_wrong_size_class); + #endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 203c8cbb9983..e9be9430faa0 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3385,7 +3385,9 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) enum btrfs_loop_type { LOOP_CACHING_NOWAIT, LOOP_CACHING_WAIT, + LOOP_UNSET_SIZE_CLASS, LOOP_ALLOC_CHUNK, + LOOP_WRONG_SIZE_CLASS, LOOP_NO_EMPTY_SIZE, }; @@ -3950,24 +3952,6 @@ static int can_allocate_chunk(struct btrfs_fs_info *fs_info, } } -static int chunk_allocation_failed(struct find_free_extent_ctl *ffe_ctl) -{ - switch (ffe_ctl->policy) { - case BTRFS_EXTENT_ALLOC_CLUSTERED: - /* - * If we can't allocate a new chunk we've already looped through - * at least once, move on to the NO_EMPTY_SIZE case. - */ - ffe_ctl->loop = LOOP_NO_EMPTY_SIZE; - return 0; - case BTRFS_EXTENT_ALLOC_ZONED: - /* Give up here */ - return -ENOSPC; - default: - BUG(); - } -} - /* * Return >0 means caller needs to re-search for free extent * Return 0 means we have the needed free extent. @@ -4001,31 +3985,28 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking * caching kthreads as we move along * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching + * LOOP_UNSET_SIZE_CLASS, allow unset size class * LOOP_ALLOC_CHUNK, force a chunk allocation and try again * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try * again */ if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) { ffe_ctl->index = 0; - if (ffe_ctl->loop == LOOP_CACHING_NOWAIT) { - /* - * We want to skip the LOOP_CACHING_WAIT step if we - * don't have any uncached bgs and we've already done a - * full search through. - */ - if (ffe_ctl->orig_have_caching_bg || !full_search) - ffe_ctl->loop = LOOP_CACHING_WAIT; - else - ffe_ctl->loop = LOOP_ALLOC_CHUNK; - } else { + /* + * We want to skip the LOOP_CACHING_WAIT step if we don't have + * any uncached bgs and we've already done a full search + * through. + */ + if (ffe_ctl->loop == LOOP_CACHING_NOWAIT && + (!ffe_ctl->orig_have_caching_bg && full_search)) ffe_ctl->loop++; - } + ffe_ctl->loop++; if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) { struct btrfs_trans_handle *trans; int exist = 0; - /*Check if allocation policy allows to create a new chunk */ + /* Check if allocation policy allows to create a new chunk */ ret = can_allocate_chunk(fs_info, ffe_ctl); if (ret) return ret; @@ -4045,8 +4026,10 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, CHUNK_ALLOC_FORCE_FOR_EXTENT); /* Do not bail out on ENOSPC since we can do more. */ - if (ret == -ENOSPC) - ret = chunk_allocation_failed(ffe_ctl); + if (ret == -ENOSPC) { + ret = 0; + ffe_ctl->loop++; + } else if (ret < 0) btrfs_abort_transaction(trans, ret); else @@ -4076,6 +4059,21 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, return -ENOSPC; } +static bool find_free_extent_check_size_class(struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group *bg) +{ + if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) + return true; + if (!btrfs_is_block_group_data_only(bg)) + return true; + if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) + return true; + if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS && + bg->size_class == BTRFS_BG_SZ_NONE) + return true; + return ffe_ctl->size_class == bg->size_class; +} + static int prepare_allocation_clustered(struct btrfs_fs_info *fs_info, struct find_free_extent_ctl *ffe_ctl, struct btrfs_space_info *space_info, @@ -4210,6 +4208,7 @@ static noinline int find_free_extent(struct btrfs_root *root, ffe_ctl->total_free_space = 0; ffe_ctl->found_offset = 0; ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED; + ffe_ctl->size_class = btrfs_calc_block_group_size_class(ffe_ctl->num_bytes); if (btrfs_is_zoned(fs_info)) ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED; @@ -4346,6 +4345,9 @@ have_block_group: if (unlikely(block_group->cached == BTRFS_CACHE_ERROR)) goto loop; + if (!find_free_extent_check_size_class(ffe_ctl, block_group)) + goto loop; + bg_ret = NULL; ret = do_allocation(block_group, ffe_ctl, &bg_ret); if (ret == 0) { @@ -4380,7 +4382,8 @@ have_block_group: ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes, ffe_ctl->num_bytes, - ffe_ctl->delalloc); + ffe_ctl->delalloc, + ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS); if (ret == -EAGAIN) { btrfs_add_free_space_unused(block_group, ffe_ctl->found_offset, diff --git a/fs/btrfs/extent-tree.h b/fs/btrfs/extent-tree.h index daa5e3505886..0c958fc1b3b8 100644 --- a/fs/btrfs/extent-tree.h +++ b/fs/btrfs/extent-tree.h @@ -79,6 +79,9 @@ struct find_free_extent_ctl { /* Whether or not the allocator is currently following a hint */ bool hinted; + + /* Size class of block groups to prefer in early loops */ + enum btrfs_block_group_size_class size_class; }; enum btrfs_inline_ref_type { diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index cc2eaab28d58..75d7d22c3a27 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -1349,28 +1349,33 @@ DECLARE_EVENT_CLASS(btrfs__reserve_extent, TP_STRUCT__entry_btrfs( __field( u64, bg_objectid ) __field( u64, flags ) + __field( int, bg_size_class ) __field( u64, start ) __field( u64, len ) __field( u64, loop ) __field( bool, hinted ) + __field( int, size_class ) ), TP_fast_assign_btrfs(block_group->fs_info, __entry->bg_objectid = block_group->start; __entry->flags = block_group->flags; + __entry->bg_size_class = block_group->size_class; __entry->start = ffe_ctl->search_start; __entry->len = ffe_ctl->num_bytes; __entry->loop = ffe_ctl->loop; __entry->hinted = ffe_ctl->hinted; + __entry->size_class = ffe_ctl->size_class; ), TP_printk_btrfs( -"root=%llu(%s) block_group=%llu flags=%llu(%s) start=%llu len=%llu loop=%llu hinted=%d", +"root=%llu(%s) block_group=%llu flags=%llu(%s) bg_size_class=%d start=%llu len=%llu loop=%llu hinted=%d size_class=%d", show_root_type(BTRFS_EXTENT_TREE_OBJECTID), __entry->bg_objectid, __entry->flags, __print_flags((unsigned long)__entry->flags, "|", BTRFS_GROUP_FLAGS), - __entry->start, __entry->len, __entry->loop, __entry->hinted) + __entry->bg_size_class, __entry->start, __entry->len, + __entry->loop, __entry->hinted, __entry->size_class) ); DEFINE_EVENT(btrfs__reserve_extent, btrfs_reserve_extent, From c7eec3d9aa955cbd00470f874f34bdba946bd101 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 15 Dec 2022 16:06:34 -0800 Subject: [PATCH 177/265] btrfs: load block group size class when caching Since the size class is an artifact of an arbitrary anti fragmentation strategy, it doesn't really make sense to persist it. Furthermore, most of the size class logic assumes fresh block groups. That is of course not a reasonable assumption -- we will be upgrading kernels with existing filesystems whose block groups are not classified. To work around those issues, implement logic to compute the size class of the block groups as we cache them in. To perfectly assess the state of a block group, we would have to read the entire extent tree (since the free space cache mashes together contiguous extent items) which would be prohibitively expensive for larger file systems with more extents. We can do it relatively cheaply by implementing a simple heuristic of sampling a handful of extents and picking the smallest one we see. In the happy case where the block group was classified, we will only see extents of the correct size. In the unhappy case, we will hopefully find one of the smaller extents, but there is no perfect answer anyway. Autorelocation will eventually churn up the block group if there is significant freeing anyway. There was no regression in mount performance at end state of the fsperf test suite, and the delay until the block group is marked cached is minimized by the constant number of extent samples. Signed-off-by: Boris Burkov Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 148 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 6557b1b7f89a..8a711b429515 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -540,6 +540,153 @@ u64 add_new_free_space(struct btrfs_block_group *block_group, u64 start, u64 end return total_added; } +/* + * Get an arbitrary extent item index / max_index through the block group + * + * @block_group the block group to sample from + * @index: the integral step through the block group to grab from + * @max_index: the granularity of the sampling + * @key: return value parameter for the item we find + * + * Pre-conditions on indices: + * 0 <= index <= max_index + * 0 < max_index + * + * Returns: 0 on success, 1 if the search didn't yield a useful item, negative + * error code on error. + */ +static int sample_block_group_extent_item(struct btrfs_caching_control *caching_ctl, + struct btrfs_block_group *block_group, + int index, int max_index, + struct btrfs_key *key) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *extent_root; + int ret = 0; + u64 search_offset; + u64 search_end = block_group->start + block_group->length; + struct btrfs_path *path; + + ASSERT(index >= 0); + ASSERT(index <= max_index); + ASSERT(max_index > 0); + lockdep_assert_held(&caching_ctl->mutex); + lockdep_assert_held_read(&fs_info->commit_root_sem); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + extent_root = btrfs_extent_root(fs_info, max_t(u64, block_group->start, + BTRFS_SUPER_INFO_OFFSET)); + + path->skip_locking = 1; + path->search_commit_root = 1; + path->reada = READA_FORWARD; + + search_offset = index * div_u64(block_group->length, max_index); + key->objectid = block_group->start + search_offset; + key->type = BTRFS_EXTENT_ITEM_KEY; + key->offset = 0; + + while (1) { + ret = btrfs_search_forward(extent_root, key, path, 0); + if (ret != 0) + goto out; + /* Success; sampled an extent item in the block group */ + if (key->type == BTRFS_EXTENT_ITEM_KEY && + key->objectid >= block_group->start && + key->objectid + key->offset <= search_end) + goto out; + + /* We can't possibly find a valid extent item anymore */ + if (key->objectid >= search_end) { + ret = 1; + break; + } + if (key->type < BTRFS_EXTENT_ITEM_KEY) + key->type = BTRFS_EXTENT_ITEM_KEY; + else + key->objectid++; + btrfs_release_path(path); + up_read(&fs_info->commit_root_sem); + mutex_unlock(&caching_ctl->mutex); + cond_resched(); + mutex_lock(&caching_ctl->mutex); + down_read(&fs_info->commit_root_sem); + } +out: + lockdep_assert_held(&caching_ctl->mutex); + lockdep_assert_held_read(&fs_info->commit_root_sem); + btrfs_free_path(path); + return ret; +} + +/* + * Best effort attempt to compute a block group's size class while caching it. + * + * @block_group: the block group we are caching + * + * We cannot infer the size class while adding free space extents, because that + * logic doesn't care about contiguous file extents (it doesn't differentiate + * between a 100M extent and 100 contiguous 1M extents). So we need to read the + * file extent items. Reading all of them is quite wasteful, because usually + * only a handful are enough to give a good answer. Therefore, we just grab 5 of + * them at even steps through the block group and pick the smallest size class + * we see. Since size class is best effort, and not guaranteed in general, + * inaccuracy is acceptable. + * + * To be more explicit about why this algorithm makes sense: + * + * If we are caching in a block group from disk, then there are three major cases + * to consider: + * 1. the block group is well behaved and all extents in it are the same size + * class. + * 2. the block group is mostly one size class with rare exceptions for last + * ditch allocations + * 3. the block group was populated before size classes and can have a totally + * arbitrary mix of size classes. + * + * In case 1, looking at any extent in the block group will yield the correct + * result. For the mixed cases, taking the minimum size class seems like a good + * approximation, since gaps from frees will be usable to the size class. For + * 2., a small handful of file extents is likely to yield the right answer. For + * 3, we can either read every file extent, or admit that this is best effort + * anyway and try to stay fast. + * + * Returns: 0 on success, negative error code on error. + */ +static int load_block_group_size_class(struct btrfs_caching_control *caching_ctl, + struct btrfs_block_group *block_group) +{ + struct btrfs_key key; + int i; + u64 min_size = block_group->length; + enum btrfs_block_group_size_class size_class = BTRFS_BG_SZ_NONE; + int ret; + + if (!btrfs_is_block_group_data_only(block_group)) + return 0; + + for (i = 0; i < 5; ++i) { + ret = sample_block_group_extent_item(caching_ctl, block_group, i, 5, &key); + if (ret < 0) + goto out; + if (ret > 0) + continue; + min_size = min_t(u64, min_size, key.offset); + size_class = btrfs_calc_block_group_size_class(min_size); + } + if (size_class != BTRFS_BG_SZ_NONE) { + spin_lock(&block_group->lock); + block_group->size_class = size_class; + spin_unlock(&block_group->lock); + } + +out: + return ret; +} + static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) { struct btrfs_block_group *block_group = caching_ctl->block_group; @@ -684,6 +831,7 @@ static noinline void caching_thread(struct btrfs_work *work) mutex_lock(&caching_ctl->mutex); down_read(&fs_info->commit_root_sem); + load_block_group_size_class(caching_ctl, block_group); if (btrfs_test_opt(fs_info, SPACE_CACHE)) { ret = load_free_space_cache(block_group); if (ret == 1) { From cb0922f264643f03b04352f7a04abb913c609369 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 15 Dec 2022 16:06:35 -0800 Subject: [PATCH 178/265] btrfs: don't use size classes for zoned file systems When a file system has ZNS devices which are constrained by a maximum number of active block groups, then not being able to use all the block groups for every allocation is not ideal, and could cause us to loop a ton with mixed size allocations. In general, since zoned doesn't write into gaps behind where block groups are writing, it is not susceptible to the same sort of fragmentation that size classes are designed to solve, so we can skip size classes for zoned file systems in general, even though there would probably be no harm for SMR devices. Signed-off-by: Boris Burkov Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 13 +++++++++++-- fs/btrfs/block-group.h | 1 + fs/btrfs/extent-tree.c | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 8a711b429515..45ccb25c5b1f 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -665,7 +665,7 @@ static int load_block_group_size_class(struct btrfs_caching_control *caching_ctl enum btrfs_block_group_size_class size_class = BTRFS_BG_SZ_NONE; int ret; - if (!btrfs_is_block_group_data_only(block_group)) + if (!btrfs_block_group_should_use_size_class(block_group)) return 0; for (i = 0; i < 5; ++i) { @@ -3597,7 +3597,7 @@ int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, goto out; } - if (btrfs_is_block_group_data_only(cache)) { + if (btrfs_block_group_should_use_size_class(cache)) { size_class = btrfs_calc_block_group_size_class(num_bytes); ret = btrfs_use_block_group_size_class(cache, size_class, force_wrong_size_class); if (ret) @@ -4439,3 +4439,12 @@ int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, return 0; } + +bool btrfs_block_group_should_use_size_class(struct btrfs_block_group *bg) +{ + if (btrfs_is_zoned(bg->fs_info)) + return false; + if (!btrfs_is_block_group_data_only(bg)) + return false; + return true; +} diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 2b9d328b589e..4fdc39f00a4d 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -363,5 +363,6 @@ enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size); int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, enum btrfs_block_group_size_class size_class, bool force_wrong_size_class); +bool btrfs_block_group_should_use_size_class(struct btrfs_block_group *bg); #endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index e9be9430faa0..ad6c66ff9ba8 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4064,7 +4064,7 @@ static bool find_free_extent_check_size_class(struct find_free_extent_ctl *ffe_c { if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) return true; - if (!btrfs_is_block_group_data_only(bg)) + if (!btrfs_block_group_should_use_size_class(bg)) return true; if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) return true; From 28232909ba43561887508a6ef46d7f33a648f375 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 17 Dec 2022 10:34:29 +0800 Subject: [PATCH 179/265] btrfs: scrub: improve tree block error reporting [BUG] When debugging a scrub related metadata error, it turns out that our metadata error reporting is not ideal. The only 3 error messages are: - BTRFS error (device dm-2): bdev /dev/mapper/test-scratch1 errs: wr 0, rd 0, flush 0, corrupt 0, gen 1 Showing we have metadata generation mismatch errors. - BTRFS error (device dm-2): unable to fixup (regular) error at logical 7110656 on dev /dev/mapper/test-scratch1 Showing which tree blocks are corrupted. - BTRFS warning (device dm-2): checksum/header error at logical 24772608 on dev /dev/mapper/test-scratch2, physical 3801088: metadata node (level 1) in tree 5 Showing which physical range the corrupted metadata is at. We have to combine the above 3 to know we have a corrupted metadata with generation mismatch. And this is already the better case, if we have other problems, like fsid mismatch, we can not even know the cause. [CAUSE] The problem is caused by the fact that, scrub_checksum_tree_block() never outputs any error message. It just return two bits for scrub: sblock->header_error, and sblock->generation_error. And later we report error in scrub_print_warning(), but unfortunately we only have two bits, there is not really much thing we can done to print any detailed errors. [FIX] This patch will do the following to enhance the error reporting of metadata scrub: - Add extra warning (ratelimited) for every error we hit This can help us to distinguish the different types of errors. Some errors can help us to know what's going wrong immediately, like bytenr mismatch. - Re-order the checks Currently we check bytenr first, then immediately generation. This can lead to false generation mismatch reports, while the fsid mismatches. Here is the new output for the bug I'm debugging (we forgot to writeback tree blocks for commit roots): BTRFS warning (device dm-2): tree block 24117248 mirror 1 has bad fsid, has b77cd862-f150-4c71-90ec-7baf0544d83f want 17df6abf-23cd-445f-b350-5b3e40bfd2fc BTRFS warning (device dm-2): tree block 24117248 mirror 0 has bad fsid, has b77cd862-f150-4c71-90ec-7baf0544d83f want 17df6abf-23cd-445f-b350-5b3e40bfd2fc Now we can immediately know it's some tree blocks didn't even get written back, other than the original confusing generation mismatch. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 49 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 52b346795f66..a5d026041be4 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -2053,20 +2053,33 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) * a) don't have an extent buffer and * b) the page is already kmapped */ - if (sblock->logical != btrfs_stack_header_bytenr(h)) + if (sblock->logical != btrfs_stack_header_bytenr(h)) { sblock->header_error = 1; - - if (sector->generation != btrfs_stack_header_generation(h)) { - sblock->header_error = 1; - sblock->generation_error = 1; + btrfs_warn_rl(fs_info, + "tree block %llu mirror %u has bad bytenr, has %llu want %llu", + sblock->logical, sblock->mirror_num, + btrfs_stack_header_bytenr(h), + sblock->logical); + goto out; } - if (!scrub_check_fsid(h->fsid, sector)) + if (!scrub_check_fsid(h->fsid, sector)) { sblock->header_error = 1; + btrfs_warn_rl(fs_info, + "tree block %llu mirror %u has bad fsid, has %pU want %pU", + sblock->logical, sblock->mirror_num, + h->fsid, sblock->dev->fs_devices->fsid); + goto out; + } - if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid, - BTRFS_UUID_SIZE)) + if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid, BTRFS_UUID_SIZE)) { sblock->header_error = 1; + btrfs_warn_rl(fs_info, + "tree block %llu mirror %u has bad chunk tree uuid, has %pU want %pU", + sblock->logical, sblock->mirror_num, + h->chunk_tree_uuid, fs_info->chunk_tree_uuid); + goto out; + } shash->tfm = fs_info->csum_shash; crypto_shash_init(shash); @@ -2079,9 +2092,27 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) } crypto_shash_final(shash, calculated_csum); - if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size)) + if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size)) { sblock->checksum_error = 1; + btrfs_warn_rl(fs_info, + "tree block %llu mirror %u has bad csum, has " CSUM_FMT " want " CSUM_FMT, + sblock->logical, sblock->mirror_num, + CSUM_FMT_VALUE(fs_info->csum_size, on_disk_csum), + CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum)); + goto out; + } + if (sector->generation != btrfs_stack_header_generation(h)) { + sblock->header_error = 1; + sblock->generation_error = 1; + btrfs_warn_rl(fs_info, + "tree block %llu mirror %u has bad generation, has %llu want %llu", + sblock->logical, sblock->mirror_num, + btrfs_stack_header_generation(h), + sector->generation); + } + +out: return sblock->header_error || sblock->checksum_error; } From 58e36c2a0191768ddfc0c529f2bfb1630b18c10b Mon Sep 17 00:00:00 2001 From: ye xingchen Date: Mon, 16 Jan 2023 15:58:44 +0800 Subject: [PATCH 180/265] btrfs: remove duplicate include header in extent-tree.c extent-tree.h is included more than once, added in a0231804affe ("btrfs: move extent-tree helpers into their own header file"). Signed-off-by: ye xingchen Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index ad6c66ff9ba8..688cdf816957 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -38,7 +38,6 @@ #include "dev-replace.h" #include "fs.h" #include "accessors.h" -#include "extent-tree.h" #include "root-tree.h" #include "file-item.h" #include "orphan.h" From b7625f461da6734a21c38ba6e7558538a116a2e3 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 13 Jan 2023 19:11:39 +0800 Subject: [PATCH 181/265] btrfs: sysfs: update fs features directory asynchronously [BUG] Since the introduction of per-fs feature sysfs interface (/sys/fs/btrfs//features/), the content of that directory is never updated. Thus for the following case, that directory will not show the new features like RAID56: # mkfs.btrfs -f $dev1 $dev2 $dev3 # mount $dev1 $mnt # btrfs balance start -f -mconvert=raid5 $mnt # ls /sys/fs/btrfs/$uuid/features/ extended_iref free_space_tree no_holes skinny_metadata While after unmount and mount, we got the correct features: # umount $mnt # mount $dev1 $mnt # ls /sys/fs/btrfs/$uuid/features/ extended_iref free_space_tree no_holes raid56 skinny_metadata [CAUSE] Because we never really try to update the content of per-fs features/ directory. We had an attempt to update the features directory dynamically in commit 14e46e04958d ("btrfs: synchronize incompat feature bits with sysfs files"), but unfortunately it get reverted in commit e410e34fad91 ("Revert "btrfs: synchronize incompat feature bits with sysfs files""). The problem in the original patch is, in the context of btrfs_create_chunk(), we can not afford to update the sysfs group. The exported but never utilized function, btrfs_sysfs_feature_update() is the leftover of such attempt. As even if we go sysfs_update_group(), new files will need extra memory allocation, and we have no way to specify the sysfs update to go GFP_NOFS. [FIX] This patch will address the old problem by doing asynchronous sysfs update in the cleaner thread. This involves the following changes: - Make __btrfs_(set|clear)_fs_(incompat|compat_ro) helpers to set BTRFS_FS_FEATURE_CHANGED flag when needed - Update btrfs_sysfs_feature_update() to use sysfs_update_group() And drop unnecessary arguments. - Call btrfs_sysfs_feature_update() in cleaner_kthread If we have the BTRFS_FS_FEATURE_CHANGED flag set. - Wake up cleaner_kthread in btrfs_commit_transaction if we have BTRFS_FS_FEATURE_CHANGED flag By this, all the previously dangerous call sites like btrfs_create_chunk() need no new changes, as above helpers would have already set the BTRFS_FS_FEATURE_CHANGED flag. The real work happens at cleaner_kthread, thus we pay the cost of delaying the update to sysfs directory, but the delayed time should be small enough that end user can not distinguish though it might get delayed if the cleaner thread is busy with removing subvolumes or defrag. CC: stable@vger.kernel.org # 4.14+ Reviewed-by: Anand Jain Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 3 +++ fs/btrfs/fs.c | 4 ++++ fs/btrfs/fs.h | 6 ++++++ fs/btrfs/sysfs.c | 29 ++++++++--------------------- fs/btrfs/sysfs.h | 3 +-- fs/btrfs/transaction.c | 5 +++++ 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 7586a8e9b718..a6f89ac1c086 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1914,6 +1914,9 @@ static int cleaner_kthread(void *arg) goto sleep; } + if (test_and_clear_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags)) + btrfs_sysfs_feature_update(fs_info); + btrfs_run_delayed_iputs(fs_info); again = btrfs_clean_one_deleted_snapshot(fs_info); diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c index 5553e1f8afe8..31c1648bc0b4 100644 --- a/fs/btrfs/fs.c +++ b/fs/btrfs/fs.c @@ -24,6 +24,7 @@ void __btrfs_set_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, name, flag); } spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); } } @@ -46,6 +47,7 @@ void __btrfs_clear_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, name, flag); } spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); } } @@ -68,6 +70,7 @@ void __btrfs_set_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, name, flag); } spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); } } @@ -90,5 +93,6 @@ void __btrfs_clear_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, name, flag); } spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); } } diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index 37b86acfcbcf..3d8156fc8523 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -125,6 +125,12 @@ enum { */ BTRFS_FS_NO_OVERCOMMIT, + /* + * Indicate if we have some features changed, this is mostly for + * cleaner thread to update the sysfs interface. + */ + BTRFS_FS_FEATURE_CHANGED, + #if BITS_PER_LONG == 32 /* Indicate if we have error/warn message printed on 32bit systems */ BTRFS_FS_32BIT_ERROR, diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 45615ce36498..108aa3876186 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -2272,36 +2272,23 @@ void btrfs_sysfs_del_one_qgroup(struct btrfs_fs_info *fs_info, * Change per-fs features in /sys/fs/btrfs/UUID/features to match current * values in superblock. Call after any changes to incompat/compat_ro flags */ -void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info, - u64 bit, enum btrfs_feature_set set) +void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info) { - struct btrfs_fs_devices *fs_devs; struct kobject *fsid_kobj; - u64 __maybe_unused features; - int __maybe_unused ret; + int ret; if (!fs_info) return; - /* - * See 14e46e04958df74 and e410e34fad913dd, feature bit updates are not - * safe when called from some contexts (eg. balance) - */ - features = get_features(fs_info, set); - ASSERT(bit & supported_feature_masks[set]); - - fs_devs = fs_info->fs_devices; - fsid_kobj = &fs_devs->fsid_kobj; - + fsid_kobj = &fs_info->fs_devices->fsid_kobj; if (!fsid_kobj->state_initialized) return; - /* - * FIXME: this is too heavy to update just one value, ideally we'd like - * to use sysfs_update_group but some refactoring is needed first. - */ - sysfs_remove_group(fsid_kobj, &btrfs_feature_attr_group); - ret = sysfs_create_group(fsid_kobj, &btrfs_feature_attr_group); + ret = sysfs_update_group(fsid_kobj, &btrfs_feature_attr_group); + if (ret < 0) + btrfs_warn(fs_info, + "failed to update /sys/fs/btrfs/%pU/features: %d", + fs_info->fs_devices->fsid, ret); } int __init btrfs_init_sysfs(void) diff --git a/fs/btrfs/sysfs.h b/fs/btrfs/sysfs.h index bacef43f7267..86c7eef12873 100644 --- a/fs/btrfs/sysfs.h +++ b/fs/btrfs/sysfs.h @@ -19,8 +19,7 @@ void btrfs_sysfs_remove_device(struct btrfs_device *device); int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs); void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs); void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices); -void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info, - u64 bit, enum btrfs_feature_set set); +void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info); void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action); int __init btrfs_init_sysfs(void); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 528efe559866..18329ebcb1cb 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -2464,6 +2464,11 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) wake_up(&fs_info->transaction_wait); btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + /* If we have features changed, wake up the cleaner to update sysfs. */ + if (test_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags) && + fs_info->cleaner_kthread) + wake_up_process(fs_info->cleaner_kthread); + ret = btrfs_write_and_wait_transaction(trans); if (ret) { btrfs_handle_fs_error(fs_info, ret, From e739ba307f3a467cd92b37ca92b64571fcfd7cfe Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:02 +0000 Subject: [PATCH 182/265] btrfs: send: directly return from did_overwrite_ref() and simplify it There are no resources to release before did_overwrite_ref() returns, so we don't really need the 'out' label and jumping to it when conditions are met - we can directly return and get rid of the label and jumps. Also we can deal with -ENOENT and other errors in a single if-else logic, as it's more straightforward. This helps the next patch in the series to be more simple as well. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 1c2dc8e0323d..c77508b7ed68 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -2192,48 +2192,44 @@ static int did_overwrite_ref(struct send_ctx *sctx, u64 ino, u64 ino_gen, const char *name, int name_len) { - int ret = 0; + int ret; u64 gen; u64 ow_inode; if (!sctx->parent_root) - goto out; + return 0; ret = is_inode_existent(sctx, dir, dir_gen); if (ret <= 0) - goto out; + return ret; if (dir != BTRFS_FIRST_FREE_OBJECTID) { ret = get_inode_gen(sctx->send_root, dir, &gen); - if (ret < 0 && ret != -ENOENT) - goto out; - if (ret) { - ret = 0; - goto out; - } + if (ret == -ENOENT) + return 0; + else if (ret < 0) + return ret; + if (gen != dir_gen) - goto out; + return 0; } /* check if the ref was overwritten by another ref */ ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len, &ow_inode); - if (ret < 0 && ret != -ENOENT) - goto out; - if (ret) { + if (ret == -ENOENT) { /* was never and will never be overwritten */ - ret = 0; - goto out; + return 0; + } else if (ret < 0) { + return ret; } ret = get_inode_gen(sctx->send_root, ow_inode, &gen); if (ret < 0) - goto out; + return ret; - if (ow_inode == ino && gen == ino_gen) { - ret = 0; - goto out; - } + if (ow_inode == ino && gen == ino_gen) + return 0; /* * We know that it is or will be overwritten. Check this now. @@ -2244,12 +2240,9 @@ static int did_overwrite_ref(struct send_ctx *sctx, if ((ow_inode < sctx->send_progress) || (ino != sctx->cur_ino && ow_inode == sctx->cur_ino && gen == sctx->cur_inode_gen)) - ret = 1; - else - ret = 0; + return 1; -out: - return ret; + return 0; } /* From cb689481947a36c4b0e885d21a315ff9ffbb87ab Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:03 +0000 Subject: [PATCH 183/265] btrfs: send: avoid unnecessary generation search at did_overwrite_ref() At did_overwrite_ref() we always call get_inode_gen() to find out the generation of the inode 'ow_inode'. However we don't always need to use that generation, and in fact it's very common to not use it, so we end up doing a b+tree search on the send root, allocating a path, etc, for nothing. So improve on this by getting the generation only if we need to use it. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index c77508b7ed68..73017be632ed 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -2193,8 +2193,8 @@ static int did_overwrite_ref(struct send_ctx *sctx, const char *name, int name_len) { int ret; - u64 gen; u64 ow_inode; + u64 ow_gen = 0; if (!sctx->parent_root) return 0; @@ -2204,6 +2204,8 @@ static int did_overwrite_ref(struct send_ctx *sctx, return ret; if (dir != BTRFS_FIRST_FREE_OBJECTID) { + u64 gen; + ret = get_inode_gen(sctx->send_root, dir, &gen); if (ret == -ENOENT) return 0; @@ -2224,12 +2226,15 @@ static int did_overwrite_ref(struct send_ctx *sctx, return ret; } - ret = get_inode_gen(sctx->send_root, ow_inode, &gen); - if (ret < 0) - return ret; + if (ow_inode == ino) { + ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); + if (ret < 0) + return ret; - if (ow_inode == ino && gen == ino_gen) - return 0; + /* It's the same inode, so no overwrite happened. */ + if (ow_gen == ino_gen) + return 0; + } /* * We know that it is or will be overwritten. Check this now. @@ -2237,11 +2242,19 @@ static int did_overwrite_ref(struct send_ctx *sctx, * inode 'ino' to be orphanized, therefore check if ow_inode matches * the current inode being processed. */ - if ((ow_inode < sctx->send_progress) || - (ino != sctx->cur_ino && ow_inode == sctx->cur_ino && - gen == sctx->cur_inode_gen)) + if (ow_inode < sctx->send_progress) return 1; + if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) { + if (ow_gen == 0) { + ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); + if (ret < 0) + return ret; + } + if (ow_gen == sctx->cur_inode_gen) + return 1; + } + return 0; } From b3047a42f55d8524b23d58c6ee1c75024d901823 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:04 +0000 Subject: [PATCH 184/265] btrfs: send: directly return from will_overwrite_ref() and simplify it There are no resources to release before will_overwrite_ref() returns, so we don't really need the 'out' label and jumping to it when conditions are met - we can directly return and get rid of the label and jumps. Also we can deal with -ENOENT and other errors in a single if-else logic, as it's more straightforward. This helps the next patch in the series to be more simple as well. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 73017be632ed..ec14c97ce2ee 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -2119,17 +2119,17 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, const char *name, int name_len, u64 *who_ino, u64 *who_gen, u64 *who_mode) { - int ret = 0; + int ret; u64 gen; u64 other_inode = 0; struct btrfs_inode_info info; if (!sctx->parent_root) - goto out; + return 0; ret = is_inode_existent(sctx, dir, dir_gen); if (ret <= 0) - goto out; + return 0; /* * If we have a parent root we need to verify that the parent dir was @@ -2138,24 +2138,21 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, */ if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) { ret = get_inode_gen(sctx->parent_root, dir, &gen); - if (ret < 0 && ret != -ENOENT) - goto out; - if (ret) { - ret = 0; - goto out; - } + if (ret == -ENOENT) + return 0; + else if (ret < 0) + return ret; + if (gen != dir_gen) - goto out; + return 0; } ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len, &other_inode); - if (ret < 0 && ret != -ENOENT) - goto out; - if (ret) { - ret = 0; - goto out; - } + if (ret == -ENOENT) + return 0; + else if (ret < 0) + return ret; /* * Check if the overwritten ref was already processed. If yes, the ref @@ -2166,18 +2163,15 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, is_waiting_for_move(sctx, other_inode)) { ret = get_inode_info(sctx->parent_root, other_inode, &info); if (ret < 0) - goto out; + return ret; - ret = 1; *who_ino = other_inode; *who_gen = info.gen; *who_mode = info.mode; - } else { - ret = 0; + return 1; } -out: - return ret; + return 0; } /* From 498581f33c9e60bfdfd28a8565e0ea81a1748aa3 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:05 +0000 Subject: [PATCH 185/265] btrfs: send: avoid extra b+tree searches when checking reference overrides During an incremental send, when processing the new references of an inode (either it's a new inode or an existing one renamed/moved), he will search the b+tree of the send or parent roots in order to find out the inode item of the parent directory and extract its generation. However we are doing that search twice, once with is_inode_existent() -> get_cur_inode_state() and then again at did_overwrite_ref() or will_overwrite_ref(). So avoid that and get the generation at get_cur_inode_state() and then propagate it up to did_overwrite_ref() and will_overwrite_ref(). This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 61 +++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index ec14c97ce2ee..a42282492423 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -1884,7 +1884,8 @@ enum inode_state { inode_state_did_delete, }; -static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) +static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen, + u64 *send_gen, u64 *parent_gen) { int ret; int left_ret; @@ -1898,6 +1899,8 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) goto out; left_ret = (info.nlink == 0) ? -ENOENT : ret; left_gen = info.gen; + if (send_gen) + *send_gen = ((left_ret == -ENOENT) ? 0 : info.gen); if (!sctx->parent_root) { right_ret = -ENOENT; @@ -1907,6 +1910,8 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) goto out; right_ret = (info.nlink == 0) ? -ENOENT : ret; right_gen = info.gen; + if (parent_gen) + *parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen); } if (!left_ret && !right_ret) { @@ -1951,14 +1956,15 @@ out: return ret; } -static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen) +static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen, + u64 *send_gen, u64 *parent_gen) { int ret; if (ino == BTRFS_FIRST_FREE_OBJECTID) return 1; - ret = get_cur_inode_state(sctx, ino, gen); + ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen); if (ret < 0) goto out; @@ -2120,14 +2126,14 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, u64 *who_ino, u64 *who_gen, u64 *who_mode) { int ret; - u64 gen; + u64 parent_root_dir_gen; u64 other_inode = 0; struct btrfs_inode_info info; if (!sctx->parent_root) return 0; - ret = is_inode_existent(sctx, dir, dir_gen); + ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); if (ret <= 0) return 0; @@ -2135,17 +2141,13 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, * If we have a parent root we need to verify that the parent dir was * not deleted and then re-created, if it was then we have no overwrite * and we can just unlink this entry. + * + * @parent_root_dir_gen was set to 0 if the inode does not exist in the + * parent root. */ - if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) { - ret = get_inode_gen(sctx->parent_root, dir, &gen); - if (ret == -ENOENT) - return 0; - else if (ret < 0) - return ret; - - if (gen != dir_gen) - return 0; - } + if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID && + parent_root_dir_gen != dir_gen) + return 0; ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len, &other_inode); @@ -2189,26 +2191,21 @@ static int did_overwrite_ref(struct send_ctx *sctx, int ret; u64 ow_inode; u64 ow_gen = 0; + u64 send_root_dir_gen; if (!sctx->parent_root) return 0; - ret = is_inode_existent(sctx, dir, dir_gen); + ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL); if (ret <= 0) return ret; - if (dir != BTRFS_FIRST_FREE_OBJECTID) { - u64 gen; - - ret = get_inode_gen(sctx->send_root, dir, &gen); - if (ret == -ENOENT) - return 0; - else if (ret < 0) - return ret; - - if (gen != dir_gen) - return 0; - } + /* + * @send_root_dir_gen was set to 0 if the inode does not exist in the + * send root. + */ + if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen) + return 0; /* check if the ref was overwritten by another ref */ ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len, @@ -2444,7 +2441,7 @@ static int __get_cur_name_and_parent(struct send_ctx *sctx, * This should only happen for the parent dir that we determine in * record_new_ref_if_needed(). */ - ret = is_inode_existent(sctx, ino, gen); + ret = is_inode_existent(sctx, ino, gen, NULL, NULL); if (ret < 0) goto out; @@ -4240,7 +4237,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * "testdir_2". */ list_for_each_entry(cur, &sctx->new_refs, list) { - ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen); + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); if (ret < 0) goto out; if (ret == inode_state_will_create) @@ -4356,7 +4353,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * parent directory out of order. But we need to check if this * did already happen before due to other refs in the same dir. */ - ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen); + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); if (ret < 0) goto out; if (ret == inode_state_will_create) { @@ -4562,7 +4559,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) if (cur->dir > sctx->cur_ino) continue; - ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen); + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); if (ret < 0) goto out; From 24970ccb2489c8c087d12659df7f74ef1a966a67 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:06 +0000 Subject: [PATCH 186/265] btrfs: send: remove send_progress argument from can_rmdir() All callers of can_rmdir() pass sctx->cur_ino as the value for the send_progress argument, so remove the argument and directly use sctx->cur_ino. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index a42282492423..7b3ad484cb7f 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -3210,8 +3210,7 @@ static void free_orphan_dir_info(struct send_ctx *sctx, * We check this by iterating all dir items and checking if the inode behind * the dir item was already processed. */ -static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen, - u64 send_progress) +static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) { int ret = 0; int iter_ret = 0; @@ -3267,7 +3266,7 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen, goto out; } - if (loc.objectid > send_progress) { + if (loc.objectid > sctx->cur_ino) { odi = add_orphan_dir_info(sctx, dir, dir_gen); if (IS_ERR(odi)) { ret = PTR_ERR(odi); @@ -3574,7 +3573,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) } gen = odi->gen; - ret = can_rmdir(sctx, rmdir_ino, gen, sctx->cur_ino); + ret = can_rmdir(sctx, rmdir_ino, gen); if (ret < 0) goto out; if (!ret) @@ -4465,8 +4464,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * later, we do this check again and rmdir it then if possible. * See the use of check_dirs for more details. */ - ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen, - sctx->cur_ino); + ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen); if (ret < 0) goto out; if (ret) { @@ -4571,8 +4569,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) goto out; } else if (ret == inode_state_did_delete && cur->dir != last_dir_ino_rm) { - ret = can_rmdir(sctx, cur->dir, cur->dir_gen, - sctx->cur_ino); + ret = can_rmdir(sctx, cur->dir, cur->dir_gen); if (ret < 0) goto out; if (ret) { From d921b9cf91e1f84a39f4beab5c824b9c8ad87c0c Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:07 +0000 Subject: [PATCH 187/265] btrfs: send: avoid duplicated orphan dir allocation and initialization At can_rmdir() we are allocating and initializing an orphan dir object twice. This can be deduplicated outside of the loop that iterates over the dir index keys. So deduplicate that code, even because other patch in the series will need to add more initialization code and another one will add one more condition. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 7b3ad484cb7f..859036048462 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -3253,13 +3253,6 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) dm = get_waiting_dir_move(sctx, loc.objectid); if (dm) { - odi = add_orphan_dir_info(sctx, dir, dir_gen); - if (IS_ERR(odi)) { - ret = PTR_ERR(odi); - goto out; - } - odi->gen = dir_gen; - odi->last_dir_index_offset = found_key.offset; dm->rmdir_ino = dir; dm->rmdir_gen = dir_gen; ret = 0; @@ -3267,13 +3260,6 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) } if (loc.objectid > sctx->cur_ino) { - odi = add_orphan_dir_info(sctx, dir, dir_gen); - if (IS_ERR(odi)) { - ret = PTR_ERR(odi); - goto out; - } - odi->gen = dir_gen; - odi->last_dir_index_offset = found_key.offset; ret = 0; goto out; } @@ -3288,7 +3274,18 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) out: btrfs_free_path(path); - return ret; + + if (ret) + return ret; + + odi = add_orphan_dir_info(sctx, dir, dir_gen); + if (IS_ERR(odi)) + return PTR_ERR(odi); + + odi->gen = dir_gen; + odi->last_dir_index_offset = found_key.offset; + + return 0; } static int is_waiting_for_move(struct send_ctx *sctx, u64 ino) From 78cf1a954da52a4b290992f2c32dfbf0ffc9ba45 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:08 +0000 Subject: [PATCH 188/265] btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() At can_rmdir() we start by searching the orphan dirs rbtree for an orphan dir object for the target directory. Later when iterating over the dir index keys, if we find that any dir entry points to inode for which there is a pending dir move or the inode was not yet processed, we exit because we can't remove the directory yet. However we end up always calling add_orphan_dir_info(), which will iterate again the rbtree and if there is already an orphan dir object (created by the first call to can_rmdir()), it returns the existing object. This is unnecessary work because in case there is already an existing orphan dir object, we got a reference to it at the start of can_rmdir(). So skip the call to add_orphan_dir_info() if we already have a reference for an orphan dir object. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 859036048462..11e84add6076 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -3278,11 +3278,14 @@ out: if (ret) return ret; - odi = add_orphan_dir_info(sctx, dir, dir_gen); - if (IS_ERR(odi)) - return PTR_ERR(odi); + if (!odi) { + odi = add_orphan_dir_info(sctx, dir, dir_gen); + if (IS_ERR(odi)) + return PTR_ERR(odi); + + odi->gen = dir_gen; + } - odi->gen = dir_gen; odi->last_dir_index_offset = found_key.offset; return 0; From 474e4761f6e8b700612d409327661290d7e58d77 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:09 +0000 Subject: [PATCH 189/265] btrfs: send: reduce searches on parent root when checking if dir can be removed During an incremental send, every time we remove a reference (dentry) for an inode and the parent directory does not exists anymore in the send root, we go check if we can remove the directory by making a call to can_rmdir(). This helper can only return true (value 1) if all dentries were already removed, and for that it always does a search on the parent root for dir index keys - if it finds any dentry referring to an inode with a number higher then the inode currently being processed, then the directory can not be removed and it must return false (value 0). However that means if a directory that was deleted had 1000 dentries, and each one pointed to an inode with a number higher then the number of the directory's inode, we end up doing 1000 searches on the parent root. Typically files are created in a directory after the directory was created and therefore they get an higher inode number than the directory. It's also common to have the each dentry pointing to an inode with a higher number then the inodes the previous dentries point to, for example when creating a series of files inside a directory, a very common pattern. So improve on that by having the first call to can_rmdir() for a directory to check the number of the inode that the last dentry points to and cache that inode number in the orphan dir structure. Then every subsequent call to can_rmdir() can avoid doing a search on the parent root if the number of the inode currently being processed is smaller than cached inode number at the directory's orphan dir structure. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 65 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 11e84add6076..b8d80e5aeddd 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -321,6 +321,7 @@ struct orphan_dir_info { u64 ino; u64 gen; u64 last_dir_index_offset; + u64 dir_high_seq_ino; }; struct name_cache_entry { @@ -3161,6 +3162,7 @@ static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx, odi->ino = dir_ino; odi->gen = dir_gen; odi->last_dir_index_offset = 0; + odi->dir_high_seq_ino = 0; rb_link_node(&odi->node, parent, p); rb_insert_color(&odi->node, &sctx->orphan_dirs); @@ -3221,6 +3223,8 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) struct btrfs_key loc; struct btrfs_dir_item *di; struct orphan_dir_info *odi = NULL; + u64 dir_high_seq_ino = 0; + u64 last_dir_index_offset = 0; /* * Don't try to rmdir the top/root subvolume dir. @@ -3228,17 +3232,62 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) if (dir == BTRFS_FIRST_FREE_OBJECTID) return 0; + odi = get_orphan_dir_info(sctx, dir, dir_gen); + if (odi && sctx->cur_ino < odi->dir_high_seq_ino) + return 0; + path = alloc_path_for_send(); if (!path) return -ENOMEM; + if (!odi) { + /* + * Find the inode number associated with the last dir index + * entry. This is very likely the inode with the highest number + * of all inodes that have an entry in the directory. We can + * then use it to avoid future calls to can_rmdir(), when + * processing inodes with a lower number, from having to search + * the parent root b+tree for dir index keys. + */ + key.objectid = dir; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (ret > 0) { + /* Can't happen, the root is never empty. */ + ASSERT(path->slots[0] > 0); + if (WARN_ON(path->slots[0] == 0)) { + ret = -EUCLEAN; + goto out; + } + path->slots[0]--; + } + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) { + /* No index keys, dir can be removed. */ + ret = 1; + goto out; + } + + di = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_item); + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc); + dir_high_seq_ino = loc.objectid; + if (sctx->cur_ino < dir_high_seq_ino) { + ret = 0; + goto out; + } + + btrfs_release_path(path); + } + key.objectid = dir; key.type = BTRFS_DIR_INDEX_KEY; - key.offset = 0; - - odi = get_orphan_dir_info(sctx, dir, dir_gen); - if (odi) - key.offset = odi->last_dir_index_offset; + key.offset = (odi ? odi->last_dir_index_offset : 0); btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { struct waiting_dir_move *dm; @@ -3251,6 +3300,9 @@ static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) struct btrfs_dir_item); btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc); + dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid); + last_dir_index_offset = found_key.offset; + dm = get_waiting_dir_move(sctx, loc.objectid); if (dm) { dm->rmdir_ino = dir; @@ -3286,7 +3338,8 @@ out: odi->gen = dir_gen; } - odi->last_dir_index_offset = found_key.offset; + odi->last_dir_index_offset = last_dir_index_offset; + odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino); return 0; } From 8c139e1d780665648c74811d36719ef467c32306 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:10 +0000 Subject: [PATCH 190/265] btrfs: send: iterate waiting dir move rbtree only once when processing refs When processing the new references for an inode, we unnecessarily iterate twice the waiting dir moves rbtree, once with is_waiting_for_move() and if we found an entry in the rbtree, we iterate it again with a call to get_waiting_dir_move(). This is pointless, we can make this simpler and more efficient by calling only get_waiting_dir_move(), so just do that. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index b8d80e5aeddd..0623d30fa2dc 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -4335,12 +4335,9 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * the source path when performing its rename * operation. */ - if (is_waiting_for_move(sctx, ow_inode)) { - wdm = get_waiting_dir_move(sctx, - ow_inode); - ASSERT(wdm); + wdm = get_waiting_dir_move(sctx, ow_inode); + if (wdm) wdm->orphanized = true; - } /* * Make sure we clear our orphanized inode's From d307d2f35ca5c57ef2966d2a7ccb2de1118f61f4 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:12 +0000 Subject: [PATCH 191/265] btrfs: send: initialize all the red black trees earlier After we allocate the send context object and before we initialize all the red black trees, we can jump to the 'out' label if some errors happen, and then under the 'out' label we use RB_EMPTY_ROOT() against some of the those trees, which we have not yet initialized. This happens to work out ok because the send context object was initialized to zeroes with kzalloc and the RB_ROOT initializer just happens to have the following definition: #define RB_ROOT (struct rb_root) { NULL, } But it's really neither clean nor a good practice as RB_ROOT is supposed to be opaque and in case it changes or we change those red black trees to some other data structure, it leaves us in a precarious situation. So initialize all the red black trees immediately after allocating the send context and before any jump into the 'out' label. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 0623d30fa2dc..44fc1facbd1f 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -8142,6 +8142,12 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) INIT_LIST_HEAD(&sctx->backref_cache.lru_list); mt_init(&sctx->backref_cache.entries); + sctx->pending_dir_moves = RB_ROOT; + sctx->waiting_dir_moves = RB_ROOT; + sctx->orphan_dirs = RB_ROOT; + sctx->rbtree_new_refs = RB_ROOT; + sctx->rbtree_deleted_refs = RB_ROOT; + sctx->flags = arg->flags; if (arg->flags & BTRFS_SEND_FLAG_VERSION) { @@ -8207,12 +8213,6 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) goto out; } - sctx->pending_dir_moves = RB_ROOT; - sctx->waiting_dir_moves = RB_ROOT; - sctx->orphan_dirs = RB_ROOT; - sctx->rbtree_new_refs = RB_ROOT; - sctx->rbtree_deleted_refs = RB_ROOT; - sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots), arg->clone_sources_count + 1, GFP_KERNEL); From 90b90d4ac03cbaaa87a81ee04e9cdd9b4225cdd4 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:13 +0000 Subject: [PATCH 192/265] btrfs: send: genericize the backref cache to allow it to be reused The backref cache is a cache backed by a maple tree and a linked list to keep track of temporal access to cached entries (the LRU entry always at the head of the list). This type of caching method is going to be useful in other scenarios, so make the cache implementation more generic and move it into its own header and source files. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/Makefile | 3 +- fs/btrfs/lru_cache.c | 97 ++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/lru_cache.h | 44 ++++++++++++++++++++ fs/btrfs/send.c | 80 +++++++++++------------------------- 4 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 fs/btrfs/lru_cache.c create mode 100644 fs/btrfs/lru_cache.h diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index 460eced3f5bd..90d53209755b 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -32,7 +32,8 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ uuid-tree.o props.o free-space-tree.o tree-checker.o space-info.o \ block-rsv.o delalloc-space.o block-group.o discard.o reflink.o \ - subpage.o tree-mod-log.o extent-io-tree.o fs.o messages.o bio.o + subpage.o tree-mod-log.o extent-io-tree.o fs.o messages.o bio.o \ + lru_cache.o btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o btrfs-$(CONFIG_BTRFS_FS_CHECK_INTEGRITY) += check-integrity.o diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c new file mode 100644 index 000000000000..177e7e705363 --- /dev/null +++ b/fs/btrfs/lru_cache.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "lru_cache.h" +#include "messages.h" + +/* + * Initialize a cache object. + * + * @cache: The cache. + * @max_size: Maximum size (number of entries) for the cache. + */ +void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) +{ + INIT_LIST_HEAD(&cache->lru_list); + mt_init(&cache->entries); + cache->size = 0; + cache->max_size = max_size; +} + +/* + * Lookup for an entry in the cache. + * + * @cache: The cache. + * @key: The key of the entry we are looking for. + * + * Returns the entry associated with the key or NULL if none found. + */ +struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, + u64 key) +{ + struct btrfs_lru_cache_entry *entry; + + entry = mtree_load(&cache->entries, key); + if (entry) + list_move_tail(&entry->lru_list, &cache->lru_list); + + return entry; +} + +/* + * Store an entry in the cache. + * + * @cache: The cache. + * @entry: The entry to store. + * + * Returns 0 on success and < 0 on error. + */ +int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *new_entry, + gfp_t gfp) +{ + int ret; + + if (cache->size == cache->max_size) { + struct btrfs_lru_cache_entry *lru_entry; + struct btrfs_lru_cache_entry *mt_entry; + + lru_entry = list_first_entry(&cache->lru_list, + struct btrfs_lru_cache_entry, + lru_list); + mt_entry = mtree_erase(&cache->entries, lru_entry->key); + ASSERT(mt_entry == lru_entry); + list_del(&mt_entry->lru_list); + kfree(mt_entry); + cache->size--; + } + + ret = mtree_insert(&cache->entries, new_entry->key, new_entry, gfp); + if (ret < 0) + return ret; + + list_add_tail(&new_entry->lru_list, &cache->lru_list); + cache->size++; + + return 0; +} + +/* + * Empty a cache. + * + * @cache: The cache to empty. + * + * Removes all entries from the cache. + */ +void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache) +{ + struct btrfs_lru_cache_entry *entry; + struct btrfs_lru_cache_entry *tmp; + + list_for_each_entry_safe(entry, tmp, &cache->lru_list, lru_list) + kfree(entry); + + INIT_LIST_HEAD(&cache->lru_list); + mtree_destroy(&cache->entries); + cache->size = 0; +} diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h new file mode 100644 index 000000000000..44a7b41236ce --- /dev/null +++ b/fs/btrfs/lru_cache.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_LRU_CACHE_H +#define BTRFS_LRU_CACHE_H + +#include +#include + +/* + * A cache entry. This is meant to be embedded in a structure of a user of + * this module. Similar to how struct list_head and struct rb_node are used. + * + * Note: it should be embedded as the first element in a struct (offset 0), and + * this module assumes it was allocated with kmalloc(), so it calls kfree() when + * it needs to free an entry. + */ +struct btrfs_lru_cache_entry { + struct list_head lru_list; + u64 key; +}; + +struct btrfs_lru_cache { + struct list_head lru_list; + struct maple_tree entries; + /* Number of entries stored in the cache. */ + unsigned int size; + /* Maximum number of entries the cache can have. */ + unsigned int max_size; +}; + +static inline unsigned int btrfs_lru_cache_size(const struct btrfs_lru_cache *cache) +{ + return cache->size; +} + +void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size); +struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, + u64 key); +int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *new_entry, + gfp_t gfp); +void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache); + +#endif diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 44fc1facbd1f..9c5a2c96e544 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -32,6 +32,7 @@ #include "file-item.h" #include "ioctl.h" #include "verity.h" +#include "lru_cache.h" /* * Maximum number of references an extent can have in order for us to attempt to @@ -107,15 +108,15 @@ struct clone_root { * x86_64). */ struct backref_cache_entry { - /* List to link to the cache's lru list. */ - struct list_head list; - /* The key for this entry in the cache. */ - u64 key; + struct btrfs_lru_cache_entry entry; u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS]; /* Number of valid elements in the root_ids array. */ int num_roots; }; +/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */ +static_assert(offsetof(struct backref_cache_entry, entry) == 0); + struct send_ctx { struct file *send_filp; loff_t send_off; @@ -285,13 +286,8 @@ struct send_ctx { struct rb_root rbtree_new_refs; struct rb_root rbtree_deleted_refs; - struct { - u64 last_reloc_trans; - struct list_head lru_list; - struct maple_tree entries; - /* Number of entries stored in the cache. */ - int size; - } backref_cache; + struct btrfs_lru_cache backref_cache; + u64 backref_cache_last_reloc_trans; }; struct pending_dir_move { @@ -1387,19 +1383,6 @@ static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id, return 0; } -static void empty_backref_cache(struct send_ctx *sctx) -{ - struct backref_cache_entry *entry; - struct backref_cache_entry *tmp; - - list_for_each_entry_safe(entry, tmp, &sctx->backref_cache.lru_list, list) - kfree(entry); - - INIT_LIST_HEAD(&sctx->backref_cache.lru_list); - mtree_destroy(&sctx->backref_cache.entries); - sctx->backref_cache.size = 0; -} - static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx, const u64 **root_ids_ret, int *root_count_ret) { @@ -1407,9 +1390,10 @@ static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx, struct send_ctx *sctx = bctx->sctx; struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; const u64 key = leaf_bytenr >> fs_info->sectorsize_bits; + struct btrfs_lru_cache_entry *raw_entry; struct backref_cache_entry *entry; - if (sctx->backref_cache.size == 0) + if (btrfs_lru_cache_size(&sctx->backref_cache) == 0) return false; /* @@ -1423,18 +1407,18 @@ static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx, * transaction handle or holding fs_info->commit_root_sem, so no need * to take any lock here. */ - if (fs_info->last_reloc_trans > sctx->backref_cache.last_reloc_trans) { - empty_backref_cache(sctx); + if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) { + btrfs_lru_cache_clear(&sctx->backref_cache); return false; } - entry = mtree_load(&sctx->backref_cache.entries, key); - if (!entry) + raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key); + if (!raw_entry) return false; + entry = container_of(raw_entry, struct backref_cache_entry, entry); *root_ids_ret = entry->root_ids; *root_count_ret = entry->num_roots; - list_move_tail(&entry->list, &sctx->backref_cache.lru_list); return true; } @@ -1460,7 +1444,7 @@ static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids, if (!new_entry) return; - new_entry->key = leaf_bytenr >> fs_info->sectorsize_bits; + new_entry->entry.key = leaf_bytenr >> fs_info->sectorsize_bits; new_entry->num_roots = 0; ULIST_ITER_INIT(&uiter); while ((node = ulist_next(root_ids, &uiter)) != NULL) { @@ -1488,23 +1472,12 @@ static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids, * none of the roots is part of the list of roots from which we are * allowed to clone. Cache the new entry as it's still useful to avoid * backref walking to determine which roots have a path to the leaf. + * + * Also use GFP_NOFS because we're called while holding a transaction + * handle or while holding fs_info->commit_root_sem. */ - - if (sctx->backref_cache.size >= SEND_MAX_BACKREF_CACHE_SIZE) { - struct backref_cache_entry *lru_entry; - struct backref_cache_entry *mt_entry; - - lru_entry = list_first_entry(&sctx->backref_cache.lru_list, - struct backref_cache_entry, list); - mt_entry = mtree_erase(&sctx->backref_cache.entries, lru_entry->key); - ASSERT(mt_entry == lru_entry); - list_del(&mt_entry->list); - kfree(mt_entry); - sctx->backref_cache.size--; - } - - ret = mtree_insert(&sctx->backref_cache.entries, new_entry->key, - new_entry, GFP_NOFS); + ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry, + GFP_NOFS); ASSERT(ret == 0 || ret == -ENOMEM); if (ret) { /* Caching is optional, no worries. */ @@ -1512,17 +1485,13 @@ static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids, return; } - list_add_tail(&new_entry->list, &sctx->backref_cache.lru_list); - /* * We are called from iterate_extent_inodes() while either holding a * transaction handle or holding fs_info->commit_root_sem, so no need * to take any lock here. */ - if (sctx->backref_cache.size == 0) - sctx->backref_cache.last_reloc_trans = fs_info->last_reloc_trans; - - sctx->backref_cache.size++; + if (btrfs_lru_cache_size(&sctx->backref_cache) == 1) + sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans; } static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei, @@ -8139,8 +8108,7 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) INIT_RADIX_TREE(&sctx->name_cache, GFP_KERNEL); INIT_LIST_HEAD(&sctx->name_cache_list); - INIT_LIST_HEAD(&sctx->backref_cache.lru_list); - mt_init(&sctx->backref_cache.entries); + btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); sctx->pending_dir_moves = RB_ROOT; sctx->waiting_dir_moves = RB_ROOT; @@ -8404,7 +8372,7 @@ out: close_current_inode(sctx); - empty_backref_cache(sctx); + btrfs_lru_cache_clear(&sctx->backref_cache); kfree(sctx); } From 6273ee621f3d5829667943c1f53d8115752b45aa Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:14 +0000 Subject: [PATCH 193/265] btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems The lru cache is backed by a maple tree, which uses the unsigned long type for keys, and that type has a width of 32 bits on 32 bits systems and a width of 64 bits on 64 bits systems. Currently there is only one user of the lru cache, the send backref cache, which uses a sector number as a key, a logical address right shifted by fs_info->sectorsize_bits, so a 32 bits width is not yet a problem (the same happens with the radix tree we use to track extent buffers, fs_info->buffer_radix). However the next patches in the series will start using the lru cache for cases where inode numbers are the keys, and the inode numbers are always 64 bits, even if we are running on a 32 bits system. So adapt the lru cache to allow multiple values under the same key, by having the maple tree store a head entry that points to a list of entries instead of pointing to a single entry. This is a similar approach to what we currently do for the name cache in send (which uses a radix tree that has indexes with an unsigned long type as well), and will allow later to use the lru cache for the send name cache as well. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/lru_cache.c | 87 ++++++++++++++++++++++++++++++++++++-------- fs/btrfs/lru_cache.h | 12 ++++++ 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c index 177e7e705363..6012bceedffc 100644 --- a/fs/btrfs/lru_cache.c +++ b/fs/btrfs/lru_cache.c @@ -18,6 +18,18 @@ void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) cache->max_size = max_size; } +static struct btrfs_lru_cache_entry *match_entry(struct list_head *head, u64 key) +{ + struct btrfs_lru_cache_entry *entry; + + list_for_each_entry(entry, head, list) { + if (entry->key == key) + return entry; + } + + return NULL; +} + /* * Lookup for an entry in the cache. * @@ -29,15 +41,48 @@ void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, u64 key) { + struct list_head *head; struct btrfs_lru_cache_entry *entry; - entry = mtree_load(&cache->entries, key); + head = mtree_load(&cache->entries, key); + if (!head) + return NULL; + + entry = match_entry(head, key); if (entry) list_move_tail(&entry->lru_list, &cache->lru_list); return entry; } +static void delete_entry(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *entry) +{ + struct list_head *prev = entry->list.prev; + + ASSERT(cache->size > 0); + ASSERT(!mtree_empty(&cache->entries)); + + list_del(&entry->list); + list_del(&entry->lru_list); + + if (list_empty(prev)) { + struct list_head *head; + + /* + * If previous element in the list entry->list is now empty, it + * means it's a head entry not pointing to any cached entries, + * so remove it from the maple tree and free it. + */ + head = mtree_erase(&cache->entries, entry->key); + ASSERT(head == prev); + kfree(head); + } + + kfree(entry); + cache->size--; +} + /* * Store an entry in the cache. * @@ -50,26 +95,39 @@ int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, struct btrfs_lru_cache_entry *new_entry, gfp_t gfp) { + const u64 key = new_entry->key; + struct list_head *head; int ret; + head = kmalloc(sizeof(*head), gfp); + if (!head) + return -ENOMEM; + + ret = mtree_insert(&cache->entries, key, head, gfp); + if (ret == 0) { + INIT_LIST_HEAD(head); + list_add_tail(&new_entry->list, head); + } else if (ret == -EEXIST) { + kfree(head); + head = mtree_load(&cache->entries, key); + ASSERT(head != NULL); + if (match_entry(head, key) != NULL) + return -EEXIST; + list_add_tail(&new_entry->list, head); + } else if (ret < 0) { + kfree(head); + return ret; + } + if (cache->size == cache->max_size) { struct btrfs_lru_cache_entry *lru_entry; - struct btrfs_lru_cache_entry *mt_entry; lru_entry = list_first_entry(&cache->lru_list, struct btrfs_lru_cache_entry, lru_list); - mt_entry = mtree_erase(&cache->entries, lru_entry->key); - ASSERT(mt_entry == lru_entry); - list_del(&mt_entry->lru_list); - kfree(mt_entry); - cache->size--; + delete_entry(cache, lru_entry); } - ret = mtree_insert(&cache->entries, new_entry->key, new_entry, gfp); - if (ret < 0) - return ret; - list_add_tail(&new_entry->lru_list, &cache->lru_list); cache->size++; @@ -89,9 +147,8 @@ void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache) struct btrfs_lru_cache_entry *tmp; list_for_each_entry_safe(entry, tmp, &cache->lru_list, lru_list) - kfree(entry); + delete_entry(cache, entry); - INIT_LIST_HEAD(&cache->lru_list); - mtree_destroy(&cache->entries); - cache->size = 0; + ASSERT(cache->size == 0); + ASSERT(mtree_empty(&cache->entries)); } diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h index 44a7b41236ce..7a255cd2958e 100644 --- a/fs/btrfs/lru_cache.h +++ b/fs/btrfs/lru_cache.h @@ -17,6 +17,18 @@ struct btrfs_lru_cache_entry { struct list_head lru_list; u64 key; + /* + * The maple tree uses unsigned long type for the keys, which is 32 bits + * on 32 bits systems, and 64 bits on 64 bits systems. So if we want to + * use something like inode numbers as keys, which are always a u64, we + * have to deal with this in a special way - we store the key in the + * entry itself, as a u64, and the values inserted into the maple tree + * are linked lists of entries - so in case we are on a 64 bits system, + * that list always has a single entry, while on 32 bits systems it + * may have more than one, with each entry having the same value for + * their lower 32 bits of the u64 key. + */ + struct list_head list; }; struct btrfs_lru_cache { From e8a7f49d9bfce2fc359cc698d98c8b95dce24c6f Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:15 +0000 Subject: [PATCH 194/265] btrfs: send: cache information about created directories During an incremental send, when processing the reference for an inode we need to check if the directory where the new reference is located was already created before creating the new reference. This check, which is done by the helper did_create_dir(), can be expensive if the directory has many entries, since it consists in searching the send root's b+tree and visiting every single dir index key until we either find one which points to an inode with a number smaller than the current inode's number or until we visited all index keys. So it doesn't scale well for very large directories. So improve on this by caching created directories using a lru cache, and limiting its size to 64 entries, which results in using at most 4096 bytes of memory. The caching is optional, if we fail to allocate memory, we just proceed as before and use the existing slower path. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 9c5a2c96e544..1cde74638939 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -117,6 +117,14 @@ struct backref_cache_entry { /* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */ static_assert(offsetof(struct backref_cache_entry, entry) == 0); +/* + * Max number of entries in the cache that stores directories that were already + * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses + * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 40 bytes, but + * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). + */ +#define SEND_MAX_DIR_CREATED_CACHE_SIZE 64 + struct send_ctx { struct file *send_filp; loff_t send_off; @@ -288,6 +296,8 @@ struct send_ctx { struct btrfs_lru_cache backref_cache; u64 backref_cache_last_reloc_trans; + + struct btrfs_lru_cache dir_created_cache; }; struct pending_dir_move { @@ -2936,6 +2946,22 @@ out: return ret; } +static void cache_dir_created(struct send_ctx *sctx, u64 dir) +{ + struct btrfs_lru_cache_entry *entry; + int ret; + + /* Caching is optional, ignore any failures. */ + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return; + + entry->key = dir; + ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL); + if (ret < 0) + kfree(entry); +} + /* * We need some special handling for inodes that get processed before the parent * directory got created. See process_recorded_refs for details. @@ -2951,6 +2977,9 @@ static int did_create_dir(struct send_ctx *sctx, u64 dir) struct btrfs_key di_key; struct btrfs_dir_item *di; + if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir)) + return 1; + path = alloc_path_for_send(); if (!path) return -ENOMEM; @@ -2974,6 +3003,7 @@ static int did_create_dir(struct send_ctx *sctx, u64 dir) if (di_key.type != BTRFS_ROOT_ITEM_KEY && di_key.objectid < sctx->send_progress) { ret = 1; + cache_dir_created(sctx, dir); break; } } @@ -3003,7 +3033,12 @@ static int send_create_inode_if_needed(struct send_ctx *sctx) return 0; } - return send_create_inode(sctx, sctx->cur_ino); + ret = send_create_inode(sctx, sctx->cur_ino); + + if (ret == 0 && S_ISDIR(sctx->cur_inode_mode)) + cache_dir_created(sctx, sctx->cur_ino); + + return ret; } struct recorded_ref { @@ -4401,6 +4436,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) ret = send_create_inode(sctx, cur->dir); if (ret < 0) goto out; + cache_dir_created(sctx, cur->dir); } } @@ -8109,6 +8145,8 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) INIT_LIST_HEAD(&sctx->name_cache_list); btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); + btrfs_lru_cache_init(&sctx->dir_created_cache, + SEND_MAX_DIR_CREATED_CACHE_SIZE); sctx->pending_dir_moves = RB_ROOT; sctx->waiting_dir_moves = RB_ROOT; @@ -8373,6 +8411,7 @@ out: close_current_inode(sctx); btrfs_lru_cache_clear(&sctx->backref_cache); + btrfs_lru_cache_clear(&sctx->dir_created_cache); kfree(sctx); } From 0da0c5605ed1e6c8138d9c40e5fcc84d3c99ad7b Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:16 +0000 Subject: [PATCH 195/265] btrfs: allow a generation number to be associated with lru cache entries This allows an optional generation number to be associated to each entry of the lru cache. Entries with the same key but different generations, are stored in the linked list to which the maple tree points to. This is meant to be used when there's a small number of different generations, so the impact of searching a linked list is negligible. The goal is to get rid of the open coded name cache in the send code (which uses a radix tree and a similar linked list of values/entries) and use instead the lru cache module. For that particular use case we have at most 2 generations that are associated to each key (inode number): one generation for the send root and another generation for the parent root. The actual migration of the send name cache is done in the next patch in the series. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/lru_cache.c | 12 +++++++----- fs/btrfs/lru_cache.h | 9 ++++++++- fs/btrfs/send.c | 8 +++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c index 6012bceedffc..01821d66a8a2 100644 --- a/fs/btrfs/lru_cache.c +++ b/fs/btrfs/lru_cache.c @@ -18,12 +18,13 @@ void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) cache->max_size = max_size; } -static struct btrfs_lru_cache_entry *match_entry(struct list_head *head, u64 key) +static struct btrfs_lru_cache_entry *match_entry(struct list_head *head, u64 key, + u64 gen) { struct btrfs_lru_cache_entry *entry; list_for_each_entry(entry, head, list) { - if (entry->key == key) + if (entry->key == key && entry->gen == gen) return entry; } @@ -35,11 +36,12 @@ static struct btrfs_lru_cache_entry *match_entry(struct list_head *head, u64 key * * @cache: The cache. * @key: The key of the entry we are looking for. + * @gen: Generation associated to the key. * * Returns the entry associated with the key or NULL if none found. */ struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, - u64 key) + u64 key, u64 gen) { struct list_head *head; struct btrfs_lru_cache_entry *entry; @@ -48,7 +50,7 @@ struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cac if (!head) return NULL; - entry = match_entry(head, key); + entry = match_entry(head, key, gen); if (entry) list_move_tail(&entry->lru_list, &cache->lru_list); @@ -111,7 +113,7 @@ int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, kfree(head); head = mtree_load(&cache->entries, key); ASSERT(head != NULL); - if (match_entry(head, key) != NULL) + if (match_entry(head, key, new_entry->gen) != NULL) return -EEXIST; list_add_tail(&new_entry->list, head); } else if (ret < 0) { diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h index 7a255cd2958e..c1f20f4b7038 100644 --- a/fs/btrfs/lru_cache.h +++ b/fs/btrfs/lru_cache.h @@ -17,6 +17,13 @@ struct btrfs_lru_cache_entry { struct list_head lru_list; u64 key; + /* + * Optional generation associated to a key. Use 0 if not needed/used. + * Entries with the same key and different generations are stored in a + * linked list, so use this only for cases where there's a small number + * of different generations. + */ + u64 gen; /* * The maple tree uses unsigned long type for the keys, which is 32 bits * on 32 bits systems, and 64 bits on 64 bits systems. So if we want to @@ -47,7 +54,7 @@ static inline unsigned int btrfs_lru_cache_size(const struct btrfs_lru_cache *ca void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size); struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, - u64 key); + u64 key, u64 gen); int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, struct btrfs_lru_cache_entry *new_entry, gfp_t gfp); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 1cde74638939..8a7d8979bf57 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -120,7 +120,7 @@ static_assert(offsetof(struct backref_cache_entry, entry) == 0); /* * Max number of entries in the cache that stores directories that were already * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses - * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 40 bytes, but + * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). */ #define SEND_MAX_DIR_CREATED_CACHE_SIZE 64 @@ -1422,7 +1422,7 @@ static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx, return false; } - raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key); + raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0); if (!raw_entry) return false; @@ -1455,6 +1455,7 @@ static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids, return; new_entry->entry.key = leaf_bytenr >> fs_info->sectorsize_bits; + new_entry->entry.gen = 0; new_entry->num_roots = 0; ULIST_ITER_INIT(&uiter); while ((node = ulist_next(root_ids, &uiter)) != NULL) { @@ -2957,6 +2958,7 @@ static void cache_dir_created(struct send_ctx *sctx, u64 dir) return; entry->key = dir; + entry->gen = 0; ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL); if (ret < 0) kfree(entry); @@ -2977,7 +2979,7 @@ static int did_create_dir(struct send_ctx *sctx, u64 dir) struct btrfs_key di_key; struct btrfs_dir_item *di; - if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir)) + if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0)) return 1; path = alloc_path_for_send(); From d588adae3b97241ee2e837a4dd74674b9a209cd6 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:17 +0000 Subject: [PATCH 196/265] btrfs: add an api to delete a specific entry from the lru cache In order to replace the open coded name cache in send with the lru cache, we need an API for the lru cache to delete a specific entry for which we did a previous lookup. This adds the API for it, and a next patch in the series will use it. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/lru_cache.c | 16 ++++++++++++---- fs/btrfs/lru_cache.h | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c index 01821d66a8a2..38722dc07676 100644 --- a/fs/btrfs/lru_cache.c +++ b/fs/btrfs/lru_cache.c @@ -57,8 +57,16 @@ struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cac return entry; } -static void delete_entry(struct btrfs_lru_cache *cache, - struct btrfs_lru_cache_entry *entry) +/* + * Remove an entry from the cache. + * + * @cache: The cache to remove from. + * @entry: The entry to remove from the cache. + * + * Note: this also frees the memory used by the entry. + */ +void btrfs_lru_cache_remove(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *entry) { struct list_head *prev = entry->list.prev; @@ -127,7 +135,7 @@ int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, lru_entry = list_first_entry(&cache->lru_list, struct btrfs_lru_cache_entry, lru_list); - delete_entry(cache, lru_entry); + btrfs_lru_cache_remove(cache, lru_entry); } list_add_tail(&new_entry->lru_list, &cache->lru_list); @@ -149,7 +157,7 @@ void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache) struct btrfs_lru_cache_entry *tmp; list_for_each_entry_safe(entry, tmp, &cache->lru_list, lru_list) - delete_entry(cache, entry); + btrfs_lru_cache_remove(cache, entry); ASSERT(cache->size == 0); ASSERT(mtree_empty(&cache->entries)); diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h index c1f20f4b7038..a97206f04990 100644 --- a/fs/btrfs/lru_cache.h +++ b/fs/btrfs/lru_cache.h @@ -58,6 +58,8 @@ struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cac int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, struct btrfs_lru_cache_entry *new_entry, gfp_t gfp); +void btrfs_lru_cache_remove(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *entry); void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache); #endif From c48545debfff5624a1dd2581f6f8f2fa7fb5f287 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:18 +0000 Subject: [PATCH 197/265] btrfs: send: use the lru cache to implement the name cache The name cache in send is basically a lru cache implemented with a radix tree and linked lists, very similar to the lru cache module which is used for the send backref cache and the cache of previously created directories during a send operation. So remove all the custom caching code for the name cache and make it use the lru cache instead. One particular detail to note is that the current cache behaves a bit differently when it comes to eviction of entries. Namely when after inserting a new name in the cache, if the cache now has 256 entries, we evict the last 128 LRU entries. The lru_cache.{c,h} module behaves a bit differently in that once we reach the cache limit, we evict a single LRU entry. In practice this doesn't make much difference, but it's actually better to evict just one entry instead of half of the entries, as there's always a chance we will need a name stored in one of that last 128 removed entries. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 168 ++++++++---------------------------------------- 1 file changed, 27 insertions(+), 141 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 8a7d8979bf57..bac9b498f3f0 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -81,8 +81,7 @@ struct clone_root { bool found_ref; }; -#define SEND_CTX_MAX_NAME_CACHE_SIZE 128 -#define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2) +#define SEND_MAX_NAME_CACHE_SIZE 256 /* * Limit the root_ids array of struct backref_cache_entry to 12 elements. @@ -183,9 +182,7 @@ struct send_ctx { struct list_head new_refs; struct list_head deleted_refs; - struct radix_tree_root name_cache; - struct list_head name_cache_list; - int name_cache_size; + struct btrfs_lru_cache name_cache; /* * The inode we are currently processing. It's not NULL only when we @@ -331,18 +328,11 @@ struct orphan_dir_info { }; struct name_cache_entry { - struct list_head list; /* - * radix_tree has only 32bit entries but we need to handle 64bit inums. - * We use the lower 32bit of the 64bit inum to store it in the tree. If - * more then one inum would fall into the same entry, we use radix_list - * to store the additional entries. radix_list is also used to store - * entries where two entries have the same inum but different - * generations. + * The key in the entry is an inode number, and the generation matches + * the inode's generation. */ - struct list_head radix_list; - u64 ino; - u64 gen; + struct btrfs_lru_cache_entry entry; u64 parent_ino; u64 parent_gen; int ret; @@ -351,6 +341,9 @@ struct name_cache_entry { char name[]; }; +/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */ +static_assert(offsetof(struct name_cache_entry, entry) == 0); + #define ADVANCE 1 #define ADVANCE_ONLY_NEXT -1 @@ -2261,113 +2254,16 @@ out: return ret; } -/* - * Insert a name cache entry. On 32bit kernels the radix tree index is 32bit, - * so we need to do some special handling in case we have clashes. This function - * takes care of this with the help of name_cache_entry::radix_list. - * In case of error, nce is kfreed. - */ -static int name_cache_insert(struct send_ctx *sctx, - struct name_cache_entry *nce) +static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, + u64 ino, u64 gen) { - int ret = 0; - struct list_head *nce_head; + struct btrfs_lru_cache_entry *entry; - nce_head = radix_tree_lookup(&sctx->name_cache, - (unsigned long)nce->ino); - if (!nce_head) { - nce_head = kmalloc(sizeof(*nce_head), GFP_KERNEL); - if (!nce_head) { - kfree(nce); - return -ENOMEM; - } - INIT_LIST_HEAD(nce_head); - - ret = radix_tree_insert(&sctx->name_cache, nce->ino, nce_head); - if (ret < 0) { - kfree(nce_head); - kfree(nce); - return ret; - } - } - list_add_tail(&nce->radix_list, nce_head); - list_add_tail(&nce->list, &sctx->name_cache_list); - sctx->name_cache_size++; - - return ret; -} - -static void name_cache_delete(struct send_ctx *sctx, - struct name_cache_entry *nce) -{ - struct list_head *nce_head; - - nce_head = radix_tree_lookup(&sctx->name_cache, - (unsigned long)nce->ino); - if (!nce_head) { - btrfs_err(sctx->send_root->fs_info, - "name_cache_delete lookup failed ino %llu cache size %d, leaking memory", - nce->ino, sctx->name_cache_size); - } - - list_del(&nce->radix_list); - list_del(&nce->list); - sctx->name_cache_size--; - - /* - * We may not get to the final release of nce_head if the lookup fails - */ - if (nce_head && list_empty(nce_head)) { - radix_tree_delete(&sctx->name_cache, (unsigned long)nce->ino); - kfree(nce_head); - } -} - -static struct name_cache_entry *name_cache_search(struct send_ctx *sctx, - u64 ino, u64 gen) -{ - struct list_head *nce_head; - struct name_cache_entry *cur; - - nce_head = radix_tree_lookup(&sctx->name_cache, (unsigned long)ino); - if (!nce_head) + entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen); + if (!entry) return NULL; - list_for_each_entry(cur, nce_head, radix_list) { - if (cur->ino == ino && cur->gen == gen) - return cur; - } - return NULL; -} - -/* - * Remove some entries from the beginning of name_cache_list. - */ -static void name_cache_clean_unused(struct send_ctx *sctx) -{ - struct name_cache_entry *nce; - - if (sctx->name_cache_size < SEND_CTX_NAME_CACHE_CLEAN_SIZE) - return; - - while (sctx->name_cache_size > SEND_CTX_MAX_NAME_CACHE_SIZE) { - nce = list_entry(sctx->name_cache_list.next, - struct name_cache_entry, list); - name_cache_delete(sctx, nce); - kfree(nce); - } -} - -static void name_cache_free(struct send_ctx *sctx) -{ - struct name_cache_entry *nce; - - while (!list_empty(&sctx->name_cache_list)) { - nce = list_entry(sctx->name_cache_list.next, - struct name_cache_entry, list); - name_cache_delete(sctx, nce); - kfree(nce); - } + return container_of(entry, struct name_cache_entry, entry); } /* @@ -2386,7 +2282,7 @@ static int __get_cur_name_and_parent(struct send_ctx *sctx, { int ret; int nce_ret; - struct name_cache_entry *nce = NULL; + struct name_cache_entry *nce; /* * First check if we already did a call to this function with the same @@ -2396,17 +2292,9 @@ static int __get_cur_name_and_parent(struct send_ctx *sctx, nce = name_cache_search(sctx, ino, gen); if (nce) { if (ino < sctx->send_progress && nce->need_later_update) { - name_cache_delete(sctx, nce); - kfree(nce); + btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry); nce = NULL; } else { - /* - * Removes the entry from the list and adds it back to - * the end. This marks the entry as recently used so - * that name_cache_clean_unused does not remove it. - */ - list_move_tail(&nce->list, &sctx->name_cache_list); - *parent_ino = nce->parent_ino; *parent_gen = nce->parent_gen; ret = fs_path_add(dest, nce->name, nce->name_len); @@ -2473,8 +2361,8 @@ out_cache: goto out; } - nce->ino = ino; - nce->gen = gen; + nce->entry.key = ino; + nce->entry.gen = gen; nce->parent_ino = *parent_ino; nce->parent_gen = *parent_gen; nce->name_len = fs_path_len(dest); @@ -2486,10 +2374,11 @@ out_cache: else nce->need_later_update = 1; - nce_ret = name_cache_insert(sctx, nce); - if (nce_ret < 0) + nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL); + if (nce_ret < 0) { + kfree(nce); ret = nce_ret; - name_cache_clean_unused(sctx); + } out: return ret; @@ -4356,10 +4245,9 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) * and get instead the orphan name. */ nce = name_cache_search(sctx, ow_inode, ow_gen); - if (nce) { - name_cache_delete(sctx, nce); - kfree(nce); - } + if (nce) + btrfs_lru_cache_remove(&sctx->name_cache, + &nce->entry); /* * ow_inode might currently be an ancestor of @@ -8143,9 +8031,8 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) INIT_LIST_HEAD(&sctx->new_refs); INIT_LIST_HEAD(&sctx->deleted_refs); - INIT_RADIX_TREE(&sctx->name_cache, GFP_KERNEL); - INIT_LIST_HEAD(&sctx->name_cache_list); + btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE); btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); btrfs_lru_cache_init(&sctx->dir_created_cache, SEND_MAX_DIR_CREATED_CACHE_SIZE); @@ -8408,10 +8295,9 @@ out: kvfree(sctx->send_buf); kvfree(sctx->verity_descriptor); - name_cache_free(sctx); - close_current_inode(sctx); + btrfs_lru_cache_clear(&sctx->name_cache); btrfs_lru_cache_clear(&sctx->backref_cache); btrfs_lru_cache_clear(&sctx->dir_created_cache); From ace79df8a44ae1f668615c7177fa1a7bdc04af7e Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:19 +0000 Subject: [PATCH 198/265] btrfs: send: update size of roots array for backref cache entries Currently we limit the size of the roots array, for backref cache entries, to 12 elements. This is because that number is enough for most cases and to make the backref cache entry size to be exactly 128 bytes, so that memory is allocated from the kmalloc-128 slab and no space is wasted. However recent changes in the series refactored the backref cache to be more generic and allow it to be reused for other purposes, which resulted in increasing the size of the embedded structure btrfs_lru_cache_entry in order to allow for supporting inode numbers as keys on 32 bits system and allow multiple generations per key. This resulted in increasing the size of struct backref_cache_entry from 128 bytes to 152 bytes. Since the cache entries are allocated with kmalloc(), it means we end up using the slab kmalloc-192, so we end up wasting 40 bytes of memory. So bump the size of the roots array from 12 elements to 17 elements, so we end up using 192 bytes for each backref cache entry. This patch is part of a larger patchset and the changelog of the last patch in the series contains a sample performance test and results. The patches that comprise the patchset are the following: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/send.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index bac9b498f3f0..83c737ab19be 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -84,19 +84,20 @@ struct clone_root { #define SEND_MAX_NAME_CACHE_SIZE 256 /* - * Limit the root_ids array of struct backref_cache_entry to 12 elements. - * This makes the size of a cache entry to be exactly 128 bytes on x86_64. + * Limit the root_ids array of struct backref_cache_entry to 17 elements. + * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which + * can be satisfied from the kmalloc-192 slab, without wasting any space. * The most common case is to have a single root for cloning, which corresponds - * to the send root. Having the user specify more than 11 clone roots is not + * to the send root. Having the user specify more than 16 clone roots is not * common, and in such rare cases we simply don't use caching if the number of - * cloning roots that lead down to a leaf is more than 12. + * cloning roots that lead down to a leaf is more than 17. */ -#define SEND_MAX_BACKREF_CACHE_ROOTS 12 +#define SEND_MAX_BACKREF_CACHE_ROOTS 17 /* * Max number of entries in the cache. - * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, the size in bytes, excluding - * maple tree's internal nodes, is 16K. + * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding + * maple tree's internal nodes, is 24K. */ #define SEND_MAX_BACKREF_CACHE_SIZE 128 From 3e49363be6330f49e670240e8f46e6fe0bf5628a Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 11 Jan 2023 11:36:20 +0000 Subject: [PATCH 199/265] btrfs: send: cache utimes operations for directories if possible Whenever we add or remove an entry to a directory, we issue an utimes command for the directory. If we add 1000 entries to a directory (create 1000 files under it or move 1000 files to it), then we issue the same utimes command 1000 times, which increases the send stream size, results in more pipe IO, one search in the send b+tree, allocating one path for the search, etc, as well as making the receiver do a system call for each duplicated utimes command. We also issue an utimes command when we create a new directory, but later we might add entries to it corresponding to inodes with an higher inode number, so it's pointless to issue the utimes command before we create the last inode under the directory. So use a lru cache to track directories for which we must send a utimes command. When we need to remove an entry from the cache, we issue the utimes command for the respective directory. When finishing the send operation, we go over each cache element and issue the respective utimes command. Finally the caching is entirely optional, just a performance optimization, meaning that if we fail to cache (due to memory allocation failure), we issue the utimes command right away, that is, we fallback to the previous, unoptimized, behaviour. This patch belongs to a patchset comprised of the following patches: btrfs: send: directly return from did_overwrite_ref() and simplify it btrfs: send: avoid unnecessary generation search at did_overwrite_ref() btrfs: send: directly return from will_overwrite_ref() and simplify it btrfs: send: avoid extra b+tree searches when checking reference overrides btrfs: send: remove send_progress argument from can_rmdir() btrfs: send: avoid duplicated orphan dir allocation and initialization btrfs: send: avoid unnecessary orphan dir rbtree search at can_rmdir() btrfs: send: reduce searches on parent root when checking if dir can be removed btrfs: send: iterate waiting dir move rbtree only once when processing refs btrfs: send: initialize all the red black trees earlier btrfs: send: genericize the backref cache to allow it to be reused btrfs: adapt lru cache to allow for 64 bits keys on 32 bits systems btrfs: send: cache information about created directories btrfs: allow a generation number to be associated with lru cache entries btrfs: add an api to delete a specific entry from the lru cache btrfs: send: use the lru cache to implement the name cache btrfs: send: update size of roots array for backref cache entries btrfs: send: cache utimes operations for directories if possible The following test was run before and after applying the whole patchset, and on a non-debug kernel (Debian's default kernel config): #!/bin/bash MNT=/mnt/sdi DEV=/dev/sdi mkfs.btrfs -f $DEV > /dev/null mount $DEV $MNT mkdir $MNT/A for ((i = 1; i <= 20000; i++)); do echo -n > $MNT/A/file_$i done btrfs subvolume snapshot -r $MNT $MNT/snap1 mkdir $MNT/B for ((i = 20000; i <= 40000; i++)); do echo -n > $MNT/B/file_$i done mv $MNT/A/file_* $MNT/B/ btrfs subvolume snapshot -r $MNT $MNT/snap2 start=$(date +%s%N) btrfs send -p $MNT/snap1 $MNT/snap2 > /dev/null end=$(date +%s%N) dur=$(( (end - start) / 1000000 )) echo "Incremental send took $dur milliseconds" umount $MNT Before the whole patchset: 18408 milliseconds After the whole patchset: 1942 milliseconds (9.5x speedup) Using 60000 files instead of 40000: Before the whole patchset: 39764 milliseconds After the whole patchset: 3076 milliseconds (12.9x speedup) Using 20000 files instead of 40000: Before the whole patchset: 5072 milliseconds After the whole patchset: 916 milliseconds (5.5x speedup) Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/lru_cache.c | 4 +- fs/btrfs/lru_cache.h | 15 +++++++ fs/btrfs/send.c | 104 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 117 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c index 38722dc07676..0fe0ae54ac67 100644 --- a/fs/btrfs/lru_cache.c +++ b/fs/btrfs/lru_cache.c @@ -9,6 +9,8 @@ * * @cache: The cache. * @max_size: Maximum size (number of entries) for the cache. + * Use 0 for unlimited size, it's the user's responsability to + * trim the cache in that case. */ void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) { @@ -129,7 +131,7 @@ int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, return ret; } - if (cache->size == cache->max_size) { + if (cache->max_size > 0 && cache->size == cache->max_size) { struct btrfs_lru_cache_entry *lru_entry; lru_entry = list_first_entry(&cache->lru_list, diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h index a97206f04990..de3e18bce24a 100644 --- a/fs/btrfs/lru_cache.h +++ b/fs/btrfs/lru_cache.h @@ -47,11 +47,26 @@ struct btrfs_lru_cache { unsigned int max_size; }; +#define btrfs_lru_cache_for_each_entry_safe(cache, entry, tmp) \ + list_for_each_entry_safe_reverse((entry), (tmp), &(cache)->lru_list, lru_list) + static inline unsigned int btrfs_lru_cache_size(const struct btrfs_lru_cache *cache) { return cache->size; } +static inline bool btrfs_lru_cache_is_full(const struct btrfs_lru_cache *cache) +{ + return cache->size >= cache->max_size; +} + +static inline struct btrfs_lru_cache_entry *btrfs_lru_cache_lru_entry( + struct btrfs_lru_cache *cache) +{ + return list_first_entry_or_null(&cache->lru_list, + struct btrfs_lru_cache_entry, lru_list); +} + void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size); struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, u64 key, u64 gen); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 83c737ab19be..e5c963bb873d 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -125,6 +125,14 @@ static_assert(offsetof(struct backref_cache_entry, entry) == 0); */ #define SEND_MAX_DIR_CREATED_CACHE_SIZE 64 +/* + * Max number of entries in the cache that stores directories that were already + * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses + * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but + * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). + */ +#define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64 + struct send_ctx { struct file *send_filp; loff_t send_off; @@ -296,6 +304,7 @@ struct send_ctx { u64 backref_cache_last_reloc_trans; struct btrfs_lru_cache dir_created_cache; + struct btrfs_lru_cache dir_utimes_cache; }; struct pending_dir_move { @@ -2749,6 +2758,63 @@ out: return ret; } +/* + * If the cache is full, we can't remove entries from it and do a call to + * send_utimes() for each respective inode, because we might be finishing + * processing an inode that is a directory and it just got renamed, and existing + * entries in the cache may refer to inodes that have the directory in their + * full path - in which case we would generate outdated paths (pre-rename) + * for the inodes that the cache entries point to. Instead of prunning the + * cache when inserting, do it after we finish processing each inode at + * finish_inode_if_needed(). + */ +static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen) +{ + struct btrfs_lru_cache_entry *entry; + int ret; + + entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen); + if (entry != NULL) + return 0; + + /* Caching is optional, don't fail if we can't allocate memory. */ + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return send_utimes(sctx, dir, gen); + + entry->key = dir; + entry->gen = gen; + + ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL); + ASSERT(ret != -EEXIST); + if (ret) { + kfree(entry); + return send_utimes(sctx, dir, gen); + } + + return 0; +} + +static int trim_dir_utimes_cache(struct send_ctx *sctx) +{ + while (btrfs_lru_cache_size(&sctx->dir_utimes_cache) > + SEND_MAX_DIR_UTIMES_CACHE_SIZE) { + struct btrfs_lru_cache_entry *lru; + int ret; + + lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache); + ASSERT(lru != NULL); + + ret = send_utimes(sctx, lru->key, lru->gen); + if (ret) + return ret; + + btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru); + } + + return 0; +} + /* * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have * a valid path yet because we did not process the refs yet. So, the inode @@ -3542,7 +3608,7 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) } finish: - ret = send_utimes(sctx, pm->ino, pm->gen); + ret = cache_dir_utimes(sctx, pm->ino, pm->gen); if (ret < 0) goto out; @@ -3562,7 +3628,7 @@ finish: if (ret < 0) goto out; - ret = send_utimes(sctx, cur->dir, cur->dir_gen); + ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); if (ret < 0) goto out; } @@ -4509,8 +4575,7 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) if (ret == inode_state_did_create || ret == inode_state_no_change) { - /* TODO delayed utimes */ - ret = send_utimes(sctx, cur->dir, cur->dir_gen); + ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); if (ret < 0) goto out; } else if (ret == inode_state_did_delete && @@ -6692,12 +6757,26 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) * it's moved/renamed, therefore we don't need to do it here. */ sctx->send_progress = sctx->cur_ino + 1; - ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); + + /* + * If the current inode is a non-empty directory, delay issuing + * the utimes command for it, as it's very likely we have inodes + * with an higher number inside it. We want to issue the utimes + * command only after adding all dentries to it. + */ + if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0) + ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); + else + ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); + if (ret < 0) goto out; } out: + if (!ret) + ret = trim_dir_utimes_cache(sctx); + return ret; } @@ -7982,6 +8061,8 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) int clone_sources_to_rollback = 0; size_t alloc_size; int sort_clone_roots = 0; + struct btrfs_lru_cache_entry *entry; + struct btrfs_lru_cache_entry *tmp; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -8037,6 +8118,11 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); btrfs_lru_cache_init(&sctx->dir_created_cache, SEND_MAX_DIR_CREATED_CACHE_SIZE); + /* + * This cache is periodically trimmed to a fixed size elsewhere, see + * cache_dir_utimes() and trim_dir_utimes_cache(). + */ + btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0); sctx->pending_dir_moves = RB_ROOT; sctx->waiting_dir_moves = RB_ROOT; @@ -8217,6 +8303,13 @@ long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg) if (ret < 0) goto out; + btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) { + ret = send_utimes(sctx, entry->key, entry->gen); + if (ret < 0) + goto out; + btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry); + } + if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) { ret = begin_cmd(sctx, BTRFS_SEND_C_END); if (ret < 0) @@ -8301,6 +8394,7 @@ out: btrfs_lru_cache_clear(&sctx->name_cache); btrfs_lru_cache_clear(&sctx->backref_cache); btrfs_lru_cache_clear(&sctx->dir_created_cache); + btrfs_lru_cache_clear(&sctx->dir_utimes_cache); kfree(sctx); } From 2b5463fcbdfb24e898916bcae2b1359042d26963 Mon Sep 17 00:00:00 2001 From: Boris Burkov Date: Thu, 12 Jan 2023 16:05:11 -0800 Subject: [PATCH 200/265] btrfs: hold block group refcount during async discard Async discard does not acquire the block group reference count while it holds a reference on the discard list. This is generally OK, as the paths which destroy block groups tend to try to synchronize on cancelling async discard work. However, relying on cancelling work requires careful analysis to be sure it is safe from races with unpinning scheduling more work. While I am unable to find a race with unpinning in the current code for either the unused bgs or relocation paths, I believe we have one in an older version of auto relocation in a Meta internal build. This suggests that this is in fact an error prone model, and could be fragile to future changes to these bg deletion paths. To make this ownership more clear, add a refcount for async discard. If work is queued for a block group, its refcount should be incremented, and when work is completed or canceled, it should be decremented. CC: stable@vger.kernel.org # 5.15+ Signed-off-by: Boris Burkov Signed-off-by: David Sterba --- fs/btrfs/discard.c | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c index ff2e524d9937..317aeff6c1da 100644 --- a/fs/btrfs/discard.c +++ b/fs/btrfs/discard.c @@ -78,6 +78,7 @@ static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl, static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, struct btrfs_block_group *block_group) { + lockdep_assert_held(&discard_ctl->lock); if (!btrfs_run_discard_work(discard_ctl)) return; @@ -89,6 +90,8 @@ static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, BTRFS_DISCARD_DELAY); block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; } + if (list_empty(&block_group->discard_list)) + btrfs_get_block_group(block_group); list_move_tail(&block_group->discard_list, get_discard_list(discard_ctl, block_group)); @@ -108,8 +111,12 @@ static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl, struct btrfs_block_group *block_group) { + bool queued; + spin_lock(&discard_ctl->lock); + queued = !list_empty(&block_group->discard_list); + if (!btrfs_run_discard_work(discard_ctl)) { spin_unlock(&discard_ctl->lock); return; @@ -121,6 +128,8 @@ static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl, block_group->discard_eligible_time = (ktime_get_ns() + BTRFS_DISCARD_UNUSED_DELAY); block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; + if (!queued) + btrfs_get_block_group(block_group); list_add_tail(&block_group->discard_list, &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]); @@ -131,6 +140,7 @@ static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl, struct btrfs_block_group *block_group) { bool running = false; + bool queued = false; spin_lock(&discard_ctl->lock); @@ -140,7 +150,16 @@ static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl, } block_group->discard_eligible_time = 0; + queued = !list_empty(&block_group->discard_list); list_del_init(&block_group->discard_list); + /* + * If the block group is currently running in the discard workfn, we + * don't want to deref it, since it's still being used by the workfn. + * The workfn will notice this case and deref the block group when it is + * finished. + */ + if (queued && !running) + btrfs_put_block_group(block_group); spin_unlock(&discard_ctl->lock); @@ -214,10 +233,12 @@ again: if (block_group && now >= block_group->discard_eligible_time) { if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED && block_group->used != 0) { - if (btrfs_is_block_group_data_only(block_group)) + if (btrfs_is_block_group_data_only(block_group)) { __add_to_discard_list(discard_ctl, block_group); - else + } else { list_del_init(&block_group->discard_list); + btrfs_put_block_group(block_group); + } goto again; } if (block_group->discard_state == BTRFS_DISCARD_RESET_CURSOR) { @@ -511,6 +532,15 @@ static void btrfs_discard_workfn(struct work_struct *work) spin_lock(&discard_ctl->lock); discard_ctl->prev_discard = trimmed; discard_ctl->prev_discard_time = now; + /* + * If the block group was removed from the discard list while it was + * running in this workfn, then we didn't deref it, since this function + * still owned that reference. But we set the discard_ctl->block_group + * back to NULL, so we can use that condition to know that now we need + * to deref the block_group. + */ + if (discard_ctl->block_group == NULL) + btrfs_put_block_group(block_group); discard_ctl->block_group = NULL; __btrfs_discard_schedule_work(discard_ctl, now, false); spin_unlock(&discard_ctl->lock); @@ -651,8 +681,12 @@ void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info) list_for_each_entry_safe(block_group, next, &fs_info->unused_bgs, bg_list) { list_del_init(&block_group->bg_list); - btrfs_put_block_group(block_group); btrfs_discard_queue_work(&fs_info->discard_ctl, block_group); + /* + * This put is for the get done by btrfs_mark_bg_unused. + * Queueing discard incremented it for discard's reference. + */ + btrfs_put_block_group(block_group); } spin_unlock(&fs_info->unused_bgs_lock); } @@ -683,6 +717,7 @@ static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl) if (block_group->used == 0) btrfs_mark_bg_unused(block_group); spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); } } spin_unlock(&discard_ctl->lock); From 4e4488d4efd56951e6c1c0f60f1fc8e0f93b7964 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 17 Jan 2023 11:21:38 +0000 Subject: [PATCH 201/265] btrfs: assert commit root semaphore is held when accessing backref cache During fiemap, when accessing the cache that stores the sharedness of an extent, we need to either be holding a transaction handle or the commit root semaphore. I left comments about this in the comment that precedes store_backref_shared_cache() and lookup_backref_shared_cache(), but have actually not enforced it through assertions. So assert that the commit root semaphore is held if we are not holding a transaction handle. Reviewed-by: Johannes Thumshirn Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 46851511b661..f846fec08c86 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1252,8 +1252,12 @@ static bool lookup_backref_shared_cache(struct btrfs_backref_share_check_ctx *ct struct btrfs_root *root, u64 bytenr, int level, bool *is_shared) { + const struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_backref_shared_cache_entry *entry; + if (!current->journal_info) + lockdep_assert_held(&fs_info->commit_root_sem); + if (!ctx->use_path_cache) return false; @@ -1288,7 +1292,7 @@ static bool lookup_backref_shared_cache(struct btrfs_backref_share_check_ctx *ct * could be a snapshot sharing this extent buffer. */ if (entry->is_shared && - entry->gen != btrfs_get_last_root_drop_gen(root->fs_info)) + entry->gen != btrfs_get_last_root_drop_gen(fs_info)) return false; *is_shared = entry->is_shared; @@ -1318,9 +1322,13 @@ static void store_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx struct btrfs_root *root, u64 bytenr, int level, bool is_shared) { + const struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_backref_shared_cache_entry *entry; u64 gen; + if (!current->journal_info) + lockdep_assert_held(&fs_info->commit_root_sem); + if (!ctx->use_path_cache) return; @@ -1336,7 +1344,7 @@ static void store_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx ASSERT(level >= 0); if (is_shared) - gen = btrfs_get_last_root_drop_gen(root->fs_info); + gen = btrfs_get_last_root_drop_gen(fs_info); else gen = btrfs_root_last_snapshot(&root->root_item); From e2fd83064a9bae368ce1c88a0cb9aee64ad4e124 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 17 Jan 2023 11:21:39 +0000 Subject: [PATCH 202/265] btrfs: skip backref walking during fiemap if we know the leaf is shared During fiemap, when checking if a data extent is shared we are doing the backref walking even if we already know the leaf is shared, which is a waste of time since if the leaf shared then the data extent is also shared. So skip the backref walking when we know we are in a shared leaf. The following test was measures the gains for a case where all leaves are shared due to a snapshot: $ cat test.sh #!/bin/bash DEV=/dev/sdj MNT=/mnt/sdj umount $DEV &> /dev/null mkfs.btrfs -f $DEV # Use compression to quickly create files with a lot of extents # (each with a size of 128K). mount -o compress=lzo $DEV $MNT # 40G gives 327680 extents, each with a size of 128K. xfs_io -f -c "pwrite -S 0xab -b 1M 0 40G" $MNT/foobar # Add some more files to increase the size of the fs and extent # trees (in the real world there's a lot of files and extents # from other files). xfs_io -f -c "pwrite -S 0xcd -b 1M 0 20G" $MNT/file1 xfs_io -f -c "pwrite -S 0xef -b 1M 0 20G" $MNT/file2 xfs_io -f -c "pwrite -S 0x73 -b 1M 0 20G" $MNT/file3 # Create a snapshot so all the extents become indirectly shared # through subtrees, with a generation less than or equals to the # generation used to create the snapshot. btrfs subvolume snapshot -r $MNT $MNT/snap1 # Unmount and mount again to clear cached metadata. umount $MNT mount -o compress=lzo $DEV $MNT start=$(date +%s%N) # The filefrag tool uses the fiemap ioctl. filefrag $MNT/foobar end=$(date +%s%N) dur=$(( (end - start) / 1000000 )) echo "fiemap took $dur milliseconds (metadata not cached)" echo start=$(date +%s%N) filefrag $MNT/foobar end=$(date +%s%N) dur=$(( (end - start) / 1000000 )) echo "fiemap took $dur milliseconds (metadata cached)" umount $MNT The results were the following on a non-debug kernel (Debian's default kernel config). Before this patch: (...) /mnt/sdi/foobar: 327680 extents found fiemap took 1821 milliseconds (metadata not cached) /mnt/sdi/foobar: 327680 extents found fiemap took 399 milliseconds (metadata cached) After this patch: (...) /mnt/sdi/foobar: 327680 extents found fiemap took 591 milliseconds (metadata not cached) /mnt/sdi/foobar: 327680 extents found fiemap took 123 milliseconds (metadata cached) That's a speedup of 3.1x and 3.2x. Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/backref.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index f846fec08c86..90e40d5ceccd 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1872,6 +1872,8 @@ int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, .have_delayed_delete_refs = false, }; int level; + bool leaf_cached; + bool leaf_is_shared; for (int i = 0; i < BTRFS_BACKREF_CTX_PREV_EXTENTS_SIZE; i++) { if (ctx->prev_extents_cache[i].bytenr == bytenr) @@ -1893,6 +1895,23 @@ int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, walk_ctx.time_seq = elem.seq; } + ctx->use_path_cache = true; + + /* + * We may have previously determined that the current leaf is shared. + * If it is, then we have a data extent that is shared due to a shared + * subtree (caused by snapshotting) and we don't need to check for data + * backrefs. If the leaf is not shared, then we must do backref walking + * to determine if the data extent is shared through reflinks. + */ + leaf_cached = lookup_backref_shared_cache(ctx, root, + ctx->curr_leaf_bytenr, 0, + &leaf_is_shared); + if (leaf_cached && leaf_is_shared) { + ret = 1; + goto out_trans; + } + walk_ctx.ignore_extent_item_pos = true; walk_ctx.trans = trans; walk_ctx.fs_info = fs_info; @@ -1901,7 +1920,6 @@ int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, /* -1 means we are in the bytenr of the data extent. */ level = -1; ULIST_ITER_INIT(&uiter); - ctx->use_path_cache = true; while (1) { bool is_shared; bool cached; @@ -1972,6 +1990,7 @@ int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, ctx->prev_extents_cache_slot = slot; } +out_trans: if (trans) { btrfs_put_tree_mod_seq(fs_info, &elem); btrfs_end_transaction(trans); From 67da05b3f28166b24316ecf675637f9b77661983 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 17 Jan 2023 10:03:21 +0000 Subject: [PATCH 203/265] btrfs: fix spelling mistakes found using codespell There quite a few spelling mistakes as found using codespell. Fix them. Signed-off-by: Colin Ian King Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 2 +- fs/btrfs/compression.c | 4 ++-- fs/btrfs/disk-io.c | 4 ++-- fs/btrfs/extent-io-tree.c | 2 +- fs/btrfs/raid56.c | 10 +++++----- fs/btrfs/raid56.h | 4 ++-- fs/btrfs/scrub.c | 2 +- fs/btrfs/super.c | 2 +- fs/btrfs/volumes.c | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 8affc88b0e0a..c3bf36fef3aa 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -283,7 +283,7 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror * RAID setup. Here we only want to write the one bad copy, so we do the * mapping ourselves and submit the bio directly. * - * The I/O is issued sychronously to block the repair read completion from + * The I/O is issued synchronously to block the repair read completion from * freeing the bio. */ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 4a5aeb8dd479..d912638abce2 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -796,7 +796,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, * Save the initial offset of this chunk, as there * is no direct correlation between compressed pages and * the original file offset. The field is only used for - * priting error messages. + * printing error messages. */ btrfs_bio(comp_bio)->file_offset = file_offset; @@ -1642,7 +1642,7 @@ static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end, * * For now is's a naive and optimistic 'return true', we'll extend the logic to * quickly (compared to direct compression) detect data characteristics - * (compressible/uncompressible) to avoid wasting CPU time on uncompressible + * (compressible/incompressible) to avoid wasting CPU time on incompressible * data. * * The following types of analysis can be performed: diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index a6f89ac1c086..dc5bf46529f0 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -767,8 +767,8 @@ static void run_one_async_free(struct btrfs_work *work) /* * Submit bio to an async queue. * - * Retrun: - * - true if the work has been succesfuly submitted + * Return: + * - true if the work has been successfully submitted * - false in case of error */ bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index b2bab7672340..29a225836e28 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -1625,7 +1625,7 @@ search: } /* - * Searche a range in the state tree for a given mask. If 'filled' == 1, this + * Search a range in the state tree for a given mask. If 'filled' == 1, this * returns 1 only if every extent in the tree has the bits set. Otherwise, 1 * is returned if any bit in the range is found set. */ diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index ff4b1d583788..ff44ec8327fd 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -998,7 +998,7 @@ static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio) } /* - * Return the total numer of errors found in the vertical stripe of @sector_nr. + * Return the total number of errors found in the vertical stripe of @sector_nr. * * @faila and @failb will also be updated to the first and second stripe * number of the errors. @@ -1183,7 +1183,7 @@ not_found: trace_info->stripe_nr = -1; } -/* Generate PQ for one veritical stripe. */ +/* Generate PQ for one vertical stripe. */ static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr) { void **pointers = rbio->finish_pointers; @@ -1357,7 +1357,7 @@ static void set_rbio_range_error(struct btrfs_raid_bio *rbio, struct bio *bio) } /* - * For subpage case, we can no longer set page Uptodate directly for + * For subpage case, we can no longer set page Up-to-date directly for * stripe_pages[], thus we need to locate the sector. */ static struct sector_ptr *find_stripe_sector(struct btrfs_raid_bio *rbio, @@ -1529,7 +1529,7 @@ static int rmw_assemble_read_bios(struct btrfs_raid_bio *rbio, /* * Build a list of bios to read all sectors (including data and P/Q). * - * This behaviro is to compensate the later csum verification and + * This behavior is to compensate the later csum verification and * recovery. */ for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; @@ -1773,7 +1773,7 @@ static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr, found_errors = get_rbio_veritical_errors(rbio, sector_nr, &faila, &failb); /* - * No errors in the veritical stripe, skip it. Can happen for recovery + * No errors in the vertical stripe, skip it. Can happen for recovery * which only part of a stripe failed csum check. */ if (!found_errors) diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index 7c73a443939e..df0e0abdeb1f 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -65,7 +65,7 @@ struct btrfs_raid_bio { /* Number of data stripes (no p/q) */ u8 nr_data; - /* Numer of all stripes (including P/Q) */ + /* Number of all stripes (including P/Q) */ u8 real_stripes; /* How many pages there are for each stripe */ @@ -132,7 +132,7 @@ struct btrfs_raid_bio { /* * Checksum buffer if the rbio is for data. The buffer should cover - * all data sectors (exlcuding P/Q sectors). + * all data sectors (excluding P/Q sectors). */ u8 *csum_buf; diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index a5d026041be4..69c93ae333f6 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -229,7 +229,7 @@ struct full_stripe_lock { }; #ifndef CONFIG_64BIT -/* This structure is for archtectures whose (void *) is smaller than u64 */ +/* This structure is for architectures whose (void *) is smaller than u64 */ struct scrub_page_private { u64 logical; }; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index e5136baef9af..581845bc206a 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -2050,7 +2050,7 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) } /* - * Metadata in mixed block goup profiles are accounted in data + * Metadata in mixed block group profiles are accounted in data */ if (!mixed && found->flags & BTRFS_BLOCK_GROUP_METADATA) { if (found->flags & BTRFS_BLOCK_GROUP_DATA) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index df43093b7a46..3510274ba195 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -728,7 +728,7 @@ static struct btrfs_fs_devices *find_fsid_reverted_metadata( /* * Handle the case where the scanned device is part of an fs whose last * metadata UUID change reverted it to the original FSID. At the same - * time * fs_devices was first created by another constitutent device + * time fs_devices was first created by another constituent device * which didn't fully observe the operation. This results in an * btrfs_fs_devices created with metadata/fsid different AND * btrfs_fs_devices::fsid_change set AND the metadata_uuid of the From c9a43aaf0971660e388b794feb0db4649746390a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 21 Jan 2023 16:06:12 +0800 Subject: [PATCH 204/265] btrfs: raid56: reduce overhead to calculate the bio length In rbio_update_error_bitmap(), we need to calculate the length of the rbio. As since it's called in the endio function, we can not directly grab the length from bi_iter. Currently we call bio_for_each_segment_all(), which will always return a range inside a page. But that's not necessary as we don't really care about anything inside the page. So use bio_for_each_bvec_all(), which can return a bvec across multiple continuous pages thus reduce the loops. Reviewed-by: Christoph Hellwig Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index ff44ec8327fd..d095c07a152d 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1425,10 +1425,9 @@ static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bi int total_sector_nr = get_bio_sector_nr(rbio, bio); u32 bio_size = 0; struct bio_vec *bvec; - struct bvec_iter_all iter_all; int i; - bio_for_each_segment_all(bvec, bio, iter_all) + bio_for_each_bvec_all(bvec, bio, i) bio_size += bvec->bv_len; /* From fd8f8ede239bc0531aca65f408ecf44d6755c558 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:49:58 +0100 Subject: [PATCH 205/265] block: export bio_split_rw bio_split_rw can be used by file systems to split and incoming write bio into multiple bios fitting the hardware limit for use as ZONE_APPEND bios. Export it for initial use in btrfs. Reviewed-by: Josef Bacik Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- block/blk-merge.c | 3 ++- include/linux/bio.h | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/block/blk-merge.c b/block/blk-merge.c index b7c193d67185..64bf7d9dd8e8 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -276,7 +276,7 @@ static bool bvec_split_segs(const struct queue_limits *lim, * responsible for ensuring that @bs is only destroyed after processing of the * split bio has finished. */ -static struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, +struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, unsigned *segs, struct bio_set *bs, unsigned max_bytes) { struct bio_vec bv, bvprv, *bvprvp = NULL; @@ -336,6 +336,7 @@ split: bio_clear_polled(bio); return bio_split(bio, bytes >> SECTOR_SHIFT, GFP_NOIO, bs); } +EXPORT_SYMBOL_GPL(bio_split_rw); /** * __bio_split_to_limits - split a bio to fit the queue limits diff --git a/include/linux/bio.h b/include/linux/bio.h index c1da63f6c808..d766be7152e1 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -12,6 +12,8 @@ #define BIO_MAX_VECS 256U +struct queue_limits; + static inline unsigned int bio_max_segs(unsigned int nr_segs) { return min(nr_segs, BIO_MAX_VECS); @@ -375,6 +377,8 @@ static inline void bip_set_seed(struct bio_integrity_payload *bip, void bio_trim(struct bio *bio, sector_t offset, sector_t size); extern struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs); +struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, + unsigned *segs, struct bio_set *bs, unsigned max_bytes); /** * bio_next_split - get next @sectors from a bio, splitting if necessary From e0cfbb2ccabbeaea9e6b3b203d83edca49c702b2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:49:59 +0100 Subject: [PATCH 206/265] btrfs: better document struct btrfs_bio Update the comments on btrfs_bio to better describe the structure. Reviewed-by: Anand Jain Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index b12f84b3b341..baaa27961cc8 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -26,9 +26,8 @@ struct btrfs_fs_info; typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); /* - * Additional info to pass along bio. - * - * Mostly for btrfs specific features like csum and mirror_num. + * Highlevel btrfs I/O structure. It is allocated by btrfs_bio_alloc and + * passed to btrfs_submit_bio for mapping to the physical devices. */ struct btrfs_bio { unsigned int mirror_num:7; @@ -42,7 +41,7 @@ struct btrfs_bio { unsigned int is_metadata:1; struct bvec_iter iter; - /* for direct I/O */ + /* File offset that this I/O operates on. */ u64 file_offset; /* @device is for stripe IO submission. */ @@ -62,7 +61,7 @@ struct btrfs_bio { btrfs_bio_end_io_t end_io; void *private; - /* For read end I/O handling */ + /* For internal use in read end I/O handling */ struct work_struct end_io_work; /* From d0e5cb2be7703172d98699275d722c4081241144 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:00 +0100 Subject: [PATCH 207/265] btrfs: add a btrfs_inode pointer to struct btrfs_bio All btrfs_bio I/Os are associated with an inode. Add a pointer to that inode, which will allow to simplify a lot of calling conventions, and which will be needed in the I/O completion path in the future. This grow the btrfs_bio structure by a pointer, but that grows will be offset by the removal of the device pointer soon. Reviewed-by: Anand Jain Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 8 ++++++-- fs/btrfs/bio.h | 5 ++++- fs/btrfs/compression.c | 3 ++- fs/btrfs/extent_io.c | 8 ++++---- fs/btrfs/inode.c | 4 +++- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index c3bf36fef3aa..b06bdf28abd7 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -22,9 +22,11 @@ static struct bio_set btrfs_bioset; * is already initialized by the block layer. */ static inline void btrfs_bio_init(struct btrfs_bio *bbio, + struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private) { memset(bbio, 0, offsetof(struct btrfs_bio, bio)); + bbio->inode = inode; bbio->end_io = end_io; bbio->private = private; } @@ -37,16 +39,18 @@ static inline void btrfs_bio_init(struct btrfs_bio *bbio, * a mempool. */ struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private) { struct bio *bio; bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); - btrfs_bio_init(btrfs_bio(bio), end_io, private); + btrfs_bio_init(btrfs_bio(bio), inode, end_io, private); return bio; } struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, + struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private) { struct bio *bio; @@ -56,7 +60,7 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, bio = bio_alloc_clone(orig->bi_bdev, orig, GFP_NOFS, &btrfs_bioset); bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, end_io, private); + btrfs_bio_init(bbio, inode, end_io, private); bio_trim(bio, offset >> 9, size >> 9); bbio->iter = bio->bi_iter; diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index baaa27961cc8..8d69d0b226d9 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -41,7 +41,8 @@ struct btrfs_bio { unsigned int is_metadata:1; struct bvec_iter iter; - /* File offset that this I/O operates on. */ + /* Inode and offset into it that this I/O operates on. */ + struct btrfs_inode *inode; u64 file_offset; /* @device is for stripe IO submission. */ @@ -80,8 +81,10 @@ int __init btrfs_bioset_init(void); void __cold btrfs_bioset_exit(void); struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private); struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, + struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private); diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index d912638abce2..81dc70b94e3c 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -344,7 +344,8 @@ static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_byte struct bio *bio; int ret; - bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, endio_func, cb); + bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, BTRFS_I(cb->inode), endio_func, + cb); bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; em = btrfs_get_chunk_map(fs_info, disk_bytenr, fs_info->sectorsize); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 3bbf8703db2a..a8e8567aa428 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -740,7 +740,8 @@ int btrfs_repair_one_sector(struct btrfs_inode *inode, struct btrfs_bio *failed_ return -EIO; } - repair_bio = btrfs_bio_alloc(1, REQ_OP_READ, failed_bbio->end_io, + repair_bio = btrfs_bio_alloc(1, REQ_OP_READ, failed_bbio->inode, + failed_bbio->end_io, failed_bbio->private); repair_bbio = btrfs_bio(repair_bio); repair_bbio->file_offset = start; @@ -1394,9 +1395,8 @@ static int alloc_new_bio(struct btrfs_inode *inode, struct bio *bio; int ret; - ASSERT(bio_ctrl->end_io_func); - - bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, bio_ctrl->end_io_func, NULL); + bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, inode, bio_ctrl->end_io_func, + NULL); /* * For compressed page range, its disk_bytenr is always @disk_bytenr * passed in, no matter if we have added any range into previous bio. diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 3c49742f0d45..0a85e42f114c 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8097,7 +8097,8 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, * the allocation is backed by btrfs_bioset. */ bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len, - btrfs_end_dio_bio, dip); + BTRFS_I(inode), btrfs_end_dio_bio, + dip); btrfs_bio(bio)->file_offset = file_offset; if (bio_op(bio) == REQ_OP_ZONE_APPEND) { @@ -10409,6 +10410,7 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, if (!bio) { bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, + inode, btrfs_encoded_read_endio, &priv); bio->bi_iter.bi_sector = From 5fa356531e33e7c7783ccd0d7938a070b5df8c22 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:01 +0100 Subject: [PATCH 208/265] btrfs: remove the direct I/O read checksum lookup optimization To prepare for pending changes drop the optimization to only look up csums once per bio that is submitted from the iomap layer. In the short run this does cause additional lookups for fragmented direct reads, but later in the series, the bio based lookup will be used on the entire bio submitted from iomap, restoring the old behavior in common code. Reviewed-by: Anand Jain Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0a85e42f114c..863a5527853c 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -100,9 +100,6 @@ struct btrfs_dio_private { */ refcount_t refs; - /* Array of checksums */ - u8 *csums; - /* This must be last */ struct bio bio; }; @@ -7907,7 +7904,6 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) dip->file_offset + dip->bytes - 1, NULL); } - kfree(dip->csums); bio_endio(&dip->bio); } @@ -7990,7 +7986,6 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, u64 file_offset, int async_submit) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_dio_private *dip = btrfs_bio(bio)->private; blk_status_t ret; /* Save the original iter for read repair */ @@ -8017,8 +8012,11 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, return; } } else { - btrfs_bio(bio)->csum = btrfs_csum_ptr(fs_info, dip->csums, - file_offset - dip->file_offset); + ret = btrfs_lookup_bio_sums(&inode->vfs_inode, bio, NULL); + if (ret) { + btrfs_bio_end_io(btrfs_bio(bio), ret); + return; + } } map: btrfs_submit_bio(fs_info, bio, 0); @@ -8030,7 +8028,6 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, struct btrfs_dio_private *dip = container_of(dio_bio, struct btrfs_dio_private, bio); struct inode *inode = iter->inode; - const bool write = (btrfs_op(dio_bio) == BTRFS_MAP_WRITE); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); const bool raid56 = (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK); @@ -8051,25 +8048,6 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, dip->file_offset = file_offset; dip->bytes = dio_bio->bi_iter.bi_size; refcount_set(&dip->refs, 1); - dip->csums = NULL; - - if (!write && !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { - unsigned int nr_sectors = - (dio_bio->bi_iter.bi_size >> fs_info->sectorsize_bits); - - /* - * Load the csums up front to reduce csum tree searches and - * contention when submitting bios. - */ - status = BLK_STS_RESOURCE; - dip->csums = kcalloc(nr_sectors, fs_info->csum_size, GFP_NOFS); - if (!dip->csums) - goto out_err; - - status = btrfs_lookup_bio_sums(inode, dio_bio, dip->csums); - if (status != BLK_STS_OK) - goto out_err; - } start_sector = dio_bio->bi_iter.bi_sector; submit_len = dio_bio->bi_iter.bi_size; From 4ae2edf12d49fdbaea2dfda0bb2ec06501bd3493 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:02 +0100 Subject: [PATCH 209/265] btrfs: simplify parameters of btrfs_lookup_bio_sums The csums argument is always NULL now, so remove it and always allocate the csums array in the btrfs_bio. Also pass the btrfs_bio instead of inode + bio to document that this function requires a btrfs_bio and not just any bio. Reviewed-by: Johannes Thumshirn Reviewed-by: Anand Jain Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 2 +- fs/btrfs/file-item.c | 52 ++++++++++++++++-------------------------- fs/btrfs/file-item.h | 2 +- fs/btrfs/inode.c | 6 ++--- 4 files changed, 25 insertions(+), 37 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 81dc70b94e3c..069b7d28ca03 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -801,7 +801,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, */ btrfs_bio(comp_bio)->file_offset = file_offset; - ret = btrfs_lookup_bio_sums(inode, comp_bio, NULL); + ret = btrfs_lookup_bio_sums(btrfs_bio(comp_bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(comp_bio), ret); break; diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 5de73466b2ca..c5324fe8f4be 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -380,32 +380,25 @@ static int search_file_offset_in_bio(struct bio *bio, struct inode *inode, /* * Lookup the checksum for the read bio in csum tree. * - * @inode: inode that the bio is for. - * @bio: bio to look up. - * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return - * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If - * NULL, the checksum buffer is allocated and returned in - * btrfs_bio(bio)->csum instead. - * * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise. */ -blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst) +blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio) { - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; - struct btrfs_bio *bbio = NULL; + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + struct bio *bio = &bbio->bio; struct btrfs_path *path; const u32 sectorsize = fs_info->sectorsize; const u32 csum_size = fs_info->csum_size; u32 orig_len = bio->bi_iter.bi_size; u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 cur_disk_bytenr; - u8 *csum; const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits; int count = 0; blk_status_t ret = BLK_STS_OK; - if ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM) || + if ((inode->flags & BTRFS_INODE_NODATASUM) || test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state)) return BLK_STS_OK; @@ -426,21 +419,14 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst if (!path) return BLK_STS_RESOURCE; - if (!dst) { - bbio = btrfs_bio(bio); - - if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { - bbio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS); - if (!bbio->csum) { - btrfs_free_path(path); - return BLK_STS_RESOURCE; - } - } else { - bbio->csum = bbio->csum_inline; + if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { + bbio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS); + if (!bbio->csum) { + btrfs_free_path(path); + return BLK_STS_RESOURCE; } - csum = bbio->csum; } else { - csum = dst; + bbio->csum = bbio->csum_inline; } /* @@ -456,7 +442,7 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst * read from the commit root and sidestep a nasty deadlock * between reading the free space cache and updating the csum tree. */ - if (btrfs_is_free_space_inode(BTRFS_I(inode))) { + if (btrfs_is_free_space_inode(inode)) { path->search_commit_root = 1; path->skip_locking = 1; } @@ -479,14 +465,15 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst ASSERT(cur_disk_bytenr - orig_disk_bytenr < UINT_MAX); sector_offset = (cur_disk_bytenr - orig_disk_bytenr) >> fs_info->sectorsize_bits; - csum_dst = csum + sector_offset * csum_size; + csum_dst = bbio->csum + sector_offset * csum_size; count = search_csum_tree(fs_info, path, cur_disk_bytenr, search_len, csum_dst); if (count < 0) { ret = errno_to_blk_status(count); - if (bbio) - btrfs_bio_free_csum(bbio); + if (bbio->csum != bbio->csum_inline) + kfree(bbio->csum); + bbio->csum = NULL; break; } @@ -504,12 +491,13 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst memset(csum_dst, 0, csum_size); count = 1; - if (BTRFS_I(inode)->root->root_key.objectid == + if (inode->root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID) { u64 file_offset; int ret; - ret = search_file_offset_in_bio(bio, inode, + ret = search_file_offset_in_bio(bio, + &inode->vfs_inode, cur_disk_bytenr, &file_offset); if (ret) set_extent_bits(io_tree, file_offset, diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h index 031225668434..a2f9747adf3a 100644 --- a/fs/btrfs/file-item.h +++ b/fs/btrfs/file-item.h @@ -38,7 +38,7 @@ static inline u32 btrfs_file_extent_calc_inline_size(u32 datasize) int btrfs_del_csums(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 bytenr, u64 len); -blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst); +blk_status_t btrfs_lookup_bio_sums(struct btrfs_bio *bbio); int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 objectid, u64 pos, u64 num_bytes); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 863a5527853c..7c8f5349ed7a 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2780,7 +2780,7 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, * Lookup bio sums does extra checks around whether we need to csum or * not, which is why we ignore skip_sum here. */ - ret = btrfs_lookup_bio_sums(&inode->vfs_inode, bio, NULL); + ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(bio), ret); return; @@ -8012,7 +8012,7 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, return; } } else { - ret = btrfs_lookup_bio_sums(&inode->vfs_inode, bio, NULL); + ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(bio), ret); return; @@ -10279,7 +10279,7 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, blk_status_t ret; if (!priv->skip_csum) { - ret = btrfs_lookup_bio_sums(&inode->vfs_inode, bio, NULL); + ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); if (ret) return ret; } From 9ba0004bd95e059983b4ca8defad82ab41627e51 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:03 +0100 Subject: [PATCH 210/265] btrfs: refactor error handling in btrfs_submit_bio Add a bbio local variable and to prepare for calling functions that return a blk_status_t, rename the existing int used for error handling so that ret can be reused for the blk_status_t, and a label that can be reused for failing the passed in bio. Reviewed-by: Anand Jain Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index b06bdf28abd7..088817618f57 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -230,20 +230,21 @@ static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) { + struct btrfs_bio *bbio = btrfs_bio(bio); u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; u64 map_length = length; struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; - int ret; + blk_status_t ret; + int error; btrfs_bio_counter_inc_blocked(fs_info); - ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, - &bioc, &smap, &mirror_num, 1); - if (ret) { - btrfs_bio_counter_dec(fs_info); - btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret)); - return; + error = __btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, + &bioc, &smap, &mirror_num, 1); + if (error) { + ret = errno_to_blk_status(error); + goto fail; } if (map_length < length) { @@ -255,8 +256,8 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror if (!bioc) { /* Single mirror read/write fast path */ - btrfs_bio(bio)->mirror_num = mirror_num; - btrfs_bio(bio)->device = smap.dev; + bbio->mirror_num = mirror_num; + bbio->device = smap.dev; bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; bio->bi_private = fs_info; bio->bi_end_io = btrfs_simple_end_io; @@ -278,6 +279,11 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror for (dev_nr = 0; dev_nr < total_devs; dev_nr++) btrfs_submit_mirrored_bio(bioc, dev_nr); } + return; + +fail: + btrfs_bio_counter_dec(fs_info); + btrfs_bio_end_io(bbio, ret); } /* From 7276aa7d38255b40e578267c3634ebc05f5d5236 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:04 +0100 Subject: [PATCH 211/265] btrfs: save the bio iter for checksum validation in common code All callers of btrfs_submit_bio that want to validate checksums currently have to store a copy of the iter in the btrfs_bio. Move the assignment into common code. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 5 ++++- fs/btrfs/compression.c | 4 ---- fs/btrfs/extent_io.c | 1 - fs/btrfs/inode.c | 7 ------- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 088817618f57..8285aa49586a 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -63,7 +63,6 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, btrfs_bio_init(bbio, inode, end_io, private); bio_trim(bio, offset >> 9, size >> 9); - bbio->iter = bio->bi_iter; return bio; } @@ -254,6 +253,10 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror BUG(); } + /* Save the iter for the end_io handler for data reads. */ + if (bio_op(bio) == REQ_OP_READ && !(bio->bi_opf & REQ_META)) + bbio->iter = bio->bi_iter; + if (!bioc) { /* Single mirror read/write fast path */ bbio->mirror_num = mirror_num; diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 069b7d28ca03..3e504a21e756 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -789,10 +789,6 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, submit = true; if (submit) { - /* Save the original iter for read repair */ - if (bio_op(comp_bio) == REQ_OP_READ) - btrfs_bio(comp_bio)->iter = comp_bio->bi_iter; - /* * Save the initial offset of this chunk, as there * is no direct correlation between compressed pages and diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index a8e8567aa428..1e28ae62d0ce 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -756,7 +756,6 @@ int btrfs_repair_one_sector(struct btrfs_inode *inode, struct btrfs_bio *failed_ } bio_add_page(repair_bio, page, failrec->len, pgoff); - repair_bbio->iter = repair_bio->bi_iter; btrfs_debug(fs_info, "repair read error: submitting new read to mirror %d", diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 7c8f5349ed7a..c368a45bc079 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2773,9 +2773,6 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, return; } - /* Save the original iter for read repair */ - btrfs_bio(bio)->iter = bio->bi_iter; - /* * Lookup bio sums does extra checks around whether we need to csum or * not, which is why we ignore skip_sum here. @@ -7988,10 +7985,6 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, struct btrfs_fs_info *fs_info = inode->root->fs_info; blk_status_t ret; - /* Save the original iter for read repair */ - if (btrfs_op(bio) == BTRFS_MAP_READ) - btrfs_bio(bio)->iter = bio->bi_iter; - if (inode->flags & BTRFS_INODE_NODATASUM) goto map; From 1c2b3ee3b0ec4bc971e23fe18d4c92333a6ad18a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:05 +0100 Subject: [PATCH 212/265] btrfs: pre-load data checksum for reads in btrfs_submit_bio Instead of calling btrfs_lookup_bio_sums in every caller of btrfs_submit_bio that reads data, do the call once in btrfs_submit_bio. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 12 ++++++++++-- fs/btrfs/compression.c | 6 ------ fs/btrfs/inode.c | 24 ------------------------ 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 8285aa49586a..3e23b97129f8 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -14,6 +14,7 @@ #include "dev-replace.h" #include "rcu-string.h" #include "zoned.h" +#include "file-item.h" static struct bio_set btrfs_bioset; @@ -253,9 +254,16 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror BUG(); } - /* Save the iter for the end_io handler for data reads. */ - if (bio_op(bio) == REQ_OP_READ && !(bio->bi_opf & REQ_META)) + /* + * Save the iter for the end_io handler and preload the checksums for + * data reads. + */ + if (bio_op(bio) == REQ_OP_READ && !(bio->bi_opf & REQ_META)) { bbio->iter = bio->bi_iter; + ret = btrfs_lookup_bio_sums(bbio); + if (ret) + goto fail; + } if (!bioc) { /* Single mirror read/write fast path */ diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 3e504a21e756..585f32e7efe2 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -797,12 +797,6 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, */ btrfs_bio(comp_bio)->file_offset = file_offset; - ret = btrfs_lookup_bio_sums(btrfs_bio(comp_bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(comp_bio), ret); - break; - } - ASSERT(comp_bio->bi_iter.bi_size); btrfs_submit_bio(fs_info, comp_bio, mirror_num); comp_bio = NULL; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index c368a45bc079..598897b0d661 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2762,7 +2762,6 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - blk_status_t ret; if (compress_type != BTRFS_COMPRESS_NONE) { /* @@ -2773,16 +2772,6 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, return; } - /* - * Lookup bio sums does extra checks around whether we need to csum or - * not, which is why we ignore skip_sum here. - */ - ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - return; - } - btrfs_submit_bio(fs_info, bio, mirror_num); } @@ -8004,12 +7993,6 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, btrfs_bio_end_io(btrfs_bio(bio), ret); return; } - } else { - ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - return; - } } map: btrfs_submit_bio(fs_info, bio, 0); @@ -10269,13 +10252,6 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, { struct btrfs_encoded_read_private *priv = btrfs_bio(bio)->private; struct btrfs_fs_info *fs_info = inode->root->fs_info; - blk_status_t ret; - - if (!priv->skip_csum) { - ret = btrfs_lookup_bio_sums(btrfs_bio(bio)); - if (ret) - return ret; - } atomic_inc(&priv->pending); btrfs_submit_bio(fs_info, bio, mirror_num); From e52190441bd6b268aed6ecc0efe3614c4222014e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:06 +0100 Subject: [PATCH 213/265] btrfs: add a btrfs_data_csum_ok helper Add a new checksumming helper that wraps btrfs_check_data_csum and does all the checks to if we're dealing with some form of nodatacsum I/O. This helper will be used by the new storage layer checksum validation and repair code. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 2 ++ fs/btrfs/inode.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 195c09e20609..3faabcef9898 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -423,6 +423,8 @@ int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, u32 pgoff, u8 *csum, const u8 * const csum_expected); int btrfs_check_data_csum(struct btrfs_inode *inode, struct btrfs_bio *bbio, u32 bio_offset, struct page *page, u32 pgoff); +bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, + u32 bio_offset, struct bio_vec *bv); unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, u32 bio_offset, struct page *page, u64 start, u64 end); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 598897b0d661..ba90f90e3d87 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3495,6 +3495,44 @@ zeroit: return -EIO; } +/* + * Verify the checksum of a single data sector. + * + * @bbio: btrfs_io_bio which contains the csum + * @dev: device the sector is on + * @bio_offset: offset to the beginning of the bio (in bytes) + * @bv: bio_vec to check + * + * Check if the checksum on a data block is valid. When a checksum mismatch is + * detected, report the error and fill the corrupted range with zero. + * + * Return %true if the sector is ok or had no checksum to start with, else %false. + */ +bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, + u32 bio_offset, struct bio_vec *bv) +{ + struct btrfs_inode *inode = bbio->inode; + u64 file_offset = bbio->file_offset + bio_offset; + u64 end = file_offset + bv->bv_len - 1; + + if (!bbio->csum) + return true; + + if (btrfs_is_data_reloc_root(inode->root) && + test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM, + 1, NULL)) { + /* Skip the range without csum for data reloc inode */ + clear_extent_bits(&inode->io_tree, file_offset, end, + EXTENT_NODATASUM); + return true; + } + + if (btrfs_check_data_csum(inode, bbio, bio_offset, bv->bv_page, + bv->bv_offset) < 0) + return false; + return true; +} + /* * When reads are done, we need to check csums to verify the data is correct. * if there's a match, we allow the bio to finish. If not, the code in From 7609afac677546b225d8327d726cc558d3666496 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:07 +0100 Subject: [PATCH 214/265] btrfs: handle checksum validation and repair at the storage layer Currently btrfs handles checksum validation and repair in the end I/O handler for the btrfs_bio. This leads to a lot of duplicate code plus issues with varying semantics or bugs, e.g. - the until recently broken repair for compressed extents - the fact that encoded reads validate the checksums but do not kick of read repair - the inconsistent checking of the BTRFS_FS_STATE_NO_CSUMS flag This commit revamps the checksum validation and repair code to instead work below the btrfs_submit_bio interfaces. In case of a checksum failure (or a plain old I/O error), the repair is now kicked off before the upper level ->end_io handler is invoked. Progress of an in-progress repair is tracked by a small structure that is allocated using a mempool for each original bio with failed sectors, which holds a reference to the original bio. This new structure is allocated using a mempool to guarantee forward progress even under memory pressure. The mempool will be replenished when the repair completes, just as the mempools backing the bios. There is one significant behavior change here: If repair fails or is impossible to start with, the whole bio will be failed to the upper layer. This is the behavior that all I/O submitters except for buffered I/O already emulated in their end_io handler. For buffered I/O this now means that a large readahead request can fail due to a single bad sector, but as readahead errors are ignored the following readpage if the sector is actually accessed will still be able to read. This also matches the I/O failure handling in other file systems. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 190 ++++++++++++++++++++++++++++++++++++++++- fs/btrfs/compression.c | 41 +-------- fs/btrfs/extent_io.c | 124 ++------------------------- fs/btrfs/inode.c | 81 +----------------- 4 files changed, 202 insertions(+), 234 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 3e23b97129f8..9bfdb90bd897 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -17,6 +17,14 @@ #include "file-item.h" static struct bio_set btrfs_bioset; +static struct bio_set btrfs_repair_bioset; +static mempool_t btrfs_failed_bio_pool; + +struct btrfs_failed_bio { + struct btrfs_bio *bbio; + int num_copies; + atomic_t repair_count; +}; /* * Initialize a btrfs_bio structure. This skips the embedded bio itself as it @@ -67,6 +75,162 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, return bio; } +static int next_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror) +{ + if (cur_mirror == fbio->num_copies) + return cur_mirror + 1 - fbio->num_copies; + return cur_mirror + 1; +} + +static int prev_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror) +{ + if (cur_mirror == 1) + return fbio->num_copies; + return cur_mirror - 1; +} + +static void btrfs_repair_done(struct btrfs_failed_bio *fbio) +{ + if (atomic_dec_and_test(&fbio->repair_count)) { + fbio->bbio->end_io(fbio->bbio); + mempool_free(fbio, &btrfs_failed_bio_pool); + } +} + +static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, + struct btrfs_device *dev) +{ + struct btrfs_failed_bio *fbio = repair_bbio->private; + struct btrfs_inode *inode = repair_bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct bio_vec *bv = bio_first_bvec_all(&repair_bbio->bio); + int mirror = repair_bbio->mirror_num; + + if (repair_bbio->bio.bi_status || + !btrfs_data_csum_ok(repair_bbio, dev, 0, bv)) { + bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ); + repair_bbio->bio.bi_iter = repair_bbio->iter; + + mirror = next_repair_mirror(fbio, mirror); + if (mirror == fbio->bbio->mirror_num) { + btrfs_debug(fs_info, "no mirror left"); + fbio->bbio->bio.bi_status = BLK_STS_IOERR; + goto done; + } + + btrfs_submit_bio(fs_info, &repair_bbio->bio, mirror); + return; + } + + do { + mirror = prev_repair_mirror(fbio, mirror); + btrfs_repair_io_failure(fs_info, btrfs_ino(inode), + repair_bbio->file_offset, fs_info->sectorsize, + repair_bbio->iter.bi_sector << SECTOR_SHIFT, + bv->bv_page, bv->bv_offset, mirror); + } while (mirror != fbio->bbio->mirror_num); + +done: + btrfs_repair_done(fbio); + bio_put(&repair_bbio->bio); +} + +/* + * Try to kick off a repair read to the next available mirror for a bad sector. + * + * This primarily tries to recover good data to serve the actual read request, + * but also tries to write the good data back to the bad mirror(s) when a + * read succeeded to restore the redundancy. + */ +static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, + u32 bio_offset, + struct bio_vec *bv, + struct btrfs_failed_bio *fbio) +{ + struct btrfs_inode *inode = failed_bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 sectorsize = fs_info->sectorsize; + const u64 logical = (failed_bbio->iter.bi_sector << SECTOR_SHIFT); + struct btrfs_bio *repair_bbio; + struct bio *repair_bio; + int num_copies; + int mirror; + + btrfs_debug(fs_info, "repair read error: read error at %llu", + failed_bbio->file_offset + bio_offset); + + num_copies = btrfs_num_copies(fs_info, logical, sectorsize); + if (num_copies == 1) { + btrfs_debug(fs_info, "no copy to repair from"); + failed_bbio->bio.bi_status = BLK_STS_IOERR; + return fbio; + } + + if (!fbio) { + fbio = mempool_alloc(&btrfs_failed_bio_pool, GFP_NOFS); + fbio->bbio = failed_bbio; + fbio->num_copies = num_copies; + atomic_set(&fbio->repair_count, 1); + } + + atomic_inc(&fbio->repair_count); + + repair_bio = bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, + &btrfs_repair_bioset); + repair_bio->bi_iter.bi_sector = failed_bbio->iter.bi_sector; + bio_add_page(repair_bio, bv->bv_page, bv->bv_len, bv->bv_offset); + + repair_bbio = btrfs_bio(repair_bio); + btrfs_bio_init(repair_bbio, failed_bbio->inode, NULL, fbio); + repair_bbio->file_offset = failed_bbio->file_offset + bio_offset; + + mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); + btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); + btrfs_submit_bio(fs_info, repair_bio, mirror); + return fbio; +} + +static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *dev) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u32 sectorsize = fs_info->sectorsize; + struct bvec_iter *iter = &bbio->iter; + blk_status_t status = bbio->bio.bi_status; + struct btrfs_failed_bio *fbio = NULL; + u32 offset = 0; + + /* + * Hand off repair bios to the repair code as there is no upper level + * submitter for them. + */ + if (bbio->bio.bi_pool == &btrfs_repair_bioset) { + btrfs_end_repair_bio(bbio, dev); + return; + } + + /* Clear the I/O error. A failed repair will reset it. */ + bbio->bio.bi_status = BLK_STS_OK; + + while (iter->bi_size) { + struct bio_vec bv = bio_iter_iovec(&bbio->bio, *iter); + + bv.bv_len = min(bv.bv_len, sectorsize); + if (status || !btrfs_data_csum_ok(bbio, dev, offset, &bv)) + fbio = repair_one_sector(bbio, offset, &bv, fbio); + + bio_advance_iter_single(&bbio->bio, iter, sectorsize); + offset += sectorsize; + } + + btrfs_bio_free_csum(bbio); + + if (fbio) + btrfs_repair_done(fbio); + else + bbio->end_io(bbio); +} + static void btrfs_log_dev_io_error(struct bio *bio, struct btrfs_device *dev) { if (!dev || !dev->bdev) @@ -94,7 +258,11 @@ static void btrfs_end_bio_work(struct work_struct *work) { struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); - bbio->end_io(bbio); + /* Metadata reads are checked and repaired by the submitter. */ + if (bbio->bio.bi_opf & REQ_META) + bbio->end_io(bbio); + else + btrfs_check_read_bio(bbio, bbio->device); } static void btrfs_simple_end_io(struct bio *bio) @@ -122,7 +290,10 @@ static void btrfs_raid56_end_io(struct bio *bio) btrfs_bio_counter_dec(bioc->fs_info); bbio->mirror_num = bioc->mirror_num; - bbio->end_io(bbio); + if (bio_op(bio) == REQ_OP_READ && !(bbio->bio.bi_opf & REQ_META)) + btrfs_check_read_bio(bbio, NULL); + else + bbio->end_io(bbio); btrfs_put_bioc(bioc); } @@ -402,10 +573,25 @@ int __init btrfs_bioset_init(void) offsetof(struct btrfs_bio, bio), BIOSET_NEED_BVECS)) return -ENOMEM; + if (bioset_init(&btrfs_repair_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), + BIOSET_NEED_BVECS)) + goto out_free_bioset; + if (mempool_init_kmalloc_pool(&btrfs_failed_bio_pool, BIO_POOL_SIZE, + sizeof(struct btrfs_failed_bio))) + goto out_free_repair_bioset; return 0; + +out_free_repair_bioset: + bioset_exit(&btrfs_repair_bioset); +out_free_bioset: + bioset_exit(&btrfs_bioset); + return -ENOMEM; } void __cold btrfs_bioset_exit(void) { + mempool_exit(&btrfs_failed_bio_pool); + bioset_exit(&btrfs_repair_bioset); bioset_exit(&btrfs_bioset); } diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 585f32e7efe2..9c2d3f5b4028 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -164,52 +164,15 @@ static void finish_compressed_bio_read(struct compressed_bio *cb) kfree(cb); } -/* - * Verify the checksums and kick off repair if needed on the uncompressed data - * before decompressing it into the original bio and freeing the uncompressed - * pages. - */ static void end_compressed_bio_read(struct btrfs_bio *bbio) { struct compressed_bio *cb = bbio->private; - struct inode *inode = cb->inode; - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct btrfs_inode *bi = BTRFS_I(inode); - bool csum = !(bi->flags & BTRFS_INODE_NODATASUM) && - !test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state); - blk_status_t status = bbio->bio.bi_status; - struct bvec_iter iter; - struct bio_vec bv; - u32 offset; - btrfs_bio_for_each_sector(fs_info, bv, bbio, iter, offset) { - u64 start = bbio->file_offset + offset; - - if (!status && - (!csum || !btrfs_check_data_csum(bi, bbio, offset, - bv.bv_page, bv.bv_offset))) { - btrfs_clean_io_failure(bi, start, bv.bv_page, - bv.bv_offset); - } else { - int ret; - - refcount_inc(&cb->pending_ios); - ret = btrfs_repair_one_sector(BTRFS_I(inode), bbio, offset, - bv.bv_page, bv.bv_offset, - true); - if (ret) { - refcount_dec(&cb->pending_ios); - status = errno_to_blk_status(ret); - } - } - } - - if (status) - cb->status = status; + if (bbio->bio.bi_status) + cb->status = bbio->bio.bi_status; if (refcount_dec_and_test(&cb->pending_ios)) finish_compressed_bio_read(cb); - btrfs_bio_free_csum(bbio); bio_put(&bbio->bio); } diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 1e28ae62d0ce..88def7fd8598 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -803,79 +803,6 @@ static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) btrfs_subpage_end_reader(fs_info, page, start, len); } -static void end_sector_io(struct page *page, u64 offset, bool uptodate) -{ - struct btrfs_inode *inode = BTRFS_I(page->mapping->host); - const u32 sectorsize = inode->root->fs_info->sectorsize; - - end_page_read(page, uptodate, offset, sectorsize); - unlock_extent(&inode->io_tree, offset, offset + sectorsize - 1, NULL); -} - -static void submit_data_read_repair(struct inode *inode, - struct btrfs_bio *failed_bbio, - u32 bio_offset, const struct bio_vec *bvec, - unsigned int error_bitmap) -{ - const unsigned int pgoff = bvec->bv_offset; - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct page *page = bvec->bv_page; - const u64 start = page_offset(bvec->bv_page) + bvec->bv_offset; - const u64 end = start + bvec->bv_len - 1; - const u32 sectorsize = fs_info->sectorsize; - const int nr_bits = (end + 1 - start) >> fs_info->sectorsize_bits; - int i; - - BUG_ON(bio_op(&failed_bbio->bio) == REQ_OP_WRITE); - - /* This repair is only for data */ - ASSERT(is_data_inode(inode)); - - /* We're here because we had some read errors or csum mismatch */ - ASSERT(error_bitmap); - - /* - * We only get called on buffered IO, thus page must be mapped and bio - * must not be cloned. - */ - ASSERT(page->mapping && !bio_flagged(&failed_bbio->bio, BIO_CLONED)); - - /* Iterate through all the sectors in the range */ - for (i = 0; i < nr_bits; i++) { - const unsigned int offset = i * sectorsize; - bool uptodate = false; - int ret; - - if (!(error_bitmap & (1U << i))) { - /* - * This sector has no error, just end the page read - * and unlock the range. - */ - uptodate = true; - goto next; - } - - ret = btrfs_repair_one_sector(BTRFS_I(inode), failed_bbio, - bio_offset + offset, page, pgoff + offset, - true); - if (!ret) { - /* - * We have submitted the read repair, the page release - * will be handled by the endio function of the - * submitted repair bio. - * Thus we don't need to do any thing here. - */ - continue; - } - /* - * Continue on failed repair, otherwise the remaining sectors - * will not be properly unlocked. - */ -next: - end_sector_io(page, start + offset, uptodate); - } -} - /* lots and lots of room for performance fixes in the end_bio funcs */ void end_extent_writepage(struct page *page, int err, u64 start, u64 end) @@ -1093,8 +1020,6 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio) struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); const u32 sectorsize = fs_info->sectorsize; - unsigned int error_bitmap = (unsigned int)-1; - bool repair = false; u64 start; u64 end; u32 len; @@ -1126,25 +1051,14 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio) len = bvec->bv_len; mirror = bbio->mirror_num; - if (likely(uptodate)) { - if (is_data_inode(inode)) { - error_bitmap = btrfs_verify_data_csum(bbio, - bio_offset, page, start, end); - if (error_bitmap) - uptodate = false; - } else { - if (btrfs_validate_metadata_buffer(bbio, - page, start, end, mirror)) - uptodate = false; - } - } + if (uptodate && !is_data_inode(inode) && + btrfs_validate_metadata_buffer(bbio, page, start, end, mirror)) + uptodate = false; if (likely(uptodate)) { loff_t i_size = i_size_read(inode); pgoff_t end_index = i_size >> PAGE_SHIFT; - btrfs_clean_io_failure(BTRFS_I(inode), start, page, 0); - /* * Zero out the remaining part if this range straddles * i_size. @@ -1161,19 +1075,7 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio) zero_user_segment(page, zero_start, offset_in_page(end) + 1); } - } else if (is_data_inode(inode)) { - /* - * Only try to repair bios that actually made it to a - * device. If the bio failed to be submitted mirror - * is 0 and we need to fail it without retrying. - * - * This also includes the high level bios for compressed - * extents - these never make it to a device and repair - * is already handled on the lower compressed bio. - */ - if (mirror > 0) - repair = true; - } else { + } else if (!is_data_inode(inode)) { struct extent_buffer *eb; eb = find_extent_buffer_readpage(fs_info, page, start); @@ -1182,19 +1084,10 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio) atomic_dec(&eb->io_pages); } - if (repair) { - /* - * submit_data_read_repair() will handle all the good - * and bad sectors, we just continue to the next bvec. - */ - submit_data_read_repair(inode, bbio, bio_offset, bvec, - error_bitmap); - } else { - /* Update page status and unlock */ - end_page_read(page, uptodate, start, len); - endio_readpage_release_extent(&processed, BTRFS_I(inode), - start, end, PageUptodate(page)); - } + /* Update page status and unlock. */ + end_page_read(page, uptodate, start, len); + endio_readpage_release_extent(&processed, BTRFS_I(inode), + start, end, PageUptodate(page)); ASSERT(bio_offset + len > bio_offset); bio_offset += len; @@ -1202,7 +1095,6 @@ static void end_bio_extent_readpage(struct btrfs_bio *bbio) } /* Release the last extent */ endio_readpage_release_extent(&processed, NULL, 0, 0, false); - btrfs_bio_free_csum(bbio); bio_put(bio); } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index ba90f90e3d87..237513508fcc 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7941,39 +7941,6 @@ void btrfs_submit_dio_repair_bio(struct btrfs_inode *inode, struct bio *bio, int btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); } -static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip, - struct btrfs_bio *bbio, - const bool uptodate) -{ - struct inode *inode = &dip->inode->vfs_inode; - struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; - const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); - blk_status_t err = BLK_STS_OK; - struct bvec_iter iter; - struct bio_vec bv; - u32 offset; - - btrfs_bio_for_each_sector(fs_info, bv, bbio, iter, offset) { - u64 start = bbio->file_offset + offset; - - if (uptodate && - (!csum || !btrfs_check_data_csum(BTRFS_I(inode), bbio, offset, - bv.bv_page, bv.bv_offset))) { - btrfs_clean_io_failure(BTRFS_I(inode), start, - bv.bv_page, bv.bv_offset); - } else { - int ret; - - ret = btrfs_repair_one_sector(BTRFS_I(inode), bbio, offset, - bv.bv_page, bv.bv_offset, false); - if (ret) - err = errno_to_blk_status(ret); - } - } - - return err; -} - blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, struct bio *bio, u64 dio_file_offset) @@ -7987,18 +7954,14 @@ static void btrfs_end_dio_bio(struct btrfs_bio *bbio) struct bio *bio = &bbio->bio; blk_status_t err = bio->bi_status; - if (err) + if (err) { btrfs_warn(dip->inode->root->fs_info, "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", btrfs_ino(dip->inode), bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, err); - - if (bio_op(bio) == REQ_OP_READ) - err = btrfs_check_read_dio_bio(dip, bbio, !err); - - if (err) dip->bio.bi_status = err; + } btrfs_record_physical_zoned(&dip->inode->vfs_inode, bbio->file_offset, bio); @@ -10282,7 +10245,6 @@ struct btrfs_encoded_read_private { wait_queue_head_t wait; atomic_t pending; blk_status_t status; - bool skip_csum; }; static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, @@ -10296,44 +10258,11 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, return BLK_STS_OK; } -static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio) -{ - const bool uptodate = (bbio->bio.bi_status == BLK_STS_OK); - struct btrfs_encoded_read_private *priv = bbio->private; - struct btrfs_inode *inode = priv->inode; - struct btrfs_fs_info *fs_info = inode->root->fs_info; - u32 sectorsize = fs_info->sectorsize; - struct bio_vec *bvec; - struct bvec_iter_all iter_all; - u32 bio_offset = 0; - - if (priv->skip_csum || !uptodate) - return bbio->bio.bi_status; - - bio_for_each_segment_all(bvec, &bbio->bio, iter_all) { - unsigned int i, nr_sectors, pgoff; - - nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec->bv_len); - pgoff = bvec->bv_offset; - for (i = 0; i < nr_sectors; i++) { - ASSERT(pgoff < PAGE_SIZE); - if (btrfs_check_data_csum(inode, bbio, bio_offset, - bvec->bv_page, pgoff)) - return BLK_STS_IOERR; - bio_offset += sectorsize; - pgoff += sectorsize; - } - } - return BLK_STS_OK; -} - static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) { struct btrfs_encoded_read_private *priv = bbio->private; - blk_status_t status; - status = btrfs_encoded_read_verify_csum(bbio); - if (status) { + if (bbio->bio.bi_status) { /* * The memory barrier implied by the atomic_dec_return() here * pairs with the memory barrier implied by the @@ -10342,11 +10271,10 @@ static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) * write is observed before the load of status in * btrfs_encoded_read_regular_fill_pages(). */ - WRITE_ONCE(priv->status, status); + WRITE_ONCE(priv->status, bbio->bio.bi_status); } if (!atomic_dec_return(&priv->pending)) wake_up(&priv->wait); - btrfs_bio_free_csum(bbio); bio_put(&bbio->bio); } @@ -10359,7 +10287,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, .inode = inode, .file_offset = file_offset, .pending = ATOMIC_INIT(1), - .skip_csum = (inode->flags & BTRFS_INODE_NODATASUM), }; unsigned long i = 0; u64 cur = 0; From 7ab0fdfc810209fc493f2f67bdd41d5aafbc3a05 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:08 +0100 Subject: [PATCH 215/265] btrfs: open code btrfs_bio_free_csum btrfs_bio_free_csum has only one caller left, and that caller is always for an data inode and doesn't need zeroing of the csum pointer as that pointer will never be touched again. Just open code the conditional kfree there. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 3 ++- fs/btrfs/bio.h | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 9bfdb90bd897..6ef953234068 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -223,7 +223,8 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de offset += sectorsize; } - btrfs_bio_free_csum(bbio); + if (bbio->csum != bbio->csum_inline) + kfree(bbio->csum); if (fbio) btrfs_repair_done(fbio); diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 8d69d0b226d9..996275eb1062 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -94,16 +94,6 @@ static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) bbio->end_io(bbio); } -static inline void btrfs_bio_free_csum(struct btrfs_bio *bbio) -{ - if (bbio->is_metadata) - return; - if (bbio->csum != bbio->csum_inline) { - kfree(bbio->csum); - bbio->csum = NULL; - } -} - /* * Iterate through a btrfs_bio (@bbio) on a per-sector basis. * From ac9f942e86941e8e615b04f791214cc04acd09d0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:09 +0100 Subject: [PATCH 216/265] btrfs: remove btrfs_bio_for_each_sector btrfs_bio_for_each_sector is unused now, so remove it. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.h | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 996275eb1062..2e799c3348d5 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -94,22 +94,6 @@ static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) bbio->end_io(bbio); } -/* - * Iterate through a btrfs_bio (@bbio) on a per-sector basis. - * - * bvl - struct bio_vec - * bbio - struct btrfs_bio - * iters - struct bvec_iter - * bio_offset - unsigned int - */ -#define btrfs_bio_for_each_sector(fs_info, bvl, bbio, iter, bio_offset) \ - for ((iter) = (bbio)->iter, (bio_offset) = 0; \ - (iter).bi_size && \ - (((bvl) = bio_iter_iovec((&(bbio)->bio), (iter))), 1); \ - (bio_offset) += fs_info->sectorsize, \ - bio_advance_iter_single(&(bbio)->bio, &(iter), \ - (fs_info)->sectorsize)) - void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num); int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, From 3d49d0d31237d11268959c7873c56aab166be07b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:10 +0100 Subject: [PATCH 217/265] btrfs: remove now unused checksumming helpers Remove the unused btrfs_verify_data_csum helper, and fold btrfs_check_data_csum into its only caller. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 5 -- fs/btrfs/inode.c | 124 ++++++----------------------------------- 2 files changed, 16 insertions(+), 113 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 3faabcef9898..99430d0ebbb5 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -421,13 +421,8 @@ blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, u64 dio_file_offset); int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, u32 pgoff, u8 *csum, const u8 * const csum_expected); -int btrfs_check_data_csum(struct btrfs_inode *inode, struct btrfs_bio *bbio, - u32 bio_offset, struct page *page, u32 pgoff); bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, u32 bio_offset, struct bio_vec *bv); -unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, - u32 bio_offset, struct page *page, - u64 start, u64 end); noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, u64 *orig_start, u64 *orig_block_len, u64 *ram_bytes, bool nowait, bool strict); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 237513508fcc..9987d16626f7 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3456,45 +3456,6 @@ static u8 *btrfs_csum_ptr(const struct btrfs_fs_info *fs_info, u8 *csums, u64 of return csums + offset_in_sectors * fs_info->csum_size; } -/* - * check_data_csum - verify checksum of one sector of uncompressed data - * @inode: inode - * @bbio: btrfs_bio which contains the csum - * @bio_offset: offset to the beginning of the bio (in bytes) - * @page: page where is the data to be verified - * @pgoff: offset inside the page - * - * The length of such check is always one sector size. - * - * When csum mismatch is detected, we will also report the error and fill the - * corrupted range with zero. (Thus it needs the extra parameters) - */ -int btrfs_check_data_csum(struct btrfs_inode *inode, struct btrfs_bio *bbio, - u32 bio_offset, struct page *page, u32 pgoff) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - u32 len = fs_info->sectorsize; - u8 *csum_expected; - u8 csum[BTRFS_CSUM_SIZE]; - - ASSERT(pgoff + len <= PAGE_SIZE); - - csum_expected = btrfs_csum_ptr(fs_info, bbio->csum, bio_offset); - - if (btrfs_check_sector_csum(fs_info, page, pgoff, csum, csum_expected)) - goto zeroit; - return 0; - -zeroit: - btrfs_print_data_csum_error(inode, bbio->file_offset + bio_offset, - csum, csum_expected, bbio->mirror_num); - if (bbio->device) - btrfs_dev_stat_inc_and_print(bbio->device, - BTRFS_DEV_STAT_CORRUPTION_ERRS); - memzero_page(page, pgoff, len); - return -EIO; -} - /* * Verify the checksum of a single data sector. * @@ -3512,8 +3473,13 @@ bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, u32 bio_offset, struct bio_vec *bv) { struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; u64 file_offset = bbio->file_offset + bio_offset; u64 end = file_offset + bv->bv_len - 1; + u8 *csum_expected; + u8 csum[BTRFS_CSUM_SIZE]; + + ASSERT(bv->bv_len == fs_info->sectorsize); if (!bbio->csum) return true; @@ -3527,77 +3493,19 @@ bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, return true; } - if (btrfs_check_data_csum(inode, bbio, bio_offset, bv->bv_page, - bv->bv_offset) < 0) - return false; + csum_expected = btrfs_csum_ptr(fs_info, bbio->csum, bio_offset); + if (btrfs_check_sector_csum(fs_info, bv->bv_page, bv->bv_offset, csum, + csum_expected)) + goto zeroit; return true; -} -/* - * When reads are done, we need to check csums to verify the data is correct. - * if there's a match, we allow the bio to finish. If not, the code in - * extent_io.c will try to find good copies for us. - * - * @bio_offset: offset to the beginning of the bio (in bytes) - * @start: file offset of the range start - * @end: file offset of the range end (inclusive) - * - * Return a bitmap where bit set means a csum mismatch, and bit not set means - * csum match. - */ -unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, - u32 bio_offset, struct page *page, - u64 start, u64 end) -{ - struct btrfs_inode *inode = BTRFS_I(page->mapping->host); - struct btrfs_root *root = inode->root; - struct btrfs_fs_info *fs_info = root->fs_info; - struct extent_io_tree *io_tree = &inode->io_tree; - const u32 sectorsize = root->fs_info->sectorsize; - u32 pg_off; - unsigned int result = 0; - - /* - * This only happens for NODATASUM or compressed read. - * Normally this should be covered by above check for compressed read - * or the next check for NODATASUM. Just do a quicker exit here. - */ - if (bbio->csum == NULL) - return 0; - - if (inode->flags & BTRFS_INODE_NODATASUM) - return 0; - - if (unlikely(test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state))) - return 0; - - ASSERT(page_offset(page) <= start && - end <= page_offset(page) + PAGE_SIZE - 1); - for (pg_off = offset_in_page(start); - pg_off < offset_in_page(end); - pg_off += sectorsize, bio_offset += sectorsize) { - u64 file_offset = pg_off + page_offset(page); - int ret; - - if (btrfs_is_data_reloc_root(root) && - test_range_bit(io_tree, file_offset, - file_offset + sectorsize - 1, - EXTENT_NODATASUM, 1, NULL)) { - /* Skip the range without csum for data reloc inode */ - clear_extent_bits(io_tree, file_offset, - file_offset + sectorsize - 1, - EXTENT_NODATASUM); - continue; - } - ret = btrfs_check_data_csum(inode, bbio, bio_offset, page, pg_off); - if (ret < 0) { - const int nr_bit = (pg_off - offset_in_page(start)) >> - root->fs_info->sectorsize_bits; - - result |= (1U << nr_bit); - } - } - return result; +zeroit: + btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected, + bbio->mirror_num); + if (dev) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); + memzero_bvec(bv); + return false; } /* From 860c8c451661c72f05713ffdef3707f1fa9184d1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:11 +0100 Subject: [PATCH 218/265] btrfs: remove struct btrfs_bio::device field The device field is only used by the simple end I/O handler, and for that it can simply be stored in the bi_private field of the bio, which is currently used for the fs_info that can be retrieved through bbio->inode as well. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 10 +++++----- fs/btrfs/bio.h | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 6ef953234068..5e545adf0aea 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -263,18 +263,19 @@ static void btrfs_end_bio_work(struct work_struct *work) if (bbio->bio.bi_opf & REQ_META) bbio->end_io(bbio); else - btrfs_check_read_bio(bbio, bbio->device); + btrfs_check_read_bio(bbio, bbio->bio.bi_private); } static void btrfs_simple_end_io(struct bio *bio) { - struct btrfs_fs_info *fs_info = bio->bi_private; struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_device *dev = bio->bi_private; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; btrfs_bio_counter_dec(fs_info); if (bio->bi_status) - btrfs_log_dev_io_error(bio, bbio->device); + btrfs_log_dev_io_error(bio, dev); if (bio_op(bio) == REQ_OP_READ) { INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work); @@ -440,9 +441,8 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror if (!bioc) { /* Single mirror read/write fast path */ bbio->mirror_num = mirror_num; - bbio->device = smap.dev; bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; - bio->bi_private = fs_info; + bio->bi_private = smap.dev; bio->bi_end_io = btrfs_simple_end_io; btrfs_submit_dev_bio(smap.dev, bio); } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 2e799c3348d5..61a791cf5360 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -45,8 +45,6 @@ struct btrfs_bio { struct btrfs_inode *inode; u64 file_offset; - /* @device is for stripe IO submission. */ - struct btrfs_device *device; union { /* For data checksum verification. */ struct { From 0571b6357c5e414cd5db8e03150074a5ca1c5c12 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:12 +0100 Subject: [PATCH 219/265] btrfs: remove the io_failure_record infrastructure struct io_failure_record and the io_failure_tree tree are unused now, so remove them. This in turn makes struct btrfs_inode smaller by 16 bytes. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 7 - fs/btrfs/extent-io-tree.h | 1 - fs/btrfs/extent_io.c | 260 -------------------------------------- fs/btrfs/extent_io.h | 31 ----- fs/btrfs/inode.c | 16 --- 5 files changed, 315 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 99430d0ebbb5..78c7979b8dca 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -93,12 +93,6 @@ struct btrfs_inode { /* the io_tree does range state (DIRTY, LOCKED etc) */ struct extent_io_tree io_tree; - /* special utility tree used to record which mirrors have already been - * tried when checksums fail for a given block - */ - struct rb_root io_failure_tree; - spinlock_t io_failure_lock; - /* * Keep track of where the inode has extent items mapped in order to * make sure the i_size adjustments are accurate @@ -414,7 +408,6 @@ static inline void btrfs_inode_split_flags(u64 inode_item_flags, void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num); void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type); -void btrfs_submit_dio_repair_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num); blk_status_t btrfs_submit_bio_start(struct btrfs_inode *inode, struct bio *bio); blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, struct bio *bio, diff --git a/fs/btrfs/extent-io-tree.h b/fs/btrfs/extent-io-tree.h index e3eeec380844..21766e49ec02 100644 --- a/fs/btrfs/extent-io-tree.h +++ b/fs/btrfs/extent-io-tree.h @@ -6,7 +6,6 @@ #include "misc.h" struct extent_changeset; -struct io_failure_record; /* Bits for the extent state */ enum { diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 88def7fd8598..41bcc0bfe56a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -515,266 +515,6 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, start, end, page_ops, NULL); } -static int insert_failrec(struct btrfs_inode *inode, - struct io_failure_record *failrec) -{ - struct rb_node *exist; - - spin_lock(&inode->io_failure_lock); - exist = rb_simple_insert(&inode->io_failure_tree, failrec->bytenr, - &failrec->rb_node); - spin_unlock(&inode->io_failure_lock); - - return (exist == NULL) ? 0 : -EEXIST; -} - -static struct io_failure_record *get_failrec(struct btrfs_inode *inode, u64 start) -{ - struct rb_node *node; - struct io_failure_record *failrec = ERR_PTR(-ENOENT); - - spin_lock(&inode->io_failure_lock); - node = rb_simple_search(&inode->io_failure_tree, start); - if (node) - failrec = rb_entry(node, struct io_failure_record, rb_node); - spin_unlock(&inode->io_failure_lock); - return failrec; -} - -static void free_io_failure(struct btrfs_inode *inode, - struct io_failure_record *rec) -{ - spin_lock(&inode->io_failure_lock); - rb_erase(&rec->rb_node, &inode->io_failure_tree); - spin_unlock(&inode->io_failure_lock); - - kfree(rec); -} - -static int next_mirror(const struct io_failure_record *failrec, int cur_mirror) -{ - if (cur_mirror == failrec->num_copies) - return cur_mirror + 1 - failrec->num_copies; - return cur_mirror + 1; -} - -static int prev_mirror(const struct io_failure_record *failrec, int cur_mirror) -{ - if (cur_mirror == 1) - return failrec->num_copies; - return cur_mirror - 1; -} - -/* - * each time an IO finishes, we do a fast check in the IO failure tree - * to see if we need to process or clean up an io_failure_record - */ -int btrfs_clean_io_failure(struct btrfs_inode *inode, u64 start, - struct page *page, unsigned int pg_offset) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_io_tree *io_tree = &inode->io_tree; - u64 ino = btrfs_ino(inode); - u64 locked_start, locked_end; - struct io_failure_record *failrec; - int mirror; - int ret; - - failrec = get_failrec(inode, start); - if (IS_ERR(failrec)) - return 0; - - BUG_ON(!failrec->this_mirror); - - if (sb_rdonly(fs_info->sb)) - goto out; - - ret = find_first_extent_bit(io_tree, failrec->bytenr, &locked_start, - &locked_end, EXTENT_LOCKED, NULL); - if (ret || locked_start > failrec->bytenr || - locked_end < failrec->bytenr + failrec->len - 1) - goto out; - - mirror = failrec->this_mirror; - do { - mirror = prev_mirror(failrec, mirror); - btrfs_repair_io_failure(fs_info, ino, start, failrec->len, - failrec->logical, page, pg_offset, mirror); - } while (mirror != failrec->failed_mirror); - -out: - free_io_failure(inode, failrec); - return 0; -} - -/* - * Can be called when - * - hold extent lock - * - under ordered extent - * - the inode is freeing - */ -void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end) -{ - struct io_failure_record *failrec; - struct rb_node *node, *next; - - if (RB_EMPTY_ROOT(&inode->io_failure_tree)) - return; - - spin_lock(&inode->io_failure_lock); - node = rb_simple_search_first(&inode->io_failure_tree, start); - while (node) { - failrec = rb_entry(node, struct io_failure_record, rb_node); - if (failrec->bytenr > end) - break; - - next = rb_next(node); - rb_erase(&failrec->rb_node, &inode->io_failure_tree); - kfree(failrec); - - node = next; - } - spin_unlock(&inode->io_failure_lock); -} - -static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode, - struct btrfs_bio *bbio, - unsigned int bio_offset) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - u64 start = bbio->file_offset + bio_offset; - struct io_failure_record *failrec; - const u32 sectorsize = fs_info->sectorsize; - int ret; - - failrec = get_failrec(BTRFS_I(inode), start); - if (!IS_ERR(failrec)) { - btrfs_debug(fs_info, - "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu", - failrec->logical, failrec->bytenr, failrec->len); - /* - * when data can be on disk more than twice, add to failrec here - * (e.g. with a list for failed_mirror) to make - * clean_io_failure() clean all those errors at once. - */ - ASSERT(failrec->this_mirror == bbio->mirror_num); - ASSERT(failrec->len == fs_info->sectorsize); - return failrec; - } - - failrec = kzalloc(sizeof(*failrec), GFP_NOFS); - if (!failrec) - return ERR_PTR(-ENOMEM); - - RB_CLEAR_NODE(&failrec->rb_node); - failrec->bytenr = start; - failrec->len = sectorsize; - failrec->failed_mirror = bbio->mirror_num; - failrec->this_mirror = bbio->mirror_num; - failrec->logical = (bbio->iter.bi_sector << SECTOR_SHIFT) + bio_offset; - - btrfs_debug(fs_info, - "new io failure record logical %llu start %llu", - failrec->logical, start); - - failrec->num_copies = btrfs_num_copies(fs_info, failrec->logical, sectorsize); - if (failrec->num_copies == 1) { - /* - * We only have a single copy of the data, so don't bother with - * all the retry and error correction code that follows. No - * matter what the error is, it is very likely to persist. - */ - btrfs_debug(fs_info, - "cannot repair logical %llu num_copies %d", - failrec->logical, failrec->num_copies); - kfree(failrec); - return ERR_PTR(-EIO); - } - - /* Set the bits in the private failure tree */ - ret = insert_failrec(BTRFS_I(inode), failrec); - if (ret) { - kfree(failrec); - return ERR_PTR(ret); - } - - return failrec; -} - -int btrfs_repair_one_sector(struct btrfs_inode *inode, struct btrfs_bio *failed_bbio, - u32 bio_offset, struct page *page, unsigned int pgoff, - bool submit_buffered) -{ - u64 start = failed_bbio->file_offset + bio_offset; - struct io_failure_record *failrec; - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct bio *failed_bio = &failed_bbio->bio; - const int icsum = bio_offset >> fs_info->sectorsize_bits; - struct bio *repair_bio; - struct btrfs_bio *repair_bbio; - - btrfs_debug(fs_info, - "repair read error: read error at %llu", start); - - BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); - - failrec = btrfs_get_io_failure_record(&inode->vfs_inode, failed_bbio, bio_offset); - if (IS_ERR(failrec)) - return PTR_ERR(failrec); - - /* - * There are two premises: - * a) deliver good data to the caller - * b) correct the bad sectors on disk - * - * Since we're only doing repair for one sector, we only need to get - * a good copy of the failed sector and if we succeed, we have setup - * everything for btrfs_repair_io_failure to do the rest for us. - */ - failrec->this_mirror = next_mirror(failrec, failrec->this_mirror); - if (failrec->this_mirror == failrec->failed_mirror) { - btrfs_debug(fs_info, - "failed to repair num_copies %d this_mirror %d failed_mirror %d", - failrec->num_copies, failrec->this_mirror, failrec->failed_mirror); - free_io_failure(inode, failrec); - return -EIO; - } - - repair_bio = btrfs_bio_alloc(1, REQ_OP_READ, failed_bbio->inode, - failed_bbio->end_io, - failed_bbio->private); - repair_bbio = btrfs_bio(repair_bio); - repair_bbio->file_offset = start; - repair_bio->bi_iter.bi_sector = failrec->logical >> 9; - - if (failed_bbio->csum) { - const u32 csum_size = fs_info->csum_size; - - repair_bbio->csum = repair_bbio->csum_inline; - memcpy(repair_bbio->csum, - failed_bbio->csum + csum_size * icsum, csum_size); - } - - bio_add_page(repair_bio, page, failrec->len, pgoff); - - btrfs_debug(fs_info, - "repair read error: submitting new read to mirror %d", - failrec->this_mirror); - - /* - * At this point we have a bio, so any errors from bio submission will - * be handled by the endio on the repair_bio, so we can't return an - * error here. - */ - if (submit_buffered) - btrfs_submit_data_read_bio(inode, repair_bio, - failrec->this_mirror, 0); - else - btrfs_submit_dio_repair_bio(inode, repair_bio, failrec->this_mirror); - - return BLK_STS_OK; -} - static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) { struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index a2c82448b2e0..1b311cd69783 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -60,11 +60,9 @@ enum { #define BITMAP_LAST_BYTE_MASK(nbits) \ (BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1))) -struct btrfs_bio; struct btrfs_root; struct btrfs_inode; struct btrfs_fs_info; -struct io_failure_record; struct extent_io_tree; struct btrfs_tree_parent_check; @@ -279,35 +277,6 @@ int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array); void end_extent_writepage(struct page *page, int err, u64 start, u64 end); -/* - * When IO fails, either with EIO or csum verification fails, we - * try other mirrors that might have a good copy of the data. This - * io_failure_record is used to record state as we go through all the - * mirrors. If another mirror has good data, the sector is set up to date - * and things continue. If a good mirror can't be found, the original - * bio end_io callback is called to indicate things have failed. - */ -struct io_failure_record { - /* Use rb_simple_node for search/insert */ - struct { - struct rb_node rb_node; - u64 bytenr; - }; - struct page *page; - u64 len; - u64 logical; - int this_mirror; - int failed_mirror; - int num_copies; -}; - -int btrfs_repair_one_sector(struct btrfs_inode *inode, struct btrfs_bio *failed_bbio, - u32 bio_offset, struct page *page, unsigned int pgoff, - bool submit_buffered); -void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end); -int btrfs_clean_io_failure(struct btrfs_inode *inode, u64 start, - struct page *page, unsigned int pg_offset); - #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, struct page *locked_page, u64 *start, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 9987d16626f7..8e1d61b731ed 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3249,8 +3249,6 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) ordered_extent->disk_num_bytes); } - btrfs_free_io_failure_record(inode, start, end); - if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { truncated = true; logical_len = ordered_extent->truncated_len; @@ -5395,8 +5393,6 @@ void btrfs_evict_inode(struct inode *inode) if (is_bad_inode(inode)) goto no_delete; - btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1); - if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) goto no_delete; @@ -7839,16 +7835,6 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) bio_endio(&dip->bio); } -void btrfs_submit_dio_repair_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) -{ - struct btrfs_dio_private *dip = btrfs_bio(bio)->private; - - BUG_ON(bio_op(bio) == REQ_OP_WRITE); - - refcount_inc(&dip->refs); - btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); -} - blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, struct bio *bio, u64 dio_file_offset) @@ -8714,7 +8700,6 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->last_log_commit = 0; spin_lock_init(&ei->lock); - spin_lock_init(&ei->io_failure_lock); ei->outstanding_extents = 0; if (sb->s_magic != BTRFS_TEST_MAGIC) btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, @@ -8734,7 +8719,6 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->io_tree.inode = ei; extent_io_tree_init(fs_info, &ei->file_extent_tree, IO_TREE_INODE_FILE_EXTENT); - ei->io_failure_tree = RB_ROOT; atomic_set(&ei->sync_writers, 0); mutex_init(&ei->log_mutex); btrfs_ordered_inode_tree_init(&ei->ordered_tree); From 0d3acb25e70d5f58f99ae9c695326a097b4b56be Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:13 +0100 Subject: [PATCH 220/265] btrfs: rename btrfs_bio::iter field Rename iter to saved_iter and move it next to the repair internals and nothing outside of bio.c should be touching it. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 12 ++++++------ fs/btrfs/bio.h | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 5e545adf0aea..d75c11808482 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -109,7 +109,7 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, if (repair_bbio->bio.bi_status || !btrfs_data_csum_ok(repair_bbio, dev, 0, bv)) { bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ); - repair_bbio->bio.bi_iter = repair_bbio->iter; + repair_bbio->bio.bi_iter = repair_bbio->saved_iter; mirror = next_repair_mirror(fbio, mirror); if (mirror == fbio->bbio->mirror_num) { @@ -126,7 +126,7 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, mirror = prev_repair_mirror(fbio, mirror); btrfs_repair_io_failure(fs_info, btrfs_ino(inode), repair_bbio->file_offset, fs_info->sectorsize, - repair_bbio->iter.bi_sector << SECTOR_SHIFT, + repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT, bv->bv_page, bv->bv_offset, mirror); } while (mirror != fbio->bbio->mirror_num); @@ -150,7 +150,7 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, struct btrfs_inode *inode = failed_bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; const u32 sectorsize = fs_info->sectorsize; - const u64 logical = (failed_bbio->iter.bi_sector << SECTOR_SHIFT); + const u64 logical = (failed_bbio->saved_iter.bi_sector << SECTOR_SHIFT); struct btrfs_bio *repair_bbio; struct bio *repair_bio; int num_copies; @@ -177,7 +177,7 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, repair_bio = bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &btrfs_repair_bioset); - repair_bio->bi_iter.bi_sector = failed_bbio->iter.bi_sector; + repair_bio->bi_iter.bi_sector = failed_bbio->saved_iter.bi_sector; bio_add_page(repair_bio, bv->bv_page, bv->bv_len, bv->bv_offset); repair_bbio = btrfs_bio(repair_bio); @@ -195,7 +195,7 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; u32 sectorsize = fs_info->sectorsize; - struct bvec_iter *iter = &bbio->iter; + struct bvec_iter *iter = &bbio->saved_iter; blk_status_t status = bbio->bio.bi_status; struct btrfs_failed_bio *fbio = NULL; u32 offset = 0; @@ -432,7 +432,7 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror * data reads. */ if (bio_op(bio) == REQ_OP_READ && !(bio->bi_opf & REQ_META)) { - bbio->iter = bio->bi_iter; + bbio->saved_iter = bio->bi_iter; ret = btrfs_lookup_bio_sums(bbio); if (ret) goto fail; diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 61a791cf5360..c232148348df 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -39,17 +39,20 @@ struct btrfs_bio { * it's a metadata bio. */ unsigned int is_metadata:1; - struct bvec_iter iter; /* Inode and offset into it that this I/O operates on. */ struct btrfs_inode *inode; u64 file_offset; union { - /* For data checksum verification. */ + /* + * Data checksumming and original I/O information for internal + * use in the btrfs_submit_bio machinery. + */ struct { u8 *csum; u8 csum_inline[BTRFS_BIO_INLINE_CSUM_SIZE]; + struct bvec_iter saved_iter; }; /* For metadata parentness verification. */ From 295fe46ff19b990c19b84b0d39a88d7e73710b19 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:14 +0100 Subject: [PATCH 221/265] btrfs: remove struct btrfs_bio::is_metadata flag This flag is unused now, so remove it. Re-expand the mirror_num field to 8 bits, and move it to the I/O completion internal section of the structure. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.h | 11 +---------- fs/btrfs/disk-io.c | 1 - 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index c232148348df..a96bcb3f36f6 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -30,16 +30,6 @@ typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); * passed to btrfs_submit_bio for mapping to the physical devices. */ struct btrfs_bio { - unsigned int mirror_num:7; - - /* - * Extra indicator for metadata bios. - * For some btrfs bios they use pages without a mapping, thus - * we can not rely on page->mapping->host to determine if - * it's a metadata bio. - */ - unsigned int is_metadata:1; - /* Inode and offset into it that this I/O operates on. */ struct btrfs_inode *inode; u64 file_offset; @@ -64,6 +54,7 @@ struct btrfs_bio { void *private; /* For internal use in read end I/O handling */ + unsigned int mirror_num; struct work_struct end_io_work; /* diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index dc5bf46529f0..cb8e495f981b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -846,7 +846,6 @@ void btrfs_submit_metadata_bio(struct btrfs_inode *inode, struct bio *bio, int m blk_status_t ret; bio->bi_opf |= REQ_META; - bbio->is_metadata = 1; if (btrfs_op(bio) != BTRFS_MAP_WRITE) { btrfs_submit_bio(fs_info, bio, mirror_num); From deb6216fa0b6b66304fc81e19b509af1b8203f98 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:15 +0100 Subject: [PATCH 222/265] btrfs: open code the submit_bio_start helpers The submit helpers are now trivial and can be called directly. Note that btree_csum_one_bio has to be moved up in the file a bit to avoid a forward declaration. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 4 ---- fs/btrfs/disk-io.c | 54 ++++++++++++++++++------------------------ fs/btrfs/disk-io.h | 1 - fs/btrfs/inode.c | 20 ---------------- 4 files changed, 23 insertions(+), 56 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 78c7979b8dca..ba5f023aaf55 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -408,10 +408,6 @@ static inline void btrfs_inode_split_flags(u64 inode_item_flags, void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num); void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type); -blk_status_t btrfs_submit_bio_start(struct btrfs_inode *inode, struct bio *bio); -blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, - struct bio *bio, - u64 dio_file_offset); int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, u32 pgoff, u8 *csum, const u8 * const csum_expected); bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index cb8e495f981b..764ee9338637 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -52,6 +52,7 @@ #include "relocation.h" #include "scrub.h" #include "super.h" +#include "file-item.h" #define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ BTRFS_HEADER_FLAG_RELOC |\ @@ -455,6 +456,24 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec return csum_one_extent_buffer(eb); } +static blk_status_t btree_csum_one_bio(struct bio *bio) +{ + struct bio_vec *bvec; + struct btrfs_root *root; + struct bvec_iter_all iter_all; + int ret = 0; + + ASSERT(!bio_flagged(bio, BIO_CLONED)); + bio_for_each_segment_all(bvec, bio, iter_all) { + root = BTRFS_I(bvec->bv_page->mapping->host)->root; + ret = csum_dirty_buffer(root->fs_info, bvec); + if (ret) + break; + } + + return errno_to_blk_status(ret); +} + static int check_tree_block_fsid(struct extent_buffer *eb) { struct btrfs_fs_info *fs_info = eb->fs_info; @@ -708,14 +727,14 @@ static void run_one_async_start(struct btrfs_work *work) async = container_of(work, struct async_submit_bio, work); switch (async->submit_cmd) { case WQ_SUBMIT_METADATA: - ret = btree_submit_bio_start(async->bio); + ret = btree_csum_one_bio(async->bio); break; case WQ_SUBMIT_DATA: - ret = btrfs_submit_bio_start(async->inode, async->bio); + ret = btrfs_csum_one_bio(async->inode, async->bio, (u64)-1, false); break; case WQ_SUBMIT_DATA_DIO: - ret = btrfs_submit_bio_start_direct_io(async->inode, - async->bio, async->dio_file_offset); + ret = btrfs_csum_one_bio(async->inode, async->bio, + async->dio_file_offset, false); break; default: /* Can't happen so return something that would prevent the IO. */ @@ -800,33 +819,6 @@ bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_ return true; } -static blk_status_t btree_csum_one_bio(struct bio *bio) -{ - struct bio_vec *bvec; - struct btrfs_root *root; - int ret = 0; - struct bvec_iter_all iter_all; - - ASSERT(!bio_flagged(bio, BIO_CLONED)); - bio_for_each_segment_all(bvec, bio, iter_all) { - root = BTRFS_I(bvec->bv_page->mapping->host)->root; - ret = csum_dirty_buffer(root->fs_info, bvec); - if (ret) - break; - } - - return errno_to_blk_status(ret); -} - -blk_status_t btree_submit_bio_start(struct bio *bio) -{ - /* - * when we're called for a write, we're already in the async - * submission context. Just jump into btrfs_submit_bio. - */ - return btree_csum_one_bio(bio); -} - static bool should_async_write(struct btrfs_fs_info *fs_info, struct btrfs_inode *bi) { diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index f2f295eb6103..5898beb64848 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -122,7 +122,6 @@ enum btrfs_wq_submit_cmd { bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, u64 dio_file_offset, enum btrfs_wq_submit_cmd cmd); -blk_status_t btree_submit_bio_start(struct bio *bio); int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, struct btrfs_root *root); int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 8e1d61b731ed..cdb0f8cb0d4f 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2532,19 +2532,6 @@ void btrfs_clear_delalloc_extent(struct btrfs_inode *inode, } } -/* - * in order to insert checksums into the metadata in large chunks, - * we wait until bio submission time. All the pages in the bio are - * checksummed and sums are attached onto the ordered extent record. - * - * At IO completion time the cums attached on the ordered extent record - * are inserted into the btree - */ -blk_status_t btrfs_submit_bio_start(struct btrfs_inode *inode, struct bio *bio) -{ - return btrfs_csum_one_bio(inode, bio, (u64)-1, false); -} - /* * Split an extent_map at [start, start + len] * @@ -7835,13 +7822,6 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) bio_endio(&dip->bio); } -blk_status_t btrfs_submit_bio_start_direct_io(struct btrfs_inode *inode, - struct bio *bio, - u64 dio_file_offset) -{ - return btrfs_csum_one_bio(inode, bio, dio_file_offset, false); -} - static void btrfs_end_dio_bio(struct btrfs_bio *bbio) { struct btrfs_dio_private *dip = bbio->private; From f8c44673e5a5f5131773d4a6974fb8ea4db033f8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:16 +0100 Subject: [PATCH 223/265] btrfs: simplify the btrfs_csum_one_bio calling convention To prepare for further bio submission changes btrfs_csum_one_bio should be able to take all it's arguments from the btrfs_bio structure. It can always use the bbio->inode already, and once the compression code is updated to set ->file_offset that one can be used unconditionally as well instead of looking at the page mapping now that btrfs doesn't allow ordered extents to span discontiguous data ranges. The only slightly tricky bit is the one_ordered flag set by the compressed writes. Replace that one with the driver private bio flag, which gets cleared before the bio is handed off to the block layer so that we don't get in the way of driver use. Note: this leaves an argument and a flag to btrfs_wq_submit_bio unused. But that whole mechanism will be removed in its current form in the next patch. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 3 +++ fs/btrfs/bio.h | 3 +++ fs/btrfs/compression.c | 6 ++++-- fs/btrfs/disk-io.c | 5 +---- fs/btrfs/file-item.c | 20 +++++--------------- fs/btrfs/file-item.h | 6 ++++-- fs/btrfs/inode.c | 4 ++-- 7 files changed, 22 insertions(+), 25 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index d75c11808482..6af0a788c542 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -438,6 +438,9 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror goto fail; } + /* Do not leak our private flag into the block layer. */ + bio->bi_opf &= ~REQ_BTRFS_ONE_ORDERED; + if (!bioc) { /* Single mirror read/write fast path */ bbio->mirror_num = mirror_num; diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index a96bcb3f36f6..4142004ce053 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -86,6 +86,9 @@ static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) bbio->end_io(bbio); } +/* Bio only refers to one ordered extent. */ +#define REQ_BTRFS_ONE_ORDERED REQ_DRV + void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num); int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 9c2d3f5b4028..7999c28aa5e1 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -357,7 +357,8 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, blk_status_t ret = BLK_STS_OK; int skip_sum = inode->flags & BTRFS_INODE_NODATASUM; const bool use_append = btrfs_use_zone_append(inode, disk_start); - const enum req_op bio_op = use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE; + const enum req_op bio_op = REQ_BTRFS_ONE_ORDERED | + (use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE); ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && IS_ALIGNED(len, fs_info->sectorsize)); @@ -395,6 +396,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, ret = errno_to_blk_status(PTR_ERR(bio)); break; } + btrfs_bio(bio)->file_offset = start; if (blkcg_css) bio->bi_opf |= REQ_CGROUP_PUNT; } @@ -436,7 +438,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, if (submit) { if (!skip_sum) { - ret = btrfs_csum_one_bio(inode, bio, start, true); + ret = btrfs_csum_one_bio(btrfs_bio(bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(bio), ret); break; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 764ee9338637..0270226389a7 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -730,11 +730,8 @@ static void run_one_async_start(struct btrfs_work *work) ret = btree_csum_one_bio(async->bio); break; case WQ_SUBMIT_DATA: - ret = btrfs_csum_one_bio(async->inode, async->bio, (u64)-1, false); - break; case WQ_SUBMIT_DATA_DIO: - ret = btrfs_csum_one_bio(async->inode, async->bio, - async->dio_file_offset, false); + ret = btrfs_csum_one_bio(btrfs_bio(async->bio)); break; default: /* Can't happen so return something that would prevent the IO. */ diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index c5324fe8f4be..41c77a100853 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -772,23 +772,16 @@ fail: /* * Calculate checksums of the data contained inside a bio. - * - * @inode: Owner of the data inside the bio - * @bio: Contains the data to be checksummed - * @offset: If (u64)-1, @bio may contain discontiguous bio vecs, so the - * file offsets are determined from the page offsets in the bio. - * Otherwise, this is the starting file offset of the bio vecs in - * @bio, which must be contiguous. - * @one_ordered: If true, @bio only refers to one ordered extent. */ -blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, - u64 offset, bool one_ordered) +blk_status_t btrfs_csum_one_bio(struct btrfs_bio *bbio) { + struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct bio *bio = &bbio->bio; + u64 offset = bbio->file_offset; struct btrfs_ordered_sum *sums; struct btrfs_ordered_extent *ordered = NULL; - const bool use_page_offsets = (offset == (u64)-1); char *data; struct bvec_iter iter; struct bio_vec bvec; @@ -816,9 +809,6 @@ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, shash->tfm = fs_info->csum_shash; bio_for_each_segment(bvec, bio, iter) { - if (use_page_offsets) - offset = page_offset(bvec.bv_page) + bvec.bv_offset; - if (!ordered) { ordered = btrfs_lookup_ordered_extent(inode, offset); /* @@ -840,7 +830,7 @@ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, - 1); for (i = 0; i < blockcount; i++) { - if (!one_ordered && + if (!(bio->bi_opf & REQ_BTRFS_ONE_ORDERED) && !in_range(offset, ordered->file_offset, ordered->num_bytes)) { unsigned long bytes_left; diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h index a2f9747adf3a..cd7f2ae515c0 100644 --- a/fs/btrfs/file-item.h +++ b/fs/btrfs/file-item.h @@ -49,8 +49,10 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_ordered_sum *sums); -blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, - u64 offset, bool one_ordered); +blk_status_t btrfs_csum_one_bio(struct btrfs_bio *bbio); +int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, + struct list_head *list, int search_commit, + bool nowait); int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end, struct list_head *list, int search_commit, bool nowait); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index cdb0f8cb0d4f..f542d539b831 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2736,7 +2736,7 @@ void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int btrfs_wq_submit_bio(inode, bio, mirror_num, 0, WQ_SUBMIT_DATA)) return; - ret = btrfs_csum_one_bio(inode, bio, (u64)-1, false); + ret = btrfs_csum_one_bio(btrfs_bio(bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(bio), ret); return; @@ -7863,7 +7863,7 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, * If we aren't doing async submit, calculate the csum of the * bio now. */ - ret = btrfs_csum_one_bio(inode, bio, file_offset, false); + ret = btrfs_csum_one_bio(btrfs_bio(bio)); if (ret) { btrfs_bio_end_io(btrfs_bio(bio), ret); return; From f8a53bb58ec7e2150f9b03f210675ba3e6d8b919 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:17 +0100 Subject: [PATCH 224/265] btrfs: handle checksum generation in the storage layer Instead of letting the callers of btrfs_submit_bio deal with checksumming the (meta)data in the bio and making decisions on when to offload the checksumming to the bio, leave that to btrfs_submit_bio. Do do so the existing btrfs_submit_bio function is split into an upper and a lower half, so that the lower half can be offloaded to a workqueue. Note that this changes the behavior for direct writes to raid56 volumes so that async checksum offloading is not skipped when more I/O is expected. This runs counter to the argument explaining why it was done, although I can't measure any affects of the change. Commits later in this series will make sure the entire direct writes is offloaded to the workqueue at once and thus make sure it is sent to the raid56 code from a single thread. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 202 ++++++++++++++++++++++++++++++++++++----- fs/btrfs/compression.c | 9 -- fs/btrfs/disk-io.c | 155 +------------------------------ fs/btrfs/disk-io.h | 9 +- fs/btrfs/inode.c | 67 +------------- 5 files changed, 182 insertions(+), 260 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 6af0a788c542..0856440d3adf 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -401,6 +401,166 @@ static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio); } +static void __btrfs_submit_bio(struct bio *bio, struct btrfs_io_context *bioc, + struct btrfs_io_stripe *smap, int mirror_num) +{ + /* Do not leak our private flag into the block layer. */ + bio->bi_opf &= ~REQ_BTRFS_ONE_ORDERED; + + if (!bioc) { + /* Single mirror read/write fast path. */ + btrfs_bio(bio)->mirror_num = mirror_num; + bio->bi_iter.bi_sector = smap->physical >> SECTOR_SHIFT; + bio->bi_private = smap->dev; + bio->bi_end_io = btrfs_simple_end_io; + btrfs_submit_dev_bio(smap->dev, bio); + } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + /* Parity RAID write or read recovery. */ + bio->bi_private = bioc; + bio->bi_end_io = btrfs_raid56_end_io; + if (bio_op(bio) == REQ_OP_READ) + raid56_parity_recover(bio, bioc, mirror_num); + else + raid56_parity_write(bio, bioc); + } else { + /* Write to multiple mirrors. */ + int total_devs = bioc->num_stripes; + + bioc->orig_bio = bio; + for (int dev_nr = 0; dev_nr < total_devs; dev_nr++) + btrfs_submit_mirrored_bio(bioc, dev_nr); + } +} + +static blk_status_t btrfs_bio_csum(struct btrfs_bio *bbio) +{ + if (bbio->bio.bi_opf & REQ_META) + return btree_csum_one_bio(&bbio->bio); + return btrfs_csum_one_bio(bbio); +} + +/* + * Async submit bios are used to offload expensive checksumming onto the worker + * threads. + */ +struct async_submit_bio { + struct btrfs_bio *bbio; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe smap; + int mirror_num; + struct btrfs_work work; +}; + +/* + * In order to insert checksums into the metadata in large chunks, we wait + * until bio submission time. All the pages in the bio are checksummed and + * sums are attached onto the ordered extent record. + * + * At IO completion time the csums attached on the ordered extent record are + * inserted into the btree. + */ +static void run_one_async_start(struct btrfs_work *work) +{ + struct async_submit_bio *async = + container_of(work, struct async_submit_bio, work); + blk_status_t ret; + + ret = btrfs_bio_csum(async->bbio); + if (ret) + async->bbio->bio.bi_status = ret; +} + +/* + * In order to insert checksums into the metadata in large chunks, we wait + * until bio submission time. All the pages in the bio are checksummed and + * sums are attached onto the ordered extent record. + * + * At IO completion time the csums attached on the ordered extent record are + * inserted into the tree. + */ +static void run_one_async_done(struct btrfs_work *work) +{ + struct async_submit_bio *async = + container_of(work, struct async_submit_bio, work); + struct bio *bio = &async->bbio->bio; + + /* If an error occurred we just want to clean up the bio and move on. */ + if (bio->bi_status) { + btrfs_bio_end_io(async->bbio, bio->bi_status); + return; + } + + /* + * All of the bios that pass through here are from async helpers. + * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context. + * This changes nothing when cgroups aren't in use. + */ + bio->bi_opf |= REQ_CGROUP_PUNT; + __btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num); +} + +static void run_one_async_free(struct btrfs_work *work) +{ + kfree(container_of(work, struct async_submit_bio, work)); +} + +static bool should_async_write(struct btrfs_bio *bbio) +{ + /* + * If the I/O is not issued by fsync and friends, (->sync_writers != 0), + * then try to defer the submission to a workqueue to parallelize the + * checksum calculation. + */ + if (atomic_read(&bbio->inode->sync_writers)) + return false; + + /* + * Submit metadata writes synchronously if the checksum implementation + * is fast, or we are on a zoned device that wants I/O to be submitted + * in order. + */ + if (bbio->bio.bi_opf & REQ_META) { + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + + if (btrfs_is_zoned(fs_info)) + return false; + if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) + return false; + } + + return true; +} + +/* + * Submit bio to an async queue. + * + * Return true if the work has been succesfuly submitted, else false. + */ +static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, + struct btrfs_io_context *bioc, + struct btrfs_io_stripe *smap, int mirror_num) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct async_submit_bio *async; + + async = kmalloc(sizeof(*async), GFP_NOFS); + if (!async) + return false; + + async->bbio = bbio; + async->bioc = bioc; + async->smap = *smap; + async->mirror_num = mirror_num; + + btrfs_init_work(&async->work, run_one_async_start, run_one_async_done, + run_one_async_free); + if (op_is_sync(bbio->bio.bi_opf)) + btrfs_queue_work(fs_info->hipri_workers, &async->work); + else + btrfs_queue_work(fs_info->workers, &async->work); + return true; +} + void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) { struct btrfs_bio *bbio = btrfs_bio(bio); @@ -438,33 +598,25 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror goto fail; } - /* Do not leak our private flag into the block layer. */ - bio->bi_opf &= ~REQ_BTRFS_ONE_ORDERED; + if (btrfs_op(bio) == BTRFS_MAP_WRITE) { + /* + * Csum items for reloc roots have already been cloned at this + * point, so they are handled as part of the no-checksum case. + */ + if (!(bbio->inode->flags & BTRFS_INODE_NODATASUM) && + !test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state) && + !btrfs_is_data_reloc_root(bbio->inode->root)) { + if (should_async_write(bbio) && + btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num)) + return; - if (!bioc) { - /* Single mirror read/write fast path */ - bbio->mirror_num = mirror_num; - bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; - bio->bi_private = smap.dev; - bio->bi_end_io = btrfs_simple_end_io; - btrfs_submit_dev_bio(smap.dev, bio); - } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { - /* Parity RAID write or read recovery */ - bio->bi_private = bioc; - bio->bi_end_io = btrfs_raid56_end_io; - if (bio_op(bio) == REQ_OP_READ) - raid56_parity_recover(bio, bioc, mirror_num); - else - raid56_parity_write(bio, bioc); - } else { - /* Write to multiple mirrors */ - int total_devs = bioc->num_stripes; - int dev_nr; - - bioc->orig_bio = bio; - for (dev_nr = 0; dev_nr < total_devs; dev_nr++) - btrfs_submit_mirrored_bio(bioc, dev_nr); + ret = btrfs_bio_csum(bbio); + if (ret) + goto fail; + } } + + __btrfs_submit_bio(bio, bioc, &smap, mirror_num); return; fail: diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 7999c28aa5e1..d6a8fcd46044 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -355,7 +355,6 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, u64 cur_disk_bytenr = disk_start; u64 next_stripe_start; blk_status_t ret = BLK_STS_OK; - int skip_sum = inode->flags & BTRFS_INODE_NODATASUM; const bool use_append = btrfs_use_zone_append(inode, disk_start); const enum req_op bio_op = REQ_BTRFS_ONE_ORDERED | (use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE); @@ -437,14 +436,6 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, submit = true; if (submit) { - if (!skip_sum) { - ret = btrfs_csum_one_bio(btrfs_bio(bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - break; - } - } - ASSERT(bio->bi_iter.bi_size); btrfs_submit_bio(fs_info, bio, 0); bio = NULL; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 0270226389a7..6278530d5986 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -52,7 +52,6 @@ #include "relocation.h" #include "scrub.h" #include "super.h" -#include "file-item.h" #define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ BTRFS_HEADER_FLAG_RELOC |\ @@ -79,23 +78,6 @@ static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info) crypto_free_shash(fs_info->csum_shash); } -/* - * async submit bios are used to offload expensive checksumming - * onto the worker threads. They checksum file and metadata bios - * just before they are sent down the IO stack. - */ -struct async_submit_bio { - struct btrfs_inode *inode; - struct bio *bio; - enum btrfs_wq_submit_cmd submit_cmd; - int mirror_num; - - /* Optional parameter for used by direct io */ - u64 dio_file_offset; - struct btrfs_work work; - blk_status_t status; -}; - /* * Compute the csum of a btree block and store the result to provided buffer. */ @@ -456,7 +438,7 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec return csum_one_extent_buffer(eb); } -static blk_status_t btree_csum_one_bio(struct bio *bio) +blk_status_t btree_csum_one_bio(struct bio *bio) { struct bio_vec *bvec; struct btrfs_root *root; @@ -719,143 +701,10 @@ err: return ret; } -static void run_one_async_start(struct btrfs_work *work) -{ - struct async_submit_bio *async; - blk_status_t ret; - - async = container_of(work, struct async_submit_bio, work); - switch (async->submit_cmd) { - case WQ_SUBMIT_METADATA: - ret = btree_csum_one_bio(async->bio); - break; - case WQ_SUBMIT_DATA: - case WQ_SUBMIT_DATA_DIO: - ret = btrfs_csum_one_bio(btrfs_bio(async->bio)); - break; - default: - /* Can't happen so return something that would prevent the IO. */ - ret = BLK_STS_IOERR; - ASSERT(0); - } - if (ret) - async->status = ret; -} - -/* - * In order to insert checksums into the metadata in large chunks, we wait - * until bio submission time. All the pages in the bio are checksummed and - * sums are attached onto the ordered extent record. - * - * At IO completion time the csums attached on the ordered extent record are - * inserted into the tree. - */ -static void run_one_async_done(struct btrfs_work *work) -{ - struct async_submit_bio *async = - container_of(work, struct async_submit_bio, work); - struct btrfs_inode *inode = async->inode; - struct btrfs_bio *bbio = btrfs_bio(async->bio); - - /* If an error occurred we just want to clean up the bio and move on */ - if (async->status) { - btrfs_bio_end_io(bbio, async->status); - return; - } - - /* - * All of the bios that pass through here are from async helpers. - * Use REQ_CGROUP_PUNT to issue them from the owning cgroup's context. - * This changes nothing when cgroups aren't in use. - */ - async->bio->bi_opf |= REQ_CGROUP_PUNT; - btrfs_submit_bio(inode->root->fs_info, async->bio, async->mirror_num); -} - -static void run_one_async_free(struct btrfs_work *work) -{ - struct async_submit_bio *async; - - async = container_of(work, struct async_submit_bio, work); - kfree(async); -} - -/* - * Submit bio to an async queue. - * - * Return: - * - true if the work has been successfully submitted - * - false in case of error - */ -bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, - u64 dio_file_offset, enum btrfs_wq_submit_cmd cmd) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct async_submit_bio *async; - - async = kmalloc(sizeof(*async), GFP_NOFS); - if (!async) - return false; - - async->inode = inode; - async->bio = bio; - async->mirror_num = mirror_num; - async->submit_cmd = cmd; - - btrfs_init_work(&async->work, run_one_async_start, run_one_async_done, - run_one_async_free); - - async->dio_file_offset = dio_file_offset; - - async->status = 0; - - if (op_is_sync(bio->bi_opf)) - btrfs_queue_work(fs_info->hipri_workers, &async->work); - else - btrfs_queue_work(fs_info->workers, &async->work); - return true; -} - -static bool should_async_write(struct btrfs_fs_info *fs_info, - struct btrfs_inode *bi) -{ - if (btrfs_is_zoned(fs_info)) - return false; - if (atomic_read(&bi->sync_writers)) - return false; - if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) - return false; - return true; -} - void btrfs_submit_metadata_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_bio *bbio = btrfs_bio(bio); - blk_status_t ret; - bio->bi_opf |= REQ_META; - - if (btrfs_op(bio) != BTRFS_MAP_WRITE) { - btrfs_submit_bio(fs_info, bio, mirror_num); - return; - } - - /* - * Kthread helpers are used to submit writes so that checksumming can - * happen in parallel across all CPUs. - */ - if (should_async_write(fs_info, inode) && - btrfs_wq_submit_bio(inode, bio, mirror_num, 0, WQ_SUBMIT_METADATA)) - return; - - ret = btree_csum_one_bio(bio); - if (ret) { - btrfs_bio_end_io(bbio, ret); - return; - } - - btrfs_submit_bio(fs_info, bio, mirror_num); + btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); } #ifdef CONFIG_MIGRATION diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 5898beb64848..ac55f8ec3a31 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -114,14 +114,7 @@ int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, int btrfs_read_extent_buffer(struct extent_buffer *buf, struct btrfs_tree_parent_check *check); -enum btrfs_wq_submit_cmd { - WQ_SUBMIT_METADATA, - WQ_SUBMIT_DATA, - WQ_SUBMIT_DATA_DIO, -}; - -bool btrfs_wq_submit_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, - u64 dio_file_offset, enum btrfs_wq_submit_cmd cmd); +blk_status_t btree_csum_one_bio(struct bio *bio); int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, struct btrfs_root *root); int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index f542d539b831..b9cd088ded82 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2721,27 +2721,6 @@ void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int } } - /* - * If we need to checksum, and the I/O is not issued by fsync and - * friends, that is ->sync_writers != 0, defer the submission to a - * workqueue to parallelize it. - * - * Csum items for reloc roots have already been cloned at this point, - * so they are handled as part of the no-checksum case. - */ - if (!(inode->flags & BTRFS_INODE_NODATASUM) && - !test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state) && - !btrfs_is_data_reloc_root(inode->root)) { - if (!atomic_read(&inode->sync_writers) && - btrfs_wq_submit_bio(inode, bio, mirror_num, 0, WQ_SUBMIT_DATA)) - return; - - ret = btrfs_csum_one_bio(btrfs_bio(bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - return; - } - } btrfs_submit_bio(fs_info, bio, mirror_num); } @@ -7843,36 +7822,6 @@ static void btrfs_end_dio_bio(struct btrfs_bio *bbio) btrfs_dio_private_put(dip); } -static void btrfs_submit_dio_bio(struct bio *bio, struct btrfs_inode *inode, - u64 file_offset, int async_submit) -{ - struct btrfs_fs_info *fs_info = inode->root->fs_info; - blk_status_t ret; - - if (inode->flags & BTRFS_INODE_NODATASUM) - goto map; - - if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - /* Check btrfs_submit_data_write_bio() for async submit rules */ - if (async_submit && !atomic_read(&inode->sync_writers) && - btrfs_wq_submit_bio(inode, bio, 0, file_offset, - WQ_SUBMIT_DATA_DIO)) - return; - - /* - * If we aren't doing async submit, calculate the csum of the - * bio now. - */ - ret = btrfs_csum_one_bio(btrfs_bio(bio)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - return; - } - } -map: - btrfs_submit_bio(fs_info, bio, 0); -} - static void btrfs_submit_direct(const struct iomap_iter *iter, struct bio *dio_bio, loff_t file_offset) { @@ -7880,11 +7829,8 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, container_of(dio_bio, struct btrfs_dio_private, bio); struct inode *inode = iter->inode; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - const bool raid56 = (btrfs_data_alloc_profile(fs_info) & - BTRFS_BLOCK_GROUP_RAID56_MASK); struct bio *bio; u64 start_sector; - int async_submit = 0; u64 submit_len; u64 clone_offset = 0; u64 clone_len; @@ -7951,19 +7897,10 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, * We transfer the initial reference to the last bio, so we * don't need to increment the reference count for the last one. */ - if (submit_len > 0) { + if (submit_len > 0) refcount_inc(&dip->refs); - /* - * If we are submitting more than one bio, submit them - * all asynchronously. The exception is RAID 5 or 6, as - * asynchronous checksums make it difficult to collect - * full stripe writes. - */ - if (!raid56) - async_submit = 1; - } - btrfs_submit_dio_bio(bio, BTRFS_I(inode), file_offset, async_submit); + btrfs_submit_bio(fs_info, bio, 0); dio_data->submitted += clone_len; clone_offset += clone_len; From 69ccf3f4244abc5f6d73ca5d8caf6b42a1db42c6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:18 +0100 Subject: [PATCH 225/265] btrfs: handle recording of zoned writes in the storage layer Move the code that splits the ordered extents and records the physical location for them to the storage layer so that the higher level consumers don't have to care about physical block numbers at all. This will also allow to eventually remove accounting for the zone append write sizes in the upper layer with a little bit more block layer work. Reviewed-by: Naohiro Aota Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 8 ++++++++ fs/btrfs/btrfs_inode.h | 1 + fs/btrfs/compression.c | 1 - fs/btrfs/extent_io.c | 6 ------ fs/btrfs/inode.c | 37 +++++++------------------------------ fs/btrfs/zoned.c | 13 +++++-------- fs/btrfs/zoned.h | 6 ++---- 7 files changed, 23 insertions(+), 49 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 0856440d3adf..c49a4d7d4a57 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -281,6 +281,8 @@ static void btrfs_simple_end_io(struct bio *bio) INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work); queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); } else { + if (bio_op(bio) == REQ_OP_ZONE_APPEND) + btrfs_record_physical_zoned(bbio); bbio->end_io(bbio); } } @@ -599,6 +601,12 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + ret = btrfs_extract_ordered_extent(btrfs_bio(bio)); + if (ret) + goto fail; + } + /* * Csum items for reloc roots have already been cloned at this * point, so they are handled as part of the no-checksum case. diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index ba5f023aaf55..b83b731c63e1 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -410,6 +410,7 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type); int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, u32 pgoff, u8 *csum, const u8 * const csum_expected); +blk_status_t btrfs_extract_ordered_extent(struct btrfs_bio *bbio); bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, u32 bio_offset, struct bio_vec *bv); noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index d6a8fcd46044..301a83960bce 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -273,7 +273,6 @@ static void end_compressed_bio_write(struct btrfs_bio *bbio) if (refcount_dec_and_test(&cb->pending_ios)) { struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - btrfs_record_physical_zoned(cb->inode, cb->start, &bbio->bio); queue_work(fs_info->compressed_write_workers, &cb->write_end_work); } bio_put(&bbio->bio); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 41bcc0bfe56a..2983fe2e10fe 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -586,7 +586,6 @@ static void end_bio_extent_writepage(struct btrfs_bio *bbio) u64 start; u64 end; struct bvec_iter_all iter_all; - bool first_bvec = true; ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { @@ -608,11 +607,6 @@ static void end_bio_extent_writepage(struct btrfs_bio *bbio) start = page_offset(page) + bvec->bv_offset; end = start + bvec->bv_len - 1; - if (first_bvec) { - btrfs_record_physical_zoned(inode, start, bio); - first_bvec = false; - } - end_extent_writepage(page, error, start, end); btrfs_page_clear_writeback(fs_info, page, start, bvec->bv_len); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index b9cd088ded82..90e3fd7e10ea 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2647,19 +2647,19 @@ out: return ret; } -static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, - struct bio *bio, loff_t file_offset) +blk_status_t btrfs_extract_ordered_extent(struct btrfs_bio *bbio) { + u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + u64 len = bbio->bio.bi_iter.bi_size; + struct btrfs_inode *inode = bbio->inode; struct btrfs_ordered_extent *ordered; - u64 start = (u64)bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 file_len; - u64 len = bio->bi_iter.bi_size; u64 end = start + len; u64 ordered_end; u64 pre, post; int ret = 0; - ordered = btrfs_lookup_ordered_extent(inode, file_offset); + ordered = btrfs_lookup_ordered_extent(inode, bbio->file_offset); if (WARN_ON_ONCE(!ordered)) return BLK_STS_IOERR; @@ -2699,7 +2699,7 @@ static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, ret = btrfs_split_ordered_extent(ordered, pre, post); if (ret) goto out; - ret = split_zoned_em(inode, file_offset, file_len, pre, post); + ret = split_zoned_em(inode, bbio->file_offset, file_len, pre, post); out: btrfs_put_ordered_extent(ordered); @@ -2709,19 +2709,7 @@ out: void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; - blk_status_t ret; - - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - ret = extract_ordered_extent(inode, bio, - page_offset(bio_first_bvec_all(bio)->bv_page)); - if (ret) { - btrfs_bio_end_io(btrfs_bio(bio), ret); - return; - } - } - - btrfs_submit_bio(fs_info, bio, mirror_num); + btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); } void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, @@ -7816,8 +7804,6 @@ static void btrfs_end_dio_bio(struct btrfs_bio *bbio) dip->bio.bi_status = err; } - btrfs_record_physical_zoned(&dip->inode->vfs_inode, bbio->file_offset, bio); - bio_put(bio); btrfs_dio_private_put(dip); } @@ -7876,15 +7862,6 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, dip); btrfs_bio(bio)->file_offset = file_offset; - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - status = extract_ordered_extent(BTRFS_I(inode), bio, - file_offset); - if (status) { - bio_put(bio); - goto out_err; - } - } - ASSERT(submit_len >= clone_len); submit_len -= clone_len; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index d46701a77b17..5bf67c3c9f84 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -17,6 +17,7 @@ #include "space-info.h" #include "fs.h" #include "accessors.h" +#include "bio.h" /* Maximum number of zones to report per blkdev_report_zones() call */ #define BTRFS_REPORT_NR_ZONES 4096 @@ -1660,21 +1661,17 @@ bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start) return ret; } -void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, - struct bio *bio) +void btrfs_record_physical_zoned(struct btrfs_bio *bbio) { + const u64 physical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; struct btrfs_ordered_extent *ordered; - const u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; - if (bio_op(bio) != REQ_OP_ZONE_APPEND) - return; - - ordered = btrfs_lookup_ordered_extent(BTRFS_I(inode), file_offset); + ordered = btrfs_lookup_ordered_extent(bbio->inode, bbio->file_offset); if (WARN_ON(!ordered)) return; ordered->physical = physical; - ordered->bdev = bio->bi_bdev; + ordered->bdev = bbio->bio.bi_bdev; btrfs_put_ordered_extent(ordered); } diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index f43990985d80..bc93a740e7cf 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -57,8 +57,7 @@ void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb); void btrfs_free_redirty_list(struct btrfs_transaction *trans); bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start); -void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, - struct bio *bio); +void btrfs_record_physical_zoned(struct btrfs_bio *bbio); void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered); bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, struct extent_buffer *eb, @@ -190,8 +189,7 @@ static inline bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start) return false; } -static inline void btrfs_record_physical_zoned(struct inode *inode, - u64 file_offset, struct bio *bio) +static inline void btrfs_record_physical_zoned(struct btrfs_bio *bbio) { } From 542e300e8398ea529a4a20a125c77d234d5ca37e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:19 +0100 Subject: [PATCH 226/265] btrfs: support cloned bios in btree_csum_one_bio To allow splitting bios in btrfs_submit_bio, btree_csum_one_bio needs to be able to handle cloned bios. As btree_csum_one_bio is always called before handing the bio to the block layer that is trivially done by using bio_for_each_segment instead of bio_for_each_segment_all. Also switch the function to take a btrfs_bio and use that to derive the fs_info. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 2 +- fs/btrfs/disk-io.c | 14 ++++++-------- fs/btrfs/disk-io.h | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index c49a4d7d4a57..27e46757fdd8 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -437,7 +437,7 @@ static void __btrfs_submit_bio(struct bio *bio, struct btrfs_io_context *bioc, static blk_status_t btrfs_bio_csum(struct btrfs_bio *bbio) { if (bbio->bio.bi_opf & REQ_META) - return btree_csum_one_bio(&bbio->bio); + return btree_csum_one_bio(bbio); return btrfs_csum_one_bio(bbio); } diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 6278530d5986..2ae329b5ce98 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -438,17 +438,15 @@ static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec return csum_one_extent_buffer(eb); } -blk_status_t btree_csum_one_bio(struct bio *bio) +blk_status_t btree_csum_one_bio(struct btrfs_bio *bbio) { - struct bio_vec *bvec; - struct btrfs_root *root; - struct bvec_iter_all iter_all; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct bvec_iter iter; + struct bio_vec bv; int ret = 0; - ASSERT(!bio_flagged(bio, BIO_CLONED)); - bio_for_each_segment_all(bvec, bio, iter_all) { - root = BTRFS_I(bvec->bv_page->mapping->host)->root; - ret = csum_dirty_buffer(root->fs_info, bvec); + bio_for_each_segment(bv, &bbio->bio, iter) { + ret = csum_dirty_buffer(fs_info, &bv); if (ret) break; } diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index ac55f8ec3a31..f2dd4c6d9c25 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -114,7 +114,7 @@ int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, int btrfs_read_extent_buffer(struct extent_buffer *buf, struct btrfs_tree_parent_check *check); -blk_status_t btree_csum_one_bio(struct bio *bio); +blk_status_t btree_csum_one_bio(struct btrfs_bio *bbio); int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, struct btrfs_root *root); int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, From 852eee62d31abd695cd43e1b875d664ed292a8ca Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:20 +0100 Subject: [PATCH 227/265] btrfs: allow btrfs_submit_bio to split bios Currently the I/O submitters have to split bios according to the chunk stripe boundaries. This leads to extra lookups in the extent trees and a lot of boilerplate code. To drop this requirement, split the bio when __btrfs_map_block returns a mapping that is smaller than the requested size and keep a count of pending bios in the original btrfs_bio so that the upper level completion is only invoked when all clones have completed. Based on a patch from Qu Wenruo. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Reviewed-by: Qu Wenruo Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 110 +++++++++++++++++++++++++++++++++++++++++-------- fs/btrfs/bio.h | 1 + 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 27e46757fdd8..521a909292a0 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -17,6 +17,7 @@ #include "file-item.h" static struct bio_set btrfs_bioset; +static struct bio_set btrfs_clone_bioset; static struct bio_set btrfs_repair_bioset; static mempool_t btrfs_failed_bio_pool; @@ -38,6 +39,7 @@ static inline void btrfs_bio_init(struct btrfs_bio *bbio, bbio->inode = inode; bbio->end_io = end_io; bbio->private = private; + atomic_set(&bbio->pending_ios, 1); } /* @@ -75,6 +77,59 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, return bio; } +static struct bio *btrfs_split_bio(struct bio *orig, u64 map_length) +{ + struct btrfs_bio *orig_bbio = btrfs_bio(orig); + struct bio *bio; + + bio = bio_split(orig, map_length >> SECTOR_SHIFT, GFP_NOFS, + &btrfs_clone_bioset); + btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, NULL, orig_bbio); + + btrfs_bio(bio)->file_offset = orig_bbio->file_offset; + if (!(orig->bi_opf & REQ_BTRFS_ONE_ORDERED)) + orig_bbio->file_offset += map_length; + + atomic_inc(&orig_bbio->pending_ios); + return bio; +} + +static void btrfs_orig_write_end_io(struct bio *bio); + +static void btrfs_bbio_propagate_error(struct btrfs_bio *bbio, + struct btrfs_bio *orig_bbio) +{ + /* + * For writes we tolerate nr_mirrors - 1 write failures, so we can't + * just blindly propagate a write failure here. Instead increment the + * error count in the original I/O context so that it is guaranteed to + * be larger than the error tolerance. + */ + if (bbio->bio.bi_end_io == &btrfs_orig_write_end_io) { + struct btrfs_io_stripe *orig_stripe = orig_bbio->bio.bi_private; + struct btrfs_io_context *orig_bioc = orig_stripe->bioc; + + atomic_add(orig_bioc->max_errors, &orig_bioc->error); + } else { + orig_bbio->bio.bi_status = bbio->bio.bi_status; + } +} + +static void btrfs_orig_bbio_end_io(struct btrfs_bio *bbio) +{ + if (bbio->bio.bi_pool == &btrfs_clone_bioset) { + struct btrfs_bio *orig_bbio = bbio->private; + + if (bbio->bio.bi_status) + btrfs_bbio_propagate_error(bbio, orig_bbio); + bio_put(&bbio->bio); + bbio = orig_bbio; + } + + if (atomic_dec_and_test(&bbio->pending_ios)) + bbio->end_io(bbio); +} + static int next_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror) { if (cur_mirror == fbio->num_copies) @@ -92,7 +147,7 @@ static int prev_repair_mirror(struct btrfs_failed_bio *fbio, int cur_mirror) static void btrfs_repair_done(struct btrfs_failed_bio *fbio) { if (atomic_dec_and_test(&fbio->repair_count)) { - fbio->bbio->end_io(fbio->bbio); + btrfs_orig_bbio_end_io(fbio->bbio); mempool_free(fbio, &btrfs_failed_bio_pool); } } @@ -229,7 +284,7 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de if (fbio) btrfs_repair_done(fbio); else - bbio->end_io(bbio); + btrfs_orig_bbio_end_io(bbio); } static void btrfs_log_dev_io_error(struct bio *bio, struct btrfs_device *dev) @@ -283,7 +338,7 @@ static void btrfs_simple_end_io(struct bio *bio) } else { if (bio_op(bio) == REQ_OP_ZONE_APPEND) btrfs_record_physical_zoned(bbio); - bbio->end_io(bbio); + btrfs_orig_bbio_end_io(bbio); } } @@ -297,7 +352,7 @@ static void btrfs_raid56_end_io(struct bio *bio) if (bio_op(bio) == REQ_OP_READ && !(bbio->bio.bi_opf & REQ_META)) btrfs_check_read_bio(bbio, NULL); else - bbio->end_io(bbio); + btrfs_orig_bbio_end_io(bbio); btrfs_put_bioc(bioc); } @@ -324,7 +379,7 @@ static void btrfs_orig_write_end_io(struct bio *bio) else bio->bi_status = BLK_STS_OK; - bbio->end_io(bbio); + btrfs_orig_bbio_end_io(bbio); btrfs_put_bioc(bioc); } @@ -488,7 +543,7 @@ static void run_one_async_done(struct btrfs_work *work) /* If an error occurred we just want to clean up the bio and move on. */ if (bio->bi_status) { - btrfs_bio_end_io(async->bbio, bio->bi_status); + btrfs_orig_bbio_end_io(async->bbio); return; } @@ -563,9 +618,11 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, return true; } -void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) +static bool btrfs_submit_chunk(struct btrfs_fs_info *fs_info, struct bio *bio, + int mirror_num) { struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_bio *orig_bbio = bbio; u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; u64 map_length = length; @@ -582,11 +639,10 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror goto fail; } + map_length = min(map_length, length); if (map_length < length) { - btrfs_crit(fs_info, - "mapping failed logical %llu bio len %llu len %llu", - logical, length, map_length); - BUG(); + bio = btrfs_split_bio(bio, map_length); + bbio = btrfs_bio(bio); } /* @@ -597,14 +653,14 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror bbio->saved_iter = bio->bi_iter; ret = btrfs_lookup_bio_sums(bbio); if (ret) - goto fail; + goto fail_put_bio; } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { if (bio_op(bio) == REQ_OP_ZONE_APPEND) { ret = btrfs_extract_ordered_extent(btrfs_bio(bio)); if (ret) - goto fail; + goto fail_put_bio; } /* @@ -616,20 +672,32 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror !btrfs_is_data_reloc_root(bbio->inode->root)) { if (should_async_write(bbio) && btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num)) - return; + goto done; ret = btrfs_bio_csum(bbio); if (ret) - goto fail; + goto fail_put_bio; } } __btrfs_submit_bio(bio, bioc, &smap, mirror_num); - return; +done: + return map_length == length; +fail_put_bio: + if (map_length < length) + bio_put(bio); fail: btrfs_bio_counter_dec(fs_info); - btrfs_bio_end_io(bbio, ret); + btrfs_bio_end_io(orig_bbio, ret); + /* Do not submit another chunk */ + return true; +} + +void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) +{ + while (!btrfs_submit_chunk(fs_info, bio, mirror_num)) + ; } /* @@ -737,10 +805,13 @@ int __init btrfs_bioset_init(void) offsetof(struct btrfs_bio, bio), BIOSET_NEED_BVECS)) return -ENOMEM; + if (bioset_init(&btrfs_clone_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), 0)) + goto out_free_bioset; if (bioset_init(&btrfs_repair_bioset, BIO_POOL_SIZE, offsetof(struct btrfs_bio, bio), BIOSET_NEED_BVECS)) - goto out_free_bioset; + goto out_free_clone_bioset; if (mempool_init_kmalloc_pool(&btrfs_failed_bio_pool, BIO_POOL_SIZE, sizeof(struct btrfs_failed_bio))) goto out_free_repair_bioset; @@ -748,6 +819,8 @@ int __init btrfs_bioset_init(void) out_free_repair_bioset: bioset_exit(&btrfs_repair_bioset); +out_free_clone_bioset: + bioset_exit(&btrfs_clone_bioset); out_free_bioset: bioset_exit(&btrfs_bioset); return -ENOMEM; @@ -757,5 +830,6 @@ void __cold btrfs_bioset_exit(void) { mempool_exit(&btrfs_failed_bio_pool); bioset_exit(&btrfs_repair_bioset); + bioset_exit(&btrfs_clone_bioset); bioset_exit(&btrfs_bioset); } diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 4142004ce053..67d416cf293d 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -55,6 +55,7 @@ struct btrfs_bio { /* For internal use in read end I/O handling */ unsigned int mirror_num; + atomic_t pending_ios; struct work_struct end_io_work; /* From 67d66982509043962cf15457051e1b840578a323 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:21 +0100 Subject: [PATCH 228/265] btrfs: pass the iomap bio to btrfs_submit_bio Now that btrfs_submit_bio splits the bio when crossing stripe boundaries, there is no need for the higher level code to do that manually. For direct I/O this is really helpful, as btrfs_submit_io can now simply take the bio allocated by iomap and send it on to btrfs_submit_bio instead of allocating clones. For that to work, the bio embedded into struct btrfs_dio_private needs to become a full btrfs_bio as expected by btrfs_submit_bio. With this change there is a single work item to offload the entire iomap bio so the heuristics to skip async processing for bios that were split isn't needed anymore either. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 22 +----- fs/btrfs/bio.h | 6 +- fs/btrfs/inode.c | 175 +++++++++++------------------------------------ 3 files changed, 43 insertions(+), 160 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 521a909292a0..de8634cf7bd8 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -31,9 +31,8 @@ struct btrfs_failed_bio { * Initialize a btrfs_bio structure. This skips the embedded bio itself as it * is already initialized by the block layer. */ -static inline void btrfs_bio_init(struct btrfs_bio *bbio, - struct btrfs_inode *inode, - btrfs_bio_end_io_t end_io, void *private) +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, + btrfs_bio_end_io_t end_io, void *private) { memset(bbio, 0, offsetof(struct btrfs_bio, bio)); bbio->inode = inode; @@ -60,23 +59,6 @@ struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, return bio; } -struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, - struct btrfs_inode *inode, - btrfs_bio_end_io_t end_io, void *private) -{ - struct bio *bio; - struct btrfs_bio *bbio; - - ASSERT(offset <= UINT_MAX && size <= UINT_MAX); - - bio = bio_alloc_clone(orig->bi_bdev, orig, GFP_NOFS, &btrfs_bioset); - bbio = btrfs_bio(bio); - btrfs_bio_init(bbio, inode, end_io, private); - - bio_trim(bio, offset >> 9, size >> 9); - return bio; -} - static struct bio *btrfs_split_bio(struct bio *orig, u64 map_length) { struct btrfs_bio *orig_bbio = btrfs_bio(orig); diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 67d416cf293d..a043c2ced5bd 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -73,13 +73,11 @@ static inline struct btrfs_bio *btrfs_bio(struct bio *bio) int __init btrfs_bioset_init(void); void __cold btrfs_bioset_exit(void); +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, + btrfs_bio_end_io_t end_io, void *private); struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, struct btrfs_inode *inode, btrfs_bio_end_io_t end_io, void *private); -struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, - struct btrfs_inode *inode, - btrfs_bio_end_io_t end_io, void *private); - static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) { diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 90e3fd7e10ea..4ac9b34ad377 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -84,24 +84,12 @@ struct btrfs_dio_data { }; struct btrfs_dio_private { - struct btrfs_inode *inode; - - /* - * Since DIO can use anonymous page, we cannot use page_offset() to - * grab the file offset, thus need a dedicated member for file offset. - */ + /* Range of I/O */ u64 file_offset; - /* Used for bio::bi_size */ u32 bytes; - /* - * References to this structure. There is one reference per in-flight - * bio plus one while we're still setting up. - */ - refcount_t refs; - /* This must be last */ - struct bio bio; + struct btrfs_bio bbio; }; static struct bio_set btrfs_dio_bioset; @@ -7767,132 +7755,47 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, return ret; } -static void btrfs_dio_private_put(struct btrfs_dio_private *dip) -{ - /* - * This implies a barrier so that stores to dio_bio->bi_status before - * this and loads of dio_bio->bi_status after this are fully ordered. - */ - if (!refcount_dec_and_test(&dip->refs)) - return; - - if (btrfs_op(&dip->bio) == BTRFS_MAP_WRITE) { - btrfs_mark_ordered_io_finished(dip->inode, NULL, - dip->file_offset, dip->bytes, - !dip->bio.bi_status); - } else { - unlock_extent(&dip->inode->io_tree, - dip->file_offset, - dip->file_offset + dip->bytes - 1, NULL); - } - - bio_endio(&dip->bio); -} - -static void btrfs_end_dio_bio(struct btrfs_bio *bbio) -{ - struct btrfs_dio_private *dip = bbio->private; - struct bio *bio = &bbio->bio; - blk_status_t err = bio->bi_status; - - if (err) { - btrfs_warn(dip->inode->root->fs_info, - "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d", - btrfs_ino(dip->inode), bio_op(bio), - bio->bi_opf, bio->bi_iter.bi_sector, - bio->bi_iter.bi_size, err); - dip->bio.bi_status = err; - } - - bio_put(bio); - btrfs_dio_private_put(dip); -} - -static void btrfs_submit_direct(const struct iomap_iter *iter, - struct bio *dio_bio, loff_t file_offset) +static void btrfs_dio_end_io(struct btrfs_bio *bbio) { struct btrfs_dio_private *dip = - container_of(dio_bio, struct btrfs_dio_private, bio); - struct inode *inode = iter->inode; - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct bio *bio; - u64 start_sector; - u64 submit_len; - u64 clone_offset = 0; - u64 clone_len; - u64 logical; - int ret; - blk_status_t status; - struct btrfs_io_geometry geom; + container_of(bbio, struct btrfs_dio_private, bbio); + struct btrfs_inode *inode = bbio->inode; + struct bio *bio = &bbio->bio; + + if (bio->bi_status) { + btrfs_warn(inode->root->fs_info, + "direct IO failed ino %llu op 0x%0x offset %#llx len %u err no %d", + btrfs_ino(inode), bio->bi_opf, + dip->file_offset, dip->bytes, bio->bi_status); + } + + if (btrfs_op(bio) == BTRFS_MAP_WRITE) + btrfs_mark_ordered_io_finished(inode, NULL, dip->file_offset, + dip->bytes, !bio->bi_status); + else + unlock_extent(&inode->io_tree, dip->file_offset, + dip->file_offset + dip->bytes - 1, NULL); + + bbio->bio.bi_private = bbio->private; + iomap_dio_bio_end_io(bio); +} + +static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, + loff_t file_offset) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_dio_private *dip = + container_of(bbio, struct btrfs_dio_private, bbio); struct btrfs_dio_data *dio_data = iter->private; - struct extent_map *em = NULL; - dip->inode = BTRFS_I(inode); + btrfs_bio_init(bbio, BTRFS_I(iter->inode), btrfs_dio_end_io, bio->bi_private); + bbio->file_offset = file_offset; + dip->file_offset = file_offset; - dip->bytes = dio_bio->bi_iter.bi_size; - refcount_set(&dip->refs, 1); + dip->bytes = bio->bi_iter.bi_size; - start_sector = dio_bio->bi_iter.bi_sector; - submit_len = dio_bio->bi_iter.bi_size; - - do { - logical = start_sector << 9; - em = btrfs_get_chunk_map(fs_info, logical, submit_len); - if (IS_ERR(em)) { - status = errno_to_blk_status(PTR_ERR(em)); - em = NULL; - goto out_err; - } - ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(dio_bio), - logical, &geom); - if (ret) { - status = errno_to_blk_status(ret); - goto out_err_em; - } - - clone_len = min(submit_len, geom.len); - ASSERT(clone_len <= UINT_MAX); - - /* - * This will never fail as it's passing GPF_NOFS and - * the allocation is backed by btrfs_bioset. - */ - bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len, - BTRFS_I(inode), btrfs_end_dio_bio, - dip); - btrfs_bio(bio)->file_offset = file_offset; - - ASSERT(submit_len >= clone_len); - submit_len -= clone_len; - - /* - * Increase the count before we submit the bio so we know - * the end IO handler won't happen before we increase the - * count. Otherwise, the dip might get freed before we're - * done setting it up. - * - * We transfer the initial reference to the last bio, so we - * don't need to increment the reference count for the last one. - */ - if (submit_len > 0) - refcount_inc(&dip->refs); - - btrfs_submit_bio(fs_info, bio, 0); - - dio_data->submitted += clone_len; - clone_offset += clone_len; - start_sector += clone_len >> 9; - file_offset += clone_len; - - free_extent_map(em); - } while (submit_len > 0); - return; - -out_err_em: - free_extent_map(em); -out_err: - dio_bio->bi_status = status; - btrfs_dio_private_put(dip); + dio_data->submitted += bio->bi_iter.bi_size; + btrfs_submit_bio(btrfs_sb(iter->inode->i_sb), bio, 0); } static const struct iomap_ops btrfs_dio_iomap_ops = { @@ -7901,7 +7804,7 @@ static const struct iomap_ops btrfs_dio_iomap_ops = { }; static const struct iomap_dio_ops btrfs_dio_ops = { - .submit_io = btrfs_submit_direct, + .submit_io = btrfs_dio_submit_io, .bio_set = &btrfs_dio_bioset, }; @@ -8736,7 +8639,7 @@ int __init btrfs_init_cachep(void) goto fail; if (bioset_init(&btrfs_dio_bioset, BIO_POOL_SIZE, - offsetof(struct btrfs_dio_private, bio), + offsetof(struct btrfs_dio_private, bbio.bio), BIOSET_NEED_BVECS)) goto fail; From 2380220e1e13b2dec4ae6f561841763ed6e1b0c0 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 21 Jan 2023 07:50:22 +0100 Subject: [PATCH 229/265] btrfs: remove stripe boundary calculation for buffered I/O Remove btrfs_bio_ctrl::len_to_stripe_boundary, so that buffer I/O will no longer limit its bio size according to stripe length now that btrfs_submit_bio can split bios at stripe boundaries. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo [hch: simplify calc_bio_boundaries a little more] Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 71 ++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 52 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2983fe2e10fe..03dac03a3083 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -99,7 +99,6 @@ struct btrfs_bio_ctrl { struct bio *bio; int mirror_num; enum btrfs_compression_type compress_type; - u32 len_to_stripe_boundary; u32 len_to_oe_boundary; btrfs_bio_end_io_t end_io_func; @@ -900,7 +899,7 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, ASSERT(bio); /* The limit should be calculated when bio_ctrl->bio is allocated */ - ASSERT(bio_ctrl->len_to_oe_boundary && bio_ctrl->len_to_stripe_boundary); + ASSERT(bio_ctrl->len_to_oe_boundary); if (bio_ctrl->compress_type != compress_type) return 0; @@ -936,9 +935,7 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, if (!contig) return 0; - real_size = min(bio_ctrl->len_to_oe_boundary, - bio_ctrl->len_to_stripe_boundary) - bio_size; - real_size = min(real_size, size); + real_size = min(bio_ctrl->len_to_oe_boundary - bio_size, size); /* * If real_size is 0, never call bio_add_*_page(), as even size is 0, @@ -955,58 +952,30 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, return ret; } -static int calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, - struct btrfs_inode *inode, u64 file_offset) +static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, + struct btrfs_inode *inode, u64 file_offset) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_io_geometry geom; struct btrfs_ordered_extent *ordered; - struct extent_map *em; u64 logical = (bio_ctrl->bio->bi_iter.bi_sector << SECTOR_SHIFT); - int ret; /* - * Pages for compressed extent are never submitted to disk directly, - * thus it has no real boundary, just set them to U32_MAX. - * - * The split happens for real compressed bio, which happens in - * btrfs_submit_compressed_read/write(). + * Limit the extent to the ordered boundary for Zone Append. + * Compressed bios aren't submitted directly, so it doesn't apply to + * them. */ - if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) { - bio_ctrl->len_to_oe_boundary = U32_MAX; - bio_ctrl->len_to_stripe_boundary = U32_MAX; - return 0; - } - em = btrfs_get_chunk_map(fs_info, logical, fs_info->sectorsize); - if (IS_ERR(em)) - return PTR_ERR(em); - ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio_ctrl->bio), - logical, &geom); - free_extent_map(em); - if (ret < 0) { - return ret; - } - if (geom.len > U32_MAX) - bio_ctrl->len_to_stripe_boundary = U32_MAX; - else - bio_ctrl->len_to_stripe_boundary = (u32)geom.len; - - if (bio_op(bio_ctrl->bio) != REQ_OP_ZONE_APPEND) { - bio_ctrl->len_to_oe_boundary = U32_MAX; - return 0; + if (bio_ctrl->compress_type == BTRFS_COMPRESS_NONE && + bio_op(bio_ctrl->bio) == REQ_OP_ZONE_APPEND) { + ordered = btrfs_lookup_ordered_extent(inode, file_offset); + if (ordered) { + bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, + ordered->disk_bytenr + + ordered->disk_num_bytes - logical); + btrfs_put_ordered_extent(ordered); + return; + } } - /* Ordered extent not yet created, so we're good */ - ordered = btrfs_lookup_ordered_extent(inode, file_offset); - if (!ordered) { - bio_ctrl->len_to_oe_boundary = U32_MAX; - return 0; - } - - bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, - ordered->disk_bytenr + ordered->disk_num_bytes - logical); - btrfs_put_ordered_extent(ordered); - return 0; + bio_ctrl->len_to_oe_boundary = U32_MAX; } static int alloc_new_bio(struct btrfs_inode *inode, @@ -1032,9 +1001,7 @@ static int alloc_new_bio(struct btrfs_inode *inode, bio->bi_iter.bi_sector = (disk_bytenr + offset) >> SECTOR_SHIFT; bio_ctrl->bio = bio; bio_ctrl->compress_type = compress_type; - ret = calc_bio_boundaries(bio_ctrl, inode, file_offset); - if (ret < 0) - goto error; + calc_bio_boundaries(bio_ctrl, inode, file_offset); if (wbc) { /* From 30493ff49f81d0ba42286b72498272db1a1ccae6 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 21 Jan 2023 07:50:23 +0100 Subject: [PATCH 230/265] btrfs: remove stripe boundary calculation for compressed I/O Stop looking at the stripe boundary in alloc_compressed_bio() now that that btrfs_submit_bio can split bios, open code the now trivial code from alloc_compressed_bio() in btrfs_submit_compressed_read and stop maintaining the pending_ios count for reads as there is always just a single bio now. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo [hch: remove more cruft in btrfs_submit_compressed_read, use btrfs_zoned_get_device in alloc_compressed_bio] Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/compression.c | 130 +++++++++++------------------------------ 1 file changed, 33 insertions(+), 97 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 301a83960bce..d7e767e01625 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -141,12 +141,15 @@ static int compression_decompress(int type, struct list_head *ws, static int btrfs_decompress_bio(struct compressed_bio *cb); -static void finish_compressed_bio_read(struct compressed_bio *cb) +static void end_compressed_bio_read(struct btrfs_bio *bbio) { + struct compressed_bio *cb = bbio->private; unsigned int index; struct page *page; - if (cb->status == BLK_STS_OK) + if (bbio->bio.bi_status) + cb->status = bbio->bio.bi_status; + else cb->status = errno_to_blk_status(btrfs_decompress_bio(cb)); /* Release the compressed pages */ @@ -162,17 +165,6 @@ static void finish_compressed_bio_read(struct compressed_bio *cb) /* Finally free the cb struct */ kfree(cb->compressed_pages); kfree(cb); -} - -static void end_compressed_bio_read(struct btrfs_bio *bbio) -{ - struct compressed_bio *cb = bbio->private; - - if (bbio->bio.bi_status) - cb->status = bbio->bio.bi_status; - - if (refcount_dec_and_test(&cb->pending_ios)) - finish_compressed_bio_read(cb); bio_put(&bbio->bio); } @@ -289,43 +281,30 @@ static void end_compressed_bio_write(struct btrfs_bio *bbio) * from or written to. * @endio_func: The endio function to call after the IO for compressed data * is finished. - * @next_stripe_start: Return value of logical bytenr of where next stripe starts. - * Let the caller know to only fill the bio up to the stripe - * boundary. */ - - static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_bytenr, blk_opf_t opf, - btrfs_bio_end_io_t endio_func, - u64 *next_stripe_start) + btrfs_bio_end_io_t endio_func) { - struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - struct btrfs_io_geometry geom; - struct extent_map *em; struct bio *bio; - int ret; bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, BTRFS_I(cb->inode), endio_func, cb); bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - em = btrfs_get_chunk_map(fs_info, disk_bytenr, fs_info->sectorsize); - if (IS_ERR(em)) { - bio_put(bio); - return ERR_CAST(em); - } + if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); + struct btrfs_device *device; - if (bio_op(bio) == REQ_OP_ZONE_APPEND) - bio_set_dev(bio, em->map_lookup->stripes[0].dev->bdev); + device = btrfs_zoned_get_device(fs_info, disk_bytenr, + fs_info->sectorsize); + if (IS_ERR(device)) { + bio_put(bio); + return ERR_CAST(device); + } - ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio), disk_bytenr, &geom); - free_extent_map(em); - if (ret < 0) { - bio_put(bio); - return ERR_PTR(ret); + bio_set_dev(bio, device->bdev); } - *next_stripe_start = disk_bytenr + geom.len; refcount_inc(&cb->pending_ios); return bio; } @@ -352,7 +331,6 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, struct bio *bio = NULL; struct compressed_bio *cb; u64 cur_disk_bytenr = disk_start; - u64 next_stripe_start; blk_status_t ret = BLK_STS_OK; const bool use_append = btrfs_use_zone_append(inode, disk_start); const enum req_op bio_op = REQ_BTRFS_ONE_ORDERED | @@ -388,8 +366,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, /* Allocate new bio if submitted or not yet allocated */ if (!bio) { bio = alloc_compressed_bio(cb, cur_disk_bytenr, - bio_op | write_flags, end_compressed_bio_write, - &next_stripe_start); + bio_op | write_flags, end_compressed_bio_write); if (IS_ERR(bio)) { ret = errno_to_blk_status(PTR_ERR(bio)); break; @@ -398,20 +375,12 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, if (blkcg_css) bio->bi_opf |= REQ_CGROUP_PUNT; } - /* - * We should never reach next_stripe_start start as we will - * submit comp_bio when reach the boundary immediately. - */ - ASSERT(cur_disk_bytenr != next_stripe_start); - /* * We have various limits on the real read size: - * - stripe boundary * - page boundary * - compressed length boundary */ - real_size = min_t(u64, U32_MAX, next_stripe_start - cur_disk_bytenr); - real_size = min_t(u64, real_size, PAGE_SIZE - offset_in_page(offset)); + real_size = min_t(u64, U32_MAX, PAGE_SIZE - offset_in_page(offset)); real_size = min_t(u64, real_size, compressed_len - offset); ASSERT(IS_ALIGNED(real_size, fs_info->sectorsize)); @@ -426,9 +395,6 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, submit = true; cur_disk_bytenr += added; - /* Reached stripe boundary */ - if (cur_disk_bytenr == next_stripe_start) - submit = true; /* Finished the range */ if (cur_disk_bytenr == disk_start + compressed_len) @@ -623,10 +589,9 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, struct extent_map_tree *em_tree; struct compressed_bio *cb; unsigned int compressed_len; - struct bio *comp_bio = NULL; + struct bio *comp_bio; const u64 disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT; u64 cur_disk_byte = disk_bytenr; - u64 next_stripe_start; u64 file_offset; u64 em_len; u64 em_start; @@ -693,37 +658,23 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, /* include any pages we added in add_ra-bio_pages */ cb->len = bio->bi_iter.bi_size; + comp_bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, BTRFS_I(cb->inode), + end_compressed_bio_read, cb); + comp_bio->bi_iter.bi_sector = (cur_disk_byte >> SECTOR_SHIFT); + while (cur_disk_byte < disk_bytenr + compressed_len) { u64 offset = cur_disk_byte - disk_bytenr; unsigned int index = offset >> PAGE_SHIFT; unsigned int real_size; unsigned int added; struct page *page = cb->compressed_pages[index]; - bool submit = false; - /* Allocate new bio if submitted or not yet allocated */ - if (!comp_bio) { - comp_bio = alloc_compressed_bio(cb, cur_disk_byte, - REQ_OP_READ, end_compressed_bio_read, - &next_stripe_start); - if (IS_ERR(comp_bio)) { - cb->status = errno_to_blk_status(PTR_ERR(comp_bio)); - break; - } - } - /* - * We should never reach next_stripe_start start as we will - * submit comp_bio when reach the boundary immediately. - */ - ASSERT(cur_disk_byte != next_stripe_start); /* * We have various limit on the real read size: - * - stripe boundary * - page boundary * - compressed length boundary */ - real_size = min_t(u64, U32_MAX, next_stripe_start - cur_disk_byte); - real_size = min_t(u64, real_size, PAGE_SIZE - offset_in_page(offset)); + real_size = min_t(u64, U32_MAX, PAGE_SIZE - offset_in_page(offset)); real_size = min_t(u64, real_size, compressed_len - offset); ASSERT(IS_ALIGNED(real_size, fs_info->sectorsize)); @@ -734,35 +685,20 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, */ ASSERT(added == real_size); cur_disk_byte += added; - - /* Reached stripe boundary, need to submit */ - if (cur_disk_byte == next_stripe_start) - submit = true; - - /* Has finished the range, need to submit */ - if (cur_disk_byte == disk_bytenr + compressed_len) - submit = true; - - if (submit) { - /* - * Save the initial offset of this chunk, as there - * is no direct correlation between compressed pages and - * the original file offset. The field is only used for - * printing error messages. - */ - btrfs_bio(comp_bio)->file_offset = file_offset; - - ASSERT(comp_bio->bi_iter.bi_size); - btrfs_submit_bio(fs_info, comp_bio, mirror_num); - comp_bio = NULL; - } } if (memstall) psi_memstall_leave(&pflags); - if (refcount_dec_and_test(&cb->pending_ios)) - finish_compressed_bio_read(cb); + /* + * Stash the initial offset of this chunk, as there is no direct + * correlation between compressed pages and the original file offset. + * The field is only used for printing error messages anyway. + */ + btrfs_bio(comp_bio)->file_offset = file_offset; + + ASSERT(comp_bio->bi_iter.bi_size); + btrfs_submit_bio(fs_info, comp_bio, mirror_num); return; fail: From a34e4c3f884cc592f105d214d21baee9f9c6bae8 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Sat, 21 Jan 2023 07:50:24 +0100 Subject: [PATCH 231/265] btrfs: remove stripe boundary calculation for encoded I/O Stop looking at the stripe boundary in btrfs_encoded_read_regular_fill_pages() now that btrfs_submit_bio can split bios. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/inode.c | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4ac9b34ad377..0fd26719d321 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9971,7 +9971,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 disk_io_size, struct page **pages) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_encoded_read_private priv = { .inode = inode, .file_offset = file_offset, @@ -9979,33 +9978,13 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, }; unsigned long i = 0; u64 cur = 0; - int ret; init_waitqueue_head(&priv.wait); - /* - * Submit bios for the extent, splitting due to bio or stripe limits as - * necessary. - */ + /* Submit bios for the extent, splitting due to bio limits as necessary. */ while (cur < disk_io_size) { - struct extent_map *em; - struct btrfs_io_geometry geom; struct bio *bio = NULL; - u64 remaining; + u64 remaining = disk_io_size - cur; - em = btrfs_get_chunk_map(fs_info, disk_bytenr + cur, - disk_io_size - cur); - if (IS_ERR(em)) { - ret = PTR_ERR(em); - } else { - ret = btrfs_get_io_geometry(fs_info, em, BTRFS_MAP_READ, - disk_bytenr + cur, &geom); - free_extent_map(em); - } - if (ret) { - WRITE_ONCE(priv.status, errno_to_blk_status(ret)); - break; - } - remaining = min(geom.len, disk_io_size - cur); while (bio || remaining) { size_t bytes = min_t(u64, remaining, PAGE_SIZE); From f8a02dc6fd38da3dd045509ec321b2f9130bd003 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:25 +0100 Subject: [PATCH 232/265] btrfs: remove struct btrfs_io_geometry Now that btrfs_get_io_geometry has a single caller, we can massage it into a form that is more suitable for that caller and remove the marshalling into and out of struct btrfs_io_geometry. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 114 ++++++++++++--------------------------------- fs/btrfs/volumes.h | 18 ------- 2 files changed, 31 insertions(+), 101 deletions(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 3510274ba195..7823168c08a6 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -6284,91 +6284,42 @@ static bool need_full_stripe(enum btrfs_map_op op) return (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS); } -/* - * Calculate the geometry of a particular (address, len) tuple. This - * information is used to calculate how big a particular bio can get before it - * straddles a stripe. - * - * @fs_info: the filesystem - * @em: mapping containing the logical extent - * @op: type of operation - write or read - * @logical: address that we want to figure out the geometry of - * @io_geom: pointer used to return values - * - * Returns < 0 in case a chunk for the given logical address cannot be found, - * usually shouldn't happen unless @logical is corrupted, 0 otherwise. - */ -int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, - enum btrfs_map_op op, u64 logical, - struct btrfs_io_geometry *io_geom) +static u64 btrfs_max_io_len(struct map_lookup *map, enum btrfs_map_op op, + u64 offset, u64 *stripe_nr, u64 *stripe_offset, + u64 *full_stripe_start) { - struct map_lookup *map; - u64 len; - u64 offset; - u64 stripe_offset; - u64 stripe_nr; - u32 stripe_len; - u64 raid56_full_stripe_start = (u64)-1; - int data_stripes; + u32 stripe_len = map->stripe_len; ASSERT(op != BTRFS_MAP_DISCARD); - map = em->map_lookup; - /* Offset of this logical address in the chunk */ - offset = logical - em->start; - /* Len of a stripe in a chunk */ - stripe_len = map->stripe_len; /* - * Stripe_nr is where this block falls in - * stripe_offset is the offset of this block in its stripe. + * Stripe_nr is the stripe where this block falls. stripe_offset is + * the offset of this block in its stripe. */ - stripe_nr = div64_u64_rem(offset, stripe_len, &stripe_offset); - ASSERT(stripe_offset < U32_MAX); + *stripe_nr = div64_u64_rem(offset, stripe_len, stripe_offset); + ASSERT(*stripe_offset < U32_MAX); - data_stripes = nr_data_stripes(map); + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + unsigned long full_stripe_len = stripe_len * nr_data_stripes(map); - /* Only stripe based profiles needs to check against stripe length. */ - if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) { - u64 max_len = stripe_len - stripe_offset; + *full_stripe_start = + div64_u64(offset, full_stripe_len) * full_stripe_len; /* - * In case of raid56, we need to know the stripe aligned start + * For writes to RAID56, allow to write a full stripe set, but + * no straddling of stripe sets. */ - if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { - unsigned long full_stripe_len = stripe_len * data_stripes; - raid56_full_stripe_start = offset; - - /* - * Allow a write of a full stripe, but make sure we - * don't allow straddling of stripes - */ - raid56_full_stripe_start = div64_u64(raid56_full_stripe_start, - full_stripe_len); - raid56_full_stripe_start *= full_stripe_len; - - /* - * For writes to RAID[56], allow a full stripeset across - * all disks. For other RAID types and for RAID[56] - * reads, just allow a single stripe (on a single disk). - */ - if (op == BTRFS_MAP_WRITE) { - max_len = stripe_len * data_stripes - - (offset - raid56_full_stripe_start); - } - } - len = min_t(u64, em->len - offset, max_len); - } else { - len = em->len - offset; + if (op == BTRFS_MAP_WRITE) + return full_stripe_len - (offset - *full_stripe_start); } - io_geom->len = len; - io_geom->offset = offset; - io_geom->stripe_len = stripe_len; - io_geom->stripe_nr = stripe_nr; - io_geom->stripe_offset = stripe_offset; - io_geom->raid56_stripe_offset = raid56_full_stripe_start; - - return 0; + /* + * For other RAID types and for RAID56 reads, allow a single stripe (on + * a single disk). + */ + if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) + return stripe_len - *stripe_offset; + return U64_MAX; } static void set_io_stripe(struct btrfs_io_stripe *dst, const struct map_lookup *map, @@ -6387,6 +6338,7 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, { struct extent_map *em; struct map_lookup *map; + u64 map_offset; u64 stripe_offset; u64 stripe_nr; u64 stripe_len; @@ -6405,7 +6357,7 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, int patch_the_first_stripe_for_dev_replace = 0; u64 physical_to_patch_in_first_stripe = 0; u64 raid56_full_stripe_start = (u64)-1; - struct btrfs_io_geometry geom; + u64 max_len; ASSERT(bioc_ret); ASSERT(op != BTRFS_MAP_DISCARD); @@ -6413,18 +6365,14 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, em = btrfs_get_chunk_map(fs_info, logical, *length); ASSERT(!IS_ERR(em)); - ret = btrfs_get_io_geometry(fs_info, em, op, logical, &geom); - if (ret < 0) - return ret; - map = em->map_lookup; - - *length = geom.len; - stripe_len = geom.stripe_len; - stripe_nr = geom.stripe_nr; - stripe_offset = geom.stripe_offset; - raid56_full_stripe_start = geom.raid56_stripe_offset; data_stripes = nr_data_stripes(map); + stripe_len = map->stripe_len; + + map_offset = logical - em->start; + max_len = btrfs_max_io_len(map, op, map_offset, &stripe_nr, + &stripe_offset, &raid56_full_stripe_start); + *length = min_t(u64, em->len - map_offset, max_len); down_read(&dev_replace->rwsem); dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 6b7a05f6cf82..7e51f2238f72 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -53,21 +53,6 @@ enum btrfs_raid_types { BTRFS_NR_RAID_TYPES }; -struct btrfs_io_geometry { - /* remaining bytes before crossing a stripe */ - u64 len; - /* offset of logical address in chunk */ - u64 offset; - /* length of single IO stripe */ - u32 stripe_len; - /* offset of address in stripe */ - u32 stripe_offset; - /* number of stripe where address falls */ - u64 stripe_nr; - /* offset of raid56 stripe into the chunk */ - u64 raid56_stripe_offset; -}; - /* * Use sequence counter to get consistent device stat data on * 32-bit processors. @@ -545,9 +530,6 @@ int __btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info, u64 logical, u64 *length_ret, u32 *num_stripes); -int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *map, - enum btrfs_map_op op, u64 logical, - struct btrfs_io_geometry *io_geom); int btrfs_read_sys_array(struct btrfs_fs_info *fs_info); int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info); struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, From 48253076c3a93f795fcd84ffdc97c5e763709dee Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:26 +0100 Subject: [PATCH 233/265] btrfs: open code submit_encoded_read_bio Open code the functionality in the only caller and remove the now superfluous error handling there. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/inode.c | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0fd26719d321..be122e4d4952 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9936,17 +9936,6 @@ struct btrfs_encoded_read_private { blk_status_t status; }; -static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, - struct bio *bio, int mirror_num) -{ - struct btrfs_encoded_read_private *priv = btrfs_bio(bio)->private; - struct btrfs_fs_info *fs_info = inode->root->fs_info; - - atomic_inc(&priv->pending); - btrfs_submit_bio(fs_info, bio, mirror_num); - return BLK_STS_OK; -} - static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) { struct btrfs_encoded_read_private *priv = bbio->private; @@ -9971,6 +9960,7 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 disk_io_size, struct page **pages) { + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_encoded_read_private priv = { .inode = inode, .file_offset = file_offset, @@ -9999,14 +9989,8 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, if (!bytes || bio_add_page(bio, pages[i], bytes, 0) < bytes) { - blk_status_t status; - - status = submit_encoded_read_bio(inode, bio, 0); - if (status) { - WRITE_ONCE(priv.status, status); - bio_put(bio); - goto out; - } + atomic_inc(&priv.pending); + btrfs_submit_bio(fs_info, bio, 0); bio = NULL; continue; } @@ -10017,7 +10001,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, } } -out: if (atomic_dec_return(&priv.pending)) io_wait_event(priv.wait, !atomic_read(&priv.pending)); /* See btrfs_encoded_read_endio() for ordering. */ From 285599b6fe15d642df643fd4383ab3a278374e35 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:27 +0100 Subject: [PATCH 234/265] btrfs: remove the fs_info argument to btrfs_submit_bio btrfs_submit_bio can derive it trivially from bbio->inode, so stop bothering in the callers. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 12 ++++++------ fs/btrfs/bio.h | 3 +-- fs/btrfs/compression.c | 4 ++-- fs/btrfs/disk-io.c | 2 +- fs/btrfs/inode.c | 11 ++++------- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index de8634cf7bd8..1de40e064170 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -155,7 +155,7 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, goto done; } - btrfs_submit_bio(fs_info, &repair_bbio->bio, mirror); + btrfs_submit_bio(&repair_bbio->bio, mirror); return; } @@ -223,7 +223,7 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); - btrfs_submit_bio(fs_info, repair_bio, mirror); + btrfs_submit_bio(repair_bio, mirror); return fbio; } @@ -600,10 +600,10 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, return true; } -static bool btrfs_submit_chunk(struct btrfs_fs_info *fs_info, struct bio *bio, - int mirror_num) +static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) { struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; struct btrfs_bio *orig_bbio = bbio; u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; @@ -676,9 +676,9 @@ fail: return true; } -void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) +void btrfs_submit_bio(struct bio *bio, int mirror_num) { - while (!btrfs_submit_chunk(fs_info, bio, mirror_num)) + while (!btrfs_submit_chunk(bio, mirror_num)) ; } diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index a043c2ced5bd..873ff85817f0 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -88,8 +88,7 @@ static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) /* Bio only refers to one ordered extent. */ #define REQ_BTRFS_ONE_ORDERED REQ_DRV -void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, - int mirror_num); +void btrfs_submit_bio(struct bio *bio, int mirror_num); int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, u64 length, u64 logical, struct page *page, unsigned int pg_offset, int mirror_num); diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index d7e767e01625..556ceed54d08 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -402,7 +402,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, if (submit) { ASSERT(bio->bi_iter.bi_size); - btrfs_submit_bio(fs_info, bio, 0); + btrfs_submit_bio(bio, 0); bio = NULL; } cond_resched(); @@ -698,7 +698,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, btrfs_bio(comp_bio)->file_offset = file_offset; ASSERT(comp_bio->bi_iter.bi_size); - btrfs_submit_bio(fs_info, comp_bio, mirror_num); + btrfs_submit_bio(comp_bio, mirror_num); return; fail: diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2ae329b5ce98..a84b662fdd22 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -702,7 +702,7 @@ err: void btrfs_submit_metadata_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) { bio->bi_opf |= REQ_META; - btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); + btrfs_submit_bio(bio, mirror_num); } #ifdef CONFIG_MIGRATION diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index be122e4d4952..bd83633f8ad2 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2697,14 +2697,12 @@ out: void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) { - btrfs_submit_bio(inode->root->fs_info, bio, mirror_num); + btrfs_submit_bio(bio, mirror_num); } void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; - if (compress_type != BTRFS_COMPRESS_NONE) { /* * btrfs_submit_compressed_read will handle completing the bio @@ -2714,7 +2712,7 @@ void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, return; } - btrfs_submit_bio(fs_info, bio, mirror_num); + btrfs_submit_bio(bio, mirror_num); } /* @@ -7795,7 +7793,7 @@ static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, dip->bytes = bio->bi_iter.bi_size; dio_data->submitted += bio->bi_iter.bi_size; - btrfs_submit_bio(btrfs_sb(iter->inode->i_sb), bio, 0); + btrfs_submit_bio(bio, 0); } static const struct iomap_ops btrfs_dio_iomap_ops = { @@ -9960,7 +9958,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 disk_io_size, struct page **pages) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_encoded_read_private priv = { .inode = inode, .file_offset = file_offset, @@ -9990,7 +9987,7 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, if (!bytes || bio_add_page(bio, pages[i], bytes, 0) < bytes) { atomic_inc(&priv.pending); - btrfs_submit_bio(fs_info, bio, 0); + btrfs_submit_bio(bio, 0); bio = NULL; continue; } From 35a8d7da3ca87d8612fa86a21fab4e07a70d35cb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:28 +0100 Subject: [PATCH 235/265] btrfs: remove now spurious bio submission helpers Call btrfs_submit_bio and btrfs_submit_compressed_read directly from submit_one_bio now that all additional functionality has moved into btrfs_submit_bio. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 3 --- fs/btrfs/disk-io.c | 6 ------ fs/btrfs/disk-io.h | 1 - fs/btrfs/extent_io.c | 19 ++++++++++--------- fs/btrfs/inode.c | 20 -------------------- 5 files changed, 10 insertions(+), 39 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index b83b731c63e1..49a92aa65de1 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -405,9 +405,6 @@ static inline void btrfs_inode_split_flags(u64 inode_item_flags, #define CSUM_FMT "0x%*phN" #define CSUM_FMT_VALUE(size, bytes) size, bytes -void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num); -void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, - int mirror_num, enum btrfs_compression_type compress_type); int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, u32 pgoff, u8 *csum, const u8 * const csum_expected); blk_status_t btrfs_extract_ordered_extent(struct btrfs_bio *bbio); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index a84b662fdd22..0da0bde347e5 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -699,12 +699,6 @@ err: return ret; } -void btrfs_submit_metadata_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) -{ - bio->bi_opf |= REQ_META; - btrfs_submit_bio(bio, mirror_num); -} - #ifdef CONFIG_MIGRATION static int btree_migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode) diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index f2dd4c6d9c25..3b53fc29a858 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -86,7 +86,6 @@ void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, int btrfs_validate_metadata_buffer(struct btrfs_bio *bbio, struct page *page, u64 start, u64 end, int mirror); -void btrfs_submit_metadata_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info); #endif diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 03dac03a3083..34c044501f8a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -125,7 +125,7 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) { struct bio *bio; struct bio_vec *bv; - struct btrfs_inode *inode; + struct inode *inode; int mirror_num; if (!bio_ctrl->bio) @@ -133,7 +133,7 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) bio = bio_ctrl->bio; bv = bio_first_bvec_all(bio); - inode = BTRFS_I(bv->bv_page->mapping->host); + inode = bv->bv_page->mapping->host; mirror_num = bio_ctrl->mirror_num; /* Caller should ensure the bio has at least some range added */ @@ -141,7 +141,7 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) btrfs_bio(bio)->file_offset = page_offset(bv->bv_page) + bv->bv_offset; - if (!is_data_inode(&inode->vfs_inode)) { + if (!is_data_inode(inode)) { if (btrfs_op(bio) != BTRFS_MAP_WRITE) { /* * For metadata read, we should have the parent_check, @@ -152,14 +152,15 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) bio_ctrl->parent_check, sizeof(struct btrfs_tree_parent_check)); } - btrfs_submit_metadata_bio(inode, bio, mirror_num); - } else if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - btrfs_submit_data_write_bio(inode, bio, mirror_num); - } else { - btrfs_submit_data_read_bio(inode, bio, mirror_num, - bio_ctrl->compress_type); + bio->bi_opf |= REQ_META; } + if (btrfs_op(bio) == BTRFS_MAP_READ && + bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) + btrfs_submit_compressed_read(inode, bio, mirror_num); + else + btrfs_submit_bio(bio, mirror_num); + /* The bio is owned by the end_io handler now */ bio_ctrl->bio = NULL; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index bd83633f8ad2..74b45b2a3bee 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2695,26 +2695,6 @@ out: return errno_to_blk_status(ret); } -void btrfs_submit_data_write_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) -{ - btrfs_submit_bio(bio, mirror_num); -} - -void btrfs_submit_data_read_bio(struct btrfs_inode *inode, struct bio *bio, - int mirror_num, enum btrfs_compression_type compress_type) -{ - if (compress_type != BTRFS_COMPRESS_NONE) { - /* - * btrfs_submit_compressed_read will handle completing the bio - * if there were any errors, so just return here. - */ - btrfs_submit_compressed_read(&inode->vfs_inode, bio, mirror_num); - return; - } - - btrfs_submit_bio(bio, mirror_num); -} - /* * given a list of ordered sums record them in the inode. This happens * at IO completion time based on sums calculated at bio submission time. From 243cf8d1b6737d4b53ac16b211987bbd299478e6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:29 +0100 Subject: [PATCH 236/265] btrfs: calculate file system wide queue limit for zoned mode To be able to split a write into properly sized zone append commands, we need a queue_limits structure that contains the least common denominator suitable for all devices. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/fs.h | 5 ++++- fs/btrfs/zoned.c | 52 ++++++++++++++++++++++++------------------------ fs/btrfs/zoned.h | 1 - 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index 3d8156fc8523..4c477eae6891 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -3,6 +3,7 @@ #ifndef BTRFS_FS_H #define BTRFS_FS_H +#include #include #include #include @@ -748,8 +749,10 @@ struct btrfs_fs_info { */ u64 zone_size; - /* Max size to emit ZONE_APPEND write command */ + /* Constraints for ZONE_APPEND commands: */ + struct queue_limits limits; u64 max_zone_append_size; + struct mutex zoned_meta_io_lock; spinlock_t treelog_bg_lock; u64 treelog_bg; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 5bf67c3c9f84..bcbe63ebd693 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -421,25 +421,6 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) nr_sectors = bdev_nr_sectors(bdev); zone_info->zone_size_shift = ilog2(zone_info->zone_size); zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors); - /* - * We limit max_zone_append_size also by max_segments * - * PAGE_SIZE. Technically, we can have multiple pages per segment. But, - * since btrfs adds the pages one by one to a bio, and btrfs cannot - * increase the metadata reservation even if it increases the number of - * extents, it is safe to stick with the limit. - * - * With the zoned emulation, we can have non-zoned device on the zoned - * mode. In this case, we don't have a valid max zone append size. So, - * use max_segments * PAGE_SIZE as the pseudo max_zone_append_size. - */ - if (bdev_is_zoned(bdev)) { - zone_info->max_zone_append_size = min_t(u64, - (u64)bdev_max_zone_append_sectors(bdev) << SECTOR_SHIFT, - (u64)bdev_max_segments(bdev) << PAGE_SHIFT); - } else { - zone_info->max_zone_append_size = - (u64)bdev_max_segments(bdev) << PAGE_SHIFT; - } if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; @@ -719,9 +700,9 @@ static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info) int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) { + struct queue_limits *lim = &fs_info->limits; struct btrfs_device *device; u64 zone_size = 0; - u64 max_zone_append_size = 0; int ret; /* @@ -731,6 +712,8 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) if (!btrfs_fs_incompat(fs_info, ZONED)) return btrfs_check_for_zoned_device(fs_info); + blk_set_stacking_limits(lim); + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { struct btrfs_zoned_device_info *zone_info = device->zone_info; @@ -745,10 +728,17 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) zone_info->zone_size, zone_size); return -EINVAL; } - if (!max_zone_append_size || - (zone_info->max_zone_append_size && - zone_info->max_zone_append_size < max_zone_append_size)) - max_zone_append_size = zone_info->max_zone_append_size; + + /* + * With the zoned emulation, we can have non-zoned device on the + * zoned mode. In this case, we don't have a valid max zone + * append size. + */ + if (bdev_is_zoned(device->bdev)) { + blk_stack_limits(lim, + &bdev_get_queue(device->bdev)->limits, + 0); + } } /* @@ -769,8 +759,18 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) } fs_info->zone_size = zone_size; - fs_info->max_zone_append_size = ALIGN_DOWN(max_zone_append_size, - fs_info->sectorsize); + /* + * Also limit max_zone_append_size by max_segments * PAGE_SIZE. + * Technically, we can have multiple pages per segment. But, since + * we add the pages one by one to a bio, and cannot increase the + * metadata reservation even if it increases the number of extents, it + * is safe to stick with the limit. + */ + fs_info->max_zone_append_size = ALIGN_DOWN( + min3((u64)lim->max_zone_append_sectors << SECTOR_SHIFT, + (u64)lim->max_sectors << SECTOR_SHIFT, + (u64)lim->max_segments << PAGE_SHIFT), + fs_info->sectorsize); fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED; if (fs_info->max_zone_append_size < fs_info->max_extent_size) fs_info->max_extent_size = fs_info->max_zone_append_size; diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index bc93a740e7cf..f25f332b7728 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -20,7 +20,6 @@ struct btrfs_zoned_device_info { */ u64 zone_size; u8 zone_size_shift; - u64 max_zone_append_size; u32 nr_zones; unsigned int max_active_zones; atomic_t active_zones_left; From d5e4377d505189c30df50d54f9944d7fb8d528bb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:30 +0100 Subject: [PATCH 237/265] btrfs: split zone append bios in btrfs_submit_bio The current btrfs zoned device support is a little cumbersome in the data I/O path as it requires the callers to not issue I/O larger than the supported ZONE_APPEND size of the underlying device. This leads to a lot of extra accounting. Instead change btrfs_submit_bio so that it can take write bios of arbitrary size and form from the upper layers, and just split them internally to the ZONE_APPEND queue limits. Then remove all the upper layer warts catering to limited write sized on zoned devices, including the extra refcount in the compressed_bio. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/bio.c | 44 ++++++++++------- fs/btrfs/compression.c | 110 ++++++++--------------------------------- fs/btrfs/compression.h | 3 -- fs/btrfs/extent_io.c | 72 ++++++--------------------- fs/btrfs/inode.c | 4 -- fs/btrfs/zoned.c | 20 -------- fs/btrfs/zoned.h | 9 ---- 7 files changed, 63 insertions(+), 199 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 1de40e064170..8445df47495a 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -59,13 +59,22 @@ struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, return bio; } -static struct bio *btrfs_split_bio(struct bio *orig, u64 map_length) +static struct bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, + struct bio *orig, u64 map_length, + bool use_append) { struct btrfs_bio *orig_bbio = btrfs_bio(orig); struct bio *bio; - bio = bio_split(orig, map_length >> SECTOR_SHIFT, GFP_NOFS, - &btrfs_clone_bioset); + if (use_append) { + unsigned int nr_segs; + + bio = bio_split_rw(orig, &fs_info->limits, &nr_segs, + &btrfs_clone_bioset, map_length); + } else { + bio = bio_split(orig, map_length >> SECTOR_SHIFT, GFP_NOFS, + &btrfs_clone_bioset); + } btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, NULL, orig_bbio); btrfs_bio(bio)->file_offset = orig_bbio->file_offset; @@ -397,16 +406,10 @@ static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) */ if (bio_op(bio) == REQ_OP_ZONE_APPEND) { u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + u64 zone_start = round_down(physical, dev->fs_info->zone_size); - if (btrfs_dev_is_sequential(dev, physical)) { - u64 zone_start = round_down(physical, - dev->fs_info->zone_size); - - bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; - } else { - bio->bi_opf &= ~REQ_OP_ZONE_APPEND; - bio->bi_opf |= REQ_OP_WRITE; - } + ASSERT(btrfs_dev_is_sequential(dev, physical)); + bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; } btrfs_debug_in_rcu(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", @@ -603,11 +606,13 @@ static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) { struct btrfs_bio *bbio = btrfs_bio(bio); - struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_bio *orig_bbio = bbio; u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; u64 map_length = length; + bool use_append = btrfs_use_zone_append(inode, logical); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t ret; @@ -622,8 +627,11 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) } map_length = min(map_length, length); + if (use_append) + map_length = min(map_length, fs_info->max_zone_append_size); + if (map_length < length) { - bio = btrfs_split_bio(bio, map_length); + bio = btrfs_split_bio(fs_info, bio, map_length, use_append); bbio = btrfs_bio(bio); } @@ -639,7 +647,9 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) } if (btrfs_op(bio) == BTRFS_MAP_WRITE) { - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + if (use_append) { + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; ret = btrfs_extract_ordered_extent(btrfs_bio(bio)); if (ret) goto fail_put_bio; @@ -649,9 +659,9 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) * Csum items for reloc roots have already been cloned at this * point, so they are handled as part of the no-checksum case. */ - if (!(bbio->inode->flags & BTRFS_INODE_NODATASUM) && + if (!(inode->flags & BTRFS_INODE_NODATASUM) && !test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state) && - !btrfs_is_data_reloc_root(bbio->inode->root)) { + !btrfs_is_data_reloc_root(inode->root)) { if (should_async_write(bbio) && btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num)) goto done; diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 556ceed54d08..f42f31f22d13 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -258,57 +258,14 @@ static void btrfs_finish_compressed_write_work(struct work_struct *work) static void end_compressed_bio_write(struct btrfs_bio *bbio) { struct compressed_bio *cb = bbio->private; + struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - if (bbio->bio.bi_status) - cb->status = bbio->bio.bi_status; + cb->status = bbio->bio.bi_status; + queue_work(fs_info->compressed_write_workers, &cb->write_end_work); - if (refcount_dec_and_test(&cb->pending_ios)) { - struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - - queue_work(fs_info->compressed_write_workers, &cb->write_end_work); - } bio_put(&bbio->bio); } -/* - * Allocate a compressed_bio, which will be used to read/write on-disk - * (aka, compressed) * data. - * - * @cb: The compressed_bio structure, which records all the needed - * information to bind the compressed data to the uncompressed - * page cache. - * @disk_byten: The logical bytenr where the compressed data will be read - * from or written to. - * @endio_func: The endio function to call after the IO for compressed data - * is finished. - */ -static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_bytenr, - blk_opf_t opf, - btrfs_bio_end_io_t endio_func) -{ - struct bio *bio; - - bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, BTRFS_I(cb->inode), endio_func, - cb); - bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - struct btrfs_device *device; - - device = btrfs_zoned_get_device(fs_info, disk_bytenr, - fs_info->sectorsize); - if (IS_ERR(device)) { - bio_put(bio); - return ERR_CAST(device); - } - - bio_set_dev(bio, device->bdev); - } - refcount_inc(&cb->pending_ios); - return bio; -} - /* * worker function to build and submit bios for previously compressed pages. * The corresponding pages in the inode should be marked for writeback @@ -332,16 +289,12 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, struct compressed_bio *cb; u64 cur_disk_bytenr = disk_start; blk_status_t ret = BLK_STS_OK; - const bool use_append = btrfs_use_zone_append(inode, disk_start); - const enum req_op bio_op = REQ_BTRFS_ONE_ORDERED | - (use_append ? REQ_OP_ZONE_APPEND : REQ_OP_WRITE); ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && IS_ALIGNED(len, fs_info->sectorsize)); cb = kmalloc(sizeof(struct compressed_bio), GFP_NOFS); if (!cb) return BLK_STS_RESOURCE; - refcount_set(&cb->pending_ios, 1); cb->status = BLK_STS_OK; cb->inode = &inode->vfs_inode; cb->start = start; @@ -352,8 +305,16 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, INIT_WORK(&cb->write_end_work, btrfs_finish_compressed_write_work); cb->nr_pages = nr_pages; - if (blkcg_css) + if (blkcg_css) { kthread_associate_blkcg(blkcg_css); + write_flags |= REQ_CGROUP_PUNT; + } + + write_flags |= REQ_BTRFS_ONE_ORDERED; + bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_WRITE | write_flags, + BTRFS_I(cb->inode), end_compressed_bio_write, cb); + bio->bi_iter.bi_sector = cur_disk_bytenr >> SECTOR_SHIFT; + btrfs_bio(bio)->file_offset = start; while (cur_disk_bytenr < disk_start + compressed_len) { u64 offset = cur_disk_bytenr - disk_start; @@ -361,20 +322,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, unsigned int real_size; unsigned int added; struct page *page = compressed_pages[index]; - bool submit = false; - /* Allocate new bio if submitted or not yet allocated */ - if (!bio) { - bio = alloc_compressed_bio(cb, cur_disk_bytenr, - bio_op | write_flags, end_compressed_bio_write); - if (IS_ERR(bio)) { - ret = errno_to_blk_status(PTR_ERR(bio)); - break; - } - btrfs_bio(bio)->file_offset = start; - if (blkcg_css) - bio->bi_opf |= REQ_CGROUP_PUNT; - } /* * We have various limits on the real read size: * - page boundary @@ -384,35 +332,20 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, real_size = min_t(u64, real_size, compressed_len - offset); ASSERT(IS_ALIGNED(real_size, fs_info->sectorsize)); - if (use_append) - added = bio_add_zone_append_page(bio, page, real_size, - offset_in_page(offset)); - else - added = bio_add_page(bio, page, real_size, - offset_in_page(offset)); - /* Reached zoned boundary */ - if (added == 0) - submit = true; - + added = bio_add_page(bio, page, real_size, offset_in_page(offset)); + /* + * Maximum compressed extent is smaller than bio size limit, + * thus bio_add_page() should always success. + */ + ASSERT(added == real_size); cur_disk_bytenr += added; - - /* Finished the range */ - if (cur_disk_bytenr == disk_start + compressed_len) - submit = true; - - if (submit) { - ASSERT(bio->bi_iter.bi_size); - btrfs_submit_bio(bio, 0); - bio = NULL; - } - cond_resched(); } + /* Finished the range. */ + ASSERT(bio->bi_iter.bi_size); + btrfs_submit_bio(bio, 0); if (blkcg_css) kthread_associate_blkcg(NULL); - - if (refcount_dec_and_test(&cb->pending_ios)) - finish_compressed_bio_write(cb); return ret; } @@ -624,7 +557,6 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, goto out; } - refcount_set(&cb->pending_ios, 1); cb->status = BLK_STS_OK; cb->inode = inode; diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 6209d40a1e08..a5e3377db9ad 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -31,9 +31,6 @@ static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0); #define BTRFS_ZLIB_DEFAULT_LEVEL 3 struct compressed_bio { - /* Number of outstanding bios */ - refcount_t pending_ios; - /* Number of compressed pages in the array */ unsigned int nr_pages; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 34c044501f8a..e2816b86311a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -896,7 +896,6 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, u32 real_size; const sector_t sector = disk_bytenr >> SECTOR_SHIFT; bool contig = false; - int ret; ASSERT(bio); /* The limit should be calculated when bio_ctrl->bio is allocated */ @@ -945,12 +944,7 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, if (real_size == 0) return 0; - if (bio_op(bio) == REQ_OP_ZONE_APPEND) - ret = bio_add_zone_append_page(bio, page, real_size, pg_offset); - else - ret = bio_add_page(bio, page, real_size, pg_offset); - - return ret; + return bio_add_page(bio, page, real_size, pg_offset); } static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, @@ -965,7 +959,7 @@ static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, * them. */ if (bio_ctrl->compress_type == BTRFS_COMPRESS_NONE && - bio_op(bio_ctrl->bio) == REQ_OP_ZONE_APPEND) { + btrfs_use_zone_append(inode, logical)) { ordered = btrfs_lookup_ordered_extent(inode, file_offset); if (ordered) { bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, @@ -979,16 +973,14 @@ static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, bio_ctrl->len_to_oe_boundary = U32_MAX; } -static int alloc_new_bio(struct btrfs_inode *inode, - struct btrfs_bio_ctrl *bio_ctrl, - struct writeback_control *wbc, - blk_opf_t opf, - u64 disk_bytenr, u32 offset, u64 file_offset, - enum btrfs_compression_type compress_type) +static void alloc_new_bio(struct btrfs_inode *inode, + struct btrfs_bio_ctrl *bio_ctrl, + struct writeback_control *wbc, blk_opf_t opf, + u64 disk_bytenr, u32 offset, u64 file_offset, + enum btrfs_compression_type compress_type) { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct bio *bio; - int ret; bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, inode, bio_ctrl->end_io_func, NULL); @@ -1006,40 +998,14 @@ static int alloc_new_bio(struct btrfs_inode *inode, if (wbc) { /* - * For Zone append we need the correct block_device that we are - * going to write to set in the bio to be able to respect the - * hardware limitation. Look it up here: + * Pick the last added device to support cgroup writeback. For + * multi-device file systems this means blk-cgroup policies have + * to always be set on the last added/replaced device. + * This is a bit odd but has been like that for a long time. */ - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - struct btrfs_device *dev; - - dev = btrfs_zoned_get_device(fs_info, disk_bytenr, - fs_info->sectorsize); - if (IS_ERR(dev)) { - ret = PTR_ERR(dev); - goto error; - } - - bio_set_dev(bio, dev->bdev); - } else { - /* - * Otherwise pick the last added device to support - * cgroup writeback. For multi-device file systems this - * means blk-cgroup policies have to always be set on the - * last added/replaced device. This is a bit odd but has - * been like that for a long time. - */ - bio_set_dev(bio, fs_info->fs_devices->latest_dev->bdev); - } + bio_set_dev(bio, fs_info->fs_devices->latest_dev->bdev); wbc_init_bio(wbc, bio); - } else { - ASSERT(bio_op(bio) != REQ_OP_ZONE_APPEND); } - return 0; -error: - bio_ctrl->bio = NULL; - btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret)); - return ret; } /* @@ -1065,7 +1031,6 @@ static int submit_extent_page(blk_opf_t opf, enum btrfs_compression_type compress_type, bool force_bio_submit) { - int ret = 0; struct btrfs_inode *inode = BTRFS_I(page->mapping->host); unsigned int cur = pg_offset; @@ -1085,12 +1050,9 @@ static int submit_extent_page(blk_opf_t opf, /* Allocate new bio if needed */ if (!bio_ctrl->bio) { - ret = alloc_new_bio(inode, bio_ctrl, wbc, opf, - disk_bytenr, offset, - page_offset(page) + cur, - compress_type); - if (ret < 0) - return ret; + alloc_new_bio(inode, bio_ctrl, wbc, opf, disk_bytenr, + offset, page_offset(page) + cur, + compress_type); } /* * We must go through btrfs_bio_add_page() to ensure each @@ -1647,10 +1609,6 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, * find_next_dirty_byte() are all exclusive */ iosize = min(min(em_end, end + 1), dirty_range_end) - cur; - - if (btrfs_use_zone_append(inode, em->block_start)) - op = REQ_OP_ZONE_APPEND; - free_extent_map(em); em = NULL; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 74b45b2a3bee..2fd518afc4f3 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7678,10 +7678,6 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, iomap->offset = start; iomap->bdev = fs_info->fs_devices->latest_dev->bdev; iomap->length = len; - - if (write && btrfs_use_zone_append(BTRFS_I(inode), em->block_start)) - iomap->flags |= IOMAP_F_ZONE_APPEND; - free_extent_map(em); return 0; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index bcbe63ebd693..06ed1eeb5981 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1845,26 +1845,6 @@ int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); } -struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info, - u64 logical, u64 length) -{ - struct btrfs_device *device; - struct extent_map *em; - struct map_lookup *map; - - em = btrfs_get_chunk_map(fs_info, logical, length); - if (IS_ERR(em)) - return ERR_CAST(em); - - map = em->map_lookup; - /* We only support single profile for now */ - device = map->stripes[0].dev; - - free_extent_map(em); - - return device; -} - /* * Activate block group and underlying device zones * diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index f25f332b7728..157f46132c56 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -66,8 +66,6 @@ void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length); int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, u64 physical_start, u64 physical_pos); -struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info, - u64 logical, u64 length); bool btrfs_zone_activate(struct btrfs_block_group *block_group); int btrfs_zone_finish(struct btrfs_block_group *block_group); bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags); @@ -221,13 +219,6 @@ static inline int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, return -EOPNOTSUPP; } -static inline struct btrfs_device *btrfs_zoned_get_device( - struct btrfs_fs_info *fs_info, - u64 logical, u64 length) -{ - return ERR_PTR(-EOPNOTSUPP); -} - static inline bool btrfs_zone_activate(struct btrfs_block_group *block_group) { return true; From 8e81aa16a42169faae1ba15cd648cc8bb83eaa48 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 21 Jan 2023 07:50:31 +0100 Subject: [PATCH 238/265] iomap: remove IOMAP_F_ZONE_APPEND No users left now that btrfs takes REQ_OP_WRITE bios from iomap and splits and converts them to REQ_OP_ZONE_APPEND internally. Reviewed-by: Josef Bacik Reviewed-by: Damien Le Moal Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/iomap/direct-io.c | 10 ++-------- include/linux/iomap.h | 3 +-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 9804714b1751..f771001574d0 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -217,16 +217,10 @@ static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio, { blk_opf_t opflags = REQ_SYNC | REQ_IDLE; - if (!(dio->flags & IOMAP_DIO_WRITE)) { - WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND); + if (!(dio->flags & IOMAP_DIO_WRITE)) return REQ_OP_READ; - } - - if (iomap->flags & IOMAP_F_ZONE_APPEND) - opflags |= REQ_OP_ZONE_APPEND; - else - opflags |= REQ_OP_WRITE; + opflags |= REQ_OP_WRITE; if (use_fua) opflags |= REQ_FUA; else diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 0983dfc9a203..fca43a4bd96b 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -58,8 +58,7 @@ struct vm_fault; #define IOMAP_F_SHARED (1U << 2) #define IOMAP_F_MERGED (1U << 3) #define IOMAP_F_BUFFER_HEAD (1U << 4) -#define IOMAP_F_ZONE_APPEND (1U << 5) -#define IOMAP_F_XATTR (1U << 6) +#define IOMAP_F_XATTR (1U << 5) /* * Flags set by the core iomap code during operations: From d3fb66150c05b1b082984c88e6895e663119ac4e Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:54 -0500 Subject: [PATCH 239/265] btrfs: always lock the block before calling btrfs_clean_tree_block We want to clean up the dirty handling for extent buffers so it's a little more consistent, so skip the check for generation == transid and simply always lock the extent buffer before calling btrfs_clean_tree_block. Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 688cdf816957..97f2c10ad654 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -5469,8 +5469,7 @@ static noinline int walk_up_proc(struct btrfs_trans_handle *trans, } } /* make block locked assertion in btrfs_clean_tree_block happy */ - if (!path->locks[level] && - btrfs_header_generation(eb) == trans->transid) { + if (!path->locks[level]) { btrfs_tree_lock(eb); path->locks[level] = BTRFS_WRITE_LOCK; } From ed25dab3a0d1b14b59a3ad74b9d6bb4f4dca03b8 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:55 -0500 Subject: [PATCH 240/265] btrfs: add trans argument to btrfs_clean_tree_block We check the header generation in the extent buffer against the current running transaction id to see if it's safe to clear DIRTY on this buffer. Generally speaking if we're clearing the buffer dirty we're holding the transaction open, but in the case of cleaning up an aborted transaction we don't, so we have extra checks in that path to check the transid. To allow for a future cleanup go ahead and pass in the trans handle so we don't have to rely on ->running_transaction being set. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 31 ++++++++++++++++--------------- fs/btrfs/disk-io.c | 6 +++--- fs/btrfs/disk-io.h | 3 ++- fs/btrfs/extent-tree.c | 4 ++-- fs/btrfs/free-space-tree.c | 2 +- fs/btrfs/ioctl.c | 2 +- fs/btrfs/qgroup.c | 2 +- fs/btrfs/tree-log.c | 6 +++--- 8 files changed, 29 insertions(+), 27 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 4754c9101a4c..cae2df92e3b0 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -484,7 +484,7 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, if (ret) return ret; } - btrfs_clean_tree_block(buf); + btrfs_clean_tree_block(trans, buf); *last_ref = 1; } return 0; @@ -1054,7 +1054,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, path->locks[level] = 0; path->nodes[level] = NULL; - btrfs_clean_tree_block(mid); + btrfs_clean_tree_block(trans, mid); btrfs_tree_unlock(mid); /* once for the path */ free_extent_buffer(mid); @@ -1115,7 +1115,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, if (wret < 0 && wret != -ENOSPC) ret = wret; if (btrfs_header_nritems(right) == 0) { - btrfs_clean_tree_block(right); + btrfs_clean_tree_block(trans, right); btrfs_tree_unlock(right); del_ptr(root, path, level + 1, pslot + 1); root_sub_used(root, right->len); @@ -1161,7 +1161,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, BUG_ON(wret == 1); } if (btrfs_header_nritems(mid) == 0) { - btrfs_clean_tree_block(mid); + btrfs_clean_tree_block(trans, mid); btrfs_tree_unlock(mid); del_ptr(root, path, level + 1, pslot); root_sub_used(root, mid->len); @@ -3041,7 +3041,8 @@ noinline int btrfs_leaf_free_space(struct extent_buffer *leaf) * min slot controls the lowest index we're willing to push to the * right. We'll push up to and including min_slot, but no lower */ -static noinline int __push_leaf_right(struct btrfs_path *path, +static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, + struct btrfs_path *path, int data_size, int empty, struct extent_buffer *right, int free_space, u32 left_nritems, @@ -3139,7 +3140,7 @@ static noinline int __push_leaf_right(struct btrfs_path *path, if (left_nritems) btrfs_mark_buffer_dirty(left); else - btrfs_clean_tree_block(left); + btrfs_clean_tree_block(trans, left); btrfs_mark_buffer_dirty(right); @@ -3151,7 +3152,7 @@ static noinline int __push_leaf_right(struct btrfs_path *path, if (path->slots[0] >= left_nritems) { path->slots[0] -= left_nritems; if (btrfs_header_nritems(path->nodes[0]) == 0) - btrfs_clean_tree_block(path->nodes[0]); + btrfs_clean_tree_block(trans, path->nodes[0]); btrfs_tree_unlock(path->nodes[0]); free_extent_buffer(path->nodes[0]); path->nodes[0] = right; @@ -3243,8 +3244,8 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root return 0; } - return __push_leaf_right(path, min_data_size, empty, - right, free_space, left_nritems, min_slot); + return __push_leaf_right(trans, path, min_data_size, empty, right, + free_space, left_nritems, min_slot); out_unlock: btrfs_tree_unlock(right); free_extent_buffer(right); @@ -3259,7 +3260,8 @@ out_unlock: * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the * items */ -static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, +static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, + struct btrfs_path *path, int data_size, int empty, struct extent_buffer *left, int free_space, u32 right_nritems, u32 max_slot) @@ -3363,7 +3365,7 @@ static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, if (right_nritems) btrfs_mark_buffer_dirty(right); else - btrfs_clean_tree_block(right); + btrfs_clean_tree_block(trans, right); btrfs_item_key(right, &disk_key, 0); fixup_low_keys(path, &disk_key, 1); @@ -3449,9 +3451,8 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root ret = -EUCLEAN; goto out; } - return __push_leaf_left(path, min_data_size, - empty, left, free_space, right_nritems, - max_slot); + return __push_leaf_left(trans, path, min_data_size, empty, left, + free_space, right_nritems, max_slot); out: btrfs_tree_unlock(left); free_extent_buffer(left); @@ -4400,7 +4401,7 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, if (leaf == root->node) { btrfs_set_header_level(leaf, 0); } else { - btrfs_clean_tree_block(leaf); + btrfs_clean_tree_block(trans, leaf); btrfs_del_leaf(trans, root, path, leaf); } } else { diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 0da0bde347e5..0b5e63929a47 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -868,11 +868,11 @@ struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, } -void btrfs_clean_tree_block(struct extent_buffer *buf) +void btrfs_clean_tree_block(struct btrfs_trans_handle *trans, + struct extent_buffer *buf) { struct btrfs_fs_info *fs_info = buf->fs_info; - if (btrfs_header_generation(buf) == - fs_info->running_transaction->transid) { + if (btrfs_header_generation(buf) == trans->transid) { btrfs_assert_tree_write_locked(buf); if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) { diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 3b53fc29a858..d5466f184687 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -39,7 +39,8 @@ struct extent_buffer *btrfs_find_create_tree_block( struct btrfs_fs_info *fs_info, u64 bytenr, u64 owner_root, int level); -void btrfs_clean_tree_block(struct extent_buffer *buf); +void btrfs_clean_tree_block(struct btrfs_trans_handle *trans, + struct extent_buffer *buf); void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info); int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info); int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 97f2c10ad654..8465327f31c8 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4838,7 +4838,7 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, btrfs_set_buffer_lockdep_class(lockdep_owner, buf, level); __btrfs_tree_lock(buf, nest); - btrfs_clean_tree_block(buf); + btrfs_clean_tree_block(trans, buf); clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); clear_bit(EXTENT_BUFFER_NO_CHECK, &buf->bflags); @@ -5473,7 +5473,7 @@ static noinline int walk_up_proc(struct btrfs_trans_handle *trans, btrfs_tree_lock(eb); path->locks[level] = BTRFS_WRITE_LOCK; } - btrfs_clean_tree_block(eb); + btrfs_clean_tree_block(trans, eb); } if (eb == root->node) { diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index c667e878ef1a..ab206af5b2f5 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1283,7 +1283,7 @@ int btrfs_clear_free_space_tree(struct btrfs_fs_info *fs_info) list_del(&free_space_root->dirty_list); btrfs_tree_lock(free_space_root->node); - btrfs_clean_tree_block(free_space_root->node); + btrfs_clean_tree_block(trans, free_space_root->node); btrfs_tree_unlock(free_space_root->node); btrfs_free_tree_block(trans, btrfs_root_id(free_space_root), free_space_root->node, 0, 1); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 7e348bd2ccde..5fdc09f7f1cb 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -707,7 +707,7 @@ static noinline int create_subvol(struct user_namespace *mnt_userns, * exists). */ btrfs_tree_lock(leaf); - btrfs_clean_tree_block(leaf); + btrfs_clean_tree_block(trans, leaf); btrfs_tree_unlock(leaf); btrfs_free_tree_block(trans, objectid, leaf, 0, 1); free_extent_buffer(leaf); diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index af97413abcf4..450d8dd858b2 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1304,7 +1304,7 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info) list_del("a_root->dirty_list); btrfs_tree_lock(quota_root->node); - btrfs_clean_tree_block(quota_root->node); + btrfs_clean_tree_block(trans, quota_root->node); btrfs_tree_unlock(quota_root->node); btrfs_free_tree_block(trans, btrfs_root_id(quota_root), quota_root->node, 0, 1); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 997ba92481cb..e683fbb9bb30 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -2625,7 +2625,7 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, if (trans) { btrfs_tree_lock(next); - btrfs_clean_tree_block(next); + btrfs_clean_tree_block(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, @@ -2695,7 +2695,7 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans, if (trans) { btrfs_tree_lock(next); - btrfs_clean_tree_block(next); + btrfs_clean_tree_block(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, @@ -2778,7 +2778,7 @@ static int walk_log_tree(struct btrfs_trans_handle *trans, if (trans) { btrfs_tree_lock(next); - btrfs_clean_tree_block(next); + btrfs_clean_tree_block(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, From c4e54a65711688e78e81d6e2720f19f0747eb176 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:56 -0500 Subject: [PATCH 241/265] btrfs: replace clearing extent buffer dirty bit with btrfs_clean_block Now that we're passing in the trans into btrfs_clean_tree_block, we can easily roll in the handling of the !trans case and replace all occurrences of if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) clear_extent_buffer_dirty(eb); with btrfs_tree_lock(eb); btrfs_clean_tree_block(eb); btrfs_tree_unlock(eb); We need the lock because if we are actually dirty we need to make sure we aren't racing with anything that's starting writeout currently. This also makes sure that we're accounting fs_info->dirty_metadata_bytes appropriately. Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 11 ++++++----- fs/btrfs/tree-log.c | 34 +++++++++++++++------------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 0b5e63929a47..be9bb8972397 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -872,7 +872,7 @@ void btrfs_clean_tree_block(struct btrfs_trans_handle *trans, struct extent_buffer *buf) { struct btrfs_fs_info *fs_info = buf->fs_info; - if (btrfs_header_generation(buf) == trans->transid) { + if (!trans || btrfs_header_generation(buf) == trans->transid) { btrfs_assert_tree_write_locked(buf); if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) { @@ -4995,11 +4995,12 @@ static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info, start += fs_info->nodesize; if (!eb) continue; - wait_on_extent_buffer_writeback(eb); - if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, - &eb->bflags)) - clear_extent_buffer_dirty(eb); + btrfs_tree_lock(eb); + wait_on_extent_buffer_writeback(eb); + btrfs_clean_tree_block(NULL, eb); + btrfs_tree_unlock(eb); + free_extent_buffer_stale(eb); } } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index e683fbb9bb30..a74acb341ef2 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -2623,11 +2623,12 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, return ret; } + btrfs_tree_lock(next); + btrfs_clean_tree_block(trans, next); + btrfs_wait_tree_block_writeback(next); + btrfs_tree_unlock(next); + if (trans) { - btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); - btrfs_wait_tree_block_writeback(next); - btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, bytenr, blocksize); if (ret) { @@ -2637,8 +2638,6 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, btrfs_redirty_list_add( trans->transaction, next); } else { - if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags)) - clear_extent_buffer_dirty(next); unaccount_log_buffer(fs_info, bytenr); } } @@ -2693,11 +2692,12 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans, next = path->nodes[*level]; + btrfs_tree_lock(next); + btrfs_clean_tree_block(trans, next); + btrfs_wait_tree_block_writeback(next); + btrfs_tree_unlock(next); + if (trans) { - btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); - btrfs_wait_tree_block_writeback(next); - btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, path->nodes[*level]->start, path->nodes[*level]->len); @@ -2706,9 +2706,6 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans, btrfs_redirty_list_add(trans->transaction, next); } else { - if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags)) - clear_extent_buffer_dirty(next); - unaccount_log_buffer(fs_info, path->nodes[*level]->start); } @@ -2776,19 +2773,18 @@ static int walk_log_tree(struct btrfs_trans_handle *trans, next = path->nodes[orig_level]; + btrfs_tree_lock(next); + btrfs_clean_tree_block(trans, next); + btrfs_wait_tree_block_writeback(next); + btrfs_tree_unlock(next); + if (trans) { - btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); - btrfs_wait_tree_block_writeback(next); - btrfs_tree_unlock(next); ret = btrfs_pin_reserved_extent(trans, next->start, next->len); if (ret) goto out; btrfs_redirty_list_add(trans->transaction, next); } else { - if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &next->bflags)) - clear_extent_buffer_dirty(next); unaccount_log_buffer(fs_info, next->start); } } From f88fd6504329ca8de0a04b6214e932c46746800d Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:57 -0500 Subject: [PATCH 242/265] btrfs: do not increment dirty_metadata_bytes in set_btree_ioerr We only add if we set the extent buffer dirty, and we subtract when we clear the extent buffer dirty. If we end up in set_btree_ioerr we have already cleared the buffer dirty, and we aren't resetting dirty on the extent buffer, so this is simply wrong. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index e2816b86311a..ada88ada49e1 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -1911,13 +1911,6 @@ static void set_btree_ioerr(struct page *page, struct extent_buffer *eb) */ mapping_set_error(page->mapping, -EIO); - /* - * If we error out, we should add back the dirty_metadata_bytes - * to make it consistent. - */ - percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, - eb->len, fs_info->dirty_metadata_batch); - /* * If writeback for a btree extent that doesn't belong to a log tree * failed, increment the counter transaction->eb_write_errors. From 190a83391bc40862538572d1313c207c348d356d Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:58 -0500 Subject: [PATCH 243/265] btrfs: rename btrfs_clean_tree_block to btrfs_clear_buffer_dirty btrfs_clean_tree_block is a misnomer, it's just clear_extent_buffer_dirty with some extra accounting around it. Rename this to btrfs_clear_buffer_dirty to make it more clear it belongs with it's setter, btrfs_mark_buffer_dirty. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 16 ++++++++-------- fs/btrfs/disk-io.c | 6 +++--- fs/btrfs/disk-io.h | 4 ++-- fs/btrfs/extent-tree.c | 6 +++--- fs/btrfs/free-space-tree.c | 2 +- fs/btrfs/ioctl.c | 2 +- fs/btrfs/qgroup.c | 2 +- fs/btrfs/tree-log.c | 6 +++--- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index cae2df92e3b0..118440857f33 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -484,7 +484,7 @@ static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, if (ret) return ret; } - btrfs_clean_tree_block(trans, buf); + btrfs_clear_buffer_dirty(trans, buf); *last_ref = 1; } return 0; @@ -1054,7 +1054,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, path->locks[level] = 0; path->nodes[level] = NULL; - btrfs_clean_tree_block(trans, mid); + btrfs_clear_buffer_dirty(trans, mid); btrfs_tree_unlock(mid); /* once for the path */ free_extent_buffer(mid); @@ -1115,7 +1115,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, if (wret < 0 && wret != -ENOSPC) ret = wret; if (btrfs_header_nritems(right) == 0) { - btrfs_clean_tree_block(trans, right); + btrfs_clear_buffer_dirty(trans, right); btrfs_tree_unlock(right); del_ptr(root, path, level + 1, pslot + 1); root_sub_used(root, right->len); @@ -1161,7 +1161,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans, BUG_ON(wret == 1); } if (btrfs_header_nritems(mid) == 0) { - btrfs_clean_tree_block(trans, mid); + btrfs_clear_buffer_dirty(trans, mid); btrfs_tree_unlock(mid); del_ptr(root, path, level + 1, pslot); root_sub_used(root, mid->len); @@ -3140,7 +3140,7 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, if (left_nritems) btrfs_mark_buffer_dirty(left); else - btrfs_clean_tree_block(trans, left); + btrfs_clear_buffer_dirty(trans, left); btrfs_mark_buffer_dirty(right); @@ -3152,7 +3152,7 @@ static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, if (path->slots[0] >= left_nritems) { path->slots[0] -= left_nritems; if (btrfs_header_nritems(path->nodes[0]) == 0) - btrfs_clean_tree_block(trans, path->nodes[0]); + btrfs_clear_buffer_dirty(trans, path->nodes[0]); btrfs_tree_unlock(path->nodes[0]); free_extent_buffer(path->nodes[0]); path->nodes[0] = right; @@ -3365,7 +3365,7 @@ static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, if (right_nritems) btrfs_mark_buffer_dirty(right); else - btrfs_clean_tree_block(trans, right); + btrfs_clear_buffer_dirty(trans, right); btrfs_item_key(right, &disk_key, 0); fixup_low_keys(path, &disk_key, 1); @@ -4401,7 +4401,7 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, if (leaf == root->node) { btrfs_set_header_level(leaf, 0); } else { - btrfs_clean_tree_block(trans, leaf); + btrfs_clear_buffer_dirty(trans, leaf); btrfs_del_leaf(trans, root, path, leaf); } } else { diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index be9bb8972397..2ef9e0b3a425 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -868,8 +868,8 @@ struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, } -void btrfs_clean_tree_block(struct btrfs_trans_handle *trans, - struct extent_buffer *buf) +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf) { struct btrfs_fs_info *fs_info = buf->fs_info; if (!trans || btrfs_header_generation(buf) == trans->transid) { @@ -4998,7 +4998,7 @@ static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info, btrfs_tree_lock(eb); wait_on_extent_buffer_writeback(eb); - btrfs_clean_tree_block(NULL, eb); + btrfs_clear_buffer_dirty(NULL, eb); btrfs_tree_unlock(eb); free_extent_buffer_stale(eb); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index d5466f184687..4d5772330110 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -39,8 +39,8 @@ struct extent_buffer *btrfs_find_create_tree_block( struct btrfs_fs_info *fs_info, u64 bytenr, u64 owner_root, int level); -void btrfs_clean_tree_block(struct btrfs_trans_handle *trans, - struct extent_buffer *buf); +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf); void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info); int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info); int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 8465327f31c8..824c657f59e8 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -4838,7 +4838,7 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, btrfs_set_buffer_lockdep_class(lockdep_owner, buf, level); __btrfs_tree_lock(buf, nest); - btrfs_clean_tree_block(trans, buf); + btrfs_clear_buffer_dirty(trans, buf); clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); clear_bit(EXTENT_BUFFER_NO_CHECK, &buf->bflags); @@ -5468,12 +5468,12 @@ static noinline int walk_up_proc(struct btrfs_trans_handle *trans, } } } - /* make block locked assertion in btrfs_clean_tree_block happy */ + /* Make block locked assertion in btrfs_clear_buffer_dirty happy. */ if (!path->locks[level]) { btrfs_tree_lock(eb); path->locks[level] = BTRFS_WRITE_LOCK; } - btrfs_clean_tree_block(trans, eb); + btrfs_clear_buffer_dirty(trans, eb); } if (eb == root->node) { diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index ab206af5b2f5..4d155a48ec59 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1283,7 +1283,7 @@ int btrfs_clear_free_space_tree(struct btrfs_fs_info *fs_info) list_del(&free_space_root->dirty_list); btrfs_tree_lock(free_space_root->node); - btrfs_clean_tree_block(trans, free_space_root->node); + btrfs_clear_buffer_dirty(trans, free_space_root->node); btrfs_tree_unlock(free_space_root->node); btrfs_free_tree_block(trans, btrfs_root_id(free_space_root), free_space_root->node, 0, 1); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 5fdc09f7f1cb..8ea557e22252 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -707,7 +707,7 @@ static noinline int create_subvol(struct user_namespace *mnt_userns, * exists). */ btrfs_tree_lock(leaf); - btrfs_clean_tree_block(trans, leaf); + btrfs_clear_buffer_dirty(trans, leaf); btrfs_tree_unlock(leaf); btrfs_free_tree_block(trans, objectid, leaf, 0, 1); free_extent_buffer(leaf); diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 450d8dd858b2..52a7d2fa2284 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -1304,7 +1304,7 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info) list_del("a_root->dirty_list); btrfs_tree_lock(quota_root->node); - btrfs_clean_tree_block(trans, quota_root->node); + btrfs_clear_buffer_dirty(trans, quota_root->node); btrfs_tree_unlock(quota_root->node); btrfs_free_tree_block(trans, btrfs_root_id(quota_root), quota_root->node, 0, 1); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index a74acb341ef2..0297379f1a70 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -2624,7 +2624,7 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, } btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); + btrfs_clear_buffer_dirty(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); @@ -2693,7 +2693,7 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans, next = path->nodes[*level]; btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); + btrfs_clear_buffer_dirty(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); @@ -2774,7 +2774,7 @@ static int walk_log_tree(struct btrfs_trans_handle *trans, next = path->nodes[orig_level]; btrfs_tree_lock(next); - btrfs_clean_tree_block(trans, next); + btrfs_clear_buffer_dirty(trans, next); btrfs_wait_tree_block_writeback(next); btrfs_tree_unlock(next); From 98c8d683c291285be35c50579ac984e71d17ddc1 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:00:59 -0500 Subject: [PATCH 244/265] btrfs: combine btrfs_clear_buffer_dirty and clear_extent_buffer_dirty btrfs_clear_buffer_dirty just does the test_clear_bit() and then calls clear_extent_buffer_dirty and does the dirty metadata accounting. Combine this into clear_extent_buffer_dirty and make the result btrfs_clear_buffer_dirty. Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 16 ---------------- fs/btrfs/extent_io.c | 16 +++++++++++++++- fs/btrfs/extent_io.h | 5 ++++- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 2ef9e0b3a425..b53f0e30ce2b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -868,22 +868,6 @@ struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, } -void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, - struct extent_buffer *buf) -{ - struct btrfs_fs_info *fs_info = buf->fs_info; - if (!trans || btrfs_header_generation(buf) == trans->transid) { - btrfs_assert_tree_write_locked(buf); - - if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)) { - percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, - -buf->len, - fs_info->dirty_metadata_batch); - clear_extent_buffer_dirty(buf); - } - } -} - static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, u64 objectid) { diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index ada88ada49e1..287e25edb3ef 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -36,6 +36,7 @@ #include "file.h" #include "dev-replace.h" #include "super.h" +#include "transaction.h" static struct kmem_cache *extent_buffer_cache; @@ -4268,12 +4269,25 @@ static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb) WARN_ON(atomic_read(&eb->refs) == 0); } -void clear_extent_buffer_dirty(const struct extent_buffer *eb) +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *eb) { + struct btrfs_fs_info *fs_info = eb->fs_info; int i; int num_pages; struct page *page; + btrfs_assert_tree_write_locked(eb); + + if (trans && btrfs_header_generation(eb) != trans->transid) + return; + + if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) + return; + + percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len, + fs_info->dirty_metadata_batch); + if (eb->fs_info->nodesize < PAGE_SIZE) return clear_subpage_extent_buffer_dirty(eb); diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 1b311cd69783..4341ad978fb8 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -11,6 +11,8 @@ #include "ulist.h" #include "misc.h" +struct btrfs_trans_handle; + enum { EXTENT_BUFFER_UPTODATE, EXTENT_BUFFER_DIRTY, @@ -260,7 +262,6 @@ void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long star void extent_buffer_bitmap_clear(const struct extent_buffer *eb, unsigned long start, unsigned long pos, unsigned long len); -void clear_extent_buffer_dirty(const struct extent_buffer *eb); bool set_extent_buffer_dirty(struct extent_buffer *eb); void set_extent_buffer_uptodate(struct extent_buffer *eb); void clear_extent_buffer_uptodate(struct extent_buffer *eb); @@ -272,6 +273,8 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, u32 bits_to_clear, unsigned long page_ops); int extent_invalidate_folio(struct extent_io_tree *tree, struct folio *folio, size_t offset); +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf); int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array); From 79b02ec1d8ce1fafc8c39f888dbba6a3aa9a35cc Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Thu, 26 Jan 2023 16:01:00 -0500 Subject: [PATCH 245/265] btrfs: replace btrfs_wait_tree_block_writeback by wait_on_extent_buffer_writeback This is used in the tree-log code and is a holdover from previous iterations of extent buffer writeback. We can simply use wait_on_extent_buffer_writeback here, and remove btrfs_wait_tree_block_writeback completely as it's equivalent (waiting on page write writeback). Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 0297379f1a70..200cea6e49e5 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -279,12 +279,6 @@ void btrfs_end_log_trans(struct btrfs_root *root) } } -static void btrfs_wait_tree_block_writeback(struct extent_buffer *buf) -{ - filemap_fdatawait_range(buf->pages[0]->mapping, - buf->start, buf->start + buf->len - 1); -} - /* * the walk control struct is used to pass state down the chain when * processing the log tree. The stage field tells us which part @@ -2625,7 +2619,7 @@ static noinline int walk_down_log_tree(struct btrfs_trans_handle *trans, btrfs_tree_lock(next); btrfs_clear_buffer_dirty(trans, next); - btrfs_wait_tree_block_writeback(next); + wait_on_extent_buffer_writeback(next); btrfs_tree_unlock(next); if (trans) { @@ -2694,7 +2688,7 @@ static noinline int walk_up_log_tree(struct btrfs_trans_handle *trans, btrfs_tree_lock(next); btrfs_clear_buffer_dirty(trans, next); - btrfs_wait_tree_block_writeback(next); + wait_on_extent_buffer_writeback(next); btrfs_tree_unlock(next); if (trans) { @@ -2775,7 +2769,7 @@ static int walk_log_tree(struct btrfs_trans_handle *trans, btrfs_tree_lock(next); btrfs_clear_buffer_dirty(trans, next); - btrfs_wait_tree_block_writeback(next); + wait_on_extent_buffer_writeback(next); btrfs_tree_unlock(next); if (trans) { From abb49e87425be0d573bdafb9a5dbe2c64719796f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:25 +0100 Subject: [PATCH 246/265] btrfs: raid56: simplify error handling and code flow in raid56_parity_write Handle the error return on alloc_rbio failure directly instead of using a goto and remove the queue_rbio goto label by moving the plugged check into the if branch. Reviewed-by: Johannes Thumshirn Reviewed-by: Qu Wenruo Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index d095c07a152d..8073fa004870 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1667,12 +1667,12 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) struct btrfs_raid_bio *rbio; struct btrfs_plug_cb *plug = NULL; struct blk_plug_cb *cb; - int ret = 0; rbio = alloc_rbio(fs_info, bioc); if (IS_ERR(rbio)) { - ret = PTR_ERR(rbio); - goto fail; + bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); + bio_endio(bio); + return; } rbio->operation = BTRFS_RBIO_WRITE; rbio_add_bio(rbio, bio); @@ -1681,31 +1681,24 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) * Don't plug on full rbios, just get them out the door * as quickly as we can */ - if (rbio_is_full(rbio)) - goto queue_rbio; - - cb = blk_check_plugged(raid_unplug, fs_info, sizeof(*plug)); - if (cb) { - plug = container_of(cb, struct btrfs_plug_cb, cb); - if (!plug->info) { - plug->info = fs_info; - INIT_LIST_HEAD(&plug->rbio_list); + if (!rbio_is_full(rbio)) { + cb = blk_check_plugged(raid_unplug, fs_info, sizeof(*plug)); + if (cb) { + plug = container_of(cb, struct btrfs_plug_cb, cb); + if (!plug->info) { + plug->info = fs_info; + INIT_LIST_HEAD(&plug->rbio_list); + } + list_add_tail(&rbio->plug_list, &plug->rbio_list); + return; } - list_add_tail(&rbio->plug_list, &plug->rbio_list); - return; } -queue_rbio: + /* * Either we don't have any existing plug, or we're doing a full stripe, - * can queue the rmw work now. + * queue the rmw work now. */ start_async_work(rbio, rmw_rbio_work); - - return; - -fail: - bio->bi_status = errno_to_blk_status(ret); - bio_endio(bio); } static int verify_one_sector(struct btrfs_raid_bio *rbio, From 4d7627010bc24f68d617c1c455e12a3dfcffcb2c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:26 +0100 Subject: [PATCH 247/265] btrfs: raid56: simplify code flow in rmw_rbio Remove the write goto label by moving the data page allocation and data read into the branch. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 8073fa004870..037dae1b7313 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2300,24 +2300,22 @@ static int rmw_rbio(struct btrfs_raid_bio *rbio) * Either full stripe write, or we have every data sector already * cached, can go to write path immediately. */ - if (rbio_is_full(rbio) || !need_read_stripe_sectors(rbio)) - goto write; + if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) { + /* + * Now we're doing sub-stripe write, also need all data stripes + * to do the full RMW. + */ + ret = alloc_rbio_data_pages(rbio); + if (ret < 0) + return ret; - /* - * Now we're doing sub-stripe write, also need all data stripes to do - * the full RMW. - */ - ret = alloc_rbio_data_pages(rbio); - if (ret < 0) - return ret; + index_rbio_pages(rbio); - index_rbio_pages(rbio); + ret = rmw_read_wait_recover(rbio); + if (ret < 0) + return ret; + } - ret = rmw_read_wait_recover(rbio); - if (ret < 0) - return ret; - -write: /* * At this stage we're not allowed to add any new bios to the * bio list any more, anyone else that wants to change this stripe From 1c76fb7b31a43ad8b63faa3c6612b04eb3dea5d6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:27 +0100 Subject: [PATCH 248/265] btrfs: raid56: wait for I/O completion in submit_read_bios In addition to setting up the end_io handler and submitting the bios in submit_read_bios, also wait for them to be completed instead of waiting for the completion manually in all three callers. Rename submit_read_bios to submit_read_wait_bio_list to make it clear it waits for the bios as well. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 037dae1b7313..6c63f6428117 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1497,7 +1497,7 @@ static void raid_wait_read_end_io(struct bio *bio) wake_up(&rbio->io_wait); } -static void submit_read_bios(struct btrfs_raid_bio *rbio, +static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio, struct bio_list *bio_list) { struct bio *bio; @@ -1514,6 +1514,8 @@ static void submit_read_bios(struct btrfs_raid_bio *rbio, } submit_bio(bio); } + + wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); } static int rmw_assemble_read_bios(struct btrfs_raid_bio *rbio, @@ -2016,8 +2018,7 @@ static int recover_rbio(struct btrfs_raid_bio *rbio) if (ret < 0) goto out; - submit_read_bios(rbio, &bio_list); - wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); + submit_read_wait_bio_list(rbio, &bio_list); ret = recover_sectors(rbio); @@ -2213,8 +2214,7 @@ static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) if (ret < 0) goto out; - submit_read_bios(rbio, &bio_list); - wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); + submit_read_wait_bio_list(rbio, &bio_list); /* * We may or may not have any corrupted sectors (including missing dev @@ -2792,8 +2792,7 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) if (ret < 0) goto cleanup; - submit_read_bios(rbio, &bio_list); - wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); + submit_read_wait_bio_list(rbio, &bio_list); /* We may have some failures, recover the failed sectors first. */ ret = recover_scrub_rbio(rbio); From 801fcfc5d790f4a9be2897713bd6dd08bed253f1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:28 +0100 Subject: [PATCH 249/265] btrfs: raid56: add a bio_list_put helper Add a helper to put all bios in a list. This does not need to be added to block layer as there are no other users of such code. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 44 ++++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 6c63f6428117..6108c06f16fc 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1183,6 +1183,14 @@ not_found: trace_info->stripe_nr = -1; } +static inline void bio_list_put(struct bio_list *bio_list) +{ + struct bio *bio; + + while ((bio = bio_list_pop(bio_list))) + bio_put(bio); +} + /* Generate PQ for one vertical stripe. */ static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr) { @@ -1228,7 +1236,6 @@ static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr) static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio, struct bio_list *bio_list) { - struct bio *bio; /* The total sector number inside the full stripe. */ int total_sector_nr; int sectornr; @@ -1317,8 +1324,7 @@ static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio, return 0; error: - while ((bio = bio_list_pop(bio_list))) - bio_put(bio); + bio_list_put(bio_list); return -EIO; } @@ -1521,7 +1527,6 @@ static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio, static int rmw_assemble_read_bios(struct btrfs_raid_bio *rbio, struct bio_list *bio_list) { - struct bio *bio; int total_sector_nr; int ret = 0; @@ -1548,8 +1553,7 @@ static int rmw_assemble_read_bios(struct btrfs_raid_bio *rbio, return 0; cleanup: - while ((bio = bio_list_pop(bio_list))) - bio_put(bio); + bio_list_put(bio_list); return ret; } @@ -1946,7 +1950,6 @@ out: static int recover_assemble_read_bios(struct btrfs_raid_bio *rbio, struct bio_list *bio_list) { - struct bio *bio; int total_sector_nr; int ret = 0; @@ -1988,16 +1991,13 @@ static int recover_assemble_read_bios(struct btrfs_raid_bio *rbio, } return 0; error: - while ((bio = bio_list_pop(bio_list))) - bio_put(bio); - + bio_list_put(bio_list); return -EIO; } static int recover_rbio(struct btrfs_raid_bio *rbio) { struct bio_list bio_list; - struct bio *bio; int ret; /* @@ -2023,9 +2023,7 @@ static int recover_rbio(struct btrfs_raid_bio *rbio) ret = recover_sectors(rbio); out: - while ((bio = bio_list_pop(&bio_list))) - bio_put(bio); - + bio_list_put(&bio_list); return ret; } @@ -2198,7 +2196,6 @@ no_csum: static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) { struct bio_list bio_list; - struct bio *bio; int ret; bio_list_init(&bio_list); @@ -2223,9 +2220,7 @@ static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) ret = recover_sectors(rbio); return ret; out: - while ((bio = bio_list_pop(&bio_list))) - bio_put(bio); - + bio_list_put(&bio_list); return ret; } @@ -2496,7 +2491,6 @@ static int finish_parity_scrub(struct btrfs_raid_bio *rbio, int need_check) struct sector_ptr p_sector = { 0 }; struct sector_ptr q_sector = { 0 }; struct bio_list bio_list; - struct bio *bio; int is_replace = 0; int ret; @@ -2627,8 +2621,7 @@ submit_write: return 0; cleanup: - while ((bio = bio_list_pop(&bio_list))) - bio_put(bio); + bio_list_put(&bio_list); return ret; } @@ -2726,7 +2719,6 @@ out: static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio, struct bio_list *bio_list) { - struct bio *bio; int total_sector_nr; int ret = 0; @@ -2767,8 +2759,7 @@ static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio, } return 0; error: - while ((bio = bio_list_pop(bio_list))) - bio_put(bio); + bio_list_put(bio_list); return ret; } @@ -2778,7 +2769,6 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) struct bio_list bio_list; int sector_nr; int ret; - struct bio *bio; bio_list_init(&bio_list); @@ -2817,9 +2807,7 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) return ret; cleanup: - while ((bio = bio_list_pop(&bio_list))) - bio_put(bio); - + bio_list_put(&bio_list); return ret; } From d838d05ea5091044957d664b9f9c38fefcaf6d97 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:29 +0100 Subject: [PATCH 250/265] btrfs: raid56: fold recover_assemble_read_bios into recover_rbio There is very little extra code in recover_rbio, and a large part of it is the superfluous extra cleanup of the bio list. Merge the two functions, and only clean up the bio list after it has been added to but before it has been emptied again by submit_read_wait_bio_list. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 61 ++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 6108c06f16fc..b8f672dc3a64 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1947,13 +1947,25 @@ out: return ret; } -static int recover_assemble_read_bios(struct btrfs_raid_bio *rbio, - struct bio_list *bio_list) +static int recover_rbio(struct btrfs_raid_bio *rbio) { + struct bio_list bio_list = BIO_EMPTY_LIST; int total_sector_nr; int ret = 0; - ASSERT(bio_list_size(bio_list) == 0); + /* + * Either we're doing recover for a read failure or degraded write, + * caller should have set error bitmap correctly. + */ + ASSERT(bitmap_weight(rbio->error_bitmap, rbio->nr_sectors)); + + /* For recovery, we need to read all sectors including P/Q. */ + ret = alloc_rbio_pages(rbio); + if (ret < 0) + return ret; + + index_rbio_pages(rbio); + /* * Read everything that hasn't failed. However this time we will * not trust any cached sector. @@ -1984,47 +1996,16 @@ static int recover_assemble_read_bios(struct btrfs_raid_bio *rbio, } sector = rbio_stripe_sector(rbio, stripe, sectornr); - ret = rbio_add_io_sector(rbio, bio_list, sector, stripe, + ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe, sectornr, REQ_OP_READ); - if (ret < 0) - goto error; + if (ret < 0) { + bio_list_put(&bio_list); + return ret; + } } - return 0; -error: - bio_list_put(bio_list); - return -EIO; -} - -static int recover_rbio(struct btrfs_raid_bio *rbio) -{ - struct bio_list bio_list; - int ret; - - /* - * Either we're doing recover for a read failure or degraded write, - * caller should have set error bitmap correctly. - */ - ASSERT(bitmap_weight(rbio->error_bitmap, rbio->nr_sectors)); - bio_list_init(&bio_list); - - /* For recovery, we need to read all sectors including P/Q. */ - ret = alloc_rbio_pages(rbio); - if (ret < 0) - goto out; - - index_rbio_pages(rbio); - - ret = recover_assemble_read_bios(rbio, &bio_list); - if (ret < 0) - goto out; submit_read_wait_bio_list(rbio, &bio_list); - - ret = recover_sectors(rbio); - -out: - bio_list_put(&bio_list); - return ret; + return recover_sectors(rbio); } static void recover_rbio_work(struct work_struct *work) From 02efa3a6baffdd753dc04034c848a5956784422d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:30 +0100 Subject: [PATCH 251/265] btrfs: raid56: fold rmw_read_wait_recover into rmw_read_bios There is very little extra code in rmw_read_bios, and a large part of it is the superfluous extra cleanup of the bio list. Merge the two functions, and only clean up the bio list after it has been added to but before it has been emptied again by submit_read_wait_bio_list. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 69 ++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 46 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index b8f672dc3a64..037d41a748ed 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1524,39 +1524,6 @@ static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio, wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); } -static int rmw_assemble_read_bios(struct btrfs_raid_bio *rbio, - struct bio_list *bio_list) -{ - int total_sector_nr; - int ret = 0; - - ASSERT(bio_list_size(bio_list) == 0); - - /* - * Build a list of bios to read all sectors (including data and P/Q). - * - * This behavior is to compensate the later csum verification and - * recovery. - */ - for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; - total_sector_nr++) { - struct sector_ptr *sector; - int stripe = total_sector_nr / rbio->stripe_nsectors; - int sectornr = total_sector_nr % rbio->stripe_nsectors; - - sector = rbio_stripe_sector(rbio, stripe, sectornr); - ret = rbio_add_io_sector(rbio, bio_list, sector, - stripe, sectornr, REQ_OP_READ); - if (ret) - goto cleanup; - } - return 0; - -cleanup: - bio_list_put(bio_list); - return ret; -} - static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio) { const int data_pages = rbio->nr_data * rbio->stripe_npages; @@ -2176,10 +2143,9 @@ no_csum: static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) { - struct bio_list bio_list; - int ret; - - bio_list_init(&bio_list); + struct bio_list bio_list = BIO_EMPTY_LIST; + int total_sector_nr; + int ret = 0; /* * Fill the data csums we need for data verification. We need to fill @@ -2188,21 +2154,32 @@ static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) */ fill_data_csums(rbio); - ret = rmw_assemble_read_bios(rbio, &bio_list); - if (ret < 0) - goto out; + /* + * Build a list of bios to read all sectors (including data and P/Q). + * + * This behavior is to compensate the later csum verification and recovery. + */ + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + struct sector_ptr *sector; + int stripe = total_sector_nr / rbio->stripe_nsectors; + int sectornr = total_sector_nr % rbio->stripe_nsectors; - submit_read_wait_bio_list(rbio, &bio_list); + sector = rbio_stripe_sector(rbio, stripe, sectornr); + ret = rbio_add_io_sector(rbio, &bio_list, sector, + stripe, sectornr, REQ_OP_READ); + if (ret) { + bio_list_put(&bio_list); + return ret; + } + } /* * We may or may not have any corrupted sectors (including missing dev * and csum mismatch), just let recover_sectors() to handle them all. */ - ret = recover_sectors(rbio); - return ret; -out: - bio_list_put(&bio_list); - return ret; + submit_read_wait_bio_list(rbio, &bio_list); + return recover_sectors(rbio); } static void raid_wait_write_end_io(struct bio *bio) From 52f0c198645c68b80536384b7e666c4c56777b5e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:31 +0100 Subject: [PATCH 252/265] btrfs: raid56: submit the read bios from scrub_assemble_read_bios Instead of filling in a bio_list and submitting the bios in the only caller, do that in scrub_assemble_read_bios. This removes the need to pass the bio_list, and also makes it clear that the extra bio_list cleanup in the caller is entirely pointless. Rename the function to scrub_read_bios to make it clear that the bios are not only assembled. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 037d41a748ed..b54c15b5f1df 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2674,14 +2674,12 @@ out: return ret; } -static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio, - struct bio_list *bio_list) +static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio) { + struct bio_list bio_list = BIO_EMPTY_LIST; int total_sector_nr; int ret = 0; - ASSERT(bio_list_size(bio_list) == 0); - /* Build a list of bios to read all the missing parts. */ for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; total_sector_nr++) { @@ -2710,42 +2708,38 @@ static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio, if (sector->uptodate) continue; - ret = rbio_add_io_sector(rbio, bio_list, sector, stripe, + ret = rbio_add_io_sector(rbio, &bio_list, sector, stripe, sectornr, REQ_OP_READ); - if (ret) - goto error; + if (ret) { + bio_list_put(&bio_list); + return ret; + } } + + submit_read_wait_bio_list(rbio, &bio_list); return 0; -error: - bio_list_put(bio_list); - return ret; } static int scrub_rbio(struct btrfs_raid_bio *rbio) { bool need_check = false; - struct bio_list bio_list; int sector_nr; int ret; - bio_list_init(&bio_list); - ret = alloc_rbio_essential_pages(rbio); if (ret) - goto cleanup; + return ret; bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); - ret = scrub_assemble_read_bios(rbio, &bio_list); + ret = scrub_assemble_read_bios(rbio); if (ret < 0) - goto cleanup; - - submit_read_wait_bio_list(rbio, &bio_list); + return ret; /* We may have some failures, recover the failed sectors first. */ ret = recover_scrub_rbio(rbio); if (ret < 0) - goto cleanup; + return ret; /* * We have every sector properly prepared. Can finish the scrub @@ -2763,10 +2757,6 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) } } return ret; - -cleanup: - bio_list_put(&bio_list); - return ret; } static void scrub_rbio_work_locked(struct work_struct *work) From 1d0ef1ca119f1a1fbcf5be5acce407147cb15245 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:32 +0100 Subject: [PATCH 253/265] btrfs: raid56: handle endio in rmw_rbio Both callers of rmv_rbio call rbio_orig_end_io right after it, so move the call into the shared function. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index b54c15b5f1df..cd85fb9b06a0 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2235,7 +2235,7 @@ static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio) return false; } -static int rmw_rbio(struct btrfs_raid_bio *rbio) +static void rmw_rbio(struct btrfs_raid_bio *rbio) { struct bio_list bio_list; int sectornr; @@ -2247,7 +2247,7 @@ static int rmw_rbio(struct btrfs_raid_bio *rbio) */ ret = alloc_rbio_parity_pages(rbio); if (ret < 0) - return ret; + goto out; /* * Either full stripe write, or we have every data sector already @@ -2260,13 +2260,13 @@ static int rmw_rbio(struct btrfs_raid_bio *rbio) */ ret = alloc_rbio_data_pages(rbio); if (ret < 0) - return ret; + goto out; index_rbio_pages(rbio); ret = rmw_read_wait_recover(rbio); if (ret < 0) - return ret; + goto out; } /* @@ -2299,7 +2299,7 @@ static int rmw_rbio(struct btrfs_raid_bio *rbio) bio_list_init(&bio_list); ret = rmw_assemble_write_bios(rbio, &bio_list); if (ret < 0) - return ret; + goto out; /* We should have at least one bio assembled. */ ASSERT(bio_list_size(&bio_list)); @@ -2316,32 +2316,22 @@ static int rmw_rbio(struct btrfs_raid_bio *rbio) break; } } - return ret; +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); } static void rmw_rbio_work(struct work_struct *work) { struct btrfs_raid_bio *rbio; - int ret; rbio = container_of(work, struct btrfs_raid_bio, work); - - ret = lock_stripe_add(rbio); - if (ret == 0) { - ret = rmw_rbio(rbio); - rbio_orig_end_io(rbio, errno_to_blk_status(ret)); - } + if (lock_stripe_add(rbio) == 0) + rmw_rbio(rbio); } static void rmw_rbio_work_locked(struct work_struct *work) { - struct btrfs_raid_bio *rbio; - int ret; - - rbio = container_of(work, struct btrfs_raid_bio, work); - - ret = rmw_rbio(rbio); - rbio_orig_end_io(rbio, errno_to_blk_status(ret)); + rmw_rbio(container_of(work, struct btrfs_raid_bio, work)); } /* From 40f87ddb5dbe43fb209901815d24e3e718aef155 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:33 +0100 Subject: [PATCH 254/265] btrfs: raid56: handle endio in recover_rbio Both callers of recover_rbio call rbio_orig_end_io right after it, so move the call into the shared function. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index cd85fb9b06a0..d06bdc20551d 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1914,7 +1914,7 @@ out: return ret; } -static int recover_rbio(struct btrfs_raid_bio *rbio) +static void recover_rbio(struct btrfs_raid_bio *rbio) { struct bio_list bio_list = BIO_EMPTY_LIST; int total_sector_nr; @@ -1929,7 +1929,7 @@ static int recover_rbio(struct btrfs_raid_bio *rbio) /* For recovery, we need to read all sectors including P/Q. */ ret = alloc_rbio_pages(rbio); if (ret < 0) - return ret; + goto out; index_rbio_pages(rbio); @@ -1967,37 +1967,28 @@ static int recover_rbio(struct btrfs_raid_bio *rbio) sectornr, REQ_OP_READ); if (ret < 0) { bio_list_put(&bio_list); - return ret; + goto out; } } submit_read_wait_bio_list(rbio, &bio_list); - return recover_sectors(rbio); + ret = recover_sectors(rbio); +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); } static void recover_rbio_work(struct work_struct *work) { struct btrfs_raid_bio *rbio; - int ret; rbio = container_of(work, struct btrfs_raid_bio, work); - - ret = lock_stripe_add(rbio); - if (ret == 0) { - ret = recover_rbio(rbio); - rbio_orig_end_io(rbio, errno_to_blk_status(ret)); - } + if (!lock_stripe_add(rbio)) + recover_rbio(rbio); } static void recover_rbio_work_locked(struct work_struct *work) { - struct btrfs_raid_bio *rbio; - int ret; - - rbio = container_of(work, struct btrfs_raid_bio, work); - - ret = recover_rbio(rbio); - rbio_orig_end_io(rbio, errno_to_blk_status(ret)); + recover_rbio(container_of(work, struct btrfs_raid_bio, work)); } static void set_rbio_raid6_extra_error(struct btrfs_raid_bio *rbio, int mirror_num) From 08241d3c745ed7c6de8a762c0737b337c68a6c87 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 11 Jan 2023 07:23:34 +0100 Subject: [PATCH 255/265] btrfs: raid56: handle endio in scrub_rbio The only caller of scrub_rbio calls rbio_orig_end_io right after it, move it into scrub_rbio to match the other work item helpers. Reviewed-by: Qu Wenruo Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index d06bdc20551d..642828c1b299 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2701,7 +2701,7 @@ static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio) return 0; } -static int scrub_rbio(struct btrfs_raid_bio *rbio) +static void scrub_rbio(struct btrfs_raid_bio *rbio) { bool need_check = false; int sector_nr; @@ -2709,18 +2709,18 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) ret = alloc_rbio_essential_pages(rbio); if (ret) - return ret; + goto out; bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); ret = scrub_assemble_read_bios(rbio); if (ret < 0) - return ret; + goto out; /* We may have some failures, recover the failed sectors first. */ ret = recover_scrub_rbio(rbio); if (ret < 0) - return ret; + goto out; /* * We have every sector properly prepared. Can finish the scrub @@ -2737,17 +2737,13 @@ static int scrub_rbio(struct btrfs_raid_bio *rbio) break; } } - return ret; +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); } static void scrub_rbio_work_locked(struct work_struct *work) { - struct btrfs_raid_bio *rbio; - int ret; - - rbio = container_of(work, struct btrfs_raid_bio, work); - ret = scrub_rbio(rbio); - rbio_orig_end_io(rbio, errno_to_blk_status(ret)); + scrub_rbio(container_of(work, struct btrfs_raid_bio, work)); } void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio) From 7b00dfffebd4f3444a3ec04d9e4203b7ac1acb47 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 8 Feb 2023 17:46:48 +0000 Subject: [PATCH 256/265] btrfs: eliminate extra call when doing binary search on extent buffer The function btrfs_bin_search() is just a wrapper around the function generic_bin_search(), which passes the same arguments plus a default low slot with a value of 0. This adds an unnecessary extra function call, since btrfs_bin_search() is not static. So improve on this by making btrfs_bin_search() an inline function that calls generic_bin_search(), renaming the later to btrfs_generic_bin_search() and exporting it. Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 16 +++------------- fs/btrfs/ctree.h | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 118440857f33..5f954642860c 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -863,8 +863,8 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans, * Slot may point to the total number of items (i.e. one position beyond the last * key) if the key is bigger than the last key in the extent buffer. */ -static noinline int generic_bin_search(struct extent_buffer *eb, int low, - const struct btrfs_key *key, int *slot) +int btrfs_generic_bin_search(struct extent_buffer *eb, int low, + const struct btrfs_key *key, int *slot) { unsigned long p; int item_size; @@ -925,16 +925,6 @@ static noinline int generic_bin_search(struct extent_buffer *eb, int low, return 1; } -/* - * Simple binary search on an extent buffer. Works for both leaves and nodes, and - * always searches over the whole range of keys (slot 0 to slot 'nritems - 1'). - */ -int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, - int *slot) -{ - return generic_bin_search(eb, 0, key, slot); -} - static void root_add_used(struct btrfs_root *root, u32 size) { spin_lock(&root->accounting_lock); @@ -1869,7 +1859,7 @@ static inline int search_for_key_slot(struct extent_buffer *eb, return 0; } - return generic_bin_search(eb, search_low_slot, key, slot); + return btrfs_generic_bin_search(eb, search_low_slot, key, slot); } static int search_leaf(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 6965703a81b6..322f2171275d 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -507,6 +507,21 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range); /* ctree.c */ int __init btrfs_ctree_init(void); void __cold btrfs_ctree_exit(void); + +int btrfs_generic_bin_search(struct extent_buffer *eb, int low, + const struct btrfs_key *key, int *slot); + +/* + * Simple binary search on an extent buffer. Works for both leaves and nodes, and + * always searches over the whole range of keys (slot 0 to slot 'nritems - 1'). + */ +static inline int btrfs_bin_search(struct extent_buffer *eb, + const struct btrfs_key *key, + int *slot) +{ + return btrfs_generic_bin_search(eb, 0, key, slot); +} + int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, int *slot); int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2); From a724f313f84beb5b63b8844d9ec42a547e4a3c18 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 8 Feb 2023 17:46:49 +0000 Subject: [PATCH 257/265] btrfs: do unsigned integer division in the extent buffer binary search loop In the search loop of the binary search function, we are doing a division by 2 of the sum of the high and low slots. Because the slots are integers, the generated assembly code for it is the following on x86_64: 0x00000000000141f1 <+145>: mov %eax,%ebx 0x00000000000141f3 <+147>: shr $0x1f,%ebx 0x00000000000141f6 <+150>: add %eax,%ebx 0x00000000000141f8 <+152>: sar %ebx It's a few more instructions than a simple right shift, because signed integer division needs to round towards zero. However we know that slots can never be negative (btrfs_header_nritems() returns an u32), so we can instead use unsigned types for the low and high slots and therefore use unsigned integer division, which results in a single instruction on x86_64: 0x00000000000141f0 <+144>: shr %ebx So use unsigned types for the slots and therefore unsigned division. This is part of a small patchset comprised of the following two patches: btrfs: eliminate extra call when doing binary search on extent buffer btrfs: do unsigned integer division in the extent buffer binary search loop The following fs_mark test was run on a non-debug kernel (Debian's default kernel config) before and after applying the patchset: $ cat test.sh #!/bin/bash DEV=/dev/sdi MNT=/mnt/sdi MOUNT_OPTIONS="-o ssd" MKFS_OPTIONS="-O no-holes -R free-space-tree" FILES=100000 THREADS=$(nproc --all) FILE_SIZE=0 umount $DEV &> /dev/null mkfs.btrfs -f $MKFS_OPTIONS $DEV mount $MOUNT_OPTIONS $DEV $MNT OPTS="-S 0 -L 6 -n $FILES -s $FILE_SIZE -t $THREADS -k" for ((i = 1; i <= $THREADS; i++)); do OPTS="$OPTS -d $MNT/d$i" done fs_mark $OPTS umount $MNT Results before applying patchset: FSUse% Count Size Files/sec App Overhead 2 1200000 0 174472.0 11549868 4 2400000 0 253503.0 11694618 4 3600000 0 257833.1 11611508 6 4800000 0 247089.5 11665983 6 6000000 0 211296.1 12121244 10 7200000 0 187330.6 12548565 Results after applying patchset: FSUse% Count Size Files/sec App Overhead 2 1200000 0 207556.0 11393252 4 2400000 0 266751.1 11347909 4 3600000 0 274397.5 11270058 6 4800000 0 259608.4 11442250 6 6000000 0 238895.8 11635921 8 7200000 0 211942.2 11873825 Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 17 +++++++++++------ fs/btrfs/ctree.h | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 5f954642860c..a5b6bb54545f 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -853,8 +853,8 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans, /* * Search for a key in the given extent_buffer. * - * The lower boundary for the search is specified by the slot number @low. Use a - * value of 0 to search over the whole extent buffer. + * The lower boundary for the search is specified by the slot number @first_slot. + * Use a value of 0 to search over the whole extent buffer. * * The slot in the extent buffer is returned via @slot. If the key exists in the * extent buffer, then @slot will point to the slot where the key is, otherwise @@ -863,18 +863,23 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans, * Slot may point to the total number of items (i.e. one position beyond the last * key) if the key is bigger than the last key in the extent buffer. */ -int btrfs_generic_bin_search(struct extent_buffer *eb, int low, +int btrfs_generic_bin_search(struct extent_buffer *eb, int first_slot, const struct btrfs_key *key, int *slot) { unsigned long p; int item_size; - int high = btrfs_header_nritems(eb); + /* + * Use unsigned types for the low and high slots, so that we get a more + * efficient division in the search loop below. + */ + u32 low = first_slot; + u32 high = btrfs_header_nritems(eb); int ret; const int key_size = sizeof(struct btrfs_disk_key); - if (low > high) { + if (unlikely(low > high)) { btrfs_err(eb->fs_info, - "%s: low (%d) > high (%d) eb %llu owner %llu level %d", + "%s: low (%u) > high (%u) eb %llu owner %llu level %d", __func__, low, high, eb->start, btrfs_header_owner(eb), btrfs_header_level(eb)); return -EINVAL; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 322f2171275d..97897107fab5 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -508,7 +508,7 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range); int __init btrfs_ctree_init(void); void __cold btrfs_ctree_exit(void); -int btrfs_generic_bin_search(struct extent_buffer *eb, int low, +int btrfs_generic_bin_search(struct extent_buffer *eb, int first_slot, const struct btrfs_key *key, int *slot); /* From 72fcf1a47b8daba82ffdeddd630688df5ac1bddb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:18 +0100 Subject: [PATCH 258/265] btrfs: use file_offset to limit bios size in calc_bio_boundaries btrfs_ordered_extent->disk_bytenr can be rewritten by the zoned I/O completion handler, and thus in general is not a good idea to limit I/O size. But the maximum bio size calculation can easily be done using the file_offset fields in the btrfs_ordered_extent and btrfs_bio structures, so switch to that instead. Reviewed-by: Josef Bacik Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 287e25edb3ef..d823994cf62a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -964,8 +964,8 @@ static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, ordered = btrfs_lookup_ordered_extent(inode, file_offset); if (ordered) { bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, - ordered->disk_bytenr + - ordered->disk_num_bytes - logical); + ordered->file_offset + + ordered->disk_num_bytes - file_offset); btrfs_put_ordered_extent(ordered); return; } From 0d495430db8d704b3a70b244b54d5ee30cf03f69 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:20 +0100 Subject: [PATCH 259/265] btrfs: set bbio->file_offset in alloc_new_bio Instead of digging into the bio_vec in submit_one_bio, set file_offset at bio allocation time from the provided parameter. This also ensures that the file_offset is available all the time when building up the bio payload. Reviewed-by: Josef Bacik Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index d823994cf62a..6a3ec0cecf51 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -140,8 +140,6 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) /* Caller should ensure the bio has at least some range added */ ASSERT(bio->bi_iter.bi_size); - btrfs_bio(bio)->file_offset = page_offset(bv->bv_page) + bv->bv_offset; - if (!is_data_inode(inode)) { if (btrfs_op(bio) != BTRFS_MAP_WRITE) { /* @@ -993,6 +991,7 @@ static void alloc_new_bio(struct btrfs_inode *inode, bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; else bio->bi_iter.bi_sector = (disk_bytenr + offset) >> SECTOR_SHIFT; + btrfs_bio(bio)->file_offset = file_offset; bio_ctrl->bio = bio; bio_ctrl->compress_type = compress_type; calc_bio_boundaries(bio_ctrl, inode, file_offset); From 921603c76246a7f716b9a244d7b1fa1653935f31 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:21 +0100 Subject: [PATCH 260/265] btrfs: pass a btrfs_bio to btrfs_use_append struct btrfs_bio has all the information needed for btrfs_use_append, so pass that instead of a btrfs_inode and file_offset. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/bio.c | 2 +- fs/btrfs/extent_io.c | 3 +-- fs/btrfs/zoned.c | 4 +++- fs/btrfs/zoned.h | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 8445df47495a..d8b90f95b157 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -612,7 +612,7 @@ static bool btrfs_submit_chunk(struct bio *bio, int mirror_num) u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - bool use_append = btrfs_use_zone_append(inode, logical); + bool use_append = btrfs_use_zone_append(bbio); struct btrfs_io_context *bioc = NULL; struct btrfs_io_stripe smap; blk_status_t ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 6a3ec0cecf51..c25fa74d7615 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -950,7 +950,6 @@ static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, struct btrfs_inode *inode, u64 file_offset) { struct btrfs_ordered_extent *ordered; - u64 logical = (bio_ctrl->bio->bi_iter.bi_sector << SECTOR_SHIFT); /* * Limit the extent to the ordered boundary for Zone Append. @@ -958,7 +957,7 @@ static void calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, * them. */ if (bio_ctrl->compress_type == BTRFS_COMPRESS_NONE && - btrfs_use_zone_append(inode, logical)) { + btrfs_use_zone_append(btrfs_bio(bio_ctrl->bio))) { ordered = btrfs_lookup_ordered_extent(inode, file_offset); if (ordered) { bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 06ed1eeb5981..8824748ee8c3 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1627,8 +1627,10 @@ void btrfs_free_redirty_list(struct btrfs_transaction *trans) spin_unlock(&trans->releasing_ebs_lock); } -bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start) +bool btrfs_use_zone_append(struct btrfs_bio *bbio) { + u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); + struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_block_group *cache; bool ret = false; diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 157f46132c56..c0570d35fea2 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -55,7 +55,7 @@ void btrfs_calc_zone_unusable(struct btrfs_block_group *cache); void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb); void btrfs_free_redirty_list(struct btrfs_transaction *trans); -bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start); +bool btrfs_use_zone_append(struct btrfs_bio *bbio); void btrfs_record_physical_zoned(struct btrfs_bio *bbio); void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered); bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, @@ -181,7 +181,7 @@ static inline void btrfs_redirty_list_add(struct btrfs_transaction *trans, struct extent_buffer *eb) { } static inline void btrfs_free_redirty_list(struct btrfs_transaction *trans) { } -static inline bool btrfs_use_zone_append(struct btrfs_inode *inode, u64 start) +static inline bool btrfs_use_zone_append(struct btrfs_bio *bbio) { return false; } From fdf9a37dcfd47e9bd18d1218f2d0b2fa3748d00d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:22 +0100 Subject: [PATCH 261/265] btrfs: never return true for reads in btrfs_use_zone_append Using Zone Append only makes sense for writes to the device, so check that in btrfs_use_zone_append. This avoids the possibility of artificially limited read size on zoned file systems. Reviewed-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/zoned.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 8824748ee8c3..442bb79ffd08 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1641,6 +1641,9 @@ bool btrfs_use_zone_append(struct btrfs_bio *bbio) if (!is_data_inode(&inode->vfs_inode)) return false; + if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) + return false; + /* * Using REQ_OP_ZONE_APPNED for relocation can break assumptions on the * extent layout the relocation code has. From 04f0847c4552b898ec5867a6b36f1e953330beae Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:23 +0100 Subject: [PATCH 262/265] btrfs: don't rely on unchanging ->bi_bdev for zone append remaps btrfs_record_physical_zoned relies on a bio->bi_bdev samples in the bio_end_io handler to find the reverse map for remapping the zone append write, but stacked block device drivers can and usually do change bi_bdev when sending on the bio to a lower device. This can happen e.g. with the nvme-multipath driver when a NVMe SSD sets the shared namespace bit. But there is no real need for the bdev in btrfs_record_physical_zoned, as it is only passed to btrfs_rmap_block, which uses it to pick the mapping to report if there are multiple reverse mappings. As zone writes can only do simple non-mirror writes right now, and anything more complex will use the stripe tree there is no chance of the multiple mappings case actually happening. Instead open code the subset of btrfs_rmap_block in btrfs_record_physical_zoned, which also removes a memory allocation and remove the bdev field in the ordered extent. Fixes: d8e3fb106f39 ("btrfs: zoned: use ZONE_APPEND write for zoned mode") Reviewed-by: Josef Bacik Signed-off-by: Christoph Hellwig Signed-off-by: David Sterba --- fs/btrfs/inode.c | 4 ++-- fs/btrfs/ordered-data.h | 1 - fs/btrfs/zoned.c | 51 +++++++++++++++++++++-------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 2fd518afc4f3..44e9acc77a74 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3162,8 +3162,8 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) goto out; } - /* A valid bdev implies a write on a sequential zone */ - if (ordered_extent->bdev) { + /* A valid ->physical implies a write on a sequential zone. */ + if (ordered_extent->physical != (u64)-1) { btrfs_rewrite_logical_zoned(ordered_extent); btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, ordered_extent->disk_num_bytes); diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index ae3ed748acaf..eb40cb39f842 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -157,7 +157,6 @@ struct btrfs_ordered_extent { * command in a workqueue context */ u64 physical; - struct block_device *bdev; }; static inline void diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index 442bb79ffd08..f95b2c94d619 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -1676,8 +1676,6 @@ void btrfs_record_physical_zoned(struct btrfs_bio *bbio) return; ordered->physical = physical; - ordered->bdev = bbio->bio.bi_bdev; - btrfs_put_ordered_extent(ordered); } @@ -1689,43 +1687,46 @@ void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered) struct extent_map *em; struct btrfs_ordered_sum *sum; u64 orig_logical = ordered->disk_bytenr; - u64 *logical = NULL; - int nr, stripe_len; + struct map_lookup *map; + u64 physical = ordered->physical; + u64 chunk_start_phys; + u64 logical; - /* Zoned devices should not have partitions. So, we can assume it is 0 */ - ASSERT(!bdev_is_partition(ordered->bdev)); - if (WARN_ON(!ordered->bdev)) + em = btrfs_get_chunk_map(fs_info, orig_logical, 1); + if (IS_ERR(em)) + return; + map = em->map_lookup; + chunk_start_phys = map->stripes[0].physical; + + if (WARN_ON_ONCE(map->num_stripes > 1) || + WARN_ON_ONCE((map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) != 0) || + WARN_ON_ONCE(physical < chunk_start_phys) || + WARN_ON_ONCE(physical > chunk_start_phys + em->orig_block_len)) { + free_extent_map(em); + return; + } + logical = em->start + (physical - map->stripes[0].physical); + free_extent_map(em); + + if (orig_logical == logical) return; - if (WARN_ON(btrfs_rmap_block(fs_info, orig_logical, ordered->bdev, - ordered->physical, &logical, &nr, - &stripe_len))) - goto out; - - WARN_ON(nr != 1); - - if (orig_logical == *logical) - goto out; - - ordered->disk_bytenr = *logical; + ordered->disk_bytenr = logical; em_tree = &inode->extent_tree; write_lock(&em_tree->lock); em = search_extent_mapping(em_tree, ordered->file_offset, ordered->num_bytes); - em->block_start = *logical; + em->block_start = logical; free_extent_map(em); write_unlock(&em_tree->lock); list_for_each_entry(sum, &ordered->list, list) { - if (*logical < orig_logical) - sum->bytenr -= orig_logical - *logical; + if (logical < orig_logical) + sum->bytenr -= orig_logical - logical; else - sum->bytenr += *logical - orig_logical; + sum->bytenr += logical - orig_logical; } - -out: - kfree(logical); } bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, From 1eb82ef87336045459929d915b8050beaea92951 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 12 Dec 2022 08:37:24 +0100 Subject: [PATCH 263/265] btrfs: remove the bdev argument to btrfs_rmap_block The only user in the zoned remap code is gone now, so remove the argument. Reviewed-by: Josef Bacik Signed-off-by: Christoph Hellwig Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 9 ++------- fs/btrfs/block-group.h | 3 +-- fs/btrfs/tests/extent-map-tests.c | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 45ccb25c5b1f..5b10401d803b 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1965,7 +1965,6 @@ static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) * * @fs_info: the filesystem * @chunk_start: logical address of block group - * @bdev: physical device to resolve, can be NULL to indicate any device * @physical: physical address to map to logical addresses * @logical: return array of logical addresses which map to @physical * @naddrs: length of @logical @@ -1976,8 +1975,7 @@ static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) * block copies. */ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, - struct block_device *bdev, u64 physical, u64 **logical, - int *naddrs, int *stripe_len) + u64 physical, u64 **logical, int *naddrs, int *stripe_len) { struct extent_map *em; struct map_lookup *map; @@ -2017,9 +2015,6 @@ int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, data_stripe_length)) continue; - if (bdev && map->stripes[i].dev->bdev != bdev) - continue; - stripe_nr = physical - map->stripes[i].physical; stripe_nr = div64_u64_rem(stripe_nr, map->stripe_len, &offset); @@ -2076,7 +2071,7 @@ static int exclude_super_stripes(struct btrfs_block_group *cache) for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { bytenr = btrfs_sb_offset(i); - ret = btrfs_rmap_block(fs_info, cache->start, NULL, + ret = btrfs_rmap_block(fs_info, cache->start, bytenr, &logical, &nr, &stripe_len); if (ret) return ret; diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 4fdc39f00a4d..6e4a0b429ac3 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -328,8 +328,7 @@ u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags); void btrfs_put_block_group_cache(struct btrfs_fs_info *info); int btrfs_free_block_groups(struct btrfs_fs_info *info); int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, - struct block_device *bdev, u64 physical, u64 **logical, - int *naddrs, int *stripe_len); + u64 physical, u64 **logical, int *naddrs, int *stripe_len); static inline u64 btrfs_data_alloc_profile(struct btrfs_fs_info *fs_info) { diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c index c5b3a631bf4f..f2f2e11dac4c 100644 --- a/fs/btrfs/tests/extent-map-tests.c +++ b/fs/btrfs/tests/extent-map-tests.c @@ -509,7 +509,7 @@ static int test_rmap_block(struct btrfs_fs_info *fs_info, goto out_free; } - ret = btrfs_rmap_block(fs_info, em->start, NULL, btrfs_sb_offset(1), + ret = btrfs_rmap_block(fs_info, em->start, btrfs_sb_offset(1), &logical, &out_ndaddrs, &out_stripe_len); if (ret || (out_ndaddrs == 0 && test->expected_mapped_addr)) { test_err("didn't rmap anything but expected %d", From 964a54e5e1a0d70cd80bd5a0885a1938463625b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Fri, 10 Feb 2023 02:13:40 +0000 Subject: [PATCH 264/265] btrfs: make kobj_type structures constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ee6d3dd4ed48 ("driver core: make kobj_type constant.") the driver core allows the usage of const struct kobj_type. Take advantage of this to constify the structure definitions to prevent modification at runtime. Reviewed-by: Anand Jain Signed-off-by: Thomas Weißschuh Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/sysfs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 108aa3876186..8c5efa5813b3 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -702,7 +702,7 @@ static void release_raid_kobj(struct kobject *kobj) kfree(to_raid_kobj(kobj)); } -static struct kobj_type btrfs_raid_ktype = { +static const struct kobj_type btrfs_raid_ktype = { .sysfs_ops = &kobj_sysfs_ops, .release = release_raid_kobj, .default_groups = raid_groups, @@ -900,7 +900,7 @@ static void space_info_release(struct kobject *kobj) kfree(sinfo); } -static struct kobj_type space_info_ktype = { +static const struct kobj_type space_info_ktype = { .sysfs_ops = &kobj_sysfs_ops, .release = space_info_release, .default_groups = space_info_groups, @@ -1259,7 +1259,7 @@ static void btrfs_release_fsid_kobj(struct kobject *kobj) complete(&fs_devs->kobj_unregister); } -static struct kobj_type btrfs_ktype = { +static const struct kobj_type btrfs_ktype = { .sysfs_ops = &kobj_sysfs_ops, .release = btrfs_release_fsid_kobj, }; @@ -1789,7 +1789,7 @@ static void btrfs_release_devid_kobj(struct kobject *kobj) complete(&device->kobj_unregister); } -static struct kobj_type devid_ktype = { +static const struct kobj_type devid_ktype = { .sysfs_ops = &kobj_sysfs_ops, .default_groups = devid_groups, .release = btrfs_release_devid_kobj, @@ -2103,7 +2103,7 @@ static void qgroups_release(struct kobject *kobj) kfree(kobj); } -static struct kobj_type qgroups_ktype = { +static const struct kobj_type qgroups_ktype = { .sysfs_ops = &kobj_sysfs_ops, .default_groups = qgroups_groups, .release = qgroups_release, @@ -2173,7 +2173,7 @@ static void qgroup_release(struct kobject *kobj) memset(&qgroup->kobj, 0, sizeof(*kobj)); } -static struct kobj_type qgroup_ktype = { +static const struct kobj_type qgroup_ktype = { .sysfs_ops = &kobj_sysfs_ops, .release = qgroup_release, .default_groups = qgroup_groups, From 7d3fd88d61a41016da01889f076fd1c60c7298fc Mon Sep 17 00:00:00 2001 From: Josh Triplett Date: Tue, 14 Feb 2023 16:42:22 -0800 Subject: [PATCH 265/265] io_uring: Support calling io_uring_register with a registered ring fd Add a new flag IORING_REGISTER_USE_REGISTERED_RING (set via the high bit of the opcode) to treat the fd as a registered index rather than a file descriptor. This makes it possible for a library to open an io_uring, register the ring fd, close the ring fd, and subsequently use the ring entirely via registered index. Signed-off-by: Josh Triplett Link: https://lore.kernel.org/r/f2396369e638284586b069dbddffb8c992afba95.1676419314.git.josh@joshtriplett.org [axboe: remove extra high bit clear] Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 6 +++++- io_uring/io_uring.c | 33 ++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 636a4c2c1294..97661a60b28c 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -472,6 +472,7 @@ struct io_uring_params { #define IORING_FEAT_RSRC_TAGS (1U << 10) #define IORING_FEAT_CQE_SKIP (1U << 11) #define IORING_FEAT_LINKED_FILE (1U << 12) +#define IORING_FEAT_REG_REG_RING (1U << 13) /* * io_uring_register(2) opcodes and arguments @@ -519,7 +520,10 @@ enum { IORING_REGISTER_FILE_ALLOC_RANGE = 25, /* this goes last */ - IORING_REGISTER_LAST + IORING_REGISTER_LAST, + + /* flag added to the opcode to use a registered ring fd */ + IORING_REGISTER_USE_REGISTERED_RING = 1U << 31 }; /* io-wq worker categories */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 0e42160a3051..3b915deb4d08 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3785,7 +3785,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED | IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS | IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP | - IORING_FEAT_LINKED_FILE; + IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING; if (copy_to_user(params, p, sizeof(*p))) { ret = -EFAULT; @@ -4306,17 +4306,36 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, struct io_ring_ctx *ctx; long ret = -EBADF; struct fd f; + bool use_registered_ring; + + use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING); + opcode &= ~IORING_REGISTER_USE_REGISTERED_RING; if (opcode >= IORING_REGISTER_LAST) return -EINVAL; - f = fdget(fd); - if (!f.file) - return -EBADF; + if (use_registered_ring) { + /* + * Ring fd has been registered via IORING_REGISTER_RING_FDS, we + * need only dereference our task private array to find it. + */ + struct io_uring_task *tctx = current->io_uring; - ret = -EOPNOTSUPP; - if (!io_is_uring_fops(f.file)) - goto out_fput; + if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX)) + return -EINVAL; + fd = array_index_nospec(fd, IO_RINGFD_REG_MAX); + f.file = tctx->registered_rings[fd]; + f.flags = 0; + if (unlikely(!f.file)) + return -EBADF; + } else { + f = fdget(fd); + if (unlikely(!f.file)) + return -EBADF; + ret = -EOPNOTSUPP; + if (!io_is_uring_fops(f.file)) + goto out_fput; + } ctx = f.file->private_data;