From 67f1e027c27054e641584655020a417eaac9cb3a Mon Sep 17 00:00:00 2001 From: Lukas Prediger Date: Tue, 14 Sep 2021 00:09:42 +0100 Subject: [PATCH 001/255] drivers/cdrom: improved ioctl for media change detection The current implementation of the CDROM_MEDIA_CHANGED ioctl relies on global state, meaning that only one process can detect a disc change while the ioctl call will return 0 for other calling processes afterwards (see bug 213267). This introduces a new cdrom ioctl, CDROM_TIMED_MEDIA_CHANGE, that works by maintaining a timestamp of the last detected disc change instead of a boolean flag: Processes calling this ioctl command can provide a timestamp of the last disc change known to them and receive an indication whether the disc was changed since then and the updated timestamp. I considered fixing the buggy behavior in the original CDROM_MEDIA_CHANGED ioctl but that would require maintaining state for each calling process in the kernel, which seems like a worse solution than introducing this new ioctl. Signed-off-by: Lukas Prediger Link: https://lore.kernel.org/all/20210912191207.74449-1-lumip@lumip.de Signed-off-by: Phillip Potter Link: https://lore.kernel.org/r/20210913230942.1188-1-phil@philpotter.co.uk Signed-off-by: Jens Axboe --- Documentation/cdrom/cdrom-standard.rst | 11 ++++ Documentation/userspace-api/ioctl/cdrom.rst | 3 ++ drivers/cdrom/cdrom.c | 59 +++++++++++++++++++-- include/linux/cdrom.h | 1 + include/uapi/linux/cdrom.h | 19 +++++++ 5 files changed, 89 insertions(+), 4 deletions(-) diff --git a/Documentation/cdrom/cdrom-standard.rst b/Documentation/cdrom/cdrom-standard.rst index 5845960ca382..52ea7b6b2fe8 100644 --- a/Documentation/cdrom/cdrom-standard.rst +++ b/Documentation/cdrom/cdrom-standard.rst @@ -907,6 +907,17 @@ commands can be identified by the underscores in their names. specifies the slot for which the information is given. The special value *CDSL_CURRENT* requests that information about the currently selected slot be returned. +`CDROM_TIMED_MEDIA_CHANGE` + Checks whether the disc has been changed since a user supplied time + and returns the time of the last disc change. + + *arg* is a pointer to a *cdrom_timed_media_change_info* struct. + *arg->last_media_change* may be set by calling code to signal + the timestamp of the last known media change (by the caller). + Upon successful return, this ioctl call will set + *arg->last_media_change* to the latest media change timestamp (in ms) + known by the kernel/driver and set *arg->has_changed* to 1 if + that timestamp is more recent than the timestamp set by the caller. `CDROM_DRIVE_STATUS` Returns the status of the drive by a call to *drive_status()*. Return values are defined in cdrom_drive_status_. diff --git a/Documentation/userspace-api/ioctl/cdrom.rst b/Documentation/userspace-api/ioctl/cdrom.rst index 3b4c0506de46..bac5bbf93ca0 100644 --- a/Documentation/userspace-api/ioctl/cdrom.rst +++ b/Documentation/userspace-api/ioctl/cdrom.rst @@ -54,6 +54,9 @@ are as follows: CDROM_SELECT_SPEED Set the CD-ROM speed CDROM_SELECT_DISC Select disc (for juke-boxes) CDROM_MEDIA_CHANGED Check is media changed + CDROM_TIMED_MEDIA_CHANGE Check if media changed + since given time + (struct cdrom_timed_media_change_info) CDROM_DRIVE_STATUS Get tray position, etc. CDROM_DISC_STATUS Get disc type, etc. CDROM_CHANGER_NSLOTS Get number of slots diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index bd2e5b1560f5..89a68457820a 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -344,6 +344,12 @@ static void cdrom_sysctl_register(void); static LIST_HEAD(cdrom_list); +static void signal_media_change(struct cdrom_device_info *cdi) +{ + cdi->mc_flags = 0x3; /* set media changed bits, on both queues */ + cdi->last_media_change_ms = ktime_to_ms(ktime_get()); +} + int cdrom_dummy_generic_packet(struct cdrom_device_info *cdi, struct packet_command *cgc) { @@ -616,6 +622,7 @@ int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi) ENSURE(cdo, generic_packet, CDC_GENERIC_PACKET); cdi->mc_flags = 0; cdi->options = CDO_USE_FFLAGS; + cdi->last_media_change_ms = ktime_to_ms(ktime_get()); if (autoclose == 1 && CDROM_CAN(CDC_CLOSE_TRAY)) cdi->options |= (int) CDO_AUTO_CLOSE; @@ -1421,8 +1428,7 @@ static int cdrom_select_disc(struct cdrom_device_info *cdi, int slot) cdi->ops->check_events(cdi, 0, slot); if (slot == CDSL_NONE) { - /* set media changed bits, on both queues */ - cdi->mc_flags = 0x3; + signal_media_change(cdi); return cdrom_load_unload(cdi, -1); } @@ -1455,7 +1461,7 @@ static int cdrom_select_disc(struct cdrom_device_info *cdi, int slot) slot = curslot; /* set media changed bits on both queues */ - cdi->mc_flags = 0x3; + signal_media_change(cdi); if ((ret = cdrom_load_unload(cdi, slot))) return ret; @@ -1521,7 +1527,7 @@ int media_changed(struct cdrom_device_info *cdi, int queue) cdi->ioctl_events = 0; if (changed) { - cdi->mc_flags = 0x3; /* set bit on both queues */ + signal_media_change(cdi); ret |= 1; cdi->media_written = 0; } @@ -2336,6 +2342,49 @@ static int cdrom_ioctl_media_changed(struct cdrom_device_info *cdi, return ret; } +/* + * Media change detection with timing information. + * + * arg is a pointer to a cdrom_timed_media_change_info struct. + * arg->last_media_change may be set by calling code to signal + * the timestamp (in ms) of the last known media change (by the caller). + * Upon successful return, ioctl call will set arg->last_media_change + * to the latest media change timestamp known by the kernel/driver + * and set arg->has_changed to 1 if that timestamp is more recent + * than the timestamp set by the caller. + */ +static int cdrom_ioctl_timed_media_change(struct cdrom_device_info *cdi, + unsigned long arg) +{ + int ret; + struct cdrom_timed_media_change_info __user *info; + struct cdrom_timed_media_change_info tmp_info; + + if (!CDROM_CAN(CDC_MEDIA_CHANGED)) + return -ENOSYS; + + info = (struct cdrom_timed_media_change_info __user *)arg; + cd_dbg(CD_DO_IOCTL, "entering CDROM_TIMED_MEDIA_CHANGE\n"); + + ret = cdrom_ioctl_media_changed(cdi, CDSL_CURRENT); + if (ret < 0) + return ret; + + if (copy_from_user(&tmp_info, info, sizeof(tmp_info)) != 0) + return -EFAULT; + + tmp_info.media_flags = 0; + if (tmp_info.last_media_change - cdi->last_media_change_ms < 0) + tmp_info.media_flags |= MEDIA_CHANGED_FLAG; + + tmp_info.last_media_change = cdi->last_media_change_ms; + + if (copy_to_user(info, &tmp_info, sizeof(*info)) != 0) + return -EFAULT; + + return 0; +} + static int cdrom_ioctl_set_options(struct cdrom_device_info *cdi, unsigned long arg) { @@ -3313,6 +3362,8 @@ int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, return cdrom_ioctl_eject_sw(cdi, arg); case CDROM_MEDIA_CHANGED: return cdrom_ioctl_media_changed(cdi, arg); + case CDROM_TIMED_MEDIA_CHANGE: + return cdrom_ioctl_timed_media_change(cdi, arg); case CDROM_SET_OPTIONS: return cdrom_ioctl_set_options(cdi, arg); case CDROM_CLEAR_OPTIONS: diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index c4fef00abdf3..0a89f111e00e 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -64,6 +64,7 @@ struct cdrom_device_info { int for_data; int (*exit)(struct cdrom_device_info *); int mrw_mode_page; + __s64 last_media_change_ms; }; struct cdrom_device_ops { diff --git a/include/uapi/linux/cdrom.h b/include/uapi/linux/cdrom.h index 6c34f6e2f1f7..804ff8d98f71 100644 --- a/include/uapi/linux/cdrom.h +++ b/include/uapi/linux/cdrom.h @@ -147,6 +147,8 @@ #define CDROM_NEXT_WRITABLE 0x5394 /* get next writable block */ #define CDROM_LAST_WRITTEN 0x5395 /* get last block written on disc */ +#define CDROM_TIMED_MEDIA_CHANGE 0x5396 /* get the timestamp of the last media change */ + /******************************************************* * CDROM IOCTL structures *******************************************************/ @@ -295,6 +297,23 @@ struct cdrom_generic_command }; }; +/* This struct is used by CDROM_TIMED_MEDIA_CHANGE */ +struct cdrom_timed_media_change_info { + __s64 last_media_change; /* Timestamp of the last detected media + * change in ms. May be set by caller, + * updated upon successful return of + * ioctl. + */ + __u64 media_flags; /* Flags returned by ioctl to indicate + * media status. + */ +}; +#define MEDIA_CHANGED_FLAG 0x1 /* Last detected media change was more + * recent than last_media_change set by + * caller. + */ +/* other bits of media_flags available for future use */ + /* * A CD-ROM physical sector size is 2048, 2052, 2056, 2324, 2332, 2336, * 2340, or 2352 bytes long. From 9c03fee7e3fa01ca86d9d6195994cd9e6d24264b Mon Sep 17 00:00:00 2001 From: Phillip Potter Date: Fri, 8 Oct 2021 00:17:20 +0100 Subject: [PATCH 002/255] cdrom: docs: reformat table in Documentation/userspace-api/ioctl/cdrom.rst Add extra space to the first column of ioctl values table, and also realign the text in the second column, to fix a documentation build warning introduced by: commit 67f1e027c270 ("drivers/cdrom: improved ioctl for media change detection") Suggested-by: Randy Dunlap Fixes: 67f1e027c270 ("drivers/cdrom: improved ioctl for media change detection") Signed-off-by: Phillip Potter Reviewed-by: Randy Dunlap Link: https://lore.kernel.org/r/20211007231720.31997-1-phil@philpotter.co.uk Signed-off-by: Jens Axboe --- Documentation/userspace-api/ioctl/cdrom.rst | 116 ++++++++++---------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/Documentation/userspace-api/ioctl/cdrom.rst b/Documentation/userspace-api/ioctl/cdrom.rst index bac5bbf93ca0..682948fc88a3 100644 --- a/Documentation/userspace-api/ioctl/cdrom.rst +++ b/Documentation/userspace-api/ioctl/cdrom.rst @@ -13,64 +13,64 @@ in drivers/cdrom/cdrom.c and drivers/block/scsi_ioctl.c ioctl values are listed in . As of this writing, they are as follows: - ====================== =============================================== - CDROMPAUSE Pause Audio Operation - CDROMRESUME Resume paused Audio Operation - CDROMPLAYMSF Play Audio MSF (struct cdrom_msf) - CDROMPLAYTRKIND Play Audio Track/index (struct cdrom_ti) - CDROMREADTOCHDR Read TOC header (struct cdrom_tochdr) - CDROMREADTOCENTRY Read TOC entry (struct cdrom_tocentry) - CDROMSTOP Stop the cdrom drive - CDROMSTART Start the cdrom drive - CDROMEJECT Ejects the cdrom media - CDROMVOLCTRL Control output volume (struct cdrom_volctrl) - CDROMSUBCHNL Read subchannel data (struct cdrom_subchnl) - CDROMREADMODE2 Read CDROM mode 2 data (2336 Bytes) - (struct cdrom_read) - CDROMREADMODE1 Read CDROM mode 1 data (2048 Bytes) - (struct cdrom_read) - CDROMREADAUDIO (struct cdrom_read_audio) - CDROMEJECT_SW enable(1)/disable(0) auto-ejecting - CDROMMULTISESSION Obtain the start-of-last-session - address of multi session disks - (struct cdrom_multisession) - CDROM_GET_MCN Obtain the "Universal Product Code" - if available (struct cdrom_mcn) - CDROM_GET_UPC Deprecated, use CDROM_GET_MCN instead. - CDROMRESET hard-reset the drive - CDROMVOLREAD Get the drive's volume setting - (struct cdrom_volctrl) - CDROMREADRAW read data in raw mode (2352 Bytes) - (struct cdrom_read) - CDROMREADCOOKED read data in cooked mode - CDROMSEEK seek msf address - CDROMPLAYBLK scsi-cd only, (struct cdrom_blk) - CDROMREADALL read all 2646 bytes - CDROMGETSPINDOWN return 4-bit spindown value - CDROMSETSPINDOWN set 4-bit spindown value - CDROMCLOSETRAY pendant of CDROMEJECT - CDROM_SET_OPTIONS Set behavior options - CDROM_CLEAR_OPTIONS Clear behavior options - CDROM_SELECT_SPEED Set the CD-ROM speed - CDROM_SELECT_DISC Select disc (for juke-boxes) - CDROM_MEDIA_CHANGED Check is media changed - CDROM_TIMED_MEDIA_CHANGE Check if media changed - since given time - (struct cdrom_timed_media_change_info) - CDROM_DRIVE_STATUS Get tray position, etc. - CDROM_DISC_STATUS Get disc type, etc. - CDROM_CHANGER_NSLOTS Get number of slots - CDROM_LOCKDOOR lock or unlock door - CDROM_DEBUG Turn debug messages on/off - CDROM_GET_CAPABILITY get capabilities - CDROMAUDIOBUFSIZ set the audio buffer size - DVD_READ_STRUCT Read structure - DVD_WRITE_STRUCT Write structure - DVD_AUTH Authentication - CDROM_SEND_PACKET send a packet to the drive - CDROM_NEXT_WRITABLE get next writable block - CDROM_LAST_WRITTEN get last block written on disc - ====================== =============================================== + ======================== =============================================== + CDROMPAUSE Pause Audio Operation + CDROMRESUME Resume paused Audio Operation + CDROMPLAYMSF Play Audio MSF (struct cdrom_msf) + CDROMPLAYTRKIND Play Audio Track/index (struct cdrom_ti) + CDROMREADTOCHDR Read TOC header (struct cdrom_tochdr) + CDROMREADTOCENTRY Read TOC entry (struct cdrom_tocentry) + CDROMSTOP Stop the cdrom drive + CDROMSTART Start the cdrom drive + CDROMEJECT Ejects the cdrom media + CDROMVOLCTRL Control output volume (struct cdrom_volctrl) + CDROMSUBCHNL Read subchannel data (struct cdrom_subchnl) + CDROMREADMODE2 Read CDROM mode 2 data (2336 Bytes) + (struct cdrom_read) + CDROMREADMODE1 Read CDROM mode 1 data (2048 Bytes) + (struct cdrom_read) + CDROMREADAUDIO (struct cdrom_read_audio) + CDROMEJECT_SW enable(1)/disable(0) auto-ejecting + CDROMMULTISESSION Obtain the start-of-last-session + address of multi session disks + (struct cdrom_multisession) + CDROM_GET_MCN Obtain the "Universal Product Code" + if available (struct cdrom_mcn) + CDROM_GET_UPC Deprecated, use CDROM_GET_MCN instead. + CDROMRESET hard-reset the drive + CDROMVOLREAD Get the drive's volume setting + (struct cdrom_volctrl) + CDROMREADRAW read data in raw mode (2352 Bytes) + (struct cdrom_read) + CDROMREADCOOKED read data in cooked mode + CDROMSEEK seek msf address + CDROMPLAYBLK scsi-cd only, (struct cdrom_blk) + CDROMREADALL read all 2646 bytes + CDROMGETSPINDOWN return 4-bit spindown value + CDROMSETSPINDOWN set 4-bit spindown value + CDROMCLOSETRAY pendant of CDROMEJECT + CDROM_SET_OPTIONS Set behavior options + CDROM_CLEAR_OPTIONS Clear behavior options + CDROM_SELECT_SPEED Set the CD-ROM speed + CDROM_SELECT_DISC Select disc (for juke-boxes) + CDROM_MEDIA_CHANGED Check is media changed + CDROM_TIMED_MEDIA_CHANGE Check if media changed + since given time + (struct cdrom_timed_media_change_info) + CDROM_DRIVE_STATUS Get tray position, etc. + CDROM_DISC_STATUS Get disc type, etc. + CDROM_CHANGER_NSLOTS Get number of slots + CDROM_LOCKDOOR lock or unlock door + CDROM_DEBUG Turn debug messages on/off + CDROM_GET_CAPABILITY get capabilities + CDROMAUDIOBUFSIZ set the audio buffer size + DVD_READ_STRUCT Read structure + DVD_WRITE_STRUCT Write structure + DVD_AUTH Authentication + CDROM_SEND_PACKET send a packet to the drive + CDROM_NEXT_WRITABLE get next writable block + CDROM_LAST_WRITTEN get last block written on disc + ======================== =============================================== The information that follows was determined from reading kernel source From 0a593fbbc245a85940ed34caa3aa1e4cb060c54b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 17 Apr 2021 09:29:49 -0600 Subject: [PATCH 003/255] null_blk: poll queue support There's currently no way to experiment with polled IO with null_blk, which seems like an oversight. This patch adds support for polled IO. We keep a list of issued IOs on submit, and then process that list when mq_ops->poll() is invoked. A new parameter is added, poll_queues. It defaults to 1 like the submit queues, meaning we'll have 1 poll queue available. Fixes-by: Bart Van Assche Fixes-by: Pavel Begunkov Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/baca710d-0f2a-16e2-60bd-b105b854e0ae@kernel.dk Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 108 ++++++++++++++++++++++++++++-- drivers/block/null_blk/null_blk.h | 4 ++ 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index e5cbcf582233..f4af95c2f9a9 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -92,6 +92,10 @@ static int g_submit_queues = 1; module_param_named(submit_queues, g_submit_queues, int, 0444); MODULE_PARM_DESC(submit_queues, "Number of submission queues"); +static int g_poll_queues = 1; +module_param_named(poll_queues, g_poll_queues, int, 0444); +MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues"); + static int g_home_node = NUMA_NO_NODE; module_param_named(home_node, g_home_node, int, 0444); MODULE_PARM_DESC(home_node, "Home node for the device"); @@ -347,6 +351,7 @@ static int nullb_apply_submit_queues(struct nullb_device *dev, NULLB_DEVICE_ATTR(size, ulong, NULL); NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL); NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues); +NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_submit_queues); NULLB_DEVICE_ATTR(home_node, uint, NULL); NULLB_DEVICE_ATTR(queue_mode, uint, NULL); NULLB_DEVICE_ATTR(blocksize, uint, NULL); @@ -466,6 +471,7 @@ static struct configfs_attribute *nullb_device_attrs[] = { &nullb_device_attr_size, &nullb_device_attr_completion_nsec, &nullb_device_attr_submit_queues, + &nullb_device_attr_poll_queues, &nullb_device_attr_home_node, &nullb_device_attr_queue_mode, &nullb_device_attr_blocksize, @@ -593,6 +599,7 @@ static struct nullb_device *null_alloc_dev(void) dev->size = g_gb * 1024; dev->completion_nsec = g_completion_nsec; dev->submit_queues = g_submit_queues; + dev->poll_queues = g_poll_queues; dev->home_node = g_home_node; dev->queue_mode = g_queue_mode; dev->blocksize = g_bs; @@ -1454,12 +1461,80 @@ static bool should_requeue_request(struct request *rq) return false; } +static int null_map_queues(struct blk_mq_tag_set *set) +{ + struct nullb *nullb = set->driver_data; + int i, qoff; + + for (i = 0, qoff = 0; i < set->nr_maps; i++) { + struct blk_mq_queue_map *map = &set->map[i]; + + switch (i) { + case HCTX_TYPE_DEFAULT: + if (nullb) + map->nr_queues = nullb->dev->submit_queues; + else + map->nr_queues = g_submit_queues; + break; + case HCTX_TYPE_READ: + map->nr_queues = 0; + continue; + case HCTX_TYPE_POLL: + if (nullb) + map->nr_queues = nullb->dev->poll_queues; + else + map->nr_queues = g_poll_queues; + break; + } + map->queue_offset = qoff; + qoff += map->nr_queues; + blk_mq_map_queues(map); + } + + return 0; +} + +static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) +{ + struct nullb_queue *nq = hctx->driver_data; + LIST_HEAD(list); + int nr = 0; + + spin_lock(&nq->poll_lock); + list_splice_init(&nq->poll_list, &list); + spin_unlock(&nq->poll_lock); + + while (!list_empty(&list)) { + struct nullb_cmd *cmd; + struct request *req; + + req = list_first_entry(&list, struct request, queuelist); + list_del_init(&req->queuelist); + cmd = blk_mq_rq_to_pdu(req); + cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req), + blk_rq_sectors(req)); + end_cmd(cmd); + nr++; + } + + return nr; +} + static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res) { + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); pr_info("rq %p timed out\n", rq); + if (hctx->type == HCTX_TYPE_POLL) { + struct nullb_queue *nq = hctx->driver_data; + + spin_lock(&nq->poll_lock); + list_del_init(&rq->queuelist); + spin_unlock(&nq->poll_lock); + } + /* * If the device is marked as blocking (i.e. memory backed or zoned * device), the submission path may be blocked waiting for resources @@ -1480,10 +1555,11 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, struct nullb_queue *nq = hctx->driver_data; sector_t nr_sectors = blk_rq_sectors(bd->rq); sector_t sector = blk_rq_pos(bd->rq); + const bool is_poll = hctx->type == HCTX_TYPE_POLL; might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); - if (nq->dev->irqmode == NULL_IRQ_TIMER) { + if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) { hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cmd->timer.function = null_cmd_timer_expired; } @@ -1507,6 +1583,13 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_OK; } } + + if (is_poll) { + spin_lock(&nq->poll_lock); + list_add_tail(&bd->rq->queuelist, &nq->poll_list); + spin_unlock(&nq->poll_lock); + return BLK_STS_OK; + } if (cmd->fake_timeout) return BLK_STS_OK; @@ -1542,6 +1625,8 @@ static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) init_waitqueue_head(&nq->wait); nq->queue_depth = nullb->queue_depth; nq->dev = nullb->dev; + INIT_LIST_HEAD(&nq->poll_list); + spin_lock_init(&nq->poll_lock); } static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data, @@ -1567,6 +1652,8 @@ static const struct blk_mq_ops null_mq_ops = { .queue_rq = null_queue_rq, .complete = null_complete_rq, .timeout = null_timeout_rq, + .poll = null_poll, + .map_queues = null_map_queues, .init_hctx = null_init_hctx, .exit_hctx = null_exit_hctx, }; @@ -1663,13 +1750,17 @@ static int setup_commands(struct nullb_queue *nq) static int setup_queues(struct nullb *nullb) { - nullb->queues = kcalloc(nr_cpu_ids, sizeof(struct nullb_queue), + int nqueues = nr_cpu_ids; + + if (g_poll_queues) + nqueues += g_poll_queues; + + nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue), GFP_KERNEL); if (!nullb->queues) return -ENOMEM; nullb->queue_depth = nullb->dev->hw_queue_depth; - return 0; } @@ -1721,9 +1812,14 @@ static int null_gendisk_register(struct nullb *nullb) static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) { + int poll_queues; + set->ops = &null_mq_ops; set->nr_hw_queues = nullb ? nullb->dev->submit_queues : g_submit_queues; + poll_queues = nullb ? nullb->dev->poll_queues : g_poll_queues; + if (poll_queues) + set->nr_hw_queues += poll_queues; set->queue_depth = nullb ? nullb->dev->hw_queue_depth : g_hw_queue_depth; set->numa_node = nullb ? nullb->dev->home_node : g_home_node; @@ -1733,7 +1829,11 @@ static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) set->flags |= BLK_MQ_F_NO_SCHED; if (g_shared_tag_bitmap) set->flags |= BLK_MQ_F_TAG_HCTX_SHARED; - set->driver_data = NULL; + set->driver_data = nullb; + if (g_poll_queues) + set->nr_maps = 3; + else + set->nr_maps = 1; if ((nullb && nullb->dev->blocking) || g_blocking) set->flags |= BLK_MQ_F_BLOCKING; diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h index 64bef125d1df..90c3eefb3ca3 100644 --- a/drivers/block/null_blk/null_blk.h +++ b/drivers/block/null_blk/null_blk.h @@ -32,6 +32,9 @@ struct nullb_queue { struct nullb_device *dev; unsigned int requeue_selection; + struct list_head poll_list; + spinlock_t poll_lock; + struct nullb_cmd *cmds; }; @@ -83,6 +86,7 @@ struct nullb_device { unsigned int zone_max_open; /* max number of open zones */ unsigned int zone_max_active; /* max number of active zones */ unsigned int submit_queues; /* number of submission queues */ + unsigned int poll_queues; /* number of IOPOLL submission queues */ unsigned int home_node; /* home node for the device */ unsigned int queue_mode; /* block interface */ unsigned int blocksize; /* block size */ From 905705f083a936e49d5259e0bb539c53d5e0a9be Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 14:59:57 -0700 Subject: [PATCH 004/255] loop: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/loop.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 7bf4686af774..00ee365ed5e0 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -2394,13 +2394,19 @@ static int loop_add(int i) disk->event_flags = DISK_EVENT_FLAG_UEVENT; sprintf(disk->disk_name, "loop%d", i); /* Make this loop device reachable from pathname. */ - add_disk(disk); + err = add_disk(disk); + if (err) + goto out_cleanup_disk; + /* Show this loop device. */ mutex_lock(&loop_ctl_mutex); lo->idr_visible = true; mutex_unlock(&loop_ctl_mutex); + return i; +out_cleanup_disk: + blk_cleanup_disk(disk); out_cleanup_tags: blk_mq_free_tag_set(&lo->tag_set); out_free_idr: From e1654f413fe08ffbc3292d8d2b8958b2cc5cb5e8 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 14:59:58 -0700 Subject: [PATCH 005/255] nbd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 1183f7872b71..f4101d3efe7d 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1762,7 +1762,9 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) disk->fops = &nbd_fops; disk->private_data = nbd; sprintf(disk->disk_name, "nbd%d", index); - add_disk(disk); + err = add_disk(disk); + if (err) + goto out_err_disk; /* * Now publish the device. @@ -1771,6 +1773,8 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) nbd_total_devices++; return nbd; +out_err_disk: + blk_cleanup_disk(disk); out_free_idr: mutex_lock(&nbd_index_mutex); idr_remove(&nbd_index_idr, index); From d9c2bd252a4578419e0b863bfe3dd97b858ccd8e Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:00:57 -0700 Subject: [PATCH 006/255] aoe: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/aoe/aoeblk.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 06b360f7123a..e436b0e8eff5 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -417,7 +417,9 @@ aoeblk_gdalloc(void *vp) spin_unlock_irqrestore(&d->lock, flags); - device_add_disk(NULL, gd, aoe_attr_groups); + err = device_add_disk(NULL, gd, aoe_attr_groups); + if (err) + goto out_disk_cleanup; aoedisk_add_debugfs(d); spin_lock_irqsave(&d->lock, flags); @@ -426,6 +428,8 @@ aoeblk_gdalloc(void *vp) spin_unlock_irqrestore(&d->lock, flags); return; +out_disk_cleanup: + blk_cleanup_disk(gd); err_tagset: blk_mq_free_tag_set(set); err_mempool: From e92ab4eda516a5bfd96c087282ebe9521deba4f4 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:00:59 -0700 Subject: [PATCH 007/255] drbd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/drbd/drbd_main.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 55234a558e98..19db80a1e409 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2794,7 +2794,9 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig goto out_idr_remove_vol; } - add_disk(disk); + err = add_disk(disk); + if (err) + goto out_cleanup_disk; /* inherit the connection state */ device->state.conn = first_connection(resource)->cstate; @@ -2808,6 +2810,8 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig drbd_debugfs_device_add(device); return NO_ERROR; +out_cleanup_disk: + blk_cleanup_disk(disk); out_idr_remove_vol: idr_remove(&connection->peer_devices, vnr); out_idr_remove_from_resource: From d1df6021b70ce7823df89941c0c97e746fa2ad92 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:02 -0700 Subject: [PATCH 008/255] n64cart: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/n64cart.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c index b168ca25b6c9..78282f01f581 100644 --- a/drivers/block/n64cart.c +++ b/drivers/block/n64cart.c @@ -115,6 +115,7 @@ static const struct block_device_operations n64cart_fops = { static int __init n64cart_probe(struct platform_device *pdev) { struct gendisk *disk; + int err = -ENOMEM; if (!start || !size) { pr_err("start or size not specified\n"); @@ -132,7 +133,7 @@ static int __init n64cart_probe(struct platform_device *pdev) disk = blk_alloc_disk(NUMA_NO_NODE); if (!disk) - return -ENOMEM; + goto out; disk->first_minor = 0; disk->flags = GENHD_FL_NO_PART_SCAN; @@ -147,11 +148,18 @@ static int __init n64cart_probe(struct platform_device *pdev) blk_queue_physical_block_size(disk->queue, 4096); blk_queue_logical_block_size(disk->queue, 4096); - add_disk(disk); + err = add_disk(disk); + if (err) + goto out_cleanup_disk; pr_info("n64cart: %u kb disk\n", size / 1024); return 0; + +out_cleanup_disk: + blk_cleanup_disk(disk); +out: + return err; } static struct platform_driver n64cart_driver = { From 7d8b72aaddd3ec5f350d3e9988d6735a7b9b18e9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Sep 2021 15:01:03 -0700 Subject: [PATCH 009/255] pcd: move the identify buffer into pcd_identify No need to pass it through a bunch of functions. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index f9cdd11f02f5..8903fdaa2046 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -630,10 +630,11 @@ static int pcd_drive_status(struct cdrom_device_info *cdi, int slot_nr) return CDS_DISC_OK; } -static int pcd_identify(struct pcd_unit *cd, char *id) +static int pcd_identify(struct pcd_unit *cd) { - int k, s; char id_cmd[12] = { 0x12, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0 }; + char id[18]; + int k, s; pcd_bufblk = -1; @@ -664,15 +665,15 @@ static int pcd_identify(struct pcd_unit *cd, char *id) * returns 0, with id set if drive is detected * -1, if drive detection failed */ -static int pcd_probe(struct pcd_unit *cd, int ms, char *id) +static int pcd_probe(struct pcd_unit *cd, int ms) { if (ms == -1) { for (cd->drive = 0; cd->drive <= 1; cd->drive++) - if (!pcd_reset(cd) && !pcd_identify(cd, id)) + if (!pcd_reset(cd) && !pcd_identify(cd)) return 0; } else { cd->drive = ms; - if (!pcd_reset(cd) && !pcd_identify(cd, id)) + if (!pcd_reset(cd) && !pcd_identify(cd)) return 0; } return -1; @@ -709,7 +710,6 @@ static void pcd_probe_capabilities(void) static int pcd_detect(void) { - char id[18]; int k, unit; struct pcd_unit *cd; @@ -727,7 +727,7 @@ static int pcd_detect(void) cd = pcd; if (cd->disk && pi_init(cd->pi, 1, -1, -1, -1, -1, -1, pcd_buffer, PI_PCD, verbose, cd->name)) { - if (!pcd_probe(cd, -1, id)) { + if (!pcd_probe(cd, -1)) { cd->present = 1; k++; } else @@ -744,7 +744,7 @@ static int pcd_detect(void) conf[D_UNI], conf[D_PRO], conf[D_DLY], pcd_buffer, PI_PCD, verbose, cd->name)) continue; - if (!pcd_probe(cd, conf[D_SLV], id)) { + if (!pcd_probe(cd, conf[D_SLV])) { cd->present = 1; k++; } else From af761f277b7fd896c27cb1100b25f11567987822 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Sep 2021 15:01:04 -0700 Subject: [PATCH 010/255] pcd: cleanup initialization Refactor the pcd initialization to have a dedicated helper to initialize a single disk. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 290 +++++++++++++++++-------------------- 1 file changed, 129 insertions(+), 161 deletions(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 8903fdaa2046..93ed63626232 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -183,8 +183,6 @@ static int pcd_audio_ioctl(struct cdrom_device_info *cdi, static int pcd_packet(struct cdrom_device_info *cdi, struct packet_command *cgc); -static int pcd_detect(void); -static void pcd_probe_capabilities(void); static void do_pcd_read_drq(void); static blk_status_t pcd_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd); @@ -302,53 +300,6 @@ static const struct blk_mq_ops pcd_mq_ops = { .queue_rq = pcd_queue_rq, }; -static void pcd_init_units(void) -{ - struct pcd_unit *cd; - int unit; - - pcd_drive_count = 0; - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - struct gendisk *disk; - - if (blk_mq_alloc_sq_tag_set(&cd->tag_set, &pcd_mq_ops, 1, - BLK_MQ_F_SHOULD_MERGE)) - continue; - - disk = blk_mq_alloc_disk(&cd->tag_set, cd); - if (IS_ERR(disk)) { - blk_mq_free_tag_set(&cd->tag_set); - continue; - } - - INIT_LIST_HEAD(&cd->rq_list); - blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); - cd->disk = disk; - cd->pi = &cd->pia; - cd->present = 0; - cd->last_sense = 0; - cd->changed = 1; - cd->drive = (*drives[unit])[D_SLV]; - if ((*drives[unit])[D_PRT]) - pcd_drive_count++; - - cd->name = &cd->info.name[0]; - snprintf(cd->name, sizeof(cd->info.name), "%s%d", name, unit); - cd->info.ops = &pcd_dops; - cd->info.handle = cd; - cd->info.speed = 0; - cd->info.capacity = 1; - cd->info.mask = 0; - disk->major = major; - disk->first_minor = unit; - disk->minors = 1; - strcpy(disk->disk_name, cd->name); /* umm... */ - disk->fops = &pcd_bdops; - disk->flags = GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; - disk->events = DISK_EVENT_MEDIA_CHANGE; - } -} - static int pcd_open(struct cdrom_device_info *cdi, int purpose) { struct pcd_unit *cd = cdi->handle; @@ -679,90 +630,31 @@ static int pcd_probe(struct pcd_unit *cd, int ms) return -1; } -static void pcd_probe_capabilities(void) +static int pcd_probe_capabilities(struct pcd_unit *cd) { - int unit, r; - char buffer[32]; char cmd[12] = { 0x5a, 1 << 3, 0x2a, 0, 0, 0, 0, 18, 0, 0, 0, 0 }; - struct pcd_unit *cd; + char buffer[32]; + int ret; - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - if (!cd->present) - continue; - r = pcd_atapi(cd, cmd, 18, buffer, "mode sense capabilities"); - if (r) - continue; - /* we should now have the cap page */ - if ((buffer[11] & 1) == 0) - cd->info.mask |= CDC_CD_R; - if ((buffer[11] & 2) == 0) - cd->info.mask |= CDC_CD_RW; - if ((buffer[12] & 1) == 0) - cd->info.mask |= CDC_PLAY_AUDIO; - if ((buffer[14] & 1) == 0) - cd->info.mask |= CDC_LOCK; - if ((buffer[14] & 8) == 0) - cd->info.mask |= CDC_OPEN_TRAY; - if ((buffer[14] >> 6) == 0) - cd->info.mask |= CDC_CLOSE_TRAY; - } -} + ret = pcd_atapi(cd, cmd, 18, buffer, "mode sense capabilities"); + if (ret) + return ret; -static int pcd_detect(void) -{ - int k, unit; - struct pcd_unit *cd; + /* we should now have the cap page */ + if ((buffer[11] & 1) == 0) + cd->info.mask |= CDC_CD_R; + if ((buffer[11] & 2) == 0) + cd->info.mask |= CDC_CD_RW; + if ((buffer[12] & 1) == 0) + cd->info.mask |= CDC_PLAY_AUDIO; + if ((buffer[14] & 1) == 0) + cd->info.mask |= CDC_LOCK; + if ((buffer[14] & 8) == 0) + cd->info.mask |= CDC_OPEN_TRAY; + if ((buffer[14] >> 6) == 0) + cd->info.mask |= CDC_CLOSE_TRAY; - printk("%s: %s version %s, major %d, nice %d\n", - name, name, PCD_VERSION, major, nice); - - par_drv = pi_register_driver(name); - if (!par_drv) { - pr_err("failed to register %s driver\n", name); - return -1; - } - - k = 0; - if (pcd_drive_count == 0) { /* nothing spec'd - so autoprobe for 1 */ - cd = pcd; - if (cd->disk && pi_init(cd->pi, 1, -1, -1, -1, -1, -1, - pcd_buffer, PI_PCD, verbose, cd->name)) { - if (!pcd_probe(cd, -1)) { - cd->present = 1; - k++; - } else - pi_release(cd->pi); - } - } else { - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - int *conf = *drives[unit]; - if (!conf[D_PRT]) - continue; - if (!cd->disk) - continue; - if (!pi_init(cd->pi, 0, conf[D_PRT], conf[D_MOD], - conf[D_UNI], conf[D_PRO], conf[D_DLY], - pcd_buffer, PI_PCD, verbose, cd->name)) - continue; - if (!pcd_probe(cd, conf[D_SLV])) { - cd->present = 1; - k++; - } else - pi_release(cd->pi); - } - } - if (k) - return 0; - - printk("%s: No CD-ROM drive found\n", name); - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - if (!cd->disk) - continue; - blk_cleanup_disk(cd->disk); - blk_mq_free_tag_set(&cd->tag_set); - } - pi_unregister_driver(par_drv); - return -1; + return 0; } /* I/O request processing */ @@ -999,43 +891,121 @@ static int pcd_get_mcn(struct cdrom_device_info *cdi, struct cdrom_mcn *mcn) return 0; } +static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, + int mode, int unit, int protocol, int delay, int ms) +{ + struct gendisk *disk; + int ret; + + ret = blk_mq_alloc_sq_tag_set(&cd->tag_set, &pcd_mq_ops, 1, + BLK_MQ_F_SHOULD_MERGE); + if (ret) + return ret; + + disk = blk_mq_alloc_disk(&cd->tag_set, cd); + if (IS_ERR(disk)) { + ret = PTR_ERR(disk); + goto out_free_tag_set; + } + + INIT_LIST_HEAD(&cd->rq_list); + blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); + cd->disk = disk; + cd->pi = &cd->pia; + cd->present = 0; + cd->last_sense = 0; + cd->changed = 1; + cd->drive = (*drives[cd - pcd])[D_SLV]; + + cd->name = &cd->info.name[0]; + snprintf(cd->name, sizeof(cd->info.name), "%s%d", name, unit); + cd->info.ops = &pcd_dops; + cd->info.handle = cd; + cd->info.speed = 0; + cd->info.capacity = 1; + cd->info.mask = 0; + disk->major = major; + disk->first_minor = unit; + disk->minors = 1; + strcpy(disk->disk_name, cd->name); /* umm... */ + disk->fops = &pcd_bdops; + disk->flags = GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE; + disk->events = DISK_EVENT_MEDIA_CHANGE; + + if (!pi_init(cd->pi, autoprobe, port, mode, unit, protocol, delay, + pcd_buffer, PI_PCD, verbose, cd->name)) + goto out_free_disk; + if (pcd_probe(cd, ms)) + goto out_pi_release; + + cd->present = 1; + pcd_probe_capabilities(cd); + register_cdrom(cd->disk, &cd->info); + add_disk(cd->disk); + return 0; + +out_pi_release: + pi_release(cd->pi); +out_free_disk: + blk_cleanup_disk(cd->disk); +out_free_tag_set: + blk_mq_free_tag_set(&cd->tag_set); + return ret; +} + static int __init pcd_init(void) { - struct pcd_unit *cd; - int unit; + int found = 0, unit; if (disable) return -EINVAL; - pcd_init_units(); - - if (pcd_detect()) - return -ENODEV; - - /* get the atapi capabilities page */ - pcd_probe_capabilities(); - - if (register_blkdev(major, name)) { - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - if (!cd->disk) - continue; - - blk_cleanup_queue(cd->disk->queue); - blk_mq_free_tag_set(&cd->tag_set); - put_disk(cd->disk); - } + if (register_blkdev(major, name)) return -EBUSY; + + pr_info("%s: %s version %s, major %d, nice %d\n", + name, name, PCD_VERSION, major, nice); + + par_drv = pi_register_driver(name); + if (!par_drv) { + pr_err("failed to register %s driver\n", name); + goto out_unregister_blkdev; } - for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - if (cd->present) { - register_cdrom(cd->disk, &cd->info); - cd->disk->private_data = cd; - add_disk(cd->disk); + for (unit = 0; unit < PCD_UNITS; unit++) { + if ((*drives[unit])[D_PRT]) + pcd_drive_count++; + } + + if (pcd_drive_count == 0) { /* nothing spec'd - so autoprobe for 1 */ + if (!pcd_init_unit(pcd, 1, -1, -1, -1, -1, -1, -1)) + found++; + } else { + for (unit = 0; unit < PCD_UNITS; unit++) { + struct pcd_unit *cd = &pcd[unit]; + int *conf = *drives[unit]; + + if (!conf[D_PRT]) + continue; + if (!pcd_init_unit(cd, 0, conf[D_PRT], conf[D_MOD], + conf[D_UNI], conf[D_PRO], conf[D_DLY], + conf[D_SLV])) + found++; } } + if (!found) { + pr_info("%s: No CD-ROM drive found\n", name); + goto out_unregister_pi_driver; + } + return 0; + +out_unregister_pi_driver: + pi_unregister_driver(par_drv); +out_unregister_blkdev: + unregister_blkdev(major, name); + return -ENODEV; } static void __exit pcd_exit(void) @@ -1044,20 +1014,18 @@ static void __exit pcd_exit(void) int unit; for (unit = 0, cd = pcd; unit < PCD_UNITS; unit++, cd++) { - if (!cd->disk) + if (!cd->present) continue; - if (cd->present) { - del_gendisk(cd->disk); - pi_release(cd->pi); - unregister_cdrom(&cd->info); - } - blk_cleanup_queue(cd->disk->queue); + del_gendisk(cd->disk); + pi_release(cd->pi); + unregister_cdrom(&cd->info); + blk_cleanup_disk(cd->disk); + blk_mq_free_tag_set(&cd->tag_set); - put_disk(cd->disk); } - unregister_blkdev(major, name); pi_unregister_driver(par_drv); + unregister_blkdev(major, name); } MODULE_LICENSE("GPL"); From fb367e6baeb0789d3f272a9aa21f5116e8ebd9dc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Sep 2021 15:01:05 -0700 Subject: [PATCH 011/255] pf: cleanup initialization Refactor the pf initialization to have a dedicated helper to initialize a single disk. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/paride/pf.c | 227 +++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 126 deletions(-) diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index d5b9c88ba76f..f471d48a87bc 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -214,7 +214,6 @@ static int pf_getgeo(struct block_device *bdev, struct hd_geometry *geo); static void pf_release(struct gendisk *disk, fmode_t mode); -static int pf_detect(void); static void do_pf_read(void); static void do_pf_read_start(void); static void do_pf_write(void); @@ -285,45 +284,6 @@ static const struct blk_mq_ops pf_mq_ops = { .queue_rq = pf_queue_rq, }; -static void __init pf_init_units(void) -{ - struct pf_unit *pf; - int unit; - - pf_drive_count = 0; - for (unit = 0, pf = units; unit < PF_UNITS; unit++, pf++) { - struct gendisk *disk; - - if (blk_mq_alloc_sq_tag_set(&pf->tag_set, &pf_mq_ops, 1, - BLK_MQ_F_SHOULD_MERGE)) - continue; - - disk = blk_mq_alloc_disk(&pf->tag_set, pf); - if (IS_ERR(disk)) { - blk_mq_free_tag_set(&pf->tag_set); - continue; - } - - INIT_LIST_HEAD(&pf->rq_list); - blk_queue_max_segments(disk->queue, cluster); - blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); - pf->disk = disk; - pf->pi = &pf->pia; - pf->media_status = PF_NM; - pf->drive = (*drives[unit])[D_SLV]; - pf->lun = (*drives[unit])[D_LUN]; - snprintf(pf->name, PF_NAMELEN, "%s%d", name, unit); - disk->major = major; - disk->first_minor = unit; - disk->minors = 1; - strcpy(disk->disk_name, pf->name); - disk->fops = &pf_fops; - disk->events = DISK_EVENT_MEDIA_CHANGE; - if (!(*drives[unit])[D_PRT]) - pf_drive_count++; - } -} - static int pf_open(struct block_device *bdev, fmode_t mode) { struct pf_unit *pf = bdev->bd_disk->private_data; @@ -718,59 +678,6 @@ static int pf_probe(struct pf_unit *pf) return -1; } -static int pf_detect(void) -{ - struct pf_unit *pf = units; - int k, unit; - - printk("%s: %s version %s, major %d, cluster %d, nice %d\n", - name, name, PF_VERSION, major, cluster, nice); - - par_drv = pi_register_driver(name); - if (!par_drv) { - pr_err("failed to register %s driver\n", name); - return -1; - } - k = 0; - if (pf_drive_count == 0) { - if (pi_init(pf->pi, 1, -1, -1, -1, -1, -1, pf_scratch, PI_PF, - verbose, pf->name)) { - if (!pf_probe(pf) && pf->disk) { - pf->present = 1; - k++; - } else - pi_release(pf->pi); - } - - } else - for (unit = 0; unit < PF_UNITS; unit++, pf++) { - int *conf = *drives[unit]; - if (!conf[D_PRT]) - continue; - if (pi_init(pf->pi, 0, conf[D_PRT], conf[D_MOD], - conf[D_UNI], conf[D_PRO], conf[D_DLY], - pf_scratch, PI_PF, verbose, pf->name)) { - if (pf->disk && !pf_probe(pf)) { - pf->present = 1; - k++; - } else - pi_release(pf->pi); - } - } - if (k) - return 0; - - printk("%s: No ATAPI disk detected\n", name); - for (pf = units, unit = 0; unit < PF_UNITS; pf++, unit++) { - if (!pf->disk) - continue; - blk_cleanup_disk(pf->disk); - blk_mq_free_tag_set(&pf->tag_set); - } - pi_unregister_driver(par_drv); - return -1; -} - /* The i/o request engine */ static int pf_start(struct pf_unit *pf, int cmd, int b, int c) @@ -1014,61 +921,129 @@ static void do_pf_write_done(void) next_request(0); } +static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, + int mode, int unit, int protocol, int delay, int ms) +{ + struct gendisk *disk; + int ret; + + ret = blk_mq_alloc_sq_tag_set(&pf->tag_set, &pf_mq_ops, 1, + BLK_MQ_F_SHOULD_MERGE); + if (ret) + return ret; + + disk = blk_mq_alloc_disk(&pf->tag_set, pf); + if (IS_ERR(disk)) { + ret = PTR_ERR(disk); + goto out_free_tag_set; + } + disk->major = major; + disk->first_minor = pf - units; + disk->minors = 1; + strcpy(disk->disk_name, pf->name); + disk->fops = &pf_fops; + disk->events = DISK_EVENT_MEDIA_CHANGE; + disk->private_data = pf; + + blk_queue_max_segments(disk->queue, cluster); + blk_queue_bounce_limit(disk->queue, BLK_BOUNCE_HIGH); + + INIT_LIST_HEAD(&pf->rq_list); + pf->disk = disk; + pf->pi = &pf->pia; + pf->media_status = PF_NM; + pf->drive = (*drives[disk->first_minor])[D_SLV]; + pf->lun = (*drives[disk->first_minor])[D_LUN]; + snprintf(pf->name, PF_NAMELEN, "%s%d", name, disk->first_minor); + + if (!pi_init(pf->pi, autoprobe, port, mode, unit, protocol, delay, + pf_scratch, PI_PF, verbose, pf->name)) + goto out_free_disk; + if (pf_probe(pf)) + goto out_pi_release; + + add_disk(disk); + pf->present = 1; + return 0; + +out_pi_release: + pi_release(pf->pi); +out_free_disk: + blk_cleanup_disk(pf->disk); +out_free_tag_set: + blk_mq_free_tag_set(&pf->tag_set); + return ret; +} + static int __init pf_init(void) { /* preliminary initialisation */ struct pf_unit *pf; - int unit; + int found = 0, unit; if (disable) return -EINVAL; - pf_init_units(); - - if (pf_detect()) - return -ENODEV; - pf_busy = 0; - - if (register_blkdev(major, name)) { - for (pf = units, unit = 0; unit < PF_UNITS; pf++, unit++) { - if (!pf->disk) - continue; - blk_cleanup_queue(pf->disk->queue); - blk_mq_free_tag_set(&pf->tag_set); - put_disk(pf->disk); - } + if (register_blkdev(major, name)) return -EBUSY; + + printk("%s: %s version %s, major %d, cluster %d, nice %d\n", + name, name, PF_VERSION, major, cluster, nice); + + par_drv = pi_register_driver(name); + if (!par_drv) { + pr_err("failed to register %s driver\n", name); + goto out_unregister_blkdev; } - for (pf = units, unit = 0; unit < PF_UNITS; pf++, unit++) { - struct gendisk *disk = pf->disk; - - if (!pf->present) - continue; - disk->private_data = pf; - add_disk(disk); + for (unit = 0; unit < PF_UNITS; unit++) { + if (!(*drives[unit])[D_PRT]) + pf_drive_count++; } + + pf = units; + if (pf_drive_count == 0) { + if (pf_init_unit(pf, 1, -1, -1, -1, -1, -1, verbose)) + found++; + } else { + for (unit = 0; unit < PF_UNITS; unit++, pf++) { + int *conf = *drives[unit]; + if (!conf[D_PRT]) + continue; + if (pf_init_unit(pf, 0, conf[D_PRT], conf[D_MOD], + conf[D_UNI], conf[D_PRO], conf[D_DLY], + verbose)) + found++; + } + } + if (!found) { + printk("%s: No ATAPI disk detected\n", name); + goto out_unregister_pi_driver; + } + pf_busy = 0; return 0; + +out_unregister_pi_driver: + pi_unregister_driver(par_drv); +out_unregister_blkdev: + unregister_blkdev(major, name); + return -ENODEV; } static void __exit pf_exit(void) { struct pf_unit *pf; int unit; - unregister_blkdev(major, name); + for (pf = units, unit = 0; unit < PF_UNITS; pf++, unit++) { - if (!pf->disk) + if (!pf->present) continue; - - if (pf->present) - del_gendisk(pf->disk); - - blk_cleanup_queue(pf->disk->queue); + del_gendisk(pf->disk); + blk_cleanup_disk(pf->disk); blk_mq_free_tag_set(&pf->tag_set); - put_disk(pf->disk); - - if (pf->present) - pi_release(pf->pi); + pi_release(pf->pi); } + + unregister_blkdev(major, name); } MODULE_LICENSE("GPL"); From 1ad392add59c2a7cc3148b2e3041696370b05e5b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Sep 2021 15:01:06 -0700 Subject: [PATCH 012/255] pd: cleanup initialization Refactor the pf initialization to have a dedicated helper to initialize a single disk. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/paride/pd.c | 140 +++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index 675327df6aff..500b89a4bdaf 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -875,9 +875,27 @@ static const struct blk_mq_ops pd_mq_ops = { .queue_rq = pd_queue_rq, }; -static void pd_probe_drive(struct pd_unit *disk) +static int pd_probe_drive(struct pd_unit *disk, int autoprobe, int port, + int mode, int unit, int protocol, int delay) { + int index = disk - pd; + int *parm = *drives[index]; struct gendisk *p; + int ret; + + disk->pi = &disk->pia; + disk->access = 0; + disk->changed = 1; + disk->capacity = 0; + disk->drive = parm[D_SLV]; + snprintf(disk->name, PD_NAMELEN, "%s%c", name, 'a' + index); + disk->alt_geom = parm[D_GEO]; + disk->standby = parm[D_SBY]; + INIT_LIST_HEAD(&disk->rq_list); + + if (!pi_init(disk->pi, autoprobe, port, mode, unit, protocol, delay, + pd_scratch, PI_PD, verbose, disk->name)) + return -ENXIO; memset(&disk->tag_set, 0, sizeof(disk->tag_set)); disk->tag_set.ops = &pd_mq_ops; @@ -887,14 +905,14 @@ static void pd_probe_drive(struct pd_unit *disk) disk->tag_set.queue_depth = 2; disk->tag_set.numa_node = NUMA_NO_NODE; disk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; - - if (blk_mq_alloc_tag_set(&disk->tag_set)) - return; + ret = blk_mq_alloc_tag_set(&disk->tag_set); + if (ret) + goto pi_release; p = blk_mq_alloc_disk(&disk->tag_set, disk); if (IS_ERR(p)) { - blk_mq_free_tag_set(&disk->tag_set); - return; + ret = PTR_ERR(p); + goto free_tag_set; } disk->gd = p; @@ -905,102 +923,84 @@ static void pd_probe_drive(struct pd_unit *disk) p->minors = 1 << PD_BITS; p->events = DISK_EVENT_MEDIA_CHANGE; p->private_data = disk; - blk_queue_max_hw_sectors(p->queue, cluster); blk_queue_bounce_limit(p->queue, BLK_BOUNCE_HIGH); if (disk->drive == -1) { - for (disk->drive = 0; disk->drive <= 1; disk->drive++) - if (pd_special_command(disk, pd_identify) == 0) - return; - } else if (pd_special_command(disk, pd_identify) == 0) - return; - disk->gd = NULL; + for (disk->drive = 0; disk->drive <= 1; disk->drive++) { + ret = pd_special_command(disk, pd_identify); + if (ret == 0) + break; + } + } else { + ret = pd_special_command(disk, pd_identify); + } + if (ret) + goto put_disk; + set_capacity(disk->gd, disk->capacity); + add_disk(disk->gd); + return 0; +put_disk: put_disk(p); + disk->gd = NULL; +free_tag_set: + blk_mq_free_tag_set(&disk->tag_set); +pi_release: + pi_release(disk->pi); + return ret; } -static int pd_detect(void) +static int __init pd_init(void) { int found = 0, unit, pd_drive_count = 0; struct pd_unit *disk; - for (unit = 0; unit < PD_UNITS; unit++) { - int *parm = *drives[unit]; - struct pd_unit *disk = pd + unit; - disk->pi = &disk->pia; - disk->access = 0; - disk->changed = 1; - disk->capacity = 0; - disk->drive = parm[D_SLV]; - snprintf(disk->name, PD_NAMELEN, "%s%c", name, 'a'+unit); - disk->alt_geom = parm[D_GEO]; - disk->standby = parm[D_SBY]; - if (parm[D_PRT]) - pd_drive_count++; - INIT_LIST_HEAD(&disk->rq_list); - } + if (disable) + return -ENODEV; + + if (register_blkdev(major, name)) + return -ENODEV; + + printk("%s: %s version %s, major %d, cluster %d, nice %d\n", + name, name, PD_VERSION, major, cluster, nice); par_drv = pi_register_driver(name); if (!par_drv) { pr_err("failed to register %s driver\n", name); - return -1; + goto out_unregister_blkdev; + } + + for (unit = 0; unit < PD_UNITS; unit++) { + int *parm = *drives[unit]; + + if (parm[D_PRT]) + pd_drive_count++; } if (pd_drive_count == 0) { /* nothing spec'd - so autoprobe for 1 */ - disk = pd; - if (pi_init(disk->pi, 1, -1, -1, -1, -1, -1, pd_scratch, - PI_PD, verbose, disk->name)) { - pd_probe_drive(disk); - if (!disk->gd) - pi_release(disk->pi); - } - + if (!pd_probe_drive(pd, 1, -1, -1, -1, -1, -1)) + found++; } else { for (unit = 0, disk = pd; unit < PD_UNITS; unit++, disk++) { int *parm = *drives[unit]; if (!parm[D_PRT]) continue; - if (pi_init(disk->pi, 0, parm[D_PRT], parm[D_MOD], - parm[D_UNI], parm[D_PRO], parm[D_DLY], - pd_scratch, PI_PD, verbose, disk->name)) { - pd_probe_drive(disk); - if (!disk->gd) - pi_release(disk->pi); - } - } - } - for (unit = 0, disk = pd; unit < PD_UNITS; unit++, disk++) { - if (disk->gd) { - set_capacity(disk->gd, disk->capacity); - add_disk(disk->gd); - found = 1; + if (!pd_probe_drive(disk, 0, parm[D_PRT], parm[D_MOD], + parm[D_UNI], parm[D_PRO], parm[D_DLY])) + found++; } } if (!found) { printk("%s: no valid drive found\n", name); - pi_unregister_driver(par_drv); + goto out_pi_unregister_driver; } - return found; -} - -static int __init pd_init(void) -{ - if (disable) - goto out1; - - if (register_blkdev(major, name)) - goto out1; - - printk("%s: %s version %s, major %d, cluster %d, nice %d\n", - name, name, PD_VERSION, major, cluster, nice); - if (!pd_detect()) - goto out2; return 0; -out2: +out_pi_unregister_driver: + pi_unregister_driver(par_drv); +out_unregister_blkdev: unregister_blkdev(major, name); -out1: return -ENODEV; } From 4dfbd1390af60765774d9565858d1a6fadacde32 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:07 -0700 Subject: [PATCH 013/255] pcd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 93ed63626232..a7fab3830d7b 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -941,9 +941,13 @@ static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, cd->present = 1; pcd_probe_capabilities(cd); register_cdrom(cd->disk, &cd->info); - add_disk(cd->disk); + ret = add_disk(cd->disk); + if (ret) + goto out_unreg_cdrom; return 0; +out_unreg_cdrom: + unregister_cdrom(&cd->info); out_pi_release: pi_release(cd->pi); out_free_disk: From 2b6cabce3954be3341e0fe7b20a27902821fd3dd Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:08 -0700 Subject: [PATCH 014/255] pcd: fix ordering of unregister_cdrom() We first register cdrom and then we add_disk() and so we we should likewise unregister the cdrom first and then del_gendisk(). Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index a7fab3830d7b..82a654fc4db8 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -1021,9 +1021,9 @@ static void __exit pcd_exit(void) if (!cd->present) continue; + unregister_cdrom(&cd->info); del_gendisk(cd->disk); pi_release(cd->pi); - unregister_cdrom(&cd->info); blk_cleanup_disk(cd->disk); blk_mq_free_tag_set(&cd->tag_set); From b6fa069971bc427e19b8f3882a808f24530994ed Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:09 -0700 Subject: [PATCH 015/255] pcd: capture errors on cdrom_register() No errors were being captured wehen cdrom_register() fails, capture the error and return the error. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 82a654fc4db8..4cc0d141db78 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -940,7 +940,9 @@ static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, cd->present = 1; pcd_probe_capabilities(cd); - register_cdrom(cd->disk, &cd->info); + ret = register_cdrom(cd->disk, &cd->info); + if (ret) + goto out_pi_release; ret = add_disk(cd->disk); if (ret) goto out_unreg_cdrom; From 3dfdd5f333bf16ec5057d508a574a3302ed84cfa Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:10 -0700 Subject: [PATCH 016/255] pd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/paride/pd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index 500b89a4bdaf..e59759bcf416 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -938,8 +938,12 @@ static int pd_probe_drive(struct pd_unit *disk, int autoprobe, int port, if (ret) goto put_disk; set_capacity(disk->gd, disk->capacity); - add_disk(disk->gd); + ret = add_disk(disk->gd); + if (ret) + goto cleanup_disk; return 0; +cleanup_disk: + blk_cleanup_disk(disk->gd); put_disk: put_disk(p); disk->gd = NULL; From 4a32e1cdb745ea9f66358810b0ce85698033f57e Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:48 -0700 Subject: [PATCH 017/255] mtip32xx: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. The read_capacity_error error label already does what we need, so just re-use that. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/mtip32xx/mtip32xx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 901855717cb5..d0b40309f47e 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3633,7 +3633,9 @@ skip_create_disk: set_capacity(dd->disk, capacity); /* Enable the block device and add it to /dev */ - device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); + rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); + if (rv) + goto read_capacity_error; if (dd->mtip_svc_handler) { set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); From 7b505627568c088b364705a86234fa1f2beb01b9 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:49 -0700 Subject: [PATCH 018/255] pktcdvd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. The out_mem2 error label already does what we need so re-use that. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/pktcdvd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index cb52cce6fb03..e48d4771d4c1 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2728,7 +2728,9 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) /* inherit events of the host device */ disk->events = pd->bdev->bd_disk->events; - add_disk(disk); + ret = add_disk(disk); + if (ret) + goto out_mem2; pkt_sysfs_dev_new(pd); pkt_debugfs_dev_new(pd); From 54494d10031b4bc043af43251bff0d10cca6857a Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:53 -0700 Subject: [PATCH 019/255] block/rsxx: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/rsxx/core.c | 4 +++- drivers/block/rsxx/dev.c | 12 +++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/block/rsxx/core.c b/drivers/block/rsxx/core.c index 83636714b8d7..8d9d69f5dfbc 100644 --- a/drivers/block/rsxx/core.c +++ b/drivers/block/rsxx/core.c @@ -935,7 +935,9 @@ static int rsxx_pci_probe(struct pci_dev *dev, card->size8 = 0; } - rsxx_attach_dev(card); + st = rsxx_attach_dev(card); + if (st) + goto failed_create_dev; /************* Setup Debugfs *************/ rsxx_debugfs_dev_new(card); diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index 268252380e88..dd33f1bdf3b8 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -191,6 +191,8 @@ static bool rsxx_discard_supported(struct rsxx_cardinfo *card) int rsxx_attach_dev(struct rsxx_cardinfo *card) { + int err = 0; + mutex_lock(&card->dev_lock); /* The block device requires the stripe size from the config. */ @@ -199,13 +201,17 @@ int rsxx_attach_dev(struct rsxx_cardinfo *card) set_capacity(card->gendisk, card->size8 >> 9); else set_capacity(card->gendisk, 0); - device_add_disk(CARD_TO_DEV(card), card->gendisk, NULL); - card->bdev_attached = 1; + err = device_add_disk(CARD_TO_DEV(card), card->gendisk, NULL); + if (err == 0) + card->bdev_attached = 1; } mutex_unlock(&card->dev_lock); - return 0; + if (err) + blk_cleanup_disk(card->gendisk); + + return err; } void rsxx_detach_dev(struct rsxx_cardinfo *card) From 637208e74a861d993c3a8eea3f9f1df4415930e0 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:55 -0700 Subject: [PATCH 020/255] block/sx8: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. A completion is used to notify the initial probe what is happening and so we must defer error handling on completion. Do this by remembering the error and using the shared cleanup function. The tags are shared and so are hanlded later for the driver already. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/sx8.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 420cd952ddc4..1c79248c4826 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -297,6 +297,7 @@ struct carm_host { struct work_struct fsm_task; + int probe_err; struct completion probe_comp; }; @@ -1181,8 +1182,11 @@ static void carm_fsm_task (struct work_struct *work) struct gendisk *disk = port->disk; set_capacity(disk, port->capacity); - add_disk(disk); - activated++; + host->probe_err = add_disk(disk); + if (!host->probe_err) + activated++; + else + break; } printk(KERN_INFO DRV_NAME "(%s): %d ports activated\n", @@ -1192,11 +1196,9 @@ static void carm_fsm_task (struct work_struct *work) reschedule = 1; break; } - case HST_PROBE_FINISHED: complete(&host->probe_comp); break; - case HST_ERROR: /* FIXME: TODO */ break; @@ -1507,7 +1509,10 @@ static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_free_irq; DPRINTK("waiting for probe_comp\n"); + host->probe_err = -ENODEV; wait_for_completion(&host->probe_comp); + if (host->probe_err) + goto err_out_free_irq; printk(KERN_INFO "%s: pci %s, ports %d, io %llx, irq %u, major %d\n", host->name, pci_name(pdev), (int) CARM_MAX_PORTS, From 4fac63f8a871bc2e38dce4944c9d964a62bac3e6 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:56 -0700 Subject: [PATCH 021/255] pf: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/paride/pf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index f471d48a87bc..380d80e507c7 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -962,7 +962,9 @@ static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, if (pf_probe(pf)) goto out_pi_release; - add_disk(disk); + ret = add_disk(disk); + if (ret) + goto out_pi_release; pf->present = 1; return 0; From d6ac27c60fec4dc59473e39abf924e430a9ea320 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:27 -0700 Subject: [PATCH 022/255] cdrom/gdrom: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/cdrom/gdrom.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index 8e1fe75af93f..d50cc1fd34d5 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c @@ -805,9 +805,14 @@ static int probe_gdrom(struct platform_device *devptr) err = -ENOMEM; goto probe_fail_free_irqs; } - add_disk(gd.disk); + err = add_disk(gd.disk); + if (err) + goto probe_fail_add_disk; + return 0; +probe_fail_add_disk: + kfree(gd.toc); probe_fail_free_irqs: free_irq(HW_EVENT_GDROM_DMA, &gd); free_irq(HW_EVENT_GDROM_CMD, &gd); From 27c97abc30e2b9ad2288977c0ecbef4d50553f57 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:28 -0700 Subject: [PATCH 023/255] rbd: add add_disk() error handling We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Signed-off-by: Jens Axboe --- drivers/block/rbd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index bf60aebd0cfb..953fa134cd3d 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -7054,7 +7054,9 @@ static ssize_t do_rbd_add(struct bus_type *bus, if (rc) goto err_out_image_lock; - device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL); + rc = device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL); + if (rc) + goto err_out_cleanup_disk; spin_lock(&rbd_dev_list_lock); list_add_tail(&rbd_dev->node, &rbd_dev_list); @@ -7068,6 +7070,8 @@ out: module_put(THIS_MODULE); return rc; +err_out_cleanup_disk: + rbd_free_disk(rbd_dev); err_out_image_lock: rbd_dev_image_unlock(rbd_dev); rbd_dev_device_release(rbd_dev); From 2d4bcf76429713c2ca0093c11b5b75072db95a50 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:49 -0700 Subject: [PATCH 024/255] block/swim3: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-2-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim3.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 965af0a3e95b..f7e3482e846b 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -1229,7 +1229,9 @@ static int swim3_attach(struct macio_dev *mdev, disk->flags |= GENHD_FL_REMOVABLE; sprintf(disk->disk_name, "fd%d", floppy_count); set_capacity(disk, 2880); - add_disk(disk); + rc = add_disk(disk); + if (rc) + goto out_cleanup_disk; disks[floppy_count++] = disk; return 0; From 2598a2bb357d64baaa94368133ddbc900b9eb246 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:50 -0700 Subject: [PATCH 025/255] floppy: fix add_disk() assumption on exit due to new developments After the patch titled "floppy: use blk_mq_alloc_disk and blk_cleanup_disk" the floppy driver was modified to allocate the blk_mq_alloc_disk() which allocates the disk with the queue. This is further clarified later with the patch titled "block: remove alloc_disk and alloc_disk_node". This clarifies that: Most drivers should use and have been converted to use blk_alloc_disk and blk_mq_alloc_disk. Only the scsi ULPs and dasd still allocate a disk separately from the request_queue so don't bother with convenience macros for something that should not see significant new users and remove these wrappers. And then we have the patch titled, "block: hold a request_queue reference for the lifetime of struct gendisk" which ensures that a queue is *always* present for sure during the entire lifetime of a disk. In the floppy driver's case then the disk always comes with the queue. So even if even if the queue was cleaned up on exit, putting the disk *is* still required, and likewise, blk_cleanup_queue() on a null queue should not happen now as disk->queue is valid from disk allocation time on. Automatic backport code scrapers should hopefully not cherry pick this patch as a stable fix candidate without full due dilligence to ensure all the work done on the block layer to make this happen is merged first. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-3-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/floppy.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 6288ce888414..2ee4d3e7ea2d 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4954,19 +4954,6 @@ static void __exit floppy_module_exit(void) blk_cleanup_queue(disks[drive][i]->queue); } blk_mq_free_tag_set(&tag_sets[drive]); - - /* - * These disks have not called add_disk(). Don't put down - * queue reference in put_disk(). - */ - if (!(allowed_drive_mask & (1 << drive)) || - fdc_state[FDC(drive)].version == FDC_NONE) { - for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { - if (disks[drive][i]) - disks[drive][i]->queue = NULL; - } - } - for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { if (disks[drive][i]) put_disk(disks[drive][i]); From 3776339ae7acaf9590c668e86f45005fc9aff014 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:51 -0700 Subject: [PATCH 026/255] floppy: use blk_cleanup_disk() Use the blk_cleanup_queue() followed by put_disk() can be replaced with blk_cleanup_disk(). No need for two separate loops. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-4-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/floppy.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 2ee4d3e7ea2d..74996728fc24 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4951,13 +4951,9 @@ static void __exit floppy_module_exit(void) } for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { if (disks[drive][i]) - blk_cleanup_queue(disks[drive][i]->queue); + blk_cleanup_disk(disks[drive][i]); } blk_mq_free_tag_set(&tag_sets[drive]); - for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { - if (disks[drive][i]) - put_disk(disks[drive][i]); - } } cancel_delayed_work_sync(&fd_timeout); From 662167e59d2f3c15a44a88088fc6c1a67c8a3650 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:52 -0700 Subject: [PATCH 027/255] floppy: fix calling platform_device_unregister() on invalid drives platform_device_unregister() should only be called when a respective platform_device_register() is called. However the floppy driver currently allows failures when registring a drive and a bail out could easily cause an invalid call to platform_device_unregister() where it was not intended. Fix this by adding a bool to keep track of when the platform device was registered for a drive. This does not fix any known panic / bug. This issue was found through code inspection while preparing the driver to use the up and coming support for device_add_disk() error handling. From what I can tell from code inspection, chances of this ever happening should be insanely small, perhaps OOM. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-5-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/floppy.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 74996728fc24..acde3cb63ef4 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4479,6 +4479,7 @@ static const struct blk_mq_ops floppy_mq_ops = { }; static struct platform_device floppy_device[N_DRIVE]; +static bool registered[N_DRIVE]; static bool floppy_available(int drive) { @@ -4694,6 +4695,8 @@ static int __init do_floppy_init(void) if (err) goto out_remove_drives; + registered[drive] = true; + device_add_disk(&floppy_device[drive].dev, disks[drive][0], NULL); } @@ -4704,7 +4707,8 @@ out_remove_drives: while (drive--) { if (floppy_available(drive)) { del_gendisk(disks[drive][0]); - platform_device_unregister(&floppy_device[drive]); + if (registered[drive]) + platform_device_unregister(&floppy_device[drive]); } } out_release_dma: @@ -4947,7 +4951,8 @@ static void __exit floppy_module_exit(void) if (disks[drive][i]) del_gendisk(disks[drive][i]); } - platform_device_unregister(&floppy_device[drive]); + if (registered[drive]) + platform_device_unregister(&floppy_device[drive]); } for (i = 0; i < ARRAY_SIZE(floppy_type); i++) { if (disks[drive][i]) From 47d34aa2d211e9cef61dd85b7b0011c9f61dd0ae Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:53 -0700 Subject: [PATCH 028/255] floppy: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-6-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/floppy.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index acde3cb63ef4..3873e789478e 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4697,8 +4697,10 @@ static int __init do_floppy_init(void) registered[drive] = true; - device_add_disk(&floppy_device[drive].dev, disks[drive][0], - NULL); + err = device_add_disk(&floppy_device[drive].dev, + disks[drive][0], NULL); + if (err) + goto out_remove_drives; } return 0; From a2379420c7d7cb14a8b214fc7c0e2f55f66393ac Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:54 -0700 Subject: [PATCH 029/255] amiflop: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. The caller for fd_alloc_disk() deals with the rest of the cleanup like the tag. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-7-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/amiflop.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index 2909fd9e72fb..bf5c124c5452 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c @@ -1780,6 +1780,7 @@ static const struct blk_mq_ops amiflop_mq_ops = { static int fd_alloc_disk(int drive, int system) { struct gendisk *disk; + int err; disk = blk_mq_alloc_disk(&unit[drive].tag_set, NULL); if (IS_ERR(disk)) @@ -1798,8 +1799,10 @@ static int fd_alloc_disk(int drive, int system) set_capacity(disk, 880 * 2); unit[drive].gendisk[system] = disk; - add_disk(disk); - return 0; + err = add_disk(disk); + if (err) + blk_cleanup_disk(disk); + return err; } static int fd_alloc_drive(int drive) From b76a30c254d987b4ae7d47415081121d4c0a7423 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:55 -0700 Subject: [PATCH 030/255] swim: simplify using blk_cleanup_disk() on swim_remove() We can simplify swim_remove() by using one call instead of two, just as other drivers do. Use that pattern. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-8-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 3911d0833e1b..868d59476065 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -912,9 +912,8 @@ static int swim_remove(struct platform_device *dev) for (drive = 0; drive < swd->floppy_count; drive++) { del_gendisk(swd->unit[drive].disk); - blk_cleanup_queue(swd->unit[drive].disk->queue); + blk_cleanup_disk(swd->unit[drive].disk); blk_mq_free_tag_set(&swd->unit[drive].tag_set); - put_disk(swd->unit[drive].disk); } unregister_blkdev(FLOPPY_MAJOR, "fd"); From 4e9abe72530a2baf5f80d60e8d0bcdb84964d2e4 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:56 -0700 Subject: [PATCH 031/255] swim: add helper for disk cleanup Disk cleanup can be shared between exit and bringup. Use a helper to do the work required. The only functional change at this point is we're being overly paraoid on exit to check for a null disk as well now, and this should be safe. We'll later expand on this, this change just makes subsequent changes easier to read. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-9-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 868d59476065..4f87d1af7c60 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -772,6 +772,17 @@ static const struct blk_mq_ops swim_mq_ops = { .queue_rq = swim_queue_rq, }; +static void swim_cleanup_floppy_disk(struct floppy_state *fs) +{ + struct gendisk *disk = fs->disk; + + if (!disk) + return; + + blk_cleanup_disk(disk); + blk_mq_free_tag_set(&fs->tag_set); +} + static int swim_floppy_init(struct swim_priv *swd) { int err; @@ -836,12 +847,7 @@ static int swim_floppy_init(struct swim_priv *swd) exit_put_disks: unregister_blkdev(FLOPPY_MAJOR, "fd"); do { - struct gendisk *disk = swd->unit[drive].disk; - - if (!disk) - continue; - blk_cleanup_disk(disk); - blk_mq_free_tag_set(&swd->unit[drive].tag_set); + swim_cleanup_floppy_disk(&swd->unit[drive]); } while (drive--); return err; } @@ -912,8 +918,7 @@ static int swim_remove(struct platform_device *dev) for (drive = 0; drive < swd->floppy_count; drive++) { del_gendisk(swd->unit[drive].disk); - blk_cleanup_disk(swd->unit[drive].disk); - blk_mq_free_tag_set(&swd->unit[drive].tag_set); + swim_cleanup_floppy_disk(&swd->unit[drive]); } unregister_blkdev(FLOPPY_MAJOR, "fd"); From 9ef41effb9b65088053e31c741c2a1ec97190117 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:57 -0700 Subject: [PATCH 032/255] swim: add a floppy registration bool which triggers del_gendisk() Instead of calling del_gendisk() on exit alone, let's add a registration bool to the floppy disk state, this way this can be done on the shared caller, swim_cleanup_floppy_disk(). This will be more useful in subsequent patches. Right now, this just shuffles functionality out to a helper in a safe way. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-10-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 4f87d1af7c60..eed453528f4c 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -185,6 +185,7 @@ struct floppy_state { int track; int ref_count; + bool registered; struct gendisk *disk; struct blk_mq_tag_set tag_set; @@ -779,6 +780,9 @@ static void swim_cleanup_floppy_disk(struct floppy_state *fs) if (!disk) return; + if (fs->registered) + del_gendisk(fs->disk); + blk_cleanup_disk(disk); blk_mq_free_tag_set(&fs->tag_set); } @@ -840,6 +844,7 @@ static int swim_floppy_init(struct swim_priv *swd) swd->unit[drive].disk->private_data = &swd->unit[drive]; set_capacity(swd->unit[drive].disk, 2880); add_disk(swd->unit[drive].disk); + swd->unit[drive].registered = true; } return 0; @@ -916,10 +921,8 @@ static int swim_remove(struct platform_device *dev) int drive; struct resource *res; - for (drive = 0; drive < swd->floppy_count; drive++) { - del_gendisk(swd->unit[drive].disk); + for (drive = 0; drive < swd->floppy_count; drive++) swim_cleanup_floppy_disk(&swd->unit[drive]); - } unregister_blkdev(FLOPPY_MAJOR, "fd"); From 625a28a7e60c7c026e4d2929c49c8461fad4b0f3 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:58 -0700 Subject: [PATCH 033/255] swim: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Since we have a caller to do our unwinding for the disk, and this is already dealt with safely we can re-use our existing error path goto label which already deals with the cleanup. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-11-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index eed453528f4c..821594cd1315 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -843,7 +843,9 @@ static int swim_floppy_init(struct swim_priv *swd) swd->unit[drive].disk->events = DISK_EVENT_MEDIA_CHANGE; swd->unit[drive].disk->private_data = &swd->unit[drive]; set_capacity(swd->unit[drive].disk, 2880); - add_disk(swd->unit[drive].disk); + err = add_disk(swd->unit[drive].disk); + if (err) + goto exit_put_disks; swd->unit[drive].registered = true; } From 44a469b6acae6ad05c4acca8429467d1d50a8b8d Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:02:59 -0700 Subject: [PATCH 034/255] block/ataflop: use the blk_cleanup_disk() helper Use the helper to replace two lines with one. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-12-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 58e921ab5729..bc8cf7a991af 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2077,8 +2077,7 @@ static int __init atari_floppy_init (void) err: while (--i >= 0) { - blk_cleanup_queue(unit[i].disk[0]->queue); - put_disk(unit[i].disk[0]); + blk_cleanup_disk(unit[i].disk[0]); blk_mq_free_tag_set(&unit[i].tag_set); } @@ -2136,8 +2135,7 @@ static void __exit atari_floppy_exit(void) if (!unit[i].disk[type]) continue; del_gendisk(unit[i].disk[type]); - blk_cleanup_queue(unit[i].disk[type]->queue); - put_disk(unit[i].disk[type]); + blk_cleanup_disk(unit[i].disk[type]); } blk_mq_free_tag_set(&unit[i].tag_set); } From 573effb298011d3fcabc9b12025cf637f8a07911 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:03:00 -0700 Subject: [PATCH 035/255] block/ataflop: add registration bool before calling del_gendisk() The ataflop assumes del_gendisk() is safe to call, this is only true because add_disk() does not return a failure, but that will change soon. And so, before we get to adding error handling for that case, let's make sure we keep track of which disks actually get registered. Then we use this to only call del_gendisk for them. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-13-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index bc8cf7a991af..fe16105ecdf5 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -299,6 +299,7 @@ static struct atari_floppy_struct { disk change detection) */ int flags; /* flags */ struct gendisk *disk[NUM_DISK_MINORS]; + bool registered[NUM_DISK_MINORS]; int ref; int type; struct blk_mq_tag_set tag_set; @@ -2001,8 +2002,10 @@ static void ataflop_probe(dev_t dev) return; mutex_lock(&ataflop_probe_lock); if (!unit[drive].disk[type]) { - if (ataflop_alloc_disk(drive, type) == 0) + if (ataflop_alloc_disk(drive, type) == 0) { add_disk(unit[drive].disk[type]); + unit[drive].registered[type] = true; + } } mutex_unlock(&ataflop_probe_lock); } @@ -2066,6 +2069,7 @@ static int __init atari_floppy_init (void) unit[i].track = -1; unit[i].flags = 0; add_disk(unit[i].disk[0]); + unit[i].registered[0] = true; } printk(KERN_INFO "Atari floppy driver: max. %cD, %strack buffering\n", @@ -2134,7 +2138,8 @@ static void __exit atari_floppy_exit(void) for (type = 0; type < NUM_DISK_MINORS; type++) { if (!unit[i].disk[type]) continue; - del_gendisk(unit[i].disk[type]); + if (unit[i].registered[type]) + del_gendisk(unit[i].disk[type]); blk_cleanup_disk(unit[i].disk[type]); } blk_mq_free_tag_set(&unit[i].tag_set); From deae1138d04758c7f8939fcb8aee330bc37e3015 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:03:01 -0700 Subject: [PATCH 036/255] block/ataflop: provide a helper for cleanup up an atari disk Instead of using two separate code paths for cleaning up an atari disk, use one. We take the more careful approach to check for *all* disk types, as is done on exit. The init path didn't have that check as the alternative disk types are only probed for later, they are not initialized by default. Yes, there is a shared tag for all disks. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-14-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index fe16105ecdf5..a07cb9b79a6c 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2010,6 +2010,20 @@ static void ataflop_probe(dev_t dev) mutex_unlock(&ataflop_probe_lock); } +static void atari_cleanup_floppy_disk(struct atari_floppy_struct *fs) +{ + int type; + + for (type = 0; type < NUM_DISK_MINORS; type++) { + if (!fs->disk[type]) + continue; + if (fs->registered[type]) + del_gendisk(fs->disk[type]); + blk_cleanup_disk(fs->disk[type]); + } + blk_mq_free_tag_set(&fs->tag_set); +} + static int __init atari_floppy_init (void) { int i; @@ -2080,10 +2094,8 @@ static int __init atari_floppy_init (void) return 0; err: - while (--i >= 0) { - blk_cleanup_disk(unit[i].disk[0]); - blk_mq_free_tag_set(&unit[i].tag_set); - } + while (--i >= 0) + atari_cleanup_floppy_disk(&unit[i]); unregister_blkdev(FLOPPY_MAJOR, "fd"); out_unlock: @@ -2132,18 +2144,10 @@ __setup("floppy=", atari_floppy_setup); static void __exit atari_floppy_exit(void) { - int i, type; + int i; - for (i = 0; i < FD_MAX_UNITS; i++) { - for (type = 0; type < NUM_DISK_MINORS; type++) { - if (!unit[i].disk[type]) - continue; - if (unit[i].registered[type]) - del_gendisk(unit[i].disk[type]); - blk_cleanup_disk(unit[i].disk[type]); - } - blk_mq_free_tag_set(&unit[i].tag_set); - } + for (i = 0; i < FD_MAX_UNITS; i++) + atari_cleanup_floppy_disk(&unit[i]); unregister_blkdev(FLOPPY_MAJOR, "fd"); del_timer_sync(&fd_timer); From 2f1510708970c873c5eee190b77071f59f67cef8 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:03:02 -0700 Subject: [PATCH 037/255] block/ataflop: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20210927220302.1073499-15-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index a07cb9b79a6c..9bc5cce6b29a 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -2082,7 +2082,9 @@ static int __init atari_floppy_init (void) for (i = 0; i < FD_MAX_UNITS; i++) { unit[i].track = -1; unit[i].flags = 0; - add_disk(unit[i].disk[0]); + ret = add_disk(unit[i].disk[0]); + if (ret) + goto err_out_dma; unit[i].registered[0] = true; } @@ -2093,6 +2095,8 @@ static int __init atari_floppy_init (void) return 0; +err_out_dma: + atari_stram_free(DMABuffer); err: while (--i >= 0) atari_cleanup_floppy_disk(&unit[i]); From db8eda9c43361023678aa23eb0dceb0a411af0f3 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Mon, 27 Sep 2021 15:01:01 -0700 Subject: [PATCH 038/255] xtensa/platforms/iss/simdisk: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Signed-off-by: Luis Chamberlain Acked-by: Max Filippov Link: https://lore.kernel.org/r/20210927220110.1066271-7-mcgrof@kernel.org Signed-off-by: Jens Axboe --- arch/xtensa/platforms/iss/simdisk.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index ddd1fe3db474..07b642c1916a 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -258,6 +258,7 @@ static int __init simdisk_setup(struct simdisk *dev, int which, struct proc_dir_entry *procdir) { char tmp[2] = { '0' + which, 0 }; + int err = -ENOMEM; dev->fd = -1; dev->filename = NULL; @@ -266,7 +267,7 @@ static int __init simdisk_setup(struct simdisk *dev, int which, dev->gd = blk_alloc_disk(NUMA_NO_NODE); if (!dev->gd) - return -ENOMEM; + goto out; dev->gd->major = simdisk_major; dev->gd->first_minor = which; dev->gd->minors = SIMDISK_MINORS; @@ -274,10 +275,18 @@ static int __init simdisk_setup(struct simdisk *dev, int which, dev->gd->private_data = dev; snprintf(dev->gd->disk_name, 32, "simdisk%d", which); set_capacity(dev->gd, 0); - add_disk(dev->gd); + err = add_disk(dev->gd); + if (err) + goto out_cleanup_disk; dev->procfile = proc_create_data(tmp, 0644, procdir, &simdisk_proc_ops, dev); + return 0; + +out_cleanup_disk: + blk_cleanup_disk(dev->gd); +out: + return err; } static int __init simdisk_init(void) From d0ac7a30e41174c794fbfa53ea986d9555e5b9f4 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 1 Oct 2021 15:26:23 +0300 Subject: [PATCH 039/255] pcd: fix error codes in pcd_init_unit() Return -ENODEV on these error paths instead of returning success. Fixes: af761f277b7f ("pcd: cleanup initialization") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20211001122623.GA2283@kili Signed-off-by: Jens Axboe --- drivers/block/paride/pcd.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 4cc0d141db78..f6b1d63e96e1 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c @@ -613,8 +613,7 @@ static int pcd_identify(struct pcd_unit *cd) } /* - * returns 0, with id set if drive is detected - * -1, if drive detection failed + * returns 0, with id set if drive is detected, otherwise an error code. */ static int pcd_probe(struct pcd_unit *cd, int ms) { @@ -627,7 +626,7 @@ static int pcd_probe(struct pcd_unit *cd, int ms) if (!pcd_reset(cd) && !pcd_identify(cd)) return 0; } - return -1; + return -ENODEV; } static int pcd_probe_capabilities(struct pcd_unit *cd) @@ -933,9 +932,12 @@ static int pcd_init_unit(struct pcd_unit *cd, bool autoprobe, int port, disk->events = DISK_EVENT_MEDIA_CHANGE; if (!pi_init(cd->pi, autoprobe, port, mode, unit, protocol, delay, - pcd_buffer, PI_PCD, verbose, cd->name)) + pcd_buffer, PI_PCD, verbose, cd->name)) { + ret = -ENODEV; goto out_free_disk; - if (pcd_probe(cd, ms)) + } + ret = pcd_probe(cd, ms); + if (ret) goto out_pi_release; cd->present = 1; From cfc03eabda8224c681087a4c6c51d1cc595ebfaf Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 1 Oct 2021 15:26:54 +0300 Subject: [PATCH 040/255] pf: fix error codes in pf_init_unit() Return a negative error code instead of success on these error paths. Fixes: fb367e6baeb0 ("pf: cleanup initialization") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20211001122654.GB2283@kili Signed-off-by: Jens Axboe --- drivers/block/paride/pf.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index 380d80e507c7..bf8d0ef41a0a 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c @@ -651,9 +651,9 @@ static int pf_identify(struct pf_unit *pf) return 0; } -/* returns 0, with id set if drive is detected - -1, if drive detection failed -*/ +/* + * returns 0, with id set if drive is detected, otherwise an error code. + */ static int pf_probe(struct pf_unit *pf) { if (pf->drive == -1) { @@ -675,7 +675,7 @@ static int pf_probe(struct pf_unit *pf) if (!pf_identify(pf)) return 0; } - return -1; + return -ENODEV; } /* The i/o request engine */ @@ -957,9 +957,12 @@ static int __init pf_init_unit(struct pf_unit *pf, bool autoprobe, int port, snprintf(pf->name, PF_NAMELEN, "%s%d", name, disk->first_minor); if (!pi_init(pf->pi, autoprobe, port, mode, unit, protocol, delay, - pf_scratch, PI_PF, verbose, pf->name)) + pf_scratch, PI_PF, verbose, pf->name)) { + ret = -ENODEV; goto out_free_disk; - if (pf_probe(pf)) + } + ret = pf_probe(pf); + if (ret) goto out_pi_release; ret = add_disk(disk); From 5deae20c552aec0750cc7b95e84ca94121aac3b3 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 1 Oct 2021 15:27:22 +0300 Subject: [PATCH 041/255] sx8: fix an error code in carm_init_one() Return a negative error code here on this error path instead of returning success. Fixes: 637208e74a86 ("block/sx8: add error handling support for add_disk()") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/20211001122722.GC2283@kili Signed-off-by: Jens Axboe --- drivers/block/sx8.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 1c79248c4826..d1676fe0da1a 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -1511,8 +1511,10 @@ static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent) DPRINTK("waiting for probe_comp\n"); host->probe_err = -ENODEV; wait_for_completion(&host->probe_comp); - if (host->probe_err) + if (host->probe_err) { + rc = host->probe_err; goto err_out_free_irq; + } printk(KERN_INFO "%s: pci %s, ports %d, io %llx, irq %u, major %d\n", host->name, pci_name(pdev), (int) CARM_MAX_PORTS, From 1f0a258f114b5b152855d31179f902cb10bdfb59 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 1 Oct 2021 19:23:26 -0600 Subject: [PATCH 042/255] swim3: add missing major.h include swim3 got this through blkdev.h previously, but blkdev.h is not including it anymore. Include it specifically for the driver, otherwise FLOPPY_MAJOR is undefined and breaks the compile on PPC if swim3 is configured. Fixes: b81e0c2372e6 ("block: drop unused includes in ") Reported-by: Naresh Kamboju Acked-by: Randy Dunlap # build-tested Signed-off-by: Jens Axboe --- drivers/block/swim3.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index f7e3482e846b..4b91c9aa5892 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include From 99457db8b40c66941072a383a5ab4e36bc53fd3d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:01 +0200 Subject: [PATCH 043/255] block: move the SECTOR_SIZE related definitions to blk_types.h Ensure these are always available for inlines in the various block layer headers. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-2-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk_types.h | 17 +++++++++++++++++ include/linux/blkdev.h | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 1e370929c89e..472e55e0e94f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -20,6 +20,23 @@ struct cgroup_subsys_state; typedef void (bio_end_io_t) (struct bio *); struct bio_crypt_ctx; +/* + * The basic unit of block I/O is a sector. It is used in a number of contexts + * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9 + * bytes. Variables of type sector_t represent an offset or size that is a + * multiple of 512 bytes. Hence these two constants. + */ +#ifndef SECTOR_SHIFT +#define SECTOR_SHIFT 9 +#endif +#ifndef SECTOR_SIZE +#define SECTOR_SIZE (1 << SECTOR_SHIFT) +#endif + +#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) +#define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT) +#define SECTOR_MASK (PAGE_SECTORS - 1) + struct block_device { sector_t bd_start_sect; struct disk_stats __percpu *bd_stats; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index fd9771a1da09..abe721591e80 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -578,23 +578,6 @@ static inline struct request_queue *bdev_get_queue(struct block_device *bdev) return bdev->bd_queue; /* this is never NULL */ } -/* - * The basic unit of block I/O is a sector. It is used in a number of contexts - * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9 - * bytes. Variables of type sector_t represent an offset or size that is a - * multiple of 512 bytes. Hence these two constants. - */ -#ifndef SECTOR_SHIFT -#define SECTOR_SHIFT 9 -#endif -#ifndef SECTOR_SIZE -#define SECTOR_SIZE (1 << SECTOR_SHIFT) -#endif - -#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) -#define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT) -#define SECTOR_MASK (PAGE_SECTORS - 1) - #ifdef CONFIG_BLK_DEV_ZONED /* Helper to convert BLK_ZONE_ZONE_XXX to its string format XXX */ From 6436bd90f76e75d2c5786a50203b05a9b7f7100d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:02 +0200 Subject: [PATCH 044/255] block: add a bdev_nr_bytes helper Add a helper to query the size of a block device in bytes. This will be used to remove open coded access to ->bd_inode. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-3-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/genhd.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/linux/genhd.h b/include/linux/genhd.h index cd4038fd5743..01d27f3a970e 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -236,9 +236,14 @@ static inline sector_t get_start_sect(struct block_device *bdev) return bdev->bd_start_sect; } +static inline loff_t bdev_nr_bytes(struct block_device *bdev) +{ + return i_size_read(bdev->bd_inode); +} + static inline sector_t bdev_nr_sectors(struct block_device *bdev) { - return i_size_read(bdev->bd_inode) >> 9; + return bdev_nr_bytes(bdev) >> SECTOR_SHIFT; } static inline sector_t get_capacity(struct gendisk *disk) From cda25b82c47496f2da0785af5a0aa72a8990cec2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:03 +0200 Subject: [PATCH 045/255] bcache: remove bdev_sectors Use the equivalent block layer helper instead. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Chaitanya Kulkarni Acked-by: Coly Li Link: https://lore.kernel.org/r/20211018101130.1838532-4-hch@lst.de Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 2 +- drivers/md/bcache/util.h | 4 ---- drivers/md/bcache/writeback.c | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index f2874c77ff79..4f89985abe4b 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1002,7 +1002,7 @@ static void calc_cached_dev_sectors(struct cache_set *c) struct cached_dev *dc; list_for_each_entry(dc, &c->cached_devs, list) - sectors += bdev_sectors(dc->bdev); + sectors += bdev_nr_sectors(dc->bdev); c->cached_dev_sectors = sectors; } diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h index b64460a76267..a7da7930a7fd 100644 --- a/drivers/md/bcache/util.h +++ b/drivers/md/bcache/util.h @@ -584,8 +584,4 @@ static inline unsigned int fract_exp_two(unsigned int x, void bch_bio_map(struct bio *bio, void *base); int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask); -static inline sector_t bdev_sectors(struct block_device *bdev) -{ - return bdev->bd_inode->i_size >> 9; -} #endif /* _BCACHE_UTIL_H */ diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 8120da278161..c7560f66dca8 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -45,7 +45,7 @@ static uint64_t __calc_target_rate(struct cached_dev *dc) * backing volume uses about 2% of the cache for dirty data. */ uint32_t bdev_share = - div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT, + div64_u64(bdev_nr_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT, c->cached_dev_sectors); uint64_t cache_dirty_target = From da7b392467da82f8b8cbfe69360e3128688c9ddb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:04 +0200 Subject: [PATCH 046/255] drbd: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Lee Duncan Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-5-hch@lst.de Signed-off-by: Jens Axboe --- drivers/block/drbd/drbd_int.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 6674a0b88341..f27d5b0f9a0b 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1826,8 +1826,7 @@ static inline sector_t drbd_md_last_sector(struct drbd_backing_dev *bdev) /* Returns the number of 512 byte sectors of the device */ static inline sector_t drbd_get_capacity(struct block_device *bdev) { - /* return bdev ? get_capacity(bdev->bd_disk) : 0; */ - return bdev ? i_size_read(bdev->bd_inode) >> 9 : 0; + return bdev ? bdev_nr_sectors(bdev) : 0; } /** From 6dcbb52cddd9e50c8f6625b02a31f6dffc0d1a7b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:05 +0200 Subject: [PATCH 047/255] dm: use bdev_nr_sectors and bdev_nr_bytes instead of open coding them Use the proper helpers to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Mike Snitzer Link: https://lore.kernel.org/r/20211018101130.1838532-6-hch@lst.de Signed-off-by: Jens Axboe --- drivers/md/dm-bufio.c | 2 +- drivers/md/dm-cache-metadata.c | 2 +- drivers/md/dm-cache-target.c | 2 +- drivers/md/dm-clone-target.c | 2 +- drivers/md/dm-dust.c | 5 ++--- drivers/md/dm-ebs-target.c | 2 +- drivers/md/dm-era-target.c | 2 +- drivers/md/dm-exception-store.h | 2 +- drivers/md/dm-flakey.c | 3 +-- drivers/md/dm-integrity.c | 6 +++--- drivers/md/dm-linear.c | 3 +-- drivers/md/dm-log-writes.c | 4 ++-- drivers/md/dm-log.c | 2 +- drivers/md/dm-mpath.c | 2 +- drivers/md/dm-raid.c | 6 +++--- drivers/md/dm-switch.c | 2 +- drivers/md/dm-table.c | 3 +-- drivers/md/dm-thin-metadata.c | 2 +- drivers/md/dm-thin.c | 2 +- drivers/md/dm-verity-target.c | 3 +-- drivers/md/dm-writecache.c | 2 +- drivers/md/dm-zoned-target.c | 2 +- 22 files changed, 28 insertions(+), 33 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 50f3e673729c..104ebc1f08dc 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1525,7 +1525,7 @@ EXPORT_SYMBOL_GPL(dm_bufio_get_block_size); sector_t dm_bufio_get_device_size(struct dm_bufio_client *c) { - sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT; + sector_t s = bdev_nr_sectors(c->bdev); if (s >= c->start) s -= c->start; else diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c index 89a73204dbf4..2874f222c313 100644 --- a/drivers/md/dm-cache-metadata.c +++ b/drivers/md/dm-cache-metadata.c @@ -334,7 +334,7 @@ static int __write_initial_superblock(struct dm_cache_metadata *cmd) int r; struct dm_block *sblock; struct cache_disk_superblock *disk_super; - sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT; + sector_t bdev_size = bdev_nr_sectors(cmd->bdev); /* FIXME: see if we can lose the max sectors limit */ if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS) diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index bdd500447dea..447d030036d1 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -1940,7 +1940,7 @@ static void cache_dtr(struct dm_target *ti) static sector_t get_dev_size(struct dm_dev *dev) { - return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(dev->bdev); } /*----------------------------------------------------------------*/ diff --git a/drivers/md/dm-clone-target.c b/drivers/md/dm-clone-target.c index edd22e4d65df..4599632d7a84 100644 --- a/drivers/md/dm-clone-target.c +++ b/drivers/md/dm-clone-target.c @@ -1514,7 +1514,7 @@ error: static sector_t get_dev_size(struct dm_dev *dev) { - return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(dev->bdev); } /*---------------------------------------------------------------------------*/ diff --git a/drivers/md/dm-dust.c b/drivers/md/dm-dust.c index 3163e2b1418e..03672204b0e3 100644 --- a/drivers/md/dm-dust.c +++ b/drivers/md/dm-dust.c @@ -415,7 +415,7 @@ static int dust_message(struct dm_target *ti, unsigned int argc, char **argv, char *result, unsigned int maxlen) { struct dust_device *dd = ti->private; - sector_t size = i_size_read(dd->dev->bdev->bd_inode) >> SECTOR_SHIFT; + sector_t size = bdev_nr_sectors(dd->dev->bdev); bool invalid_msg = false; int r = -EINVAL; unsigned long long tmp, block; @@ -544,8 +544,7 @@ static int dust_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) /* * Only pass ioctls through if the device sizes match exactly. */ - if (dd->start || - ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT) + if (dd->start || ti->len != bdev_nr_sectors(dev->bdev)) return 1; return 0; diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c index d25989660a76..7ce5d509b940 100644 --- a/drivers/md/dm-ebs-target.c +++ b/drivers/md/dm-ebs-target.c @@ -416,7 +416,7 @@ static int ebs_prepare_ioctl(struct dm_target *ti, struct block_device **bdev) * Only pass ioctls through if the device sizes match exactly. */ *bdev = dev->bdev; - return !!(ec->start || ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT); + return !!(ec->start || ti->len != bdev_nr_sectors(dev->bdev)); } static void ebs_io_hints(struct dm_target *ti, struct queue_limits *limits) diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 2a78f6874143..1f6bf152b3c7 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -1681,7 +1681,7 @@ static int era_message(struct dm_target *ti, unsigned argc, char **argv, static sector_t get_dev_size(struct dm_dev *dev) { - return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(dev->bdev); } static int era_iterate_devices(struct dm_target *ti, diff --git a/drivers/md/dm-exception-store.h b/drivers/md/dm-exception-store.h index 3f4139ac1f60..b5f20eba3641 100644 --- a/drivers/md/dm-exception-store.h +++ b/drivers/md/dm-exception-store.h @@ -168,7 +168,7 @@ static inline void dm_consecutive_chunk_count_dec(struct dm_exception *e) */ static inline sector_t get_dev_size(struct block_device *bdev) { - return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(bdev); } static inline chunk_t sector_to_chunk(struct dm_exception_store *store, diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index 4b94ffe6f2d4..345229d7e59c 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -456,8 +456,7 @@ static int flakey_prepare_ioctl(struct dm_target *ti, struct block_device **bdev /* * Only pass ioctls through if the device sizes match exactly. */ - if (fc->start || - ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) + if (fc->start || ti->len != bdev_nr_sectors((*bdev))) return 1; return 0; } diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index dc03b70f6e65..d0f788e72abf 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -4113,11 +4113,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } } - ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; + ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev); if (!ic->meta_dev) ic->meta_device_sectors = ic->data_device_sectors; else - ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT; + ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev); if (!journal_sectors) { journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, @@ -4367,7 +4367,7 @@ try_smaller_buffer: DEBUG_print(" journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections)); DEBUG_print(" journal_entries %u\n", ic->journal_entries); DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); - DEBUG_print(" data_device_sectors 0x%llx\n", i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT); + DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev)); DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 679b4c0a2eea..66ba16713f69 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -135,8 +135,7 @@ static int linear_prepare_ioctl(struct dm_target *ti, struct block_device **bdev /* * Only pass ioctls through if the device sizes match exactly. */ - if (lc->start || - ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT) + if (lc->start || ti->len != bdev_nr_sectors(dev->bdev)) return 1; return 0; } diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index d93a4db23512..46de085a9670 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c @@ -446,7 +446,7 @@ static int log_super(struct log_writes_c *lc) static inline sector_t logdev_last_sector(struct log_writes_c *lc) { - return i_size_read(lc->logdev->bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(lc->logdev->bdev); } static int log_writes_kthread(void *arg) @@ -851,7 +851,7 @@ static int log_writes_prepare_ioctl(struct dm_target *ti, /* * Only pass ioctls through if the device sizes match exactly. */ - if (ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT) + if (ti->len != bdev_nr_sectors(dev->bdev)) return 1; return 0; } diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 1ecf75ef276a..06f328928a7f 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -447,7 +447,7 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti, bdev_logical_block_size(lc->header_location. bdev)); - if (buf_size > i_size_read(dev->bdev->bd_inode)) { + if (buf_size > bdev_nr_bytes(dev->bdev)) { DMWARN("log device %s too small: need %llu bytes", dev->name, (unsigned long long)buf_size); kfree(lc); diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 694aaca4eea2..5794f5415155 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -2061,7 +2061,7 @@ static int multipath_prepare_ioctl(struct dm_target *ti, /* * Only pass ioctls through if the device sizes match exactly. */ - if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) + if (!r && ti->len != bdev_nr_sectors((*bdev))) return 1; return r; } diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index d9ef52159a22..2b26435a6946 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -1261,7 +1261,7 @@ static int parse_raid_params(struct raid_set *rs, struct dm_arg_set *as, md_rdev_init(jdev); jdev->mddev = &rs->md; jdev->bdev = rs->journal_dev.dev->bdev; - jdev->sectors = to_sector(i_size_read(jdev->bdev->bd_inode)); + jdev->sectors = bdev_nr_sectors(jdev->bdev); if (jdev->sectors < MIN_RAID456_JOURNAL_SPACE) { rs->ti->error = "No space for raid4/5/6 journal"; return -ENOSPC; @@ -1607,7 +1607,7 @@ static int _check_data_dev_sectors(struct raid_set *rs) rdev_for_each(rdev, &rs->md) if (!test_bit(Journal, &rdev->flags) && rdev->bdev) { - ds = min(ds, to_sector(i_size_read(rdev->bdev->bd_inode))); + ds = min(ds, bdev_nr_sectors(rdev->bdev)); if (ds < rs->md.dev_sectors) { rs->ti->error = "Component device(s) too small"; return -EINVAL; @@ -2662,7 +2662,7 @@ static int rs_adjust_data_offsets(struct raid_set *rs) * Make sure we got a minimum amount of free sectors per device */ if (rs->data_offset && - to_sector(i_size_read(rdev->bdev->bd_inode)) - rs->md.dev_sectors < MIN_FREE_RESHAPE_SPACE) { + bdev_nr_sectors(rdev->bdev) - rs->md.dev_sectors < MIN_FREE_RESHAPE_SPACE) { rs->ti->error = data_offset ? "No space for forward reshape" : "No space for backward reshape"; return -ENOSPC; diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c index 028a92ff6d57..534dc2ca8bb0 100644 --- a/drivers/md/dm-switch.c +++ b/drivers/md/dm-switch.c @@ -529,7 +529,7 @@ static int switch_prepare_ioctl(struct dm_target *ti, struct block_device **bdev * Only pass ioctls through if the device sizes match exactly. */ if (ti->len + sctx->path_list[path_nr].start != - i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) + bdev_nr_sectors((*bdev))) return 1; return 0; } diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 1fa4d5582dca..d95142102bd2 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -227,8 +227,7 @@ static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, { struct queue_limits *limits = data; struct block_device *bdev = dev->bdev; - sector_t dev_size = - i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; + sector_t dev_size = bdev_nr_sectors(bdev); unsigned short logical_block_size_sectors = limits->logical_block_size >> SECTOR_SHIFT; char b[BDEVNAME_SIZE]; diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index c88ed14d49e6..1a96a07cbf44 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -549,7 +549,7 @@ static int __write_initial_superblock(struct dm_pool_metadata *pmd) int r; struct dm_block *sblock; struct thin_disk_superblock *disk_super; - sector_t bdev_size = i_size_read(pmd->bdev->bd_inode) >> SECTOR_SHIFT; + sector_t bdev_size = bdev_nr_sectors(pmd->bdev); if (bdev_size > THIN_METADATA_MAX_SECTORS) bdev_size = THIN_METADATA_MAX_SECTORS; diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 4c67b77c23c1..ec119d2422d5 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -3212,7 +3212,7 @@ static int metadata_pre_commit_callback(void *context) static sector_t get_dev_size(struct block_device *bdev) { - return i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; + return bdev_nr_sectors(bdev); } static void warn_if_metadata_device_too_big(struct block_device *bdev) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index aae48a8b1a04..a7efe83aad29 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -834,8 +834,7 @@ static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev *bdev = v->data_dev->bdev; - if (v->data_start || - ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT) + if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev)) return 1; return 0; } diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 18320444fb0a..017806096b91 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -2341,7 +2341,7 @@ static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv) ti->error = "Cache data device lookup failed"; goto bad; } - wc->memory_map_size = i_size_read(wc->ssd_dev->bdev->bd_inode); + wc->memory_map_size = bdev_nr_bytes(wc->ssd_dev->bdev); /* * Parse the cache block size diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c index ae1bc48c0043..8dc21c09329f 100644 --- a/drivers/md/dm-zoned-target.c +++ b/drivers/md/dm-zoned-target.c @@ -733,7 +733,7 @@ static int dmz_get_zoned_device(struct dm_target *ti, char *path, dev->dev_idx = idx; (void)bdevname(dev->bdev, dev->name); - dev->capacity = i_size_read(bdev->bd_inode) >> SECTOR_SHIFT; + dev->capacity = bdev_nr_sectors(bdev); if (ti->begin) { ti->error = "Partial mapping is not supported"; goto err; From 0fe80347fd701a7261bea278cab692de79e83af7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:06 +0200 Subject: [PATCH 048/255] md: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Song Liu Link: https://lore.kernel.org/r/20211018101130.1838532-7-hch@lst.de Signed-off-by: Jens Axboe --- drivers/md/md.c | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 22310d5d8d41..d964da436383 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -888,8 +888,7 @@ static struct md_personality *find_pers(int level, char *clevel) /* return the offset of the super block in 512byte sectors */ static inline sector_t calc_dev_sboffset(struct md_rdev *rdev) { - sector_t num_sectors = i_size_read(rdev->bdev->bd_inode) / 512; - return MD_NEW_SIZE_SECTORS(num_sectors); + return MD_NEW_SIZE_SECTORS(bdev_nr_sectors(rdev->bdev)); } static int alloc_disk_sb(struct md_rdev *rdev) @@ -1631,8 +1630,7 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ */ switch(minor_version) { case 0: - sb_start = i_size_read(rdev->bdev->bd_inode) >> 9; - sb_start -= 8*2; + sb_start = bdev_nr_sectors(rdev->bdev) - 8 * 2; sb_start &= ~(sector_t)(4*2-1); break; case 1: @@ -1787,10 +1785,9 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ else ret = 0; } - if (minor_version) { - sectors = (i_size_read(rdev->bdev->bd_inode) >> 9); - sectors -= rdev->data_offset; - } else + if (minor_version) + sectors = bdev_nr_sectors(rdev->bdev) - rdev->data_offset; + else sectors = rdev->sb_start; if (sectors < le64_to_cpu(sb->data_size)) return -EINVAL; @@ -2168,8 +2165,7 @@ super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors) return 0; /* too confusing */ if (rdev->sb_start < rdev->data_offset) { /* minor versions 1 and 2; superblock before data */ - max_sectors = i_size_read(rdev->bdev->bd_inode) >> 9; - max_sectors -= rdev->data_offset; + max_sectors = bdev_nr_sectors(rdev->bdev) - rdev->data_offset; if (!num_sectors || num_sectors > max_sectors) num_sectors = max_sectors; } else if (rdev->mddev->bitmap_info.offset) { @@ -2178,7 +2174,7 @@ super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors) } else { /* minor version 0; superblock after data */ sector_t sb_start, bm_space; - sector_t dev_size = i_size_read(rdev->bdev->bd_inode) >> 9; + sector_t dev_size = bdev_nr_sectors(rdev->bdev); /* 8K is for superblock */ sb_start = dev_size - 8*2; @@ -3382,7 +3378,7 @@ rdev_size_store(struct md_rdev *rdev, const char *buf, size_t len) if (!sectors) return -EBUSY; } else if (!sectors) - sectors = (i_size_read(rdev->bdev->bd_inode) >> 9) - + sectors = bdev_nr_sectors(rdev->bdev) - rdev->data_offset; if (!my_mddev->pers->resize) /* Cannot change size for RAID0 or Linear etc */ @@ -3709,7 +3705,7 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe kobject_init(&rdev->kobj, &rdev_ktype); - size = i_size_read(rdev->bdev->bd_inode) >> BLOCK_SIZE_BITS; + size = bdev_nr_bytes(rdev->bdev) >> BLOCK_SIZE_BITS; if (!size) { pr_warn("md: %s has zero or unknown size, marking faulty!\n", bdevname(rdev->bdev,b)); @@ -6880,7 +6876,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) if (!mddev->persistent) { pr_debug("md: nonpersistent superblock ...\n"); - rdev->sb_start = i_size_read(rdev->bdev->bd_inode) / 512; + rdev->sb_start = bdev_nr_sectors(rdev->bdev); } else rdev->sb_start = calc_dev_sboffset(rdev); rdev->sectors = rdev->sb_start; @@ -6967,7 +6963,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) if (mddev->persistent) rdev->sb_start = calc_dev_sboffset(rdev); else - rdev->sb_start = i_size_read(rdev->bdev->bd_inode) / 512; + rdev->sb_start = bdev_nr_sectors(rdev->bdev); rdev->sectors = rdev->sb_start; From c68f3ef777939ba70e167711754d5a27bfb6e51b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:07 +0200 Subject: [PATCH 049/255] nvmet: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-8-hch@lst.de Signed-off-by: Jens Axboe --- drivers/nvme/target/io-cmd-bdev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 6139e1de50a6..70ca9dfc1771 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -87,7 +87,7 @@ int nvmet_bdev_ns_enable(struct nvmet_ns *ns) ns->bdev = NULL; return ret; } - ns->size = i_size_read(ns->bdev->bd_inode); + ns->size = bdev_nr_bytes(ns->bdev); ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev)); ns->pi_type = 0; @@ -108,7 +108,7 @@ int nvmet_bdev_ns_enable(struct nvmet_ns *ns) void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns) { - ns->size = i_size_read(ns->bdev->bd_inode); + ns->size = bdev_nr_bytes(ns->bdev); } u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts) From 64f0f42671b48ec30a3203818e26346d5b4ea5fa Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:08 +0200 Subject: [PATCH 050/255] target/iblock: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-9-hch@lst.de Signed-off-by: Jens Axboe --- drivers/target/target_core_iblock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 31df20abe141..b1ef041cacd8 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -232,9 +232,9 @@ static unsigned long long iblock_emulate_read_cap_with_block_size( struct block_device *bd, struct request_queue *q) { - unsigned long long blocks_long = (div_u64(i_size_read(bd->bd_inode), - bdev_logical_block_size(bd)) - 1); u32 block_size = bdev_logical_block_size(bd); + unsigned long long blocks_long = + div_u64(bdev_nr_bytes(bd), block_size) - 1; if (block_size == dev->dev_attrib.block_size) return blocks_long; From b86058f96cc86e415e51bd12cc3786d7cdbd8b47 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:09 +0200 Subject: [PATCH 051/255] fs: use bdev_nr_bytes instead of open coding it in blkdev_max_block Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-10-hch@lst.de Signed-off-by: Jens Axboe --- fs/buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/buffer.c b/fs/buffer.c index c615387aedca..156358977249 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -878,7 +878,7 @@ link_dev_buffers(struct page *page, struct buffer_head *head) static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size) { sector_t retval = ~((sector_t)0); - loff_t sz = i_size_read(bdev->bd_inode); + loff_t sz = bdev_nr_bytes(bdev); if (sz) { unsigned int sizebits = blksize_bits(size); From bcd1d06350e410f60518f9d778d9cc4674f57158 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:10 +0200 Subject: [PATCH 052/255] fs: simplify init_page_buffers No need to convert from bdev to inode and back. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211018101130.1838532-11-hch@lst.de Signed-off-by: Jens Axboe --- fs/buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/buffer.c b/fs/buffer.c index 156358977249..46bc589b7a03 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -897,7 +897,7 @@ init_page_buffers(struct page *page, struct block_device *bdev, struct buffer_head *head = page_buffers(page); struct buffer_head *bh = head; int uptodate = PageUptodate(page); - sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size); + sector_t end_block = blkdev_max_block(bdev, size); do { if (!buffer_mapped(bh)) { From 589aa7bc40c4f823dd6094cef51f8cff60e26e95 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:11 +0200 Subject: [PATCH 053/255] affs: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-12-hch@lst.de Signed-off-by: Jens Axboe --- fs/affs/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/affs/super.c b/fs/affs/super.c index c6c2a513ec92..c609005a9eaa 100644 --- a/fs/affs/super.c +++ b/fs/affs/super.c @@ -389,7 +389,7 @@ static int affs_fill_super(struct super_block *sb, void *data, int silent) * blocks, we will have to change it. */ - size = i_size_read(sb->s_bdev->bd_inode) >> 9; + size = bdev_nr_sectors(sb->s_bdev); pr_debug("initial blocksize=%d, #blocks=%d\n", 512, size); affs_set_blocksize(sb, PAGE_SIZE); From cda00eba022d6a0a60740989ac79fc6a258b2d7a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:12 +0200 Subject: [PATCH 054/255] btrfs: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Anand Jain Reviewed-by: Chaitanya Kulkarni Acked-by: David Sterba Link: https://lore.kernel.org/r/20211018101130.1838532-13-hch@lst.de Signed-off-by: Jens Axboe --- fs/btrfs/dev-replace.c | 3 +-- fs/btrfs/disk-io.c | 2 +- fs/btrfs/ioctl.c | 4 ++-- fs/btrfs/volumes.c | 8 ++++---- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index d029be40ea6f..fbb8b4457a72 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -283,8 +283,7 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, } - if (i_size_read(bdev->bd_inode) < - btrfs_device_get_total_bytes(srcdev)) { + if (bdev_nr_bytes(bdev) < btrfs_device_get_total_bytes(srcdev)) { btrfs_err(fs_info, "target device is smaller than source device!"); ret = -EINVAL; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 355ea88d5c5f..29e7598584c4 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -3740,7 +3740,7 @@ struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev, else if (ret) return ERR_PTR(ret); - if (bytenr + BTRFS_SUPER_INFO_SIZE >= i_size_read(bdev->bd_inode)) + if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev)) return ERR_PTR(-EINVAL); page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index cc61813213d8..36ff713da1b1 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -1730,7 +1730,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, } if (!strcmp(sizestr, "max")) - new_size = device->bdev->bd_inode->i_size; + new_size = bdev_nr_bytes(device->bdev); else { if (sizestr[0] == '-') { mod = -1; @@ -1771,7 +1771,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, ret = -EINVAL; goto out_finish; } - if (new_size > device->bdev->bd_inode->i_size) { + if (new_size > bdev_nr_bytes(device->bdev)) { ret = -EFBIG; goto out_finish; } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 2ec3b8ac8fa3..676c7c4e6e59 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1286,7 +1286,7 @@ static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev pgoff_t index; /* make sure our super fits in the device */ - if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode)) + if (bytenr + PAGE_SIZE >= bdev_nr_bytes(bdev)) return ERR_PTR(-EINVAL); /* make sure our super fits in the page */ @@ -2610,8 +2610,8 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path device->io_width = fs_info->sectorsize; device->io_align = fs_info->sectorsize; device->sector_size = fs_info->sectorsize; - device->total_bytes = round_down(i_size_read(bdev->bd_inode), - fs_info->sectorsize); + device->total_bytes = + round_down(bdev_nr_bytes(bdev), fs_info->sectorsize); device->disk_total_bytes = device->total_bytes; device->commit_total_bytes = device->total_bytes; set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); @@ -7236,7 +7236,7 @@ static int read_one_dev(struct extent_buffer *leaf, fill_device_from_item(leaf, dev_item, device); if (device->bdev) { - u64 max_total_bytes = i_size_read(device->bdev->bd_inode); + u64 max_total_bytes = bdev_nr_bytes(device->bdev); if (device->total_bytes > max_total_bytes) { btrfs_err(fs_info, From 5816e91e4a14955224ae600dfea460d22588230a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:13 +0200 Subject: [PATCH 055/255] cramfs: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-14-hch@lst.de Signed-off-by: Jens Axboe --- fs/cramfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c index 2be65269a987..666aa380011e 100644 --- a/fs/cramfs/inode.c +++ b/fs/cramfs/inode.c @@ -209,7 +209,7 @@ static void *cramfs_blkdev_read(struct super_block *sb, unsigned int offset, return read_buffers[i] + blk_offset; } - devsize = mapping->host->i_size >> PAGE_SHIFT; + devsize = bdev_nr_bytes(sb->s_bdev) >> PAGE_SHIFT; /* Ok, read in BLKS_PER_BUF pages completely first. */ for (i = 0; i < BLKS_PER_BUF; i++) { From 9e48243b6506be0970d04fe2f015b1b3520ef9f3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:14 +0200 Subject: [PATCH 056/255] fat: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-15-hch@lst.de Signed-off-by: Jens Axboe --- fs/fat/inode.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/fat/inode.c b/fs/fat/inode.c index de0c9b013a85..9f3cd03668ad 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -1536,14 +1536,11 @@ static int fat_read_static_bpb(struct super_block *sb, struct fat_bios_param_block *bpb) { static const char *notdos1x = "This doesn't look like a DOS 1.x volume"; - + sector_t bd_sects = bdev_nr_sectors(sb->s_bdev); struct fat_floppy_defaults *fdefaults = NULL; int error = -EINVAL; - sector_t bd_sects; unsigned i; - bd_sects = i_size_read(sb->s_bdev->bd_inode) / SECTOR_SIZE; - /* 16-bit DOS 1.x reliably wrote bootstrap short-jmp code */ if (b->ignored[0] != 0xeb || b->ignored[2] != 0x90) { if (!silent) From beffd16e683eb9a600f249f9e34673ada1879f8b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:15 +0200 Subject: [PATCH 057/255] hfs: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-16-hch@lst.de Signed-off-by: Jens Axboe --- fs/hfs/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c index cdf0edeeb278..5beb82652435 100644 --- a/fs/hfs/mdb.c +++ b/fs/hfs/mdb.c @@ -36,7 +36,7 @@ static int hfs_get_last_session(struct super_block *sb, /* default values */ *start = 0; - *size = i_size_read(sb->s_bdev->bd_inode) >> 9; + *size = bdev_nr_sectors(sb->s_bdev); if (HFS_SB(sb)->session >= 0) { struct cdrom_tocentry te; From 78ed961bcee16dc48ca4ab22fb7936957e4dbdf0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:16 +0200 Subject: [PATCH 058/255] hfsplus: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-17-hch@lst.de Signed-off-by: Jens Axboe --- fs/hfsplus/wrapper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/hfsplus/wrapper.c b/fs/hfsplus/wrapper.c index 0350dc7821bf..51ae6f1eb4a5 100644 --- a/fs/hfsplus/wrapper.c +++ b/fs/hfsplus/wrapper.c @@ -131,7 +131,7 @@ static int hfsplus_get_last_session(struct super_block *sb, /* default values */ *start = 0; - *size = i_size_read(sb->s_bdev->bd_inode) >> 9; + *size = bdev_nr_sectors(sb->s_bdev); if (HFSPLUS_SB(sb)->session >= 0) { struct cdrom_tocentry te; From 74e157e6a499ef47edc39cff8c37f77d01c0d155 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:17 +0200 Subject: [PATCH 059/255] jfs: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Dave Kleikamp Link: https://lore.kernel.org/r/20211018101130.1838532-18-hch@lst.de Signed-off-by: Jens Axboe --- fs/jfs/resize.c | 2 +- fs/jfs/super.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/jfs/resize.c b/fs/jfs/resize.c index bde787c354fc..a42dbb0d3d28 100644 --- a/fs/jfs/resize.c +++ b/fs/jfs/resize.c @@ -199,7 +199,7 @@ int jfs_extendfs(struct super_block *sb, s64 newLVSize, int newLogSize) txQuiesce(sb); /* Reset size of direct inode */ - sbi->direct_inode->i_size = i_size_read(sb->s_bdev->bd_inode); + sbi->direct_inode->i_size = bdev_nr_bytes(sb->s_bdev); if (sbi->mntflag & JFS_INLINELOG) { /* diff --git a/fs/jfs/super.c b/fs/jfs/super.c index 9030aeaf0f88..9241caa16116 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -551,7 +551,7 @@ static int jfs_fill_super(struct super_block *sb, void *data, int silent) ret = -ENOMEM; goto out_unload; } - inode->i_size = i_size_read(sb->s_bdev->bd_inode); + inode->i_size = bdev_nr_bytes(sb->s_bdev); inode->i_mapping->a_ops = &jfs_metapage_aops; inode_fake_hash(inode); mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); From 6e50e781fe8873610f2e4011848f8b8d7406ee96 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:18 +0200 Subject: [PATCH 060/255] nfs/blocklayout: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-19-hch@lst.de Signed-off-by: Jens Axboe --- fs/nfs/blocklayout/dev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index acb1d22907da..5e56da748b2a 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -252,7 +252,7 @@ bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d, d->bdev = bdev; - d->len = i_size_read(d->bdev->bd_inode); + d->len = bdev_nr_bytes(d->bdev); d->map = bl_map_simple; printk(KERN_INFO "pNFS: using block device %s\n", @@ -367,7 +367,7 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, return PTR_ERR(bdev); d->bdev = bdev; - d->len = i_size_read(d->bdev->bd_inode); + d->len = bdev_nr_bytes(d->bdev); d->map = bl_map_simple; d->pr_key = v->scsi.pr_key; From 4fcd69798d7f366e1155f3041caf3891ebea72c6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:19 +0200 Subject: [PATCH 061/255] nilfs2: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Ryusuke Konishi Link: https://lore.kernel.org/r/20211018101130.1838532-20-hch@lst.de Signed-off-by: Jens Axboe --- fs/nilfs2/ioctl.c | 2 +- fs/nilfs2/super.c | 2 +- fs/nilfs2/the_nilfs.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index 640ac8fe891e..1d0583cfd970 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -1107,7 +1107,7 @@ static int nilfs_ioctl_set_alloc_range(struct inode *inode, void __user *argp) goto out; ret = -ERANGE; - if (range[1] > i_size_read(inode->i_sb->s_bdev->bd_inode)) + if (range[1] > bdev_nr_bytes(inode->i_sb->s_bdev)) goto out; segbytes = nilfs->ns_blocks_per_segment * nilfs->ns_blocksize; diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index f6b2d280aab5..3134c0e42fd4 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -403,7 +403,7 @@ int nilfs_resize_fs(struct super_block *sb, __u64 newsize) int ret; ret = -ERANGE; - devsize = i_size_read(sb->s_bdev->bd_inode); + devsize = bdev_nr_bytes(sb->s_bdev); if (newsize > devsize) goto out; diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index c8bfc01da5d7..1bfcb5d3ea48 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -489,7 +489,7 @@ static int nilfs_load_super_block(struct the_nilfs *nilfs, { struct nilfs_super_block **sbp = nilfs->ns_sbp; struct buffer_head **sbh = nilfs->ns_sbh; - u64 sb2off = NILFS_SB2_OFFSET_BYTES(nilfs->ns_bdev->bd_inode->i_size); + u64 sb2off = NILFS_SB2_OFFSET_BYTES(bdev_nr_bytes(nilfs->ns_bdev)); int valid[2], swp = 0; sbp[0] = nilfs_read_super_block(sb, NILFS_SB_OFFSET_BYTES, blocksize, From d54f13a8e479adbeea4585c62dbf0cc85cedf73c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 11:37:15 -0600 Subject: [PATCH 062/255] ntfs3: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211018101130.1838532-21-hch@lst.de Signed-off-by: Jens Axboe --- fs/ntfs3/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index d41d76979e12..29813200c7af 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -921,7 +921,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) /* Parse boot. */ err = ntfs_init_from_boot(sb, rq ? queue_logical_block_size(rq) : 512, - bdev->bd_inode->i_size); + bdev_nr_bytes(bdev)); if (err) goto out; From 4646198519c9aaa1c307ec4750ac64e08507d936 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:21 +0200 Subject: [PATCH 063/255] pstore/blk: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Acked-by: Kees Cook Link: https://lore.kernel.org/r/20211018101130.1838532-22-hch@lst.de Signed-off-by: Jens Axboe --- fs/pstore/blk.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/pstore/blk.c b/fs/pstore/blk.c index 04ce58c939a0..5d1fbaffd66a 100644 --- a/fs/pstore/blk.c +++ b/fs/pstore/blk.c @@ -205,7 +205,6 @@ static ssize_t psblk_generic_blk_write(const char *buf, size_t bytes, static int __register_pstore_blk(struct pstore_device_info *dev, const char *devpath) { - struct inode *inode; int ret = -ENODEV; lockdep_assert_held(&pstore_blk_lock); @@ -217,14 +216,13 @@ static int __register_pstore_blk(struct pstore_device_info *dev, goto err; } - inode = file_inode(psblk_file); - if (!S_ISBLK(inode->i_mode)) { + if (!S_ISBLK(file_inode(psblk_file)->i_mode)) { pr_err("'%s' is not block device!\n", devpath); goto err_fput; } - inode = I_BDEV(psblk_file->f_mapping->host)->bd_inode; - dev->zone.total_size = i_size_read(inode); + dev->zone.total_size = + bdev_nr_bytes(I_BDEV(psblk_file->f_mapping->host)); ret = __register_pstore_device(dev); if (ret) From 1d5dd3b9164c77816a679d39dbd28787d3f22a3a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:22 +0200 Subject: [PATCH 064/255] reiserfs: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size and remove two cargo culted checks that can't be false. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-23-hch@lst.de Signed-off-by: Jens Axboe --- fs/reiserfs/super.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index 58481f8d63d5..8647a00434ea 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -1986,9 +1986,7 @@ static int reiserfs_fill_super(struct super_block *s, void *data, int silent) * smaller than the filesystem. If the check fails then abort and * scream, because bad stuff will happen otherwise. */ - if (s->s_bdev && s->s_bdev->bd_inode - && i_size_read(s->s_bdev->bd_inode) < - sb_block_count(rs) * sb_blocksize(rs)) { + if (bdev_nr_bytes(s->s_bdev) < sb_block_count(rs) * sb_blocksize(rs)) { SWARN(silent, s, "", "Filesystem cannot be " "mounted because it is bigger than the device"); SWARN(silent, s, "", "You may need to run fsck " From be9a7b3e15916fd3710bfd383e8ecffc0416e919 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:23 +0200 Subject: [PATCH 065/255] squashfs: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Phillip Lougher Link: https://lore.kernel.org/r/20211018101130.1838532-24-hch@lst.de Signed-off-by: Jens Axboe --- fs/squashfs/super.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index 60d6951915f4..bb44ff4c5cc6 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -179,8 +180,8 @@ static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) /* Check the filesystem does not extend beyond the end of the block device */ msblk->bytes_used = le64_to_cpu(sblk->bytes_used); - if (msblk->bytes_used < 0 || msblk->bytes_used > - i_size_read(sb->s_bdev->bd_inode)) + if (msblk->bytes_used < 0 || + msblk->bytes_used > bdev_nr_bytes(sb->s_bdev)) goto failed_mount; /* Check block size for sanity */ From 2a93ad8fcb377b9d6e05947de161e146f5be4de9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:24 +0200 Subject: [PATCH 066/255] block: use bdev_nr_bytes instead of open coding it in blkdev_fallocate Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-25-hch@lst.de Signed-off-by: Jens Axboe --- block/fops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/fops.c b/block/fops.c index 2c43e493e37c..2ae8a7bd2b84 100644 --- a/block/fops.c +++ b/block/fops.c @@ -535,7 +535,7 @@ static long blkdev_fallocate(struct file *file, int mode, loff_t start, return -EOPNOTSUPP; /* Don't go off the end of the device. */ - isize = i_size_read(bdev->bd_inode); + isize = bdev_nr_bytes(bdev); if (start >= isize) return -EINVAL; if (end >= isize) { From bcc6e2cfaa48a4ad2e17719194f6d97a8e03f6c1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:25 +0200 Subject: [PATCH 067/255] block: add a sb_bdev_nr_blocks helper Add a helper to return the size of sb->s_bdev in sb->s_blocksize_bits based unites. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20211018101130.1838532-26-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/genhd.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 01d27f3a970e..7b0326661a1e 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -251,6 +251,12 @@ static inline sector_t get_capacity(struct gendisk *disk) return bdev_nr_sectors(disk->part0); } +static inline u64 sb_bdev_nr_blocks(struct super_block *sb) +{ + return bdev_nr_sectors(sb->s_bdev) >> + (sb->s_blocksize_bits - SECTOR_SHIFT); +} + int bdev_disk_changed(struct gendisk *disk, bool invalidate); void blk_drop_partitions(struct gendisk *disk); From 5513b241b2ef903b36c2ff74976618885fc5a318 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:26 +0200 Subject: [PATCH 068/255] ext4: use sb_bdev_nr_blocks Use the sb_bdev_nr_blocks helper instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Reviewed-by: Chaitanya Kulkarni Acked-by: Theodore Ts'o Link: https://lore.kernel.org/r/20211018101130.1838532-27-hch@lst.de Signed-off-by: Jens Axboe --- fs/ext4/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 88d5d274a868..29f38251013e 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -4474,7 +4474,7 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) goto cantfind_ext4; /* check blocks count against device size */ - blocks_count = sb->s_bdev->bd_inode->i_size >> sb->s_blocksize_bits; + blocks_count = sb_bdev_nr_blocks(sb); if (blocks_count && ext4_blocks_count(es) > blocks_count) { ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " "exceeds size of device (%llu blocks)", From dd0c0bdf97a44c2e2b5541e9febde0643a9d0dbf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:27 +0200 Subject: [PATCH 069/255] jfs: use sb_bdev_nr_blocks Use the sb_bdev_nr_blocks helper instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Dave Kleikamp Link: https://lore.kernel.org/r/20211018101130.1838532-28-hch@lst.de Signed-off-by: Jens Axboe --- fs/jfs/resize.c | 3 +-- fs/jfs/super.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/jfs/resize.c b/fs/jfs/resize.c index a42dbb0d3d28..8b9a72ae5efa 100644 --- a/fs/jfs/resize.c +++ b/fs/jfs/resize.c @@ -86,8 +86,7 @@ int jfs_extendfs(struct super_block *sb, s64 newLVSize, int newLogSize) goto out; } - VolumeSize = i_size_read(sb->s_bdev->bd_inode) >> sb->s_blocksize_bits; - + VolumeSize = sb_bdev_nr_blocks(sb); if (VolumeSize) { if (newLVSize > VolumeSize) { printk(KERN_WARNING "jfs_extendfs: invalid size\n"); diff --git a/fs/jfs/super.c b/fs/jfs/super.c index 9241caa16116..24cbc9946e01 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -284,8 +284,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize, } case Opt_resize_nosize: { - *newLVSize = i_size_read(sb->s_bdev->bd_inode) >> - sb->s_blocksize_bits; + *newLVSize = sb_bdev_nr_blocks(sb); if (*newLVSize == 0) pr_err("JFS: Cannot determine volume size\n"); break; From ab70041731a6c2f153120e47746fb303aa6f237a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:28 +0200 Subject: [PATCH 070/255] ntfs: use sb_bdev_nr_blocks Use the sb_bdev_nr_blocks helper instead of open coding it and clean up ntfs_fill_super a bit by moving an assignment a little earlier that has no negative side effects. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Acked-by: Anton Altaparmakov Link: https://lore.kernel.org/r/20211018101130.1838532-29-hch@lst.de Signed-off-by: Jens Axboe --- fs/ntfs/super.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 0d7e948cb29c..5ae8de09b271 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -2772,13 +2772,12 @@ static int ntfs_fill_super(struct super_block *sb, void *opt, const int silent) ntfs_debug("Set device block size to %i bytes (block size bits %i).", blocksize, sb->s_blocksize_bits); /* Determine the size of the device in units of block_size bytes. */ - if (!i_size_read(sb->s_bdev->bd_inode)) { + vol->nr_blocks = sb_bdev_nr_blocks(sb); + if (!vol->nr_blocks) { if (!silent) ntfs_error(sb, "Unable to determine device size."); goto err_out_now; } - vol->nr_blocks = i_size_read(sb->s_bdev->bd_inode) >> - sb->s_blocksize_bits; /* Read the boot sector and return unlocked buffer head to it. */ if (!(bh = read_ntfs_boot_sector(sb, silent))) { if (!silent) @@ -2816,8 +2815,7 @@ static int ntfs_fill_super(struct super_block *sb, void *opt, const int silent) goto err_out_now; } BUG_ON(blocksize != sb->s_blocksize); - vol->nr_blocks = i_size_read(sb->s_bdev->bd_inode) >> - sb->s_blocksize_bits; + vol->nr_blocks = sb_bdev_nr_blocks(sb); ntfs_debug("Changed device block size to %i bytes (block size " "bits %i) to match volume sector size.", blocksize, sb->s_blocksize_bits); From 2ffae493dc15ee2225bd68254e1cdb1449b81ec7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:29 +0200 Subject: [PATCH 071/255] reiserfs: use sb_bdev_nr_blocks Use the sb_bdev_nr_blocks helper instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211018101130.1838532-30-hch@lst.de Signed-off-by: Jens Axboe --- fs/reiserfs/super.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index 8647a00434ea..076f9ab94306 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -1199,9 +1199,7 @@ static int reiserfs_parse_options(struct super_block *s, if (!strcmp(arg, "auto")) { /* From JFS code, to auto-get the size. */ - *blocks = - i_size_read(s->s_bdev->bd_inode) >> s-> - s_blocksize_bits; + *blocks = sb_bdev_nr_blocks(s); } else { *blocks = simple_strtoul(arg, &p, 0); if (*p != '\0') { From e4ae4735f7c2da77db57ee090b9f513bfe80d285 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 Oct 2021 12:11:30 +0200 Subject: [PATCH 072/255] udf: use sb_bdev_nr_blocks Use the sb_bdev_nr_blocks helper instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Kees Cook Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20211018101130.1838532-31-hch@lst.de Signed-off-by: Jens Axboe --- fs/udf/lowlevel.c | 5 ++--- fs/udf/super.c | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/fs/udf/lowlevel.c b/fs/udf/lowlevel.c index f1094cdcd6cd..46d697172197 100644 --- a/fs/udf/lowlevel.c +++ b/fs/udf/lowlevel.c @@ -47,8 +47,7 @@ unsigned int udf_get_last_session(struct super_block *sb) unsigned long udf_get_last_block(struct super_block *sb) { - struct block_device *bdev = sb->s_bdev; - struct cdrom_device_info *cdi = disk_to_cdi(bdev->bd_disk); + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); unsigned long lblock = 0; /* @@ -56,7 +55,7 @@ unsigned long udf_get_last_block(struct super_block *sb) * Try using the device size... */ if (!cdi || cdrom_get_last_written(cdi, &lblock) || lblock == 0) - lblock = i_size_read(bdev->bd_inode) >> sb->s_blocksize_bits; + lblock = sb_bdev_nr_blocks(sb); if (lblock) return lblock - 1; diff --git a/fs/udf/super.c b/fs/udf/super.c index b2d7c57d0688..34247fba6df9 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1175,8 +1175,7 @@ static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) struct udf_inode_info *vati; uint32_t pos; struct virtualAllocationTable20 *vat20; - sector_t blocks = i_size_read(sb->s_bdev->bd_inode) >> - sb->s_blocksize_bits; + sector_t blocks = sb_bdev_nr_blocks(sb); udf_find_vat_block(sb, p_index, type1_index, sbi->s_last_block); if (!sbi->s_vat_inode && @@ -1838,8 +1837,7 @@ static int udf_check_anchor_block(struct super_block *sb, sector_t block, int ret; if (UDF_QUERY_FLAG(sb, UDF_FLAG_VARCONV) && - udf_fixed_to_variable(block) >= - i_size_read(sb->s_bdev->bd_inode) >> sb->s_blocksize_bits) + udf_fixed_to_variable(block) >= sb_bdev_nr_blocks(sb)) return -EAGAIN; bh = udf_read_tagged(sb, block, block, &ident); @@ -1901,8 +1899,7 @@ static int udf_scan_anchors(struct super_block *sb, sector_t *lastblock, last[last_count++] = *lastblock - 152; for (i = 0; i < last_count; i++) { - if (last[i] >= i_size_read(sb->s_bdev->bd_inode) >> - sb->s_blocksize_bits) + if (last[i] >= sb_bdev_nr_blocks(sb)) continue; ret = udf_check_anchor_block(sb, last[i], fileset); if (ret != -EAGAIN) { From f09313c57a17683cbcb305989daf1d94b49fd32c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 18 Oct 2021 11:39:45 -0600 Subject: [PATCH 073/255] block: cache inode size in bdev Reading the inode size brings in a new cacheline for IO submit, and it's in the hot path being checked for every single IO. When doing millions of IOs per core per second, this is noticeable overhead. Cache the nr_sectors in the bdev itself. Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/genhd.c | 1 + block/partitions/core.c | 1 + include/linux/blk_types.h | 1 + include/linux/genhd.h | 12 ++++++------ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/block/genhd.c b/block/genhd.c index 759bc06810f8..53495e3391e3 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -58,6 +58,7 @@ void set_capacity(struct gendisk *disk, sector_t sectors) spin_lock(&bdev->bd_size_lock); i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); + bdev->bd_nr_sectors = sectors; spin_unlock(&bdev->bd_size_lock); } EXPORT_SYMBOL(set_capacity); diff --git a/block/partitions/core.c b/block/partitions/core.c index 9dbddc355b40..66ef9bc6d6a1 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -91,6 +91,7 @@ static void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) { spin_lock(&bdev->bd_size_lock); i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); + bdev->bd_nr_sectors = sectors; spin_unlock(&bdev->bd_size_lock); } diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 472e55e0e94f..fe065c394fff 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -39,6 +39,7 @@ struct bio_crypt_ctx; struct block_device { sector_t bd_start_sect; + sector_t bd_nr_sectors; struct disk_stats __percpu *bd_stats; unsigned long bd_stamp; bool bd_read_only; /* read-only policy */ diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 7b0326661a1e..900325aa5d31 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -236,14 +236,14 @@ static inline sector_t get_start_sect(struct block_device *bdev) return bdev->bd_start_sect; } -static inline loff_t bdev_nr_bytes(struct block_device *bdev) -{ - return i_size_read(bdev->bd_inode); -} - static inline sector_t bdev_nr_sectors(struct block_device *bdev) { - return bdev_nr_bytes(bdev) >> SECTOR_SHIFT; + return bdev->bd_nr_sectors; +} + +static inline loff_t bdev_nr_bytes(struct block_device *bdev) +{ + return bdev_nr_sectors(bdev) << SECTOR_SHIFT; } static inline sector_t get_capacity(struct gendisk *disk) From 9be68dd7ac0e13be2ac57770c1f921d6b3294c6e Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Wed, 1 Sep 2021 13:38:30 +0200 Subject: [PATCH 074/255] md: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. We just do the unwinding of what was not done before, and are sure to unlock prior to bailing. Signed-off-by: Luis Chamberlain Signed-off-by: Christoph Hellwig Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 22310d5d8d41..eff3d23e1fcd 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5700,7 +5700,11 @@ static int md_alloc(dev_t dev, char *name) disk->flags |= GENHD_FL_EXT_DEVT; disk->events |= DISK_EVENT_MEDIA_CHANGE; mddev->gendisk = disk; - add_disk(disk); + error = add_disk(disk); + if (error) { + blk_cleanup_disk(disk); + goto abort; + } error = kobject_add(&mddev->kobj, &disk_to_dev(disk)->kobj, "%s", "md"); if (error) { From 51238e7fbd6182e36dbc093c92ae93142c57c0f5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 1 Sep 2021 13:38:31 +0200 Subject: [PATCH 075/255] md: add the bitmap group to the default groups for the md kobject Replace the deprecated default_attrs with the default_groups mechanism, and add the always visible bitmap group to the groups created add kobject_add time. Signed-off-by: Christoph Hellwig Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index eff3d23e1fcd..e04a23dc46bf 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5490,6 +5490,10 @@ static struct attribute *md_default_attrs[] = { NULL, }; +static const struct attribute_group md_default_group = { + .attrs = md_default_attrs, +}; + static struct attribute *md_redundancy_attrs[] = { &md_scan_mode.attr, &md_last_scan_mode.attr, @@ -5512,6 +5516,12 @@ static const struct attribute_group md_redundancy_group = { .attrs = md_redundancy_attrs, }; +static const struct attribute_group *md_attr_groups[] = { + &md_default_group, + &md_bitmap_group, + NULL, +}; + static ssize_t md_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { @@ -5587,7 +5597,7 @@ static const struct sysfs_ops md_sysfs_ops = { static struct kobj_type md_ktype = { .release = md_free, .sysfs_ops = &md_sysfs_ops, - .default_attrs = md_default_attrs, + .default_groups = md_attr_groups, }; int mdp_major = 0; @@ -5596,7 +5606,6 @@ static void mddev_delayed_delete(struct work_struct *ws) { struct mddev *mddev = container_of(ws, struct mddev, del_work); - sysfs_remove_group(&mddev->kobj, &md_bitmap_group); kobject_del(&mddev->kobj); kobject_put(&mddev->kobj); } @@ -5715,9 +5724,6 @@ static int md_alloc(dev_t dev, char *name) disk->disk_name); error = 0; } - if (mddev->kobj.sd && - sysfs_create_group(&mddev->kobj, &md_bitmap_group)) - pr_debug("pointless warning\n"); abort: mutex_unlock(&disks_mutex); if (!error && mddev->kobj.sd) { From 94f3cd7d832c28681f1dea54b4dd8606e5e2bc75 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 1 Sep 2021 13:38:32 +0200 Subject: [PATCH 076/255] md: extend disks_mutex coverage disks_mutex is intended to serialize md_alloc. Extended it to also cover the kobject_uevent call and getting the sysfs dirent to help reducing error handling complexity. Signed-off-by: Christoph Hellwig Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index e04a23dc46bf..946bbedf5dcf 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5725,12 +5725,12 @@ static int md_alloc(dev_t dev, char *name) error = 0; } abort: - mutex_unlock(&disks_mutex); if (!error && mddev->kobj.sd) { kobject_uevent(&mddev->kobj, KOBJ_ADD); mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state"); mddev->sysfs_level = sysfs_get_dirent_safe(mddev->kobj.sd, "level"); } + mutex_unlock(&disks_mutex); mddev_put(mddev); return error; } From 7ad1069166c0ccdd572d27e01cc7f7f84477df1e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 1 Sep 2021 13:38:33 +0200 Subject: [PATCH 077/255] md: properly unwind when failing to add the kobject in md_alloc Add proper error handling to delete the gendisk when failing to add the md kobject and clean up the error unwinding in general. Signed-off-by: Christoph Hellwig Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 946bbedf5dcf..c6b15ff0074f 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5672,7 +5672,7 @@ static int md_alloc(dev_t dev, char *name) strcmp(mddev2->gendisk->disk_name, name) == 0) { spin_unlock(&all_mddevs_lock); error = -EEXIST; - goto abort; + goto out_unlock_disks_mutex; } spin_unlock(&all_mddevs_lock); } @@ -5685,7 +5685,7 @@ static int md_alloc(dev_t dev, char *name) error = -ENOMEM; disk = blk_alloc_disk(NUMA_NO_NODE); if (!disk) - goto abort; + goto out_unlock_disks_mutex; disk->major = MAJOR(mddev->unit); disk->first_minor = unit << shift; @@ -5710,26 +5710,23 @@ static int md_alloc(dev_t dev, char *name) disk->events |= DISK_EVENT_MEDIA_CHANGE; mddev->gendisk = disk; error = add_disk(disk); - if (error) { - blk_cleanup_disk(disk); - goto abort; - } + if (error) + goto out_cleanup_disk; error = kobject_add(&mddev->kobj, &disk_to_dev(disk)->kobj, "%s", "md"); - if (error) { - /* This isn't possible, but as kobject_init_and_add is marked - * __must_check, we must do something with the result - */ - pr_debug("md: cannot register %s/md - name in use\n", - disk->disk_name); - error = 0; - } - abort: - if (!error && mddev->kobj.sd) { - kobject_uevent(&mddev->kobj, KOBJ_ADD); - mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state"); - mddev->sysfs_level = sysfs_get_dirent_safe(mddev->kobj.sd, "level"); - } + if (error) + goto out_del_gendisk; + + kobject_uevent(&mddev->kobj, KOBJ_ADD); + mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state"); + mddev->sysfs_level = sysfs_get_dirent_safe(mddev->kobj.sd, "level"); + goto out_unlock_disks_mutex; + +out_del_gendisk: + del_gendisk(disk); +out_cleanup_disk: + blk_cleanup_disk(disk); +out_unlock_disks_mutex: mutex_unlock(&disks_mutex); mddev_put(mddev); return error; From fd3b6975e9c11c4fa00965f82a0bfbb3b7b44101 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 4 Oct 2021 23:34:48 +0800 Subject: [PATCH 078/255] md/raid1: only allocate write behind bio for WriteMostly device Commit 6607cd319b6b91bff94e90f798a61c031650b514 ("raid1: ensure write behind bio has less than BIO_MAX_VECS sectors") tried to guarantee the size of behind bio is not bigger than BIO_MAX_VECS sectors. Unfortunately the same calltrace still could happen since an array could enable write-behind without write mostly device. To match the manpage of mdadm (which says "write-behind is only attempted on drives marked as write-mostly"), we need to check WriteMostly flag to avoid such unexpected behavior. [1]. https://bugzilla.kernel.org/show_bug.cgi?id=213181#c25 Cc: stable@vger.kernel.org # v5.12+ Cc: Jens Stutte Reported-by: Jens Stutte Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/raid1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 19598bd38939..6ba12f0f0f03 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1496,7 +1496,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (!r1_bio->bios[i]) continue; - if (first_clone) { + if (first_clone && test_bit(WriteMostly, &rdev->flags)) { /* do behind I/O ? * Not if there are too many, or cannot * allocate memory, or a reader on WriteMostly From 2e94275ed582d9bfe1abc5d5ee565715cc3e6b38 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 4 Oct 2021 23:34:50 +0800 Subject: [PATCH 079/255] md/raid1: use rdev in raid1_write_request directly We already get rdev from conf->mirrors[i].rdev at the beginning of the loop, so just use it. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/raid1.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 6ba12f0f0f03..7dc8026cf6ee 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1529,13 +1529,12 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, r1_bio->bios[i] = mbio; - mbio->bi_iter.bi_sector = (r1_bio->sector + - conf->mirrors[i].rdev->data_offset); - bio_set_dev(mbio, conf->mirrors[i].rdev->bdev); + mbio->bi_iter.bi_sector = (r1_bio->sector + rdev->data_offset); + bio_set_dev(mbio, rdev->bdev); mbio->bi_end_io = raid1_end_write_request; mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA)); - if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) && - !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) && + if (test_bit(FailFast, &rdev->flags) && + !test_bit(WriteMostly, &rdev->flags) && conf->raid_disks - mddev->degraded > 1) mbio->bi_opf |= MD_FAILFAST; mbio->bi_private = r1_bio; @@ -1546,7 +1545,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, trace_block_bio_remap(mbio, disk_devt(mddev->gendisk), r1_bio->sector); /* flush_pending_writes() needs access to the rdev so...*/ - mbio->bi_bdev = (void *)conf->mirrors[i].rdev; + mbio->bi_bdev = (void *)rdev; cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug)); if (cb) From c6efe4341d1ff7719f9918ea23a34f055359eb02 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 4 Oct 2021 23:34:52 +0800 Subject: [PATCH 080/255] md/raid5: call roundup_pow_of_two in raid5_run Let's call roundup_pow_of_two here instead of open code. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/raid5.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 02ed53b20654..4ea9e7b647b0 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -7732,10 +7732,7 @@ static int raid5_run(struct mddev *mddev) * discard data disk but write parity disk */ stripe = stripe * PAGE_SIZE; - /* Round up to power of 2, as discard handling - * currently assumes that */ - while ((stripe-1) & stripe) - stripe = (stripe | (stripe-1)) + 1; + stripe = roundup_pow_of_two(stripe); mddev->queue->limits.discard_alignment = stripe; mddev->queue->limits.discard_granularity = stripe; From 5467948604baba60e9c4610de8edb8efe7f01f18 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 4 Oct 2021 23:34:53 +0800 Subject: [PATCH 081/255] md: remove unused argument from md_new_event Actually, mddev is not used by md_new_event. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 30 +++++++++++++++--------------- drivers/md/md.h | 2 +- drivers/md/raid10.c | 2 +- drivers/md/raid5.c | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index c6b15ff0074f..3db44b3408c0 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -354,7 +354,7 @@ static bool create_on_open = true; */ static DECLARE_WAIT_QUEUE_HEAD(md_event_waiters); static atomic_t md_event_count; -void md_new_event(struct mddev *mddev) +void md_new_event(void) { atomic_inc(&md_event_count); wake_up(&md_event_waiters); @@ -2886,7 +2886,7 @@ static int add_bound_rdev(struct md_rdev *rdev) if (mddev->degraded) set_bit(MD_RECOVERY_RECOVER, &mddev->recovery); set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); - md_new_event(mddev); + md_new_event(); md_wakeup_thread(mddev->thread); return 0; } @@ -3002,7 +3002,7 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); } - md_new_event(mddev); + md_new_event(); } } } else if (cmd_match(buf, "writemostly")) { @@ -4099,7 +4099,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) if (!mddev->thread) md_update_sb(mddev, 1); sysfs_notify_dirent_safe(mddev->sysfs_level); - md_new_event(mddev); + md_new_event(); rv = len; out_unlock: mddev_unlock(mddev); @@ -4620,7 +4620,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) export_rdev(rdev); mddev_unlock(mddev); if (!err) - md_new_event(mddev); + md_new_event(); return err ? err : len; } @@ -6041,7 +6041,7 @@ int md_run(struct mddev *mddev) if (mddev->sb_flags) md_update_sb(mddev, 0); - md_new_event(mddev); + md_new_event(); return 0; bitmap_abort: @@ -6431,7 +6431,7 @@ static int do_md_stop(struct mddev *mddev, int mode, if (mddev->hold_active == UNTIL_STOP) mddev->hold_active = 0; } - md_new_event(mddev); + md_new_event(); sysfs_notify_dirent_safe(mddev->sysfs_state); return 0; } @@ -6935,7 +6935,7 @@ kick_rdev: md_wakeup_thread(mddev->thread); else md_update_sb(mddev, 1); - md_new_event(mddev); + md_new_event(); return 0; busy: @@ -7008,7 +7008,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) */ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); md_wakeup_thread(mddev->thread); - md_new_event(mddev); + md_new_event(); return 0; abort_export: @@ -7982,7 +7982,7 @@ void md_error(struct mddev *mddev, struct md_rdev *rdev) md_wakeup_thread(mddev->thread); if (mddev->event_work.func) queue_work(md_misc_wq, &mddev->event_work); - md_new_event(mddev); + md_new_event(); } EXPORT_SYMBOL(md_error); @@ -8866,7 +8866,7 @@ void md_do_sync(struct md_thread *thread) mddev->curr_resync = 3; /* no longer delayed */ mddev->curr_resync_completed = j; sysfs_notify_dirent_safe(mddev->sysfs_completed); - md_new_event(mddev); + md_new_event(); update_time = jiffies; blk_start_plug(&plug); @@ -8937,7 +8937,7 @@ void md_do_sync(struct md_thread *thread) /* this is the earliest that rebuild will be * visible in /proc/mdstat */ - md_new_event(mddev); + md_new_event(); if (last_check + window > io_sectors || j == max_sectors) continue; @@ -9161,7 +9161,7 @@ static int remove_and_add_spares(struct mddev *mddev, sysfs_link_rdev(mddev, rdev); if (!test_bit(Journal, &rdev->flags)) spares++; - md_new_event(mddev); + md_new_event(); set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); } } @@ -9195,7 +9195,7 @@ static void md_start_sync(struct work_struct *ws) } else md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); - md_new_event(mddev); + md_new_event(); } /* @@ -9454,7 +9454,7 @@ void md_reap_sync_thread(struct mddev *mddev) /* flag recovery needed just to double check */ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); sysfs_notify_dirent_safe(mddev->sysfs_action); - md_new_event(mddev); + md_new_event(); if (mddev->event_work.func) queue_work(md_misc_wq, &mddev->event_work); } diff --git a/drivers/md/md.h b/drivers/md/md.h index 4c96c36bd01a..53ea7a6961de 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -731,7 +731,7 @@ extern int sync_page_io(struct md_rdev *rdev, sector_t sector, int size, struct page *page, int op, int op_flags, bool metadata_op); extern void md_do_sync(struct md_thread *thread); -extern void md_new_event(struct mddev *mddev); +extern void md_new_event(void); extern void md_allow_write(struct mddev *mddev); extern void md_wait_for_blocked_rdev(struct md_rdev *rdev, struct mddev *mddev); extern void md_set_array_sectors(struct mddev *mddev, sector_t array_sectors); diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index aa2636582841..dde98f65bd04 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -4647,7 +4647,7 @@ out: } conf->reshape_checkpoint = jiffies; md_wakeup_thread(mddev->sync_thread); - md_new_event(mddev); + md_new_event(); return 0; abort: diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 4ea9e7b647b0..9c1a5877cf9f 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -8279,7 +8279,7 @@ static int raid5_start_reshape(struct mddev *mddev) } conf->reshape_checkpoint = jiffies; md_wakeup_thread(mddev->sync_thread); - md_new_event(mddev); + md_new_event(); return 0; } From 8b9e2291e355a0eafdd5b1e21a94a6659f24b351 Mon Sep 17 00:00:00 2001 From: Xiao Ni Date: Wed, 13 Oct 2021 22:59:33 +0800 Subject: [PATCH 082/255] md: update superblock after changing rdev flags in state_store When the in memory flag is changed, we need to persist the change in the rdev superblock flags. This is needed for "writemostly" and "failfast". Reviewed-by: Li Feng Signed-off-by: Xiao Ni Signed-off-by: Song Liu Signed-off-by: Jens Axboe --- drivers/md/md.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 3db44b3408c0..e8666bdc0d28 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -2976,7 +2976,11 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) * -write_error - clears WriteErrorSeen * {,-}failfast - set/clear FailFast */ + + struct mddev *mddev = rdev->mddev; int err = -EINVAL; + bool need_update_sb = false; + if (cmd_match(buf, "faulty") && rdev->mddev->pers) { md_error(rdev->mddev, rdev); if (test_bit(Faulty, &rdev->flags)) @@ -2991,7 +2995,6 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) if (rdev->raid_disk >= 0) err = -EBUSY; else { - struct mddev *mddev = rdev->mddev; err = 0; if (mddev_is_clustered(mddev)) err = md_cluster_ops->remove_disk(mddev, rdev); @@ -3008,10 +3011,12 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) } else if (cmd_match(buf, "writemostly")) { set_bit(WriteMostly, &rdev->flags); mddev_create_serial_pool(rdev->mddev, rdev, false); + need_update_sb = true; err = 0; } else if (cmd_match(buf, "-writemostly")) { mddev_destroy_serial_pool(rdev->mddev, rdev, false); clear_bit(WriteMostly, &rdev->flags); + need_update_sb = true; err = 0; } else if (cmd_match(buf, "blocked")) { set_bit(Blocked, &rdev->flags); @@ -3037,9 +3042,11 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) err = 0; } else if (cmd_match(buf, "failfast")) { set_bit(FailFast, &rdev->flags); + need_update_sb = true; err = 0; } else if (cmd_match(buf, "-failfast")) { clear_bit(FailFast, &rdev->flags); + need_update_sb = true; err = 0; } else if (cmd_match(buf, "-insync") && rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags)) { @@ -3118,6 +3125,8 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len) clear_bit(ExternalBbl, &rdev->flags); err = 0; } + if (need_update_sb) + md_update_sb(mddev, 1); if (!err) sysfs_notify_dirent_safe(rdev->sysfs_state); return err ? err : len; From c573d586999cb7e694efad79f0cb69a8215bbef6 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 14 Oct 2021 20:07:50 +0200 Subject: [PATCH 083/255] mtip32xx: Remove redundant 'flush_workqueue()' calls 'destroy_workqueue()' already drains the queue before destroying it, so there is no need to flush it explicitly. Remove the redundant 'flush_workqueue()' calls. This was generated with coccinelle: @@ expression E; @@ - flush_workqueue(E); destroy_workqueue(E); Signed-off-by: Christophe JAILLET Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/0fea349c808c6cfbf549b0e33701320c7860c8b7.1634234221.git.christophe.jaillet@wanadoo.fr Signed-off-by: Jens Axboe --- drivers/block/mtip32xx/mtip32xx.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index d0b40309f47e..c91b9010c1a6 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -4063,7 +4063,6 @@ block_initialize_err: msi_initialize_err: if (dd->isr_workq) { - flush_workqueue(dd->isr_workq); destroy_workqueue(dd->isr_workq); drop_cpu(dd->work[0].cpu_binding); drop_cpu(dd->work[1].cpu_binding); @@ -4121,7 +4120,6 @@ static void mtip_pci_remove(struct pci_dev *pdev) mtip_block_remove(dd); if (dd->isr_workq) { - flush_workqueue(dd->isr_workq); destroy_workqueue(dd->isr_workq); drop_cpu(dd->work[0].cpu_binding); drop_cpu(dd->work[1].cpu_binding); From 4e6eef5dc25b528e08ac5b5f64f6ca9d9987241d Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:44 +0800 Subject: [PATCH 084/255] nbd: don't handle response without a corresponding request message While handling a response message from server, nbd_read_stat() will try to get request by tag, and then complete the request. However, this is problematic if nbd haven't sent a corresponding request message: t1 t2 submit_bio nbd_queue_rq blk_mq_start_request recv_work nbd_read_stat blk_mq_tag_to_rq blk_mq_complete_request nbd_send_cmd Thus add a new cmd flag 'NBD_CMD_INFLIGHT', it will be set in nbd_send_cmd() and checked in nbd_read_stat(). Noted that this patch can't fix that blk_mq_tag_to_rq() might return a freed request, and this will be fixed in following patches. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-2-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index f4101d3efe7d..d18ba557a69d 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -131,6 +131,12 @@ struct nbd_device { }; #define NBD_CMD_REQUEUED 1 +/* + * This flag will be set if nbd_queue_rq() succeed, and will be checked and + * cleared in completion. Both setting and clearing of the flag are protected + * by cmd->lock. + */ +#define NBD_CMD_INFLIGHT 2 struct nbd_cmd { struct nbd_device *nbd; @@ -405,6 +411,7 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, if (!mutex_trylock(&cmd->lock)) return BLK_EH_RESET_TIMER; + __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); if (!refcount_inc_not_zero(&nbd->config_refs)) { cmd->status = BLK_STS_TIMEOUT; mutex_unlock(&cmd->lock); @@ -734,6 +741,12 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) cmd = blk_mq_rq_to_pdu(req); mutex_lock(&cmd->lock); + if (!__test_and_clear_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { + dev_err(disk_to_dev(nbd->disk), "Suspicious reply %d (status %u flags %lu)", + tag, cmd->status, cmd->flags); + ret = -ENOENT; + goto out; + } if (cmd->cmd_cookie != nbd_handle_to_cookie(handle)) { dev_err(disk_to_dev(nbd->disk), "Double reply on req %p, cmd_cookie %u, handle cookie %u\n", req, cmd->cmd_cookie, nbd_handle_to_cookie(handle)); @@ -833,6 +846,7 @@ static bool nbd_clear_req(struct request *req, void *data, bool reserved) return true; mutex_lock(&cmd->lock); + __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); cmd->status = BLK_STS_IOERR; mutex_unlock(&cmd->lock); @@ -969,7 +983,13 @@ again: * returns EAGAIN can be retried on a different socket. */ ret = nbd_send_cmd(nbd, cmd, index); - if (ret == -EAGAIN) { + /* + * Access to this flag is protected by cmd->lock, thus it's safe to set + * the flag after nbd_send_cmd() succeed to send request to server. + */ + if (!ret) + __set_bit(NBD_CMD_INFLIGHT, &cmd->flags); + else if (ret == -EAGAIN) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Request send failed, requeueing\n"); nbd_mark_nsock_dead(nbd, nsock, 1); From 07175cb1baf4c51051b1fbd391097e349f9a02a9 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:45 +0800 Subject: [PATCH 085/255] nbd: make sure request completion won't concurrent commit cddce0116058 ("nbd: Aovid double completion of a request") try to fix that nbd_clear_que() and recv_work() can complete a request concurrently. However, the problem still exists: t1 t2 t3 nbd_disconnect_and_put flush_workqueue recv_work blk_mq_complete_request blk_mq_complete_request_remote -> this is true WRITE_ONCE(rq->state, MQ_RQ_COMPLETE) blk_mq_raise_softirq blk_done_softirq blk_complete_reqs nbd_complete_rq blk_mq_end_request blk_mq_free_request WRITE_ONCE(rq->state, MQ_RQ_IDLE) nbd_clear_que blk_mq_tagset_busy_iter nbd_clear_req __blk_mq_free_request blk_mq_put_tag blk_mq_complete_request -> complete again There are three places where request can be completed in nbd: recv_work(), nbd_clear_que() and nbd_xmit_timeout(). Since they all hold cmd->lock before completing the request, it's easy to avoid the problem by setting and checking a cmd flag. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-3-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index d18ba557a69d..0bb3c1e2d575 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -411,7 +411,11 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, if (!mutex_trylock(&cmd->lock)) return BLK_EH_RESET_TIMER; - __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); + if (!__test_and_clear_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { + mutex_unlock(&cmd->lock); + return BLK_EH_DONE; + } + if (!refcount_inc_not_zero(&nbd->config_refs)) { cmd->status = BLK_STS_TIMEOUT; mutex_unlock(&cmd->lock); @@ -846,7 +850,10 @@ static bool nbd_clear_req(struct request *req, void *data, bool reserved) return true; mutex_lock(&cmd->lock); - __clear_bit(NBD_CMD_INFLIGHT, &cmd->flags); + if (!__test_and_clear_bit(NBD_CMD_INFLIGHT, &cmd->flags)) { + mutex_unlock(&cmd->lock); + return true; + } cmd->status = BLK_STS_IOERR; mutex_unlock(&cmd->lock); From fcf3d633d8e101e84a0e072ab79957b938ac7e06 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:46 +0800 Subject: [PATCH 086/255] nbd: check sock index in nbd_read_stat() The sock that clent send request in nbd_send_cmd() and receive reply in nbd_read_stat() should be the same. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-4-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 0bb3c1e2d575..49c501cf8a8b 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -751,6 +751,10 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) ret = -ENOENT; goto out; } + if (cmd->index != index) { + dev_err(disk_to_dev(nbd->disk), "Unexpected reply %d from different sock %d (expected %d)", + tag, index, cmd->index); + } if (cmd->cmd_cookie != nbd_handle_to_cookie(handle)) { dev_err(disk_to_dev(nbd->disk), "Double reply on req %p, cmd_cookie %u, handle cookie %u\n", req, cmd->cmd_cookie, nbd_handle_to_cookie(handle)); From 0de2b7a4dd08eeb82b8cc3fe5d6b8ba49a32f7bd Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:47 +0800 Subject: [PATCH 087/255] nbd: don't start request if nbd_queue_rq() failed commit 6a468d5990ec ("nbd: don't start req until after the dead connection logic") move blk_mq_start_request() from nbd_queue_rq() to nbd_handle_cmd() to skip starting request if the connection is dead. However, request is still started in other error paths. Currently, blk_mq_end_request() will be called immediately if nbd_queue_rq() failed, thus start request in such situation is useless. So remove blk_mq_start_request() from error paths in nbd_handle_cmd(). Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-5-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 49c501cf8a8b..87066cad5711 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -939,7 +939,6 @@ static int nbd_handle_cmd(struct nbd_cmd *cmd, int index) if (!refcount_inc_not_zero(&nbd->config_refs)) { dev_err_ratelimited(disk_to_dev(nbd->disk), "Socks array is empty\n"); - blk_mq_start_request(req); return -EINVAL; } config = nbd->config; @@ -948,7 +947,6 @@ static int nbd_handle_cmd(struct nbd_cmd *cmd, int index) dev_err_ratelimited(disk_to_dev(nbd->disk), "Attempted send on invalid socket\n"); nbd_config_put(nbd); - blk_mq_start_request(req); return -EINVAL; } cmd->status = BLK_STS_OK; @@ -972,7 +970,6 @@ again: */ sock_shutdown(nbd); nbd_config_put(nbd); - blk_mq_start_request(req); return -EIO; } goto again; From f52c0e08237e7864a44311fc78bc9bf2e045611b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:48 +0800 Subject: [PATCH 088/255] nbd: clean up return value checking of sock_xmit() Check if sock_xmit() return 0 is useless because it'll never return 0, comment it and remove such checkings. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-6-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 87066cad5711..1d34560d8c1e 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -495,7 +495,8 @@ done: } /* - * Send or receive packet. + * Send or receive packet. Return a positive value on success and + * negtive value on failue, and never return 0. */ static int sock_xmit(struct nbd_device *nbd, int index, int send, struct iov_iter *iter, int msg_flags, int *sent) @@ -621,7 +622,7 @@ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index) result = sock_xmit(nbd, index, 1, &from, (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent); trace_nbd_header_sent(req, handle); - if (result <= 0) { + if (result < 0) { if (was_interrupted(result)) { /* If we havne't sent anything we can just return BUSY, * however if we have sent something we need to make @@ -665,7 +666,7 @@ send_pages: skip = 0; } result = sock_xmit(nbd, index, 1, &from, flags, &sent); - if (result <= 0) { + if (result < 0) { if (was_interrupted(result)) { /* We've already sent the header, we * have no choice but to set pending and @@ -717,7 +718,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) reply.magic = 0; iov_iter_kvec(&to, READ, &iov, 1, sizeof(reply)); result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); - if (result <= 0) { + if (result < 0) { if (!nbd_disconnected(config)) dev_err(disk_to_dev(nbd->disk), "Receive control failed (result %d)\n", result); @@ -788,7 +789,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) rq_for_each_segment(bvec, req, iter) { iov_iter_bvec(&to, READ, &bvec, 1, bvec.bv_len); result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); - if (result <= 0) { + if (result < 0) { dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n", result); /* @@ -1234,7 +1235,7 @@ static void send_disconnects(struct nbd_device *nbd) iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request)); mutex_lock(&nsock->tx_lock); ret = sock_xmit(nbd, i, 1, &from, 0, NULL); - if (ret <= 0) + if (ret < 0) dev_err(disk_to_dev(nbd->disk), "Send disconnect failed %d\n", ret); mutex_unlock(&nsock->tx_lock); From 3fe1db626a56cdf259c348404f2c5429e2f065a1 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 17:33:49 +0800 Subject: [PATCH 089/255] nbd: partition nbd_read_stat() into nbd_read_reply() and nbd_handle_reply() Prepare to fix uaf in nbd_read_stat(), no functional changes. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20210916093350.1410403-7-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 80 ++++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 1d34560d8c1e..06e292f0b0ae 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -700,38 +700,45 @@ out: return 0; } -/* NULL returned = something went wrong, inform userspace */ -static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) +static int nbd_read_reply(struct nbd_device *nbd, int index, + struct nbd_reply *reply) +{ + struct kvec iov = {.iov_base = reply, .iov_len = sizeof(*reply)}; + struct iov_iter to; + int result; + + reply->magic = 0; + iov_iter_kvec(&to, READ, &iov, 1, sizeof(*reply)); + result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); + if (result < 0) { + if (!nbd_disconnected(nbd->config)) + dev_err(disk_to_dev(nbd->disk), + "Receive control failed (result %d)\n", result); + return result; + } + + if (ntohl(reply->magic) != NBD_REPLY_MAGIC) { + dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", + (unsigned long)ntohl(reply->magic)); + return -EPROTO; + } + + return 0; +} + +/* NULL returned = something went wrong, inform userspace */ +static struct nbd_cmd *nbd_handle_reply(struct nbd_device *nbd, int index, + struct nbd_reply *reply) { - struct nbd_config *config = nbd->config; int result; - struct nbd_reply reply; struct nbd_cmd *cmd; struct request *req = NULL; u64 handle; u16 hwq; u32 tag; - struct kvec iov = {.iov_base = &reply, .iov_len = sizeof(reply)}; - struct iov_iter to; int ret = 0; - reply.magic = 0; - iov_iter_kvec(&to, READ, &iov, 1, sizeof(reply)); - result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); - if (result < 0) { - if (!nbd_disconnected(config)) - dev_err(disk_to_dev(nbd->disk), - "Receive control failed (result %d)\n", result); - return ERR_PTR(result); - } - - if (ntohl(reply.magic) != NBD_REPLY_MAGIC) { - dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", - (unsigned long)ntohl(reply.magic)); - return ERR_PTR(-EPROTO); - } - - memcpy(&handle, reply.handle, sizeof(handle)); + memcpy(&handle, reply->handle, sizeof(handle)); tag = nbd_handle_to_tag(handle); hwq = blk_mq_unique_tag_to_hwq(tag); if (hwq < nbd->tag_set.nr_hw_queues) @@ -774,9 +781,9 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) ret = -ENOENT; goto out; } - if (ntohl(reply.error)) { + if (ntohl(reply->error)) { dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n", - ntohl(reply.error)); + ntohl(reply->error)); cmd->status = BLK_STS_IOERR; goto out; } @@ -785,6 +792,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) if (rq_data_dir(req) != WRITE) { struct req_iterator iter; struct bio_vec bvec; + struct iov_iter to; rq_for_each_segment(bvec, req, iter) { iov_iter_bvec(&to, READ, &bvec, 1, bvec.bv_len); @@ -798,7 +806,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) * and let the timeout stuff handle resubmitting * this request onto another connection. */ - if (nbd_disconnected(config)) { + if (nbd_disconnected(nbd->config)) { cmd->status = BLK_STS_IOERR; goto out; } @@ -822,24 +830,30 @@ static void recv_work(struct work_struct *work) work); struct nbd_device *nbd = args->nbd; struct nbd_config *config = nbd->config; + struct nbd_sock *nsock; struct nbd_cmd *cmd; struct request *rq; while (1) { - cmd = nbd_read_stat(nbd, args->index); - if (IS_ERR(cmd)) { - struct nbd_sock *nsock = config->socks[args->index]; + struct nbd_reply reply; - mutex_lock(&nsock->tx_lock); - nbd_mark_nsock_dead(nbd, nsock, 1); - mutex_unlock(&nsock->tx_lock); + if (nbd_read_reply(nbd, args->index, &reply)) + break; + + cmd = nbd_handle_reply(nbd, args->index, &reply); + if (IS_ERR(cmd)) break; - } rq = blk_mq_rq_from_pdu(cmd); if (likely(!blk_should_fake_timeout(rq->q))) blk_mq_complete_request(rq); } + + nsock = config->socks[args->index]; + mutex_lock(&nsock->tx_lock); + nbd_mark_nsock_dead(nbd, nsock, 1); + mutex_unlock(&nsock->tx_lock); + nbd_config_put(nbd); atomic_dec(&config->recv_threads); wake_up(&config->recv_wq); From 8663b210f8c169a49aaeed3af92471a147545477 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 16 Sep 2021 22:18:10 +0800 Subject: [PATCH 090/255] nbd: fix uaf in nbd_handle_reply() There is a problem that nbd_handle_reply() might access freed request: 1) At first, a normal io is submitted and completed with scheduler: internel_tag = blk_mq_get_tag -> get tag from sched_tags blk_mq_rq_ctx_init sched_tags->rq[internel_tag] = sched_tag->static_rq[internel_tag] ... blk_mq_get_driver_tag __blk_mq_get_driver_tag -> get tag from tags tags->rq[tag] = sched_tag->static_rq[internel_tag] So, both tags->rq[tag] and sched_tags->rq[internel_tag] are pointing to the request: sched_tags->static_rq[internal_tag]. Even if the io is finished. 2) nbd server send a reply with random tag directly: recv_work nbd_handle_reply blk_mq_tag_to_rq(tags, tag) rq = tags->rq[tag] 3) if the sched_tags->static_rq is freed: blk_mq_sched_free_requests blk_mq_free_rqs(q->tag_set, hctx->sched_tags, i) -> step 2) access rq before clearing rq mapping blk_mq_clear_rq_mapping(set, tags, hctx_idx); __free_pages() -> rq is freed here 4) Then, nbd continue to use the freed request in nbd_handle_reply Fix the problem by get 'q_usage_counter' before blk_mq_tag_to_rq(), thus request is ensured not to be freed because 'q_usage_counter' is not zero. Signed-off-by: Yu Kuai Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20210916141810.2325276-1-yukuai3@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 06e292f0b0ae..0d064fab6186 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -830,6 +830,7 @@ static void recv_work(struct work_struct *work) work); struct nbd_device *nbd = args->nbd; struct nbd_config *config = nbd->config; + struct request_queue *q = nbd->disk->queue; struct nbd_sock *nsock; struct nbd_cmd *cmd; struct request *rq; @@ -840,13 +841,28 @@ static void recv_work(struct work_struct *work) if (nbd_read_reply(nbd, args->index, &reply)) break; - cmd = nbd_handle_reply(nbd, args->index, &reply); - if (IS_ERR(cmd)) + /* + * Grab .q_usage_counter so request pool won't go away, then no + * request use-after-free is possible during nbd_handle_reply(). + * If queue is frozen, there won't be any inflight requests, we + * needn't to handle the incoming garbage message. + */ + if (!percpu_ref_tryget(&q->q_usage_counter)) { + dev_err(disk_to_dev(nbd->disk), "%s: no io inflight\n", + __func__); break; + } + + cmd = nbd_handle_reply(nbd, args->index, &reply); + if (IS_ERR(cmd)) { + percpu_ref_put(&q->q_usage_counter); + break; + } rq = blk_mq_rq_from_pdu(cmd); if (likely(!blk_should_fake_timeout(rq->q))) blk_mq_complete_request(rq); + percpu_ref_put(&q->q_usage_counter); } nsock = config->socks[args->index]; From a87acfde949140946456859eafa5f15175d0f960 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sat, 11 Sep 2021 16:04:50 -0600 Subject: [PATCH 091/255] io_uring: dump sqe contents if issue fails I recently had to look at a production problem where a request ended up getting the dreaded -EINVAL error on submit. The most used and hence useless of error codes, as it just tells you that something was wrong with your request, but not more than that. Let's dump the full sqe contents if we run into an issue failure, that'll allow easier diagnosing of a wide variety of issues. Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ include/trace/events/io_uring.h | 61 +++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index d4631a55a692..5daee36c1f33 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7117,6 +7117,8 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret = io_init_req(ctx, req, sqe); if (unlikely(ret)) { fail_req: + trace_io_uring_req_failed(sqe, ret); + /* fail even hard links since we don't submit */ if (link->head) { /* diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h index 0dd30de00e5b..7346f0164cf4 100644 --- a/include/trace/events/io_uring.h +++ b/include/trace/events/io_uring.h @@ -6,6 +6,7 @@ #define _TRACE_IO_URING_H #include +#include struct io_wq_work; @@ -497,6 +498,66 @@ TRACE_EVENT(io_uring_task_run, (unsigned long long) __entry->user_data) ); +/* + * io_uring_req_failed - called when an sqe is errored dring submission + * + * @sqe: pointer to the io_uring_sqe that failed + * @error: error it failed with + * + * Allows easier diagnosing of malformed requests in production systems. + */ +TRACE_EVENT(io_uring_req_failed, + + TP_PROTO(const struct io_uring_sqe *sqe, int error), + + TP_ARGS(sqe, error), + + TP_STRUCT__entry ( + __field( u8, opcode ) + __field( u8, flags ) + __field( u8, ioprio ) + __field( u64, off ) + __field( u64, addr ) + __field( u32, len ) + __field( u32, op_flags ) + __field( u64, user_data ) + __field( u16, buf_index ) + __field( u16, personality ) + __field( u32, file_index ) + __field( u64, pad1 ) + __field( u64, pad2 ) + __field( int, error ) + ), + + TP_fast_assign( + __entry->opcode = sqe->opcode; + __entry->flags = sqe->flags; + __entry->ioprio = sqe->ioprio; + __entry->off = sqe->off; + __entry->addr = sqe->addr; + __entry->len = sqe->len; + __entry->op_flags = sqe->rw_flags; + __entry->user_data = sqe->user_data; + __entry->buf_index = sqe->buf_index; + __entry->personality = sqe->personality; + __entry->file_index = sqe->file_index; + __entry->pad1 = sqe->__pad2[0]; + __entry->pad2 = sqe->__pad2[1]; + __entry->error = error; + ), + + TP_printk("op %d, flags=0x%x, prio=%d, off=%llu, addr=%llu, " + "len=%u, rw_flags=0x%x, user_data=0x%llx, buf_index=%d, " + "personality=%d, file_index=%d, pad=0x%llx/%llx, error=%d", + __entry->opcode, __entry->flags, __entry->ioprio, + (unsigned long long)__entry->off, + (unsigned long long) __entry->addr, __entry->len, + __entry->op_flags, (unsigned long long) __entry->user_data, + __entry->buf_index, __entry->personality, __entry->file_index, + (unsigned long long) __entry->pad1, + (unsigned long long) __entry->pad2, __entry->error) +); + #endif /* _TRACE_IO_URING_H */ /* This part must be outside protection */ From 71e1cef2d794338cc7b979d4c6144e1dc12718b5 Mon Sep 17 00:00:00 2001 From: Bixuan Cui Date: Sat, 11 Sep 2021 16:58:47 +0800 Subject: [PATCH 092/255] io-wq: Remove duplicate code in io_workqueue_create() While task_work_add() in io_workqueue_create() is true, then duplicate code is executed: -> clear_bit_unlock(0, &worker->create_state); -> io_worker_release(worker); -> atomic_dec(&acct->nr_running); -> io_worker_ref_put(wq); -> return false; -> clear_bit_unlock(0, &worker->create_state); // back to io_workqueue_create() -> io_worker_release(worker); -> kfree(worker); The io_worker_release() and clear_bit_unlock() are executed twice. Fixes: 3146cba99aa2 ("io-wq: make worker creation resilient against signals") Signed-off-by: Bixuan Cui Link: https://lore.kernel.org/r/20210911085847.34849-1-cuibixuan@huawei.com Reviwed-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io-wq.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 5bf8aa81715e..811299ac9684 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -323,8 +323,10 @@ static bool io_queue_worker_create(struct io_worker *worker, init_task_work(&worker->create_work, func); worker->create_index = acct->index; - if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) + if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) { + clear_bit_unlock(0, &worker->create_state); return true; + } clear_bit_unlock(0, &worker->create_state); fail_release: io_worker_release(worker); @@ -716,11 +718,8 @@ static void io_workqueue_create(struct work_struct *work) struct io_worker *worker = container_of(work, struct io_worker, work); struct io_wqe_acct *acct = io_wqe_get_acct(worker); - if (!io_queue_worker_create(worker, acct, create_worker_cont)) { - clear_bit_unlock(0, &worker->create_state); - io_worker_release(worker); + if (!io_queue_worker_create(worker, acct, create_worker_cont)) kfree(worker); - } } static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index) From 4b628aeb69cc49adf56b9c1fde43558143e810f5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 8 Sep 2021 16:40:49 +0100 Subject: [PATCH 093/255] io_uring: kill off ios_left ->ios_left is only used to decide whether to plug or not, kill it to avoid this extra accounting, just use the initial submission number. There is no much difference in regards of enabling plugging, where this one does it in a few more cases, but all major ones should be covered well. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f13993bcf5b477f9a7d52881fc49f9457ea9870a.1631115443.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5daee36c1f33..6917f96f0e82 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -315,6 +315,7 @@ struct io_submit_state { unsigned int free_reqs; bool plug_started; + bool need_plug; /* * Batch completion logic @@ -323,8 +324,6 @@ struct io_submit_state { unsigned int compl_nr; /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; - - unsigned int ios_left; }; struct io_ring_ctx { @@ -7090,10 +7089,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, * Plug now if we have more than 1 IO left after this, and the target * is potentially a read/write to block based storage. */ - if (!state->plug_started && state->ios_left > 1 && - io_op_defs[req->opcode].plug) { + if (state->need_plug && io_op_defs[req->opcode].plug) { blk_start_plug(&state->plug); state->plug_started = true; + state->need_plug = false; } if (io_op_defs[req->opcode].needs_file) { @@ -7102,8 +7101,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (unlikely(!req->file)) ret = -EBADF; } - - state->ios_left--; return ret; } @@ -7212,7 +7209,7 @@ static void io_submit_state_start(struct io_submit_state *state, unsigned int max_ios) { state->plug_started = false; - state->ios_left = max_ios; + state->need_plug = max_ios > 2; /* set only head, no need to init link_last in advance */ state->link.head = NULL; } From 6b639522f63f82437350038a4925633f769e4ec8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 8 Sep 2021 16:40:50 +0100 Subject: [PATCH 094/255] io_uring: inline io_dismantle_req io_dismantle_req() is hot, and not _too_ huge. Inline it, there are 3 call sites, which hopefully will turn into 2 in the future. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/bdd2dc30716cac270c2403e99bccd6286e4ae201.1631115443.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6917f96f0e82..43ed9f6821fb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1960,7 +1960,7 @@ static inline void io_put_file(struct file *file) fput(file); } -static void io_dismantle_req(struct io_kiocb *req) +static inline void io_dismantle_req(struct io_kiocb *req) { unsigned int flags = req->flags; From d81499bfcd471e6b920683f8fcf06ce65f84286b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 8 Sep 2021 16:40:51 +0100 Subject: [PATCH 095/255] io_uring: inline linked part of io_req_find_next Inline part of __io_req_find_next() that returns a request but doesn't need io_disarm_next(). It's just two places, but makes links a bit faster. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4126d13f23d0e91b39b3558e16bd86cafa7fcef2.1631115443.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 43ed9f6821fb..10dd3589f8be 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2074,40 +2074,39 @@ static bool io_disarm_next(struct io_kiocb *req) return posted; } -static struct io_kiocb *__io_req_find_next(struct io_kiocb *req) +static void __io_req_find_next_prep(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + bool posted; + + spin_lock(&ctx->completion_lock); + posted = io_disarm_next(req); + if (posted) + io_commit_cqring(req->ctx); + spin_unlock(&ctx->completion_lock); + if (posted) + io_cqring_ev_posted(ctx); +} + +static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) { struct io_kiocb *nxt; + if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) + return NULL; /* * If LINK is set, we have dependent requests in this chain. If we * didn't fail this request, queue the first one up, moving any other * dependencies to the next request. In case of failure, fail the rest * of the chain. */ - if (req->flags & IO_DISARM_MASK) { - struct io_ring_ctx *ctx = req->ctx; - bool posted; - - spin_lock(&ctx->completion_lock); - posted = io_disarm_next(req); - if (posted) - io_commit_cqring(req->ctx); - spin_unlock(&ctx->completion_lock); - if (posted) - io_cqring_ev_posted(ctx); - } + if (unlikely(req->flags & IO_DISARM_MASK)) + __io_req_find_next_prep(req); nxt = req->link; req->link = NULL; return nxt; } -static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) -{ - if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK)))) - return NULL; - return __io_req_find_next(req); -} - static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) { if (!ctx) From c450178d9be9dc44327f3b526f82c36e4c5d362b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 8 Sep 2021 16:40:52 +0100 Subject: [PATCH 096/255] io_uring: dedup CQE flushing non-empty checks We don't do io_submit_flush_completions() when there is no requests enqueued, and every single caller checks for it. Hide that check into the function not forgetting about inlining. That will make it much easier for changing the empty check condition in the future. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d7ff8cef5da1b38e8ea648f5aad9a315ddfc7b57.1631115443.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 10dd3589f8be..c4ec3672da77 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1092,7 +1092,7 @@ static void __io_queue_sqe(struct io_kiocb *req); static void io_rsrc_put_work(struct work_struct *work); static void io_req_task_queue(struct io_kiocb *req); -static void io_submit_flush_completions(struct io_ring_ctx *ctx); +static void __io_submit_flush_completions(struct io_ring_ctx *ctx); static int io_req_prep_async(struct io_kiocb *req); static int io_install_fixed_file(struct io_kiocb *req, struct file *file, @@ -1164,6 +1164,12 @@ static inline void req_ref_get(struct io_kiocb *req) atomic_inc(&req->refs); } +static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) +{ + if (ctx->submit_state.compl_nr) + __io_submit_flush_completions(ctx); +} + static inline void __io_req_set_refcount(struct io_kiocb *req, int nr) { if (!(req->flags & REQ_F_REFCOUNT)) { @@ -1252,8 +1258,7 @@ static void io_fallback_req_func(struct work_struct *work) req->io_task_work.func(req, &locked); if (locked) { - if (ctx->submit_state.compl_nr) - io_submit_flush_completions(ctx); + io_submit_flush_completions(ctx); mutex_unlock(&ctx->uring_lock); } percpu_ref_put(&ctx->refs); @@ -2112,8 +2117,7 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) if (!ctx) return; if (*locked) { - if (ctx->submit_state.compl_nr) - io_submit_flush_completions(ctx); + io_submit_flush_completions(ctx); mutex_unlock(&ctx->uring_lock); *locked = false; } @@ -2130,7 +2134,7 @@ static void tctx_task_work(struct callback_head *cb) while (1) { struct io_wq_work_node *node; - if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr) + if (!tctx->task_list.first && locked) io_submit_flush_completions(ctx); spin_lock_irq(&tctx->task_lock); @@ -2315,7 +2319,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, list_add(&req->inflight_entry, &state->free_list); } -static void io_submit_flush_completions(struct io_ring_ctx *ctx) +static void __io_submit_flush_completions(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_submit_state *state = &ctx->submit_state; @@ -7195,8 +7199,7 @@ static void io_submit_state_end(struct io_submit_state *state, { if (state->link.head) io_queue_sqe(state->link.head); - if (state->compl_nr) - io_submit_flush_completions(ctx); + io_submit_flush_completions(ctx); if (state->plug_started) blk_finish_plug(&state->plug); } From d97ec6239ad8684e4d31b12b39cefca6782c09a3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 8 Sep 2021 16:40:53 +0100 Subject: [PATCH 097/255] io_uring: kill extra wake_up_process in tw add TWA_SIGNAL already wakes the thread, no need in wake_up_process() after it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7e90cf643f633e857443e0c9e72471b221735c50.1631115443.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c4ec3672da77..ead85f4d3427 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2197,8 +2197,9 @@ static void io_req_task_work_add(struct io_kiocb *req) * will do the job. */ notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL; - if (!task_work_add(tsk, &tctx->task_work, notify)) { - wake_up_process(tsk); + if (likely(!task_work_add(tsk, &tctx->task_work, notify))) { + if (notify == TWA_NONE) + wake_up_process(tsk); return; } From 83f84356bc8f2dda9d0c9c7edb94decf71a36d26 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Mon, 13 Sep 2021 21:08:54 +0800 Subject: [PATCH 098/255] io_uring: add more uring info to fdinfo for debug Developers may need some uring info to help themselves debug and address issues in production. This includes sqring/cqring head/tail and the detailed sqe/cqe info, which is very useful when an application is hung on a ring. Signed-off-by: Hao Xu Link: https://lore.kernel.org/r/20210913130854.38542-1-haoxu@linux.alibaba.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ead85f4d3427..323ef0bb4bc1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10056,8 +10056,48 @@ static int io_uring_show_cred(struct seq_file *m, unsigned int id, static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) { struct io_sq_data *sq = NULL; + struct io_overflow_cqe *ocqe; + struct io_rings *r = ctx->rings; + unsigned int sq_mask = ctx->sq_entries - 1, cq_mask = ctx->cq_entries - 1; + unsigned int cached_sq_head = ctx->cached_sq_head; + unsigned int cached_cq_tail = ctx->cached_cq_tail; + unsigned int sq_head = READ_ONCE(r->sq.head); + unsigned int sq_tail = READ_ONCE(r->sq.tail); + unsigned int cq_head = READ_ONCE(r->cq.head); + unsigned int cq_tail = READ_ONCE(r->cq.tail); bool has_lock; - int i; + unsigned int i; + + /* + * we may get imprecise sqe and cqe info if uring is actively running + * since we get cached_sq_head and cached_cq_tail without uring_lock + * and sq_tail and cq_head are changed by userspace. But it's ok since + * we usually use these info when it is stuck. + */ + seq_printf(m, "SqHead:\t%u\n", sq_head & sq_mask); + seq_printf(m, "SqTail:\t%u\n", sq_tail & sq_mask); + seq_printf(m, "CachedSqHead:\t%u\n", cached_sq_head & sq_mask); + seq_printf(m, "CqHead:\t%u\n", cq_head & cq_mask); + seq_printf(m, "CqTail:\t%u\n", cq_tail & cq_mask); + seq_printf(m, "CachedCqTail:\t%u\n", cached_cq_tail & cq_mask); + seq_printf(m, "SQEs:\t%u\n", sq_tail - cached_sq_head); + for (i = cached_sq_head; i < sq_tail; i++) { + unsigned int sq_idx = READ_ONCE(ctx->sq_array[i & sq_mask]); + + if (likely(sq_idx <= sq_mask)) { + struct io_uring_sqe *sqe = &ctx->sq_sqes[sq_idx]; + + seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n", + sq_idx, sqe->opcode, sqe->fd, sqe->flags, sqe->user_data); + } + } + seq_printf(m, "CQEs:\t%u\n", cached_cq_tail - cq_head); + for (i = cq_head; i < cached_cq_tail; i++) { + struct io_uring_cqe *cqe = &r->cqes[i & cq_mask]; + + seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n", + i & cq_mask, cqe->user_data, cqe->res, cqe->flags); + } /* * Avoid ABBA deadlock between the seq lock and the io_uring mutex, @@ -10099,7 +10139,10 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) xa_for_each(&ctx->personalities, index, cred) io_uring_show_cred(m, index, cred); } - seq_printf(m, "PollList:\n"); + if (has_lock) + mutex_unlock(&ctx->uring_lock); + + seq_puts(m, "PollList:\n"); spin_lock(&ctx->completion_lock); for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) { struct hlist_head *list = &ctx->cancel_hash[i]; @@ -10109,9 +10152,17 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) seq_printf(m, " op=%d, task_works=%d\n", req->opcode, req->task->task_works != NULL); } + + seq_puts(m, "CqOverflowList:\n"); + list_for_each_entry(ocqe, &ctx->cq_overflow_list, list) { + struct io_uring_cqe *cqe = &ocqe->cqe; + + seq_printf(m, " user_data=%llu, res=%d, flags=%x\n", + cqe->user_data, cqe->res, cqe->flags); + + } + spin_unlock(&ctx->completion_lock); - if (has_lock) - mutex_unlock(&ctx->uring_lock); } static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) From a3f349071eb0ae2aa18ac2da55060865d8190cf0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 15 Sep 2021 12:04:20 +0100 Subject: [PATCH 099/255] io_uring: remove ctx referencing from complete_post Now completions are done from task context, that means that it's either the task itself, task_work or io-wq worker. In all those cases the ctx will be staying alive by mutexing, explicit referencing or req references by iowq. Remove extra ctx pinning from io_req_complete_post(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/60a0e96434c16ab4fe587651448290d61ec9a113.1631703756.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 323ef0bb4bc1..027ca7a6df44 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1809,17 +1809,11 @@ static void io_req_complete_post(struct io_kiocb *req, long res, io_put_task(req->task, 1); list_add(&req->inflight_entry, &ctx->locked_free_list); ctx->locked_free_nr++; - } else { - if (!percpu_ref_tryget(&ctx->refs)) - req = NULL; + percpu_ref_put(&ctx->refs); } io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); - - if (req) { - io_cqring_ev_posted(ctx); - percpu_ref_put(&ctx->refs); - } + io_cqring_ev_posted(ctx); } static inline bool io_req_needs_clean(struct io_kiocb *req) From 68fe256aadc0db70cb27132a6f5583819794d867 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 15 Sep 2021 12:03:38 +0100 Subject: [PATCH 100/255] io_uring: optimise io_req_init() sqe flags checks IOSQE_IO_DRAIN is quite marginal and we don't care too much about IOSQE_BUFFER_SELECT. Save to ifs and hide both of them under SQE_VALID_FLAGS check. Now we first check whether it uses a "safe" subset, i.e. without DRAIN and BUFFER_SELECT, and only if it's not true we test the rest of the flags. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/dccfb9ab2ab0969a2d8dc59af88fa0ce44eeb1d5.1631703764.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 027ca7a6df44..3801f2e5ea1b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -103,9 +103,11 @@ #define IORING_MAX_REG_BUFFERS (1U << 14) -#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \ - IOSQE_IO_HARDLINK | IOSQE_ASYNC | \ - IOSQE_BUFFER_SELECT) +#define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \ + IOSQE_IO_HARDLINK | IOSQE_ASYNC) + +#define SQE_VALID_FLAGS (SQE_COMMON_FLAGS|IOSQE_BUFFER_SELECT|IOSQE_IO_DRAIN) + #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS) @@ -7059,20 +7061,21 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->fixed_rsrc_refs = NULL; req->task = current; - /* enforce forwards compatibility on users */ - if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) - return -EINVAL; if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; + if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) { + /* enforce forwards compatibility on users */ + if (sqe_flags & ~SQE_VALID_FLAGS) + return -EINVAL; + if ((sqe_flags & IOSQE_BUFFER_SELECT) && + !io_op_defs[req->opcode].buffer_select) + return -EOPNOTSUPP; + if (sqe_flags & IOSQE_IO_DRAIN) + ctx->drain_active = true; + } if (!io_check_restriction(ctx, req, sqe_flags)) return -EACCES; - if ((sqe_flags & IOSQE_BUFFER_SELECT) && - !io_op_defs[req->opcode].buffer_select) - return -EOPNOTSUPP; - if (unlikely(sqe_flags & IOSQE_IO_DRAIN)) - ctx->drain_active = true; - personality = READ_ONCE(sqe->personality); if (personality) { req->creds = xa_load(&ctx->personalities, personality); @@ -11017,6 +11020,8 @@ static int __init io_uring_init(void) /* should fit into one byte */ BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8)); + BUILD_BUG_ON(SQE_COMMON_FLAGS >= (1 << 8)); + BUILD_BUG_ON((SQE_VALID_FLAGS | SQE_COMMON_FLAGS) != SQE_VALID_FLAGS); BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int)); From 8d4af6857c6fb5b1922218e93052bee29eb540f4 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Wed, 22 Sep 2021 18:15:22 +0800 Subject: [PATCH 101/255] io_uring: return boolean value for io_alloc_async_data boolean value is good enough for io_alloc_async_data. Signed-off-by: Hao Xu Link: https://lore.kernel.org/r/20210922101522.9179-1-haoxu@linux.alibaba.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3801f2e5ea1b..3ced6a7bdf2b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3301,7 +3301,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, } } -static inline int io_alloc_async_data(struct io_kiocb *req) +static inline bool io_alloc_async_data(struct io_kiocb *req) { WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); From 6878b40e7b28bd780dedb7d505c13dbf82b73290 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:41 +0100 Subject: [PATCH 102/255] io_uring: mark having different creds unlikely Hint the compiler that it's not as likely to have creds different from current attached to a request. The current code generation is far from ideal, hopefully it can help to some compilers to remove duplicated jump tables and so. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e7815251ac4bf5a4a23d298c752f029ae19f3837.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 3ced6a7bdf2b..bda302bedeef 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6638,7 +6638,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) const struct cred *creds = NULL; int ret; - if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) + if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred())) creds = override_creds(req->creds); switch (req->opcode) { From 87a115fb715bf1f6765e122babbcba566ada286e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:42 +0100 Subject: [PATCH 103/255] io_uring: force_nonspin We don't really need to pass the number of requests to complete into io_do_iopoll(), a flag whether to enforce non-spin mode is enough. Should be straightforward, maybe except io_iopoll_check(). We pass !min there, because we do never enter with the number of already reaped requests is larger than the specified @min, apart from the first iteration, where nr_events is 0 and so the final check should be identical. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/782b39d1d8ec584eae15bca0a1feb6f0571fe5b8.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bda302bedeef..d60595487976 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2453,7 +2453,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, } static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, - long min) + bool force_nonspin) { struct io_kiocb *req, *tmp; unsigned int poll_flags = BLK_POLL_NOSLEEP; @@ -2462,9 +2462,9 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, /* * Only spin for completions if we don't have multiple devices hanging - * off our complete list, and we're under the requested amount. + * off our complete list. */ - if (ctx->poll_multi_queue || *nr_events >= min) + if (ctx->poll_multi_queue || force_nonspin) poll_flags |= BLK_POLL_ONESHOT; list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) { @@ -2516,7 +2516,7 @@ static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) while (!list_empty(&ctx->iopoll_list)) { unsigned int nr_events = 0; - io_do_iopoll(ctx, &nr_events, 0); + io_do_iopoll(ctx, &nr_events, true); /* let it sleep and repeat later if can't complete a request */ if (nr_events == 0) @@ -2578,7 +2578,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) list_empty(&ctx->iopoll_list)) break; } - ret = io_do_iopoll(ctx, &nr_events, min); + ret = io_do_iopoll(ctx, &nr_events, !min); } while (!ret && nr_events < min && !need_resched()); out: mutex_unlock(&ctx->uring_lock); @@ -7347,7 +7347,7 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) mutex_lock(&ctx->uring_lock); if (!list_empty(&ctx->iopoll_list)) - io_do_iopoll(ctx, &nr_events, 0); + io_do_iopoll(ctx, &nr_events, true); /* * Don't submit if refs are dying, good for io_uring_register(), From 5ba3c874eb8acb09fd8aea78ded67e023931894a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:43 +0100 Subject: [PATCH 104/255] io_uring: make io_do_iopoll return number of reqs Don't pass nr_events pointer around but return directly, it's less expensive than pointer increments. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f771a8153a86f16f12ff4272524e9e549c5de40b.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d60595487976..27ca4fb733f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2425,8 +2425,7 @@ static inline bool io_run_task_work(void) /* * Find and free completed poll iocbs */ -static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, - struct list_head *done) +static void io_iopoll_complete(struct io_ring_ctx *ctx, struct list_head *done) { struct req_batch rb; struct io_kiocb *req; @@ -2441,7 +2440,6 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, __io_cqring_fill_event(ctx, req->user_data, req->result, io_put_rw_kbuf(req)); - (*nr_events)++; if (req_ref_put_and_test(req)) io_req_free_batch(&rb, req, &ctx->submit_state); @@ -2452,12 +2450,12 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, io_req_free_batch_finish(ctx, &rb); } -static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, - bool force_nonspin) +static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_kiocb *req, *tmp; unsigned int poll_flags = BLK_POLL_NOSLEEP; DEFINE_IO_COMP_BATCH(iob); + int nr_events = 0; LIST_HEAD(done); /* @@ -2478,6 +2476,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, */ if (READ_ONCE(req->iopoll_completed)) { list_move_tail(&req->inflight_entry, &done); + nr_events++; continue; } if (!list_empty(&done)) @@ -2491,16 +2490,18 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, /* iopoll may have completed current req */ if (!rq_list_empty(iob.req_list) || - READ_ONCE(req->iopoll_completed)) + READ_ONCE(req->iopoll_completed)) { list_move_tail(&req->inflight_entry, &done); + nr_events++; + } } if (!rq_list_empty(iob.req_list)) iob.complete(&iob); if (!list_empty(&done)) - io_iopoll_complete(ctx, nr_events, &done); + io_iopoll_complete(ctx, &done); - return 0; + return nr_events; } /* @@ -2514,12 +2515,8 @@ static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) mutex_lock(&ctx->uring_lock); while (!list_empty(&ctx->iopoll_list)) { - unsigned int nr_events = 0; - - io_do_iopoll(ctx, &nr_events, true); - /* let it sleep and repeat later if can't complete a request */ - if (nr_events == 0) + if (io_do_iopoll(ctx, true) == 0) break; /* * Ensure we allow local-to-the-cpu processing to take place, @@ -2578,8 +2575,12 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) list_empty(&ctx->iopoll_list)) break; } - ret = io_do_iopoll(ctx, &nr_events, !min); - } while (!ret && nr_events < min && !need_resched()); + ret = io_do_iopoll(ctx, !min); + if (ret < 0) + break; + nr_events += ret; + ret = 0; + } while (nr_events < min && !need_resched()); out: mutex_unlock(&ctx->uring_lock); return ret; @@ -7339,7 +7340,6 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE; if (!list_empty(&ctx->iopoll_list) || to_submit) { - unsigned nr_events = 0; const struct cred *creds = NULL; if (ctx->sq_creds != current_cred()) @@ -7347,7 +7347,7 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) mutex_lock(&ctx->uring_lock); if (!list_empty(&ctx->iopoll_list)) - io_do_iopoll(ctx, &nr_events, true); + io_do_iopoll(ctx, true); /* * Don't submit if refs are dying, good for io_uring_register(), From 6f33b0bc4ea43f5c5ce7b7c9ab66051f80837862 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:44 +0100 Subject: [PATCH 105/255] io_uring: use slist for completion batching Currently we collect requests for completion batching in an array. Replace them with a singly linked list. It's as fast as arrays but doesn't take some much space in ctx, and will be used in future patches. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a666826f2854d17e9fb9417fb302edfeb750f425.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 27ca4fb733f9..0ccfd70af3e0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -322,8 +322,8 @@ struct io_submit_state { /* * Batch completion logic */ - struct io_kiocb *compl_reqs[IO_COMPL_BATCH]; - unsigned int compl_nr; + struct io_wq_work_list compl_reqs; + /* inline/task_work completion list, under ->uring_lock */ struct list_head free_list; }; @@ -882,6 +882,8 @@ struct io_kiocb { struct io_wq_work work; const struct cred *creds; + struct io_wq_work_node comp_list; + /* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu; }; @@ -1168,7 +1170,7 @@ static inline void req_ref_get(struct io_kiocb *req) static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) { - if (ctx->submit_state.compl_nr) + if (!wq_list_empty(&ctx->submit_state.compl_reqs)) __io_submit_flush_completions(ctx); } @@ -1325,6 +1327,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->submit_state.free_list); INIT_LIST_HEAD(&ctx->locked_free_list); INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); + INIT_WQ_LIST(&ctx->submit_state.compl_reqs); return ctx; err: kfree(ctx->dummy_ubuf); @@ -1826,11 +1829,16 @@ static inline bool io_req_needs_clean(struct io_kiocb *req) static void io_req_complete_state(struct io_kiocb *req, long res, unsigned int cflags) { + struct io_submit_state *state; + if (io_req_needs_clean(req)) io_clean_op(req); req->result = res; req->compl.cflags = cflags; req->flags |= REQ_F_COMPLETE_INLINE; + + state = &req->ctx->submit_state; + wq_list_add_tail(&req->comp_list, &state->compl_reqs); } static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, @@ -2319,13 +2327,14 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, static void __io_submit_flush_completions(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { + struct io_wq_work_node *node, *prev; struct io_submit_state *state = &ctx->submit_state; - int i, nr = state->compl_nr; struct req_batch rb; spin_lock(&ctx->completion_lock); - for (i = 0; i < nr; i++) { - struct io_kiocb *req = state->compl_reqs[i]; + wq_list_for_each(node, prev, &state->compl_reqs) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); __io_cqring_fill_event(ctx, req->user_data, req->result, req->compl.cflags); @@ -2335,15 +2344,18 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) io_cqring_ev_posted(ctx); io_init_req_batch(&rb); - for (i = 0; i < nr; i++) { - struct io_kiocb *req = state->compl_reqs[i]; + node = state->compl_reqs.first; + do { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); + node = req->comp_list.next; if (req_ref_put_and_test(req)) io_req_free_batch(&rb, req, &ctx->submit_state); - } + } while (node); io_req_free_batch_finish(ctx, &rb); - state->compl_nr = 0; + INIT_WQ_LIST(&state->compl_reqs); } /* @@ -2668,17 +2680,10 @@ static void io_req_task_complete(struct io_kiocb *req, bool *locked) unsigned int cflags = io_put_rw_kbuf(req); long res = req->result; - if (*locked) { - struct io_ring_ctx *ctx = req->ctx; - struct io_submit_state *state = &ctx->submit_state; - + if (*locked) io_req_complete_state(req, res, cflags); - state->compl_reqs[state->compl_nr++] = req; - if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) - io_submit_flush_completions(ctx); - } else { + else io_req_complete_post(req, res, cflags); - } } static void __io_complete_rw(struct io_kiocb *req, long res, long res2, @@ -6962,15 +6967,8 @@ issue_sqe: * doesn't support non-blocking read/write attempts */ if (likely(!ret)) { - if (req->flags & REQ_F_COMPLETE_INLINE) { - struct io_ring_ctx *ctx = req->ctx; - struct io_submit_state *state = &ctx->submit_state; - - state->compl_reqs[state->compl_nr++] = req; - if (state->compl_nr == ARRAY_SIZE(state->compl_reqs)) - io_submit_flush_completions(ctx); + if (req->flags & REQ_F_COMPLETE_INLINE) return; - } linked_timeout = io_prep_linked_timeout(req); if (linked_timeout) From 3ab665b74e59ad467b3eab5b2b95d378cea78b22 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:45 +0100 Subject: [PATCH 106/255] io_uring: remove allocation cache array We have several of request allocation layers, remove the last one, which is the submit->reqs array, and always use submit->free_reqs instead. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8547095c35f7a87bab14f6447ecd30a273ed7500.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 60 +++++++++++++++------------------------------------ 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0ccfd70af3e0..e3bfbd3f4787 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -310,12 +310,6 @@ struct io_submit_state { struct blk_plug plug; struct io_submit_link link; - /* - * io_kiocb alloc cache - */ - void *reqs[IO_REQ_CACHE_SIZE]; - unsigned int free_reqs; - bool plug_started; bool need_plug; @@ -1898,7 +1892,6 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; - int nr; /* * If we have more than a batch's worth of requests in our IRQ side @@ -1907,20 +1900,7 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) */ if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH) io_flush_cached_locked_reqs(ctx, state); - - nr = state->free_reqs; - while (!list_empty(&state->free_list)) { - struct io_kiocb *req = list_first_entry(&state->free_list, - struct io_kiocb, inflight_entry); - - list_del(&req->inflight_entry); - state->reqs[nr++] = req; - if (nr == ARRAY_SIZE(state->reqs)) - break; - } - - state->free_reqs = nr; - return nr != 0; + return !list_empty(&state->free_list); } /* @@ -1934,33 +1914,36 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; + void *reqs[IO_REQ_ALLOC_BATCH]; + struct io_kiocb *req; int ret, i; - BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH); - - if (likely(state->free_reqs || io_flush_cached_reqs(ctx))) + if (likely(!list_empty(&state->free_list) || io_flush_cached_reqs(ctx))) goto got_req; - ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH, - state->reqs); + ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs); /* * Bulk alloc is all-or-nothing. If we fail to get a batch, * retry single alloc to be on the safe side. */ if (unlikely(ret <= 0)) { - state->reqs[0] = kmem_cache_alloc(req_cachep, gfp); - if (!state->reqs[0]) + reqs[0] = kmem_cache_alloc(req_cachep, gfp); + if (!reqs[0]) return NULL; ret = 1; } - for (i = 0; i < ret; i++) - io_preinit_req(state->reqs[i], ctx); - state->free_reqs = ret; + for (i = 0; i < ret; i++) { + req = reqs[i]; + + io_preinit_req(req, ctx); + list_add(&req->inflight_entry, &state->free_list); + } got_req: - state->free_reqs--; - return state->reqs[state->free_reqs]; + req = list_first_entry(&state->free_list, struct io_kiocb, inflight_entry); + list_del(&req->inflight_entry); + return req; } static inline void io_put_file(struct file *file) @@ -2318,10 +2301,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, rb->task_refs++; rb->ctx_refs++; - if (state->free_reqs != ARRAY_SIZE(state->reqs)) - state->reqs[state->free_reqs++] = req; - else - list_add(&req->inflight_entry, &state->free_list); + list_add(&req->inflight_entry, &state->free_list); } static void __io_submit_flush_completions(struct io_ring_ctx *ctx) @@ -9228,12 +9208,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) struct io_submit_state *state = &ctx->submit_state; mutex_lock(&ctx->uring_lock); - - if (state->free_reqs) { - kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); - state->free_reqs = 0; - } - io_flush_cached_locked_reqs(ctx, state); io_req_cache_free(&state->free_list); mutex_unlock(&ctx->uring_lock); From 0d9521b9b526a264a46c42fcde5d3d9a7cb490ea Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:46 +0100 Subject: [PATCH 107/255] io-wq: add io_wq_work_node based stack Apart from just using lists (i.e. io_wq_work_list), we also want to have stacks, which are a bit faster, and have some interoperability between them. Add a stack implementation based on io_wq_work_node and some helpers. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5d3a412a5ac0d47e0f0499d70d2207d70a68925e.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.h | 57 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/fs/io-wq.h b/fs/io-wq.h index bf5c4c533760..c870062105d1 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -29,6 +29,15 @@ struct io_wq_work_list { struct io_wq_work_node *last; }; +#define wq_list_for_each(pos, prv, head) \ + for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) + +#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) +#define INIT_WQ_LIST(list) do { \ + (list)->first = NULL; \ + (list)->last = NULL; \ +} while (0) + static inline void wq_list_add_after(struct io_wq_work_node *node, struct io_wq_work_node *pos, struct io_wq_work_list *list) @@ -54,6 +63,15 @@ static inline void wq_list_add_tail(struct io_wq_work_node *node, } } +static inline void wq_list_add_head(struct io_wq_work_node *node, + struct io_wq_work_list *list) +{ + node->next = list->first; + if (!node->next) + list->last = node; + WRITE_ONCE(list->first, node); +} + static inline void wq_list_cut(struct io_wq_work_list *list, struct io_wq_work_node *last, struct io_wq_work_node *prev) @@ -69,6 +87,31 @@ static inline void wq_list_cut(struct io_wq_work_list *list, last->next = NULL; } +static inline void __wq_list_splice(struct io_wq_work_list *list, + struct io_wq_work_node *to) +{ + list->last->next = to->next; + to->next = list->first; + INIT_WQ_LIST(list); +} + +static inline bool wq_list_splice(struct io_wq_work_list *list, + struct io_wq_work_node *to) +{ + if (!wq_list_empty(list)) { + __wq_list_splice(list, to); + return true; + } + return false; +} + +static inline void wq_stack_add_head(struct io_wq_work_node *node, + struct io_wq_work_node *stack) +{ + node->next = stack->next; + stack->next = node; +} + static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work_node *node, struct io_wq_work_node *prev) @@ -76,14 +119,14 @@ static inline void wq_list_del(struct io_wq_work_list *list, wq_list_cut(list, node, prev); } -#define wq_list_for_each(pos, prv, head) \ - for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) +static inline +struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack) +{ + struct io_wq_work_node *node = stack->next; -#define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) -#define INIT_WQ_LIST(list) do { \ - (list)->first = NULL; \ - (list)->last = NULL; \ -} while (0) + stack->next = node->next; + return node; +} struct io_wq_work { struct io_wq_work_node list; From c2b6c6bc4e0d3452ac4e208c198f6062e0f1d9df Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:47 +0100 Subject: [PATCH 108/255] io_uring: replace list with stack for req caches Replace struct list_head free_list serving for caching requests with singly linked stack, which is faster. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1bc942b82422fb2624b8353bd93aca183a022846.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e3bfbd3f4787..90ad9cd254c5 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -319,7 +319,7 @@ struct io_submit_state { struct io_wq_work_list compl_reqs; /* inline/task_work completion list, under ->uring_lock */ - struct list_head free_list; + struct io_wq_work_node free_list; }; struct io_ring_ctx { @@ -379,7 +379,7 @@ struct io_ring_ctx { } ____cacheline_aligned_in_smp; /* IRQ completion list, under ->completion_lock */ - struct list_head locked_free_list; + struct io_wq_work_list locked_free_list; unsigned int locked_free_nr; const struct cred *sq_creds; /* cred used for __io_sq_thread() */ @@ -1318,8 +1318,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); INIT_LIST_HEAD(&ctx->tctx_list); - INIT_LIST_HEAD(&ctx->submit_state.free_list); - INIT_LIST_HEAD(&ctx->locked_free_list); + ctx->submit_state.free_list.next = NULL; + INIT_WQ_LIST(&ctx->locked_free_list); INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func); INIT_WQ_LIST(&ctx->submit_state.compl_reqs); return ctx; @@ -1806,7 +1806,7 @@ static void io_req_complete_post(struct io_kiocb *req, long res, } io_dismantle_req(req); io_put_task(req->task, 1); - list_add(&req->inflight_entry, &ctx->locked_free_list); + wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; percpu_ref_put(&ctx->refs); } @@ -1883,7 +1883,7 @@ static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx, struct io_submit_state *state) { spin_lock(&ctx->completion_lock); - list_splice_init(&ctx->locked_free_list, &state->free_list); + wq_list_splice(&ctx->locked_free_list, &state->free_list); ctx->locked_free_nr = 0; spin_unlock(&ctx->completion_lock); } @@ -1900,7 +1900,7 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) */ if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH) io_flush_cached_locked_reqs(ctx, state); - return !list_empty(&state->free_list); + return !!state->free_list.next; } /* @@ -1915,10 +1915,11 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) struct io_submit_state *state = &ctx->submit_state; gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; void *reqs[IO_REQ_ALLOC_BATCH]; + struct io_wq_work_node *node; struct io_kiocb *req; int ret, i; - if (likely(!list_empty(&state->free_list) || io_flush_cached_reqs(ctx))) + if (likely(state->free_list.next || io_flush_cached_reqs(ctx))) goto got_req; ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs); @@ -1938,12 +1939,11 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) req = reqs[i]; io_preinit_req(req, ctx); - list_add(&req->inflight_entry, &state->free_list); + wq_stack_add_head(&req->comp_list, &state->free_list); } got_req: - req = list_first_entry(&state->free_list, struct io_kiocb, inflight_entry); - list_del(&req->inflight_entry); - return req; + node = wq_stack_extract(&state->free_list); + return container_of(node, struct io_kiocb, comp_list); } static inline void io_put_file(struct file *file) @@ -1976,7 +1976,7 @@ static void __io_free_req(struct io_kiocb *req) io_put_task(req->task, 1); spin_lock(&ctx->completion_lock); - list_add(&req->inflight_entry, &ctx->locked_free_list); + wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; spin_unlock(&ctx->completion_lock); @@ -2300,8 +2300,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, } rb->task_refs++; rb->ctx_refs++; - - list_add(&req->inflight_entry, &state->free_list); + wq_stack_add_head(&req->comp_list, &state->free_list); } static void __io_submit_flush_completions(struct io_ring_ctx *ctx) @@ -7261,7 +7260,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) } sqe = io_get_sqe(ctx); if (unlikely(!sqe)) { - list_add(&req->inflight_entry, &ctx->submit_state.free_list); + wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); break; } /* will complete beyond this point, count as submitted */ @@ -9193,23 +9192,21 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) } } -static void io_req_cache_free(struct list_head *list) -{ - struct io_kiocb *req, *nxt; - - list_for_each_entry_safe(req, nxt, list, inflight_entry) { - list_del(&req->inflight_entry); - kmem_cache_free(req_cachep, req); - } -} - static void io_req_caches_free(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; mutex_lock(&ctx->uring_lock); io_flush_cached_locked_reqs(ctx, state); - io_req_cache_free(&state->free_list); + + while (state->free_list.next) { + struct io_wq_work_node *node; + struct io_kiocb *req; + + node = wq_stack_extract(&state->free_list); + req = container_of(node, struct io_kiocb, comp_list); + kmem_cache_free(req_cachep, req); + } mutex_unlock(&ctx->uring_lock); } From e3f721e6f6d5d916edf71e5f26ac0547a4b28e1e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:48 +0100 Subject: [PATCH 109/255] io_uring: split iopoll loop The main loop of io_do_iopoll() iterates and does ->iopoll() until it meets a first completed request, then it continues from that position and splices requests to pass them through io_iopoll_complete(). Split the loop in two for clearness, iopolling and reaping completed requests from the list. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a7f6fd27a94845e5dc925a47a4a9765a92e514fb.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 90ad9cd254c5..0e46ed6f92ea 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2441,6 +2441,12 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, struct list_head *done) io_req_free_batch_finish(ctx, &rb); } +/* same as "continue" but starts from the pos, not next to it */ +#define list_for_each_entry_safe_resume(pos, n, head, member) \ + for (n = list_next_entry(pos, member); \ + !list_entry_is_head(pos, head, member); \ + pos = n, n = list_next_entry(n, member)) + static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_kiocb *req, *tmp; @@ -2456,7 +2462,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (ctx->poll_multi_queue || force_nonspin) poll_flags |= BLK_POLL_ONESHOT; - list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) { + list_for_each_entry(req, &ctx->iopoll_list, inflight_entry) { struct kiocb *kiocb = &req->rw.kiocb; int ret; @@ -2465,12 +2471,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) * If we find a request that requires polling, break out * and complete those lists first, if we have entries there. */ - if (READ_ONCE(req->iopoll_completed)) { - list_move_tail(&req->inflight_entry, &done); - nr_events++; - continue; - } - if (!list_empty(&done)) + if (READ_ONCE(req->iopoll_completed)) break; ret = kiocb->ki_filp->f_op->iopoll(kiocb, &iob, poll_flags); @@ -2481,17 +2482,22 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) /* iopoll may have completed current req */ if (!rq_list_empty(iob.req_list) || - READ_ONCE(req->iopoll_completed)) { - list_move_tail(&req->inflight_entry, &done); - nr_events++; - } + READ_ONCE(req->iopoll_completed)) + break; } if (!rq_list_empty(iob.req_list)) iob.complete(&iob); - if (!list_empty(&done)) - io_iopoll_complete(ctx, &done); + list_for_each_entry_safe_resume(req, tmp, &ctx->iopoll_list, + inflight_entry) { + if (!READ_ONCE(req->iopoll_completed)) + break; + list_move_tail(&req->inflight_entry, &done); + nr_events++; + } + if (nr_events) + io_iopoll_complete(ctx, &done); return nr_events; } From 5eef4e87eb0b2b8482f6a77ebcad067636a867b3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:49 +0100 Subject: [PATCH 110/255] io_uring: use single linked list for iopoll Use single linked lists for keeping iopoll requests, takes less space, may be faster, but mostly will be of benefit for further patches. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/314033676b100cd485518c3bc55e1b95a0dcd71f.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.h | 3 +++ fs/io_uring.c | 54 +++++++++++++++++++++++++-------------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/fs/io-wq.h b/fs/io-wq.h index c870062105d1..87ba6a733630 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -32,6 +32,9 @@ struct io_wq_work_list { #define wq_list_for_each(pos, prv, head) \ for (pos = (head)->first, prv = NULL; pos; prv = pos, pos = (pos)->next) +#define wq_list_for_each_resume(pos, prv) \ + for (; pos; prv = pos, pos = (pos)->next) + #define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) #define INIT_WQ_LIST(list) do { \ (list)->first = NULL; \ diff --git a/fs/io_uring.c b/fs/io_uring.c index 0e46ed6f92ea..fac1d0310d66 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -412,7 +412,7 @@ struct io_ring_ctx { * For SQPOLL, only the single threaded io_sq_thread() will * manipulate the list, hence no extra locking is needed there. */ - struct list_head iopoll_list; + struct io_wq_work_list iopoll_list; struct hlist_head *cancel_hash; unsigned cancel_hash_bits; bool poll_multi_queue; @@ -1309,7 +1309,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_waitqueue_head(&ctx->cq_wait); spin_lock_init(&ctx->completion_lock); spin_lock_init(&ctx->timeout_lock); - INIT_LIST_HEAD(&ctx->iopoll_list); + INIT_WQ_LIST(&ctx->iopoll_list); INIT_LIST_HEAD(&ctx->defer_list); INIT_LIST_HEAD(&ctx->timeout_list); INIT_LIST_HEAD(&ctx->ltimeout_list); @@ -2441,15 +2441,9 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, struct list_head *done) io_req_free_batch_finish(ctx, &rb); } -/* same as "continue" but starts from the pos, not next to it */ -#define list_for_each_entry_safe_resume(pos, n, head, member) \ - for (n = list_next_entry(pos, member); \ - !list_entry_is_head(pos, head, member); \ - pos = n, n = list_next_entry(n, member)) - static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { - struct io_kiocb *req, *tmp; + struct io_wq_work_node *pos, *start, *prev; unsigned int poll_flags = BLK_POLL_NOSLEEP; DEFINE_IO_COMP_BATCH(iob); int nr_events = 0; @@ -2462,7 +2456,8 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (ctx->poll_multi_queue || force_nonspin) poll_flags |= BLK_POLL_ONESHOT; - list_for_each_entry(req, &ctx->iopoll_list, inflight_entry) { + wq_list_for_each(pos, start, &ctx->iopoll_list) { + struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); struct kiocb *kiocb = &req->rw.kiocb; int ret; @@ -2488,14 +2483,20 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (!rq_list_empty(iob.req_list)) iob.complete(&iob); - list_for_each_entry_safe_resume(req, tmp, &ctx->iopoll_list, - inflight_entry) { + else if (!pos) + return 0; + + prev = start; + wq_list_for_each_resume(pos, prev) { + struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); + if (!READ_ONCE(req->iopoll_completed)) break; - list_move_tail(&req->inflight_entry, &done); + list_add_tail(&req->inflight_entry, &done); nr_events++; } + wq_list_cut(&ctx->iopoll_list, prev, start); if (nr_events) io_iopoll_complete(ctx, &done); return nr_events; @@ -2511,7 +2512,7 @@ static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) return; mutex_lock(&ctx->uring_lock); - while (!list_empty(&ctx->iopoll_list)) { + while (!wq_list_empty(&ctx->iopoll_list)) { /* let it sleep and repeat later if can't complete a request */ if (io_do_iopoll(ctx, true) == 0) break; @@ -2560,7 +2561,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) * forever, while the workqueue is stuck trying to acquire the * very same mutex. */ - if (list_empty(&ctx->iopoll_list)) { + if (wq_list_empty(&ctx->iopoll_list)) { u32 tail = ctx->cached_cq_tail; mutex_unlock(&ctx->uring_lock); @@ -2569,7 +2570,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) /* some requests don't go through iopoll_list */ if (tail != ctx->cached_cq_tail || - list_empty(&ctx->iopoll_list)) + wq_list_empty(&ctx->iopoll_list)) break; } ret = io_do_iopoll(ctx, !min); @@ -2729,14 +2730,13 @@ static void io_iopoll_req_issued(struct io_kiocb *req) * how we do polling eventually, not spinning if we're on potentially * different devices. */ - if (list_empty(&ctx->iopoll_list)) { + if (wq_list_empty(&ctx->iopoll_list)) { ctx->poll_multi_queue = false; } else if (!ctx->poll_multi_queue) { struct io_kiocb *list_req; - list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb, - inflight_entry); - + list_req = container_of(ctx->iopoll_list.first, struct io_kiocb, + comp_list); if (list_req->file != req->file) ctx->poll_multi_queue = true; } @@ -2746,9 +2746,9 @@ static void io_iopoll_req_issued(struct io_kiocb *req) * it to the front so we find it first. */ if (READ_ONCE(req->iopoll_completed)) - list_add(&req->inflight_entry, &ctx->iopoll_list); + wq_list_add_head(&req->comp_list, &ctx->iopoll_list); else - list_add_tail(&req->inflight_entry, &ctx->iopoll_list); + wq_list_add_tail(&req->comp_list, &ctx->iopoll_list); if (unlikely(in_async)) { /* @@ -7322,14 +7322,14 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE) to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE; - if (!list_empty(&ctx->iopoll_list) || to_submit) { + if (!wq_list_empty(&ctx->iopoll_list) || to_submit) { const struct cred *creds = NULL; if (ctx->sq_creds != current_cred()) creds = override_creds(ctx->sq_creds); mutex_lock(&ctx->uring_lock); - if (!list_empty(&ctx->iopoll_list)) + if (!wq_list_empty(&ctx->iopoll_list)) io_do_iopoll(ctx, true); /* @@ -7407,7 +7407,7 @@ static int io_sq_thread(void *data) list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) { int ret = __io_sq_thread(ctx, cap_entries); - if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list))) + if (!sqt_spin && (ret > 0 || !wq_list_empty(&ctx->iopoll_list))) sqt_spin = true; } if (io_run_task_work()) @@ -7428,7 +7428,7 @@ static int io_sq_thread(void *data) io_ring_set_wakeup_flag(ctx); if ((ctx->flags & IORING_SETUP_IOPOLL) && - !list_empty_careful(&ctx->iopoll_list)) { + !wq_list_empty(&ctx->iopoll_list)) { needs_sched = false; break; } @@ -9583,7 +9583,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, /* SQPOLL thread does its own polling */ if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) || (ctx->sq_data && ctx->sq_data->thread == current)) { - while (!list_empty_careful(&ctx->iopoll_list)) { + while (!wq_list_empty(&ctx->iopoll_list)) { io_iopoll_try_reap_events(ctx); ret = true; } From 3aa83bfb6e5c3a89d0ec67e598ee04bfc1425b13 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:50 +0100 Subject: [PATCH 111/255] io_uring: add a helper for batch free Add a helper io_free_batch_list(), which takes a single linked list and puts/frees all requests from it in an efficient manner. Will be reused later. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4fc8306b542c6b1dd1d08e8021ef3bdb0ad15010.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fac1d0310d66..df122cdfc85d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2303,12 +2303,31 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, wq_stack_add_head(&req->comp_list, &state->free_list); } +static void io_free_batch_list(struct io_ring_ctx *ctx, + struct io_wq_work_list *list) + __must_hold(&ctx->uring_lock) +{ + struct io_wq_work_node *node; + struct req_batch rb; + + io_init_req_batch(&rb); + node = list->first; + do { + struct io_kiocb *req = container_of(node, struct io_kiocb, + comp_list); + + node = req->comp_list.next; + if (req_ref_put_and_test(req)) + io_req_free_batch(&rb, req, &ctx->submit_state); + } while (node); + io_req_free_batch_finish(ctx, &rb); +} + static void __io_submit_flush_completions(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_wq_work_node *node, *prev; struct io_submit_state *state = &ctx->submit_state; - struct req_batch rb; spin_lock(&ctx->completion_lock); wq_list_for_each(node, prev, &state->compl_reqs) { @@ -2322,18 +2341,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) spin_unlock(&ctx->completion_lock); io_cqring_ev_posted(ctx); - io_init_req_batch(&rb); - node = state->compl_reqs.first; - do { - struct io_kiocb *req = container_of(node, struct io_kiocb, - comp_list); - - node = req->comp_list.next; - if (req_ref_put_and_test(req)) - io_req_free_batch(&rb, req, &ctx->submit_state); - } while (node); - - io_req_free_batch_finish(ctx, &rb); + io_free_batch_list(ctx, &state->compl_reqs); INIT_WQ_LIST(&state->compl_reqs); } From b3fa03fd1b17f557e2c43736440feed66fb86741 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:51 +0100 Subject: [PATCH 112/255] io_uring: convert iopoll_completed to store_release Convert explicit barrier around iopoll_completed to smp_load_acquire() and smp_store_release(). Similar on the callback side, but replaces a single smp_rmb() with per-request smp_load_acquire(), neither imply any extra CPU ordering for x86. Use READ_ONCE as usual where it doesn't matter. Use it to move filling CQEs by iopoll earlier, that will be necessary to avoid traversing the list one extra time in the future. Suggested-by: Bart Van Assche Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8bd663cb15efdc72d6247c38ee810964e744a450.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index df122cdfc85d..fb122c2b69f2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2429,17 +2429,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, struct list_head *done) struct req_batch rb; struct io_kiocb *req; - /* order with ->result store in io_complete_rw_iopoll() */ - smp_rmb(); - io_init_req_batch(&rb); while (!list_empty(done)) { req = list_first_entry(done, struct io_kiocb, inflight_entry); list_del(&req->inflight_entry); - __io_cqring_fill_event(ctx, req->user_data, req->result, - io_put_rw_kbuf(req)); - if (req_ref_put_and_test(req)) io_req_free_batch(&rb, req, &ctx->submit_state); } @@ -2498,8 +2492,12 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) wq_list_for_each_resume(pos, prev) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - if (!READ_ONCE(req->iopoll_completed)) + /* order with io_complete_rw_iopoll(), e.g. ->result updates */ + if (!smp_load_acquire(&req->iopoll_completed)) break; + __io_cqring_fill_event(ctx, req->user_data, req->result, + io_put_rw_kbuf(req)); + list_add_tail(&req->inflight_entry, &done); nr_events++; } @@ -2712,10 +2710,9 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) } } - WRITE_ONCE(req->result, res); - /* order with io_iopoll_complete() checking ->result */ - smp_wmb(); - WRITE_ONCE(req->iopoll_completed, 1); + req->result = res; + /* order with io_iopoll_complete() checking ->iopoll_completed */ + smp_store_release(&req->iopoll_completed, 1); } /* From f5ed3bcd5b117ffe73b9dc2394bbbad26a68c086 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:52 +0100 Subject: [PATCH 113/255] io_uring: optimise batch completion First, convert rest of iopoll bits to single linked lists, and also replace per-request list_add_tail() with splicing a part of slist. With that, use io_free_batch_list() to put/free requests. The main advantage of it is that it's now the only user of struct req_batch and friends, and so they can be inlined. The main overhead there was per-request call to not-inlined io_req_free_batch(), which is expensive enough. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b37fc6d5954b241e025eead7ab92c6f44a42f229.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fb122c2b69f2..91077b56a564 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2421,35 +2421,13 @@ static inline bool io_run_task_work(void) return false; } -/* - * Find and free completed poll iocbs - */ -static void io_iopoll_complete(struct io_ring_ctx *ctx, struct list_head *done) -{ - struct req_batch rb; - struct io_kiocb *req; - - io_init_req_batch(&rb); - while (!list_empty(done)) { - req = list_first_entry(done, struct io_kiocb, inflight_entry); - list_del(&req->inflight_entry); - - if (req_ref_put_and_test(req)) - io_req_free_batch(&rb, req, &ctx->submit_state); - } - - io_commit_cqring(ctx); - io_cqring_ev_posted_iopoll(ctx); - io_req_free_batch_finish(ctx, &rb); -} - static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_wq_work_node *pos, *start, *prev; unsigned int poll_flags = BLK_POLL_NOSLEEP; + struct io_wq_work_list list; DEFINE_IO_COMP_BATCH(iob); int nr_events = 0; - LIST_HEAD(done); /* * Only spin for completions if we don't have multiple devices hanging @@ -2496,15 +2474,19 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (!smp_load_acquire(&req->iopoll_completed)) break; __io_cqring_fill_event(ctx, req->user_data, req->result, - io_put_rw_kbuf(req)); - - list_add_tail(&req->inflight_entry, &done); + io_put_rw_kbuf(req)); nr_events++; } + if (unlikely(!nr_events)) + return 0; + + io_commit_cqring(ctx); + io_cqring_ev_posted_iopoll(ctx); + list.first = start ? start->next : ctx->iopoll_list.first; + list.last = prev; wq_list_cut(&ctx->iopoll_list, prev, start); - if (nr_events) - io_iopoll_complete(ctx, &done); + io_free_batch_list(ctx, &list); return nr_events; } From d4b7a5ef2b9c06def90d12db9b99bd12d75758fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:53 +0100 Subject: [PATCH 114/255] io_uring: inline completion batching helpers We now have a single function for batched put of requests, just inline struct req_batch and all related helpers into it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/595a2917f80dd94288cd7203052c7934f5446580.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 66 +++++++++++++++++---------------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 91077b56a564..7229bf74c9e1 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2264,63 +2264,41 @@ static void io_free_req_work(struct io_kiocb *req, bool *locked) io_free_req(req); } -struct req_batch { - struct task_struct *task; - int task_refs; - int ctx_refs; -}; - -static inline void io_init_req_batch(struct req_batch *rb) -{ - rb->task_refs = 0; - rb->ctx_refs = 0; - rb->task = NULL; -} - -static void io_req_free_batch_finish(struct io_ring_ctx *ctx, - struct req_batch *rb) -{ - if (rb->ctx_refs) - percpu_ref_put_many(&ctx->refs, rb->ctx_refs); - if (rb->task) - io_put_task(rb->task, rb->task_refs); -} - -static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req, - struct io_submit_state *state) -{ - io_queue_next(req); - io_dismantle_req(req); - - if (req->task != rb->task) { - if (rb->task) - io_put_task(rb->task, rb->task_refs); - rb->task = req->task; - rb->task_refs = 0; - } - rb->task_refs++; - rb->ctx_refs++; - wq_stack_add_head(&req->comp_list, &state->free_list); -} - static void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_list *list) __must_hold(&ctx->uring_lock) { struct io_wq_work_node *node; - struct req_batch rb; + struct task_struct *task = NULL; + int task_refs = 0, ctx_refs = 0; - io_init_req_batch(&rb); node = list->first; do { struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); node = req->comp_list.next; - if (req_ref_put_and_test(req)) - io_req_free_batch(&rb, req, &ctx->submit_state); + if (!req_ref_put_and_test(req)) + continue; + + io_queue_next(req); + io_dismantle_req(req); + + if (req->task != task) { + if (task) + io_put_task(task, task_refs); + task = req->task; + task_refs = 0; + } + task_refs++; + ctx_refs++; + wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); } while (node); - io_req_free_batch_finish(ctx, &rb); + + if (ctx_refs) + percpu_ref_put_many(&ctx->refs, ctx_refs); + if (task) + io_put_task(task, task_refs); } static void __io_submit_flush_completions(struct io_ring_ctx *ctx) From 1cce17aca621c38c657410dc278a48cda982dd2e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:54 +0100 Subject: [PATCH 115/255] io_uring: don't pass tail into io_free_batch_list io_free_batch_list() iterates all requests in the passed in list, so we don't really need to know the tail but can keep iterating until meet NULL. Just pass the first node into it and it will be enough. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4a12c84b6d887d980e05f417ba4172d04c64acae.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7229bf74c9e1..693ad31082ae 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2265,14 +2265,12 @@ static void io_free_req_work(struct io_kiocb *req, bool *locked) } static void io_free_batch_list(struct io_ring_ctx *ctx, - struct io_wq_work_list *list) + struct io_wq_work_node *node) __must_hold(&ctx->uring_lock) { - struct io_wq_work_node *node; struct task_struct *task = NULL; int task_refs = 0, ctx_refs = 0; - node = list->first; do { struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); @@ -2319,7 +2317,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) spin_unlock(&ctx->completion_lock); io_cqring_ev_posted(ctx); - io_free_batch_list(ctx, &state->compl_reqs); + io_free_batch_list(ctx, state->compl_reqs.first); INIT_WQ_LIST(&state->compl_reqs); } @@ -2403,7 +2401,6 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_wq_work_node *pos, *start, *prev; unsigned int poll_flags = BLK_POLL_NOSLEEP; - struct io_wq_work_list list; DEFINE_IO_COMP_BATCH(iob); int nr_events = 0; @@ -2461,10 +2458,9 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) io_commit_cqring(ctx); io_cqring_ev_posted_iopoll(ctx); - list.first = start ? start->next : ctx->iopoll_list.first; - list.last = prev; + pos = start ? start->next : ctx->iopoll_list.first; wq_list_cut(&ctx->iopoll_list, prev, start); - io_free_batch_list(ctx, &list); + io_free_batch_list(ctx, pos); return nr_events; } From 553deffd0920fc52e8c91e7f8a42d1186a75760a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:55 +0100 Subject: [PATCH 116/255] io_uring: don't pass state to io_submit_state_end Submission state and ctx and coupled together, no need to passs Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e22d77a5786ef77e0c49b933ad74bae55cfb6ca6.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 693ad31082ae..5088b47d1bc4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7137,11 +7137,13 @@ fail_req: /* * Batched submission is done, ensure local IO is flushed out. */ -static void io_submit_state_end(struct io_submit_state *state, - struct io_ring_ctx *ctx) +static void io_submit_state_end(struct io_ring_ctx *ctx) { + struct io_submit_state *state = &ctx->submit_state; + if (state->link.head) io_queue_sqe(state->link.head); + /* flush only after queuing links as they can generate completions */ io_submit_flush_completions(ctx); if (state->plug_started) blk_finish_plug(&state->plug); @@ -7244,7 +7246,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) percpu_ref_put_many(&ctx->refs, unused); } - io_submit_state_end(&ctx->submit_state, ctx); + io_submit_state_end(ctx); /* Commit SQ ring head once we've consumed and submitted all SQEs */ io_commit_sqring(ctx); From f15a3431775a598ed89028acee334200160cc2d6 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:56 +0100 Subject: [PATCH 117/255] io_uring: deduplicate io_queue_sqe() call sites There are two call sites of io_queue_sqe() in io_submit_sqe(), combine them into one, because io_queue_sqe() is inline and we don't want to bloat binary, and will become even bigger text data bss dec hex filename 92126 13986 8 106120 19e88 ./fs/io_uring.o 91966 13986 8 105960 19de8 ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/506124b8e767f0a4576f7a459f6aea3d13fb4dda.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5088b47d1bc4..9b99ec46da25 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7117,20 +7117,18 @@ fail_req: link->last->link = req; link->last = req; + if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) + return 0; /* last request of a link, enqueue the link */ - if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) { - link->head = NULL; - io_queue_sqe(head); - } - } else { - if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { - link->head = req; - link->last = req; - } else { - io_queue_sqe(req); - } + link->head = NULL; + req = head; + } else if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) { + link->head = req; + link->last = req; + return 0; } + io_queue_sqe(req); return 0; } From 2a56a9bd64dbdff4fe5cbe00b20014da07694a78 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:57 +0100 Subject: [PATCH 118/255] io_uring: remove drain_active check from hot path req->ctx->active_drain is a bit too expensive, partially because of two dereferences. Do a trick, if we see it set in io_init_req(), set REQ_F_FORCE_ASYNC and it automatically goes through a slower path where we can catch it. It's nearly free to do in io_init_req() because there is already ->restricted check and it's in the same byte of a bitmask. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d7e7ddc63c15e8a300833132abb3eb8fd3918aef.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 53 ++++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9b99ec46da25..f0fb458e81f3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6438,23 +6438,15 @@ static bool io_drain_req(struct io_kiocb *req) int ret; u32 seq; - if (req->flags & REQ_F_FAIL) { - io_req_complete_fail_submit(req); - return true; - } - - /* - * If we need to drain a request in the middle of a link, drain the - * head request and the next request/link after the current link. - * Considering sequential execution of links, IOSQE_IO_DRAIN will be - * maintained for every request of our link. - */ - if (ctx->drain_next) { - req->flags |= REQ_F_IO_DRAIN; - ctx->drain_next = false; - } /* not interested in head, start from the first linked */ io_for_each_link(pos, req->link) { + /* + * If we need to drain a request in the middle of a link, drain + * the head request and the next request/link after the current + * link. Considering sequential execution of links, + * IOSQE_IO_DRAIN will be maintained for every request of our + * link. + */ if (pos->flags & REQ_F_IO_DRAIN) { ctx->drain_next = true; req->flags |= REQ_F_IO_DRAIN; @@ -6946,13 +6938,12 @@ issue_sqe: static inline void io_queue_sqe(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { - if (unlikely(req->ctx->drain_active) && io_drain_req(req)) - return; - if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) { __io_queue_sqe(req); } else if (req->flags & REQ_F_FAIL) { io_req_complete_fail_submit(req); + } else if (unlikely(req->ctx->drain_active) && io_drain_req(req)) { + return; } else { int ret = io_req_prep_async(req); @@ -6972,9 +6963,6 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, struct io_kiocb *req, unsigned int sqe_flags) { - if (likely(!ctx->restricted)) - return true; - if (!test_bit(req->opcode, ctx->restrictions.sqe_op)) return false; @@ -7015,11 +7003,28 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if ((sqe_flags & IOSQE_BUFFER_SELECT) && !io_op_defs[req->opcode].buffer_select) return -EOPNOTSUPP; - if (sqe_flags & IOSQE_IO_DRAIN) + if (sqe_flags & IOSQE_IO_DRAIN) { + struct io_submit_link *link = &ctx->submit_state.link; + ctx->drain_active = true; + req->flags |= REQ_F_FORCE_ASYNC; + if (link->head) + link->head->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC; + } + } + if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) { + if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags)) + return -EACCES; + /* knock it to the slow queue path, will be drained there */ + if (ctx->drain_active) + req->flags |= REQ_F_FORCE_ASYNC; + /* if there is no link, we're at "next" request and need to drain */ + if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) { + ctx->drain_next = false; + ctx->drain_active = true; + req->flags |= REQ_F_FORCE_ASYNC | IOSQE_IO_DRAIN; + } } - if (!io_check_restriction(ctx, req, sqe_flags)) - return -EACCES; personality = READ_ONCE(sqe->personality); if (personality) { From 4652fe3f10e57abc3fc6ac11c431b2ef39f78c03 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:58 +0100 Subject: [PATCH 119/255] io_uring: split slow path from io_queue_sqe We don't want the slow path of io_queue_sqe to be inlined, so extract a function from it. text data bss dec hex filename 91950 13986 8 105944 19dd8 ./fs/io_uring.o 91758 13986 8 105752 19d18 ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/fb01253911f8fb374268f65b1ba939b54ca6583f.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f0fb458e81f3..e5a414c208bd 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6935,12 +6935,10 @@ issue_sqe: } } -static inline void io_queue_sqe(struct io_kiocb *req) +static void io_queue_sqe_fallback(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { - if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) { - __io_queue_sqe(req); - } else if (req->flags & REQ_F_FAIL) { + if (req->flags & REQ_F_FAIL) { io_req_complete_fail_submit(req); } else if (unlikely(req->ctx->drain_active) && io_drain_req(req)) { return; @@ -6954,6 +6952,15 @@ static inline void io_queue_sqe(struct io_kiocb *req) } } +static inline void io_queue_sqe(struct io_kiocb *req) + __must_hold(&req->ctx->uring_lock) +{ + if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) + __io_queue_sqe(req); + else + io_queue_sqe_fallback(req); +} + /* * Check SQE restrictions (opcode and flags). * From d475a9a6226c86b7febe3863b900b820a0e6b71c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 21:59:59 +0100 Subject: [PATCH 120/255] io_uring: inline hot path of __io_queue_sqe() Extract slow paths from __io_queue_sqe() into a function and inline the hot path. With that we have everything completely inlined on the submission path up until io_issue_sqe(). -> io_submit_sqes() -> io_submit_sqe() (inlined) -> io_queue_sqe() (inlined) -> __io_queue_sqe() (inlined) -> io_issue_sqe() Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f1606864d95d7f26dc28c7eec3dc6ed6ec32618a.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e5a414c208bd..d4f8abb451c4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6891,13 +6891,38 @@ static void io_queue_linked_timeout(struct io_kiocb *req) io_put_req(req); } -static void __io_queue_sqe(struct io_kiocb *req) +static void io_queue_sqe_arm_apoll(struct io_kiocb *req) + __must_hold(&req->ctx->uring_lock) +{ + struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); + + switch (io_arm_poll_handler(req)) { + case IO_APOLL_READY: + if (linked_timeout) { + io_unprep_linked_timeout(req); + linked_timeout = NULL; + } + io_req_task_queue(req); + break; + case IO_APOLL_ABORTED: + /* + * Queued up for async execution, worker will release + * submit reference when the iocb is actually submitted. + */ + io_queue_async_work(req, NULL); + break; + } + + if (linked_timeout) + io_queue_linked_timeout(linked_timeout); +} + +static inline void __io_queue_sqe(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { struct io_kiocb *linked_timeout; int ret; -issue_sqe: ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); /* @@ -6912,24 +6937,7 @@ issue_sqe: if (linked_timeout) io_queue_linked_timeout(linked_timeout); } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) { - linked_timeout = io_prep_linked_timeout(req); - - switch (io_arm_poll_handler(req)) { - case IO_APOLL_READY: - if (linked_timeout) - io_unprep_linked_timeout(req); - goto issue_sqe; - case IO_APOLL_ABORTED: - /* - * Queued up for async execution, worker will release - * submit reference when the iocb is actually submitted. - */ - io_queue_async_work(req, NULL); - break; - } - - if (linked_timeout) - io_queue_linked_timeout(linked_timeout); + io_queue_sqe_arm_apoll(req); } else { io_req_complete_failed(req, ret); } From d9f9d2842c9156470b3f1d3dafe5684a3c036366 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 22:00:00 +0100 Subject: [PATCH 121/255] io_uring: reshuffle queue_sqe completion handling If a request completed inline the result should only be zero, it's a grave error otherwise. So, when we see REQ_F_COMPLETE_INLINE it's not even necessary to check the return code, and the flag check can be moved earlier. It's one "if" less for inline completions, and same two checks for it normally completing (ret == 0). Those are two cases we care about the most. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/ebd4e397a9c26d96c99b24447acc309741041a83.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d4f8abb451c4..bbb2a262c272 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6925,14 +6925,13 @@ static inline void __io_queue_sqe(struct io_kiocb *req) ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); + if (req->flags & REQ_F_COMPLETE_INLINE) + return; /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts */ if (likely(!ret)) { - if (req->flags & REQ_F_COMPLETE_INLINE) - return; - linked_timeout = io_prep_linked_timeout(req); if (linked_timeout) io_queue_linked_timeout(linked_timeout); From 6962980947e2b967ab26bfd34004b6b573597513 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 22:00:01 +0100 Subject: [PATCH 122/255] io_uring: restructure submit sqes to_submit checks Put an explicit check for number of requests to submit. First, we can turn while into do-while and it generates better code, and second that if can be cheaper, e.g. by using CPU flags after sub in io_sqring_entries(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5926baadd20c28feab7a5e1725fedf32e4553ff7.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bbb2a262c272..b8f60cf36156 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7225,16 +7225,19 @@ static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx) static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) __must_hold(&ctx->uring_lock) { + unsigned int entries = io_sqring_entries(ctx); int submitted = 0; + if (!entries) + return 0; /* make sure SQ entry isn't read before tail */ - nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx)); + nr = min3(nr, ctx->sq_entries, entries); if (!percpu_ref_tryget_many(&ctx->refs, nr)) return -EAGAIN; io_get_task_refs(nr); io_submit_state_start(&ctx->submit_state, nr); - while (submitted < nr) { + do { const struct io_uring_sqe *sqe; struct io_kiocb *req; @@ -7253,7 +7256,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) submitted++; if (io_submit_sqe(ctx, req, sqe)) break; - } + } while (submitted < nr); if (unlikely(submitted != nr)) { int ref_used = (submitted == -EAGAIN) ? 0 : submitted; From ef05d9ebcc927260f700a94436e7c9347657bbef Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 22:00:02 +0100 Subject: [PATCH 123/255] io_uring: kill off ->inflight_entry field ->inflight_entry is not used anymore after converting everything to single linked lists, remove it. Also adjust io_kiocb layout, so all hot bits are in first 3 cachelines. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/fd8d68087ede26c4e1707ce6b175aa1eb2381f2b.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b8f60cf36156..eff24d8eb399 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -868,18 +868,15 @@ struct io_kiocb { struct percpu_ref *fixed_rsrc_refs; /* used with ctx->iopoll_list with reads/writes */ - struct list_head inflight_entry; + struct io_wq_work_node comp_list; struct io_task_work io_task_work; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; struct async_poll *apoll; - struct io_wq_work work; - const struct cred *creds; - - struct io_wq_work_node comp_list; - /* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu; + struct io_wq_work work; + const struct cred *creds; }; struct io_tctx_node { From a1cdbb4cb5f7190c429b1127892f756b7cd32db4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 22:00:03 +0100 Subject: [PATCH 124/255] io_uring: comment why inline complete calls io_clean_op() io_req_complete_state() calls io_clean_op() and it may be not entirely obvious, leave a comment. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/21806f862151e223fdf439e5e8ed7178a8d66979.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index eff24d8eb399..5a06063d5d81 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1822,6 +1822,7 @@ static void io_req_complete_state(struct io_kiocb *req, long res, { struct io_submit_state *state; + /* clean per-opcode space, because req->compl is aliased with it */ if (io_req_needs_clean(req)) io_clean_op(req); req->result = res; From 5e371265ea1d3e0cd02236b1a6d79fe322523ae8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 24 Sep 2021 22:00:04 +0100 Subject: [PATCH 125/255] io_uring: disable draining earlier Clear ->drain_active in two more cases where we check for a need of draining. It's not a bug, but still may lead to some extra requests being punted to io-wq, and that may be not desirable. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/d20b265f77bb4e8860b15b9987252c7c711dfcba.1632516769.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5a06063d5d81..fd3dc6432a34 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6461,8 +6461,10 @@ static bool io_drain_req(struct io_kiocb *req) seq = io_get_sequence(req); /* Still a chance to pass the sequence check */ - if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) + if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) { + ctx->drain_active = false; return false; + } ret = io_req_prep_async(req); if (ret) @@ -6481,6 +6483,7 @@ fail: spin_unlock(&ctx->completion_lock); kfree(de); io_queue_async_work(req, NULL); + ctx->drain_active = false; return true; } From 22b2ca310afcea319c72e051df0371f668192b10 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 1 Oct 2021 18:07:00 +0100 Subject: [PATCH 126/255] io_uring: extra a helper for drain init Add a helper io_init_req_drain for initialising requests with IOSQE_DRAIN set. Also move bits from preambule of io_drain_req() in there, because we already modify all the bits needed inside the helper. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/dcb412825b35b1cb8891245a387d7d69f8d14cef.1633107393.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fd3dc6432a34..fb073915fa5c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6430,28 +6430,11 @@ static u32 io_get_sequence(struct io_kiocb *req) static bool io_drain_req(struct io_kiocb *req) { - struct io_kiocb *pos; struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; int ret; u32 seq; - /* not interested in head, start from the first linked */ - io_for_each_link(pos, req->link) { - /* - * If we need to drain a request in the middle of a link, drain - * the head request and the next request/link after the current - * link. Considering sequential execution of links, - * IOSQE_IO_DRAIN will be maintained for every request of our - * link. - */ - if (pos->flags & REQ_F_IO_DRAIN) { - ctx->drain_next = true; - req->flags |= REQ_F_IO_DRAIN; - break; - } - } - /* Still need defer if there is pending req in defer list. */ if (likely(list_empty_careful(&ctx->defer_list) && !(req->flags & REQ_F_IO_DRAIN))) { @@ -6992,6 +6975,25 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, return true; } +static void io_init_req_drain(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + struct io_kiocb *head = ctx->submit_state.link.head; + + ctx->drain_active = true; + if (head) { + /* + * If we need to drain a request in the middle of a link, drain + * the head request and the next request/link after the current + * link. Considering sequential execution of links, + * IOSQE_IO_DRAIN will be maintained for every request of our + * link. + */ + head->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC; + ctx->drain_next = true; + } +} + static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, const struct io_uring_sqe *sqe) __must_hold(&ctx->uring_lock) @@ -7018,14 +7020,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if ((sqe_flags & IOSQE_BUFFER_SELECT) && !io_op_defs[req->opcode].buffer_select) return -EOPNOTSUPP; - if (sqe_flags & IOSQE_IO_DRAIN) { - struct io_submit_link *link = &ctx->submit_state.link; - - ctx->drain_active = true; - req->flags |= REQ_F_FORCE_ASYNC; - if (link->head) - link->head->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC; - } + if (sqe_flags & IOSQE_IO_DRAIN) + io_init_req_drain(req); } if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) { if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags)) @@ -7037,7 +7033,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) { ctx->drain_next = false; ctx->drain_active = true; - req->flags |= REQ_F_FORCE_ASYNC | IOSQE_IO_DRAIN; + req->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC; } } From e0eb71dcfc4b862261d99f7f90169142867beb0a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 1 Oct 2021 18:07:01 +0100 Subject: [PATCH 127/255] io_uring: don't return from io_drain_req() Never return from io_drain_req() but punt to tw if we've got there but it's a false positive and we shouldn't actually drain. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/93583cee51b8783706b76c73196c155b28d9e762.1633107393.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index fb073915fa5c..b0b4c5b00f37 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6428,46 +6428,39 @@ static u32 io_get_sequence(struct io_kiocb *req) return seq; } -static bool io_drain_req(struct io_kiocb *req) +static void io_drain_req(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; int ret; - u32 seq; + u32 seq = io_get_sequence(req); /* Still need defer if there is pending req in defer list. */ - if (likely(list_empty_careful(&ctx->defer_list) && - !(req->flags & REQ_F_IO_DRAIN))) { - ctx->drain_active = false; - return false; - } - - seq = io_get_sequence(req); - /* Still a chance to pass the sequence check */ if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) { +queue: ctx->drain_active = false; - return false; + io_req_task_queue(req); + return; } ret = io_req_prep_async(req); - if (ret) - goto fail; + if (ret) { +fail: + io_req_complete_failed(req, ret); + return; + } io_prep_async_link(req); de = kmalloc(sizeof(*de), GFP_KERNEL); if (!de) { ret = -ENOMEM; -fail: - io_req_complete_failed(req, ret); - return true; + goto fail; } spin_lock(&ctx->completion_lock); if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) { spin_unlock(&ctx->completion_lock); kfree(de); - io_queue_async_work(req, NULL); - ctx->drain_active = false; - return true; + goto queue; } trace_io_uring_defer(ctx, req, req->user_data); @@ -6475,7 +6468,6 @@ fail: de->seq = seq; list_add_tail(&de->list, &ctx->defer_list); spin_unlock(&ctx->completion_lock); - return true; } static void io_clean_op(struct io_kiocb *req) @@ -6931,8 +6923,8 @@ static void io_queue_sqe_fallback(struct io_kiocb *req) { if (req->flags & REQ_F_FAIL) { io_req_complete_fail_submit(req); - } else if (unlikely(req->ctx->drain_active) && io_drain_req(req)) { - return; + } else if (unlikely(req->ctx->drain_active)) { + io_drain_req(req); } else { int ret = io_req_prep_async(req); From fc0ae0244bbbedf309a835afb278e39a3fc9bb94 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 1 Oct 2021 18:07:02 +0100 Subject: [PATCH 128/255] io_uring: init opcode in io_init_req() Move io_req_prep() call inside of io_init_req(), it simplifies a bit error handling for callers. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/a0f59291fd52da4672c323542fd56fd899e23f8f.1633107393.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b0b4c5b00f37..81292041378f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6992,7 +6992,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { struct io_submit_state *state; unsigned int sqe_flags; - int personality, ret = 0; + int personality; /* req is partially pre-initialised, see io_preinit_req() */ req->opcode = READ_ONCE(sqe->opcode); @@ -7053,9 +7053,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), (sqe_flags & IOSQE_FIXED_FILE)); if (unlikely(!req->file)) - ret = -EBADF; + return -EBADF; } - return ret; + + return io_req_prep(req, sqe); } static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, @@ -7067,7 +7068,6 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret = io_init_req(ctx, req, sqe); if (unlikely(ret)) { -fail_req: trace_io_uring_req_failed(sqe, ret); /* fail even hard links since we don't submit */ @@ -7092,10 +7092,6 @@ fail_req: return ret; } req_fail_link_node(req, ret); - } else { - ret = io_req_prep(req, sqe); - if (unlikely(ret)) - goto fail_req; } /* don't need @sqe from now on */ From 30d51dd4ad2040d4c90497287b69635af7c67502 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 1 Oct 2021 18:07:03 +0100 Subject: [PATCH 129/255] io_uring: clean up buffer select Hiding a pointer to a struct io_buffer in rw.addr is error prone. We have some place in io_kiocb, so keep kbuf's in a separate field without aliasing and risks of it being misused. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3e63a6a953b04cad81d9ea827b12344dd57b37b4.1633107393.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 46 ++++++++++++---------------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 81292041378f..5b3b52815b88 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -573,7 +573,6 @@ struct io_sr_msg { int msg_flags; int bgid; size_t len; - struct io_buffer *kbuf; }; struct io_open { @@ -877,6 +876,7 @@ struct io_kiocb { struct io_mapped_ubuf *imu; struct io_wq_work work; const struct cred *creds; + struct io_buffer *kbuf; }; struct io_tctx_node { @@ -2376,12 +2376,9 @@ static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf) static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req) { - struct io_buffer *kbuf; - if (likely(!(req->flags & REQ_F_BUFFER_SELECTED))) return 0; - kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; - return io_put_kbuf(req, kbuf); + return io_put_kbuf(req, req->kbuf); } static inline bool io_run_task_work(void) @@ -3000,9 +2997,9 @@ static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) } static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, - int bgid, struct io_buffer *kbuf, - bool needs_lock) + int bgid, bool needs_lock) { + struct io_buffer *kbuf = req->kbuf; struct io_buffer *head; if (req->flags & REQ_F_BUFFER_SELECTED) @@ -3024,12 +3021,13 @@ static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, } if (*len > kbuf->len) *len = kbuf->len; + req->flags |= REQ_F_BUFFER_SELECTED; + req->kbuf = kbuf; } else { kbuf = ERR_PTR(-ENOBUFS); } io_ring_submit_unlock(req->ctx, needs_lock); - return kbuf; } @@ -3039,13 +3037,10 @@ static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, struct io_buffer *kbuf; u16 bgid; - kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; bgid = req->buf_index; - kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); + kbuf = io_buffer_select(req, len, bgid, needs_lock); if (IS_ERR(kbuf)) return kbuf; - req->rw.addr = (u64) (unsigned long) kbuf; - req->flags |= REQ_F_BUFFER_SELECTED; return u64_to_user_ptr(kbuf->addr); } @@ -3101,9 +3096,8 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, bool needs_lock) { if (req->flags & REQ_F_BUFFER_SELECTED) { - struct io_buffer *kbuf; + struct io_buffer *kbuf = req->kbuf; - kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; iov[0].iov_base = u64_to_user_ptr(kbuf->addr); iov[0].iov_len = kbuf->len; return 0; @@ -4869,20 +4863,13 @@ static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, bool needs_lock) { struct io_sr_msg *sr = &req->sr_msg; - struct io_buffer *kbuf; - kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock); - if (IS_ERR(kbuf)) - return kbuf; - - sr->kbuf = kbuf; - req->flags |= REQ_F_BUFFER_SELECTED; - return kbuf; + return io_buffer_select(req, &sr->len, sr->bgid, needs_lock); } static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) { - return io_put_kbuf(req, req->sr_msg.kbuf); + return io_put_kbuf(req, req->kbuf); } static int io_recvmsg_prep_async(struct io_kiocb *req) @@ -6473,17 +6460,8 @@ fail: static void io_clean_op(struct io_kiocb *req) { if (req->flags & REQ_F_BUFFER_SELECTED) { - switch (req->opcode) { - case IORING_OP_READV: - case IORING_OP_READ_FIXED: - case IORING_OP_READ: - kfree((void *)(unsigned long)req->rw.addr); - break; - case IORING_OP_RECVMSG: - case IORING_OP_RECV: - kfree(req->sr_msg.kbuf); - break; - } + kfree(req->kbuf); + req->kbuf = NULL; } if (req->flags & REQ_F_NEED_CLEANUP) { From 6224590d242fc8c6b26941328e02a40b4384949b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 2 Oct 2021 19:36:14 +0100 Subject: [PATCH 130/255] io_uring: add flag to not fail link after timeout For some reason non-off IORING_OP_TIMEOUT always fails links, it's pretty inconvenient and unnecessary limits chaining after it to hard linking, which is far from ideal, e.g. doesn't pair well with timeout cancellation. Add a flag forcing it to not fail links on -ETIME. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/17c7ec0fb7a6113cc6be8cdaedcada0ba836ac0e.1633199723.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++++-- include/uapi/linux/io_uring.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5b3b52815b88..5b3b4f8f9659 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5857,7 +5857,10 @@ err: static void io_req_task_timeout(struct io_kiocb *req, bool *locked) { - req_set_fail(req); + struct io_timeout_data *data = req->async_data; + + if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS)) + req_set_fail(req); io_req_complete_post(req, -ETIME, 0); } @@ -6063,7 +6066,8 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (off && is_timeout_link) return -EINVAL; flags = READ_ONCE(sqe->timeout_flags); - if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK)) + if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK | + IORING_TIMEOUT_ETIME_SUCCESS)) return -EINVAL; /* more than one clock specified is invalid, obviously */ if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index b270a07b285e..c45b5e9a9387 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -158,6 +158,7 @@ enum { #define IORING_TIMEOUT_BOOTTIME (1U << 2) #define IORING_TIMEOUT_REALTIME (1U << 3) #define IORING_LINK_TIMEOUT_UPDATE (1U << 4) +#define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) #define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) #define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) /* From 7e3709d57651feab9c77d7a5a8024041f73f69f7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:46 +0100 Subject: [PATCH 131/255] io_uring: optimise kiocb layout We want ->comp_list in the second cacheline, which is hotter comparing to the 3rd. Swap the field with ->link, which is not as hot and controlled by flags and so not accessed unless there is a link. By the way add a couple of comments for io_kiocb fields. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9d9dde31f8f62279a5f48c575bbc27b8290edc0c.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5b3b4f8f9659..ab07a0813f20 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -863,19 +863,22 @@ struct io_kiocb { struct task_struct *task; u64 user_data; - struct io_kiocb *link; struct percpu_ref *fixed_rsrc_refs; - /* used with ctx->iopoll_list with reads/writes */ + /* used by request caches, completion batching and iopoll */ struct io_wq_work_node comp_list; + struct io_kiocb *link; struct io_task_work io_task_work; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; + /* internal polling, see IORING_FEAT_FAST_POLL */ struct async_poll *apoll; /* store used ubuf, so we can prevent reloading */ struct io_mapped_ubuf *imu; struct io_wq_work work; + /* custom credentials, valid IFF REQ_F_CREDS is set */ const struct cred *creds; + /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ struct io_buffer *kbuf; }; From 51d48dab62ed9e44e0ccc723411b0d95286a821c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:47 +0100 Subject: [PATCH 132/255] io_uring: add more likely/unlikely() annotations Add two extra unlikely() in io_submit_sqes() and one around io_req_needs_clean() to help the compiler to avoid extra jumps in hot paths. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/88e087afe657e7660194353aada9b00f11d480f9.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ab07a0813f20..bc87ed85f206 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1957,7 +1957,7 @@ static inline void io_dismantle_req(struct io_kiocb *req) { unsigned int flags = req->flags; - if (io_req_needs_clean(req)) + if (unlikely(io_req_needs_clean(req))) io_clean_op(req); if (!(flags & REQ_F_FIXED_FILE)) io_put_file(req->file); @@ -7198,11 +7198,11 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) unsigned int entries = io_sqring_entries(ctx); int submitted = 0; - if (!entries) + if (unlikely(!entries)) return 0; /* make sure SQ entry isn't read before tail */ nr = min3(nr, ctx->sq_entries, entries); - if (!percpu_ref_tryget_many(&ctx->refs, nr)) + if (unlikely(!percpu_ref_tryget_many(&ctx->refs, nr))) return -EAGAIN; io_get_task_refs(nr); From fff4e40e3094972b119e38c8f0de4e1ca9fec654 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:48 +0100 Subject: [PATCH 133/255] io_uring: delay req queueing into compl-batch list io_req_complete_state() is inlined and used in lots of places, so we want to keep it concise. Move adding a request into a completion batch list from io_req_complete_state() into the consumer, i.e. __io_queue_sqe(). before vs after text data bss dec hex filename 91894 14002 8 105904 19db0 ./fs/io_uring.o 91046 14002 8 105056 19a60 ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4afca4e11abfd4cc8e99777fdcaf4d34cf4d022d.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bc87ed85f206..045474b7b9c2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1446,6 +1446,13 @@ static void io_prep_async_link(struct io_kiocb *req) } } +static inline void io_req_add_compl_list(struct io_kiocb *req) +{ + struct io_submit_state *state = &req->ctx->submit_state; + + wq_list_add_tail(&req->comp_list, &state->compl_reqs); +} + static void io_queue_async_work(struct io_kiocb *req, bool *locked) { struct io_ring_ctx *ctx = req->ctx; @@ -1820,20 +1827,15 @@ static inline bool io_req_needs_clean(struct io_kiocb *req) return req->flags & IO_REQ_CLEAN_FLAGS; } -static void io_req_complete_state(struct io_kiocb *req, long res, - unsigned int cflags) +static inline void io_req_complete_state(struct io_kiocb *req, long res, + unsigned int cflags) { - struct io_submit_state *state; - /* clean per-opcode space, because req->compl is aliased with it */ if (io_req_needs_clean(req)) io_clean_op(req); req->result = res; req->compl.cflags = cflags; req->flags |= REQ_F_COMPLETE_INLINE; - - state = &req->ctx->submit_state; - wq_list_add_tail(&req->comp_list, &state->compl_reqs); } static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, @@ -2626,10 +2628,12 @@ static void io_req_task_complete(struct io_kiocb *req, bool *locked) unsigned int cflags = io_put_rw_kbuf(req); long res = req->result; - if (*locked) + if (*locked) { io_req_complete_state(req, res, cflags); - else + io_req_add_compl_list(req); + } else { io_req_complete_post(req, res, cflags); + } } static void __io_complete_rw(struct io_kiocb *req, long res, long res2, @@ -6886,8 +6890,10 @@ static inline void __io_queue_sqe(struct io_kiocb *req) ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); - if (req->flags & REQ_F_COMPLETE_INLINE) + if (req->flags & REQ_F_COMPLETE_INLINE) { + io_req_add_compl_list(req); return; + } /* * We async punt it if the file wasn't marked NOWAIT, or if the file * doesn't support non-blocking read/write attempts From a33ae9ce16a8ca62c5dffbe8909d185c6c5b4d77 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:49 +0100 Subject: [PATCH 134/255] io_uring: optimise request allocation Even after fully inlining io_alloc_req() my compiler does a NULL check in the path of successful allocation, no hacks like an empty dereference help it. Restructure io_alloc_req() by splitting out refilling part, so the compiler generate a slightly better binary. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/eda17571bdc7248d8e617b23e7132a5416e4680b.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 045474b7b9c2..614c982c2697 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1912,18 +1912,17 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) * Because of that, io_alloc_req() should be called only under ->uring_lock * and with extra caution to not get a request that is still worked on. */ -static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +static bool __io_alloc_req_refill(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_submit_state *state = &ctx->submit_state; gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; void *reqs[IO_REQ_ALLOC_BATCH]; - struct io_wq_work_node *node; struct io_kiocb *req; int ret, i; if (likely(state->free_list.next || io_flush_cached_reqs(ctx))) - goto got_req; + return true; ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs); @@ -1934,7 +1933,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) if (unlikely(ret <= 0)) { reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!reqs[0]) - return NULL; + return false; ret = 1; } @@ -1944,8 +1943,21 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) io_preinit_req(req, ctx); wq_stack_add_head(&req->comp_list, &state->free_list); } -got_req: - node = wq_stack_extract(&state->free_list); + return true; +} + +static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx) +{ + if (unlikely(!ctx->submit_state.free_list.next)) + return __io_alloc_req_refill(ctx); + return true; +} + +static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +{ + struct io_wq_work_node *node; + + node = wq_stack_extract(&ctx->submit_state.free_list); return container_of(node, struct io_kiocb, comp_list); } @@ -7217,12 +7229,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) const struct io_uring_sqe *sqe; struct io_kiocb *req; - req = io_alloc_req(ctx); - if (unlikely(!req)) { + if (unlikely(!io_alloc_req_refill(ctx))) { if (!submitted) submitted = -EAGAIN; break; } + req = io_alloc_req(ctx); sqe = io_get_sqe(ctx); if (unlikely(!sqe)) { wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); From 765ff496c781a982ed0a882a7784c93ce10d1155 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:50 +0100 Subject: [PATCH 135/255] io_uring: optimise INIT_WQ_LIST The invariant of io_wq_work_list is that it's empty IFF ->first is NULL, so no need to initially set ->last. With now having more users of the list it may play a role, i.e. used in each tw iteration and on every completion flushing. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c464ab5cab6e46a858c6d39c107e92b3b5291f13.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.h | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/io-wq.h b/fs/io-wq.h index 87ba6a733630..41bf37674a49 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -38,7 +38,6 @@ struct io_wq_work_list { #define wq_list_empty(list) (READ_ONCE((list)->first) == NULL) #define INIT_WQ_LIST(list) do { \ (list)->first = NULL; \ - (list)->last = NULL; \ } while (0) static inline void wq_list_add_after(struct io_wq_work_node *node, From aede728aae355d4c1d38dd02747d415af011eea7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:51 +0100 Subject: [PATCH 136/255] io_uring: don't wake sqpoll in io_cqring_ev_posted io_cqring_ev_posted() doesn't need to wake SQPOLL, it's either done by userspace or with task_work, but no action is required on request completion. Rip off bits waking it up in io_cqring_ev_posted(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b49dab27b64cf11f4c50f2f90dcaac123430e05d.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 614c982c2697..af2cbe633345 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1619,8 +1619,6 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) */ if (wq_has_sleeper(&ctx->cq_wait)) wake_up_all(&ctx->cq_wait); - if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait)) - wake_up(&ctx->sq_data->wait); if (io_should_trigger_evfd(ctx)) eventfd_signal(ctx->cq_ev_fd, 1); if (waitqueue_active(&ctx->poll_wait)) From d60aa65ba221f038404b98d8484f562f72bb807b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:52 +0100 Subject: [PATCH 137/255] io_uring: merge CQ and poll waitqueues ->cq_wait and ->poll_wait and waken up in the same manner, use a single waitqueue for both of them. CQ waiters are queued exclusively, so wake up should first go over all pollers and that's what we need. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/00fe603e50000365774cf8435ef5fe03f049c1c9.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index af2cbe633345..c9e6f4b68718 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -394,7 +394,6 @@ struct io_ring_ctx { unsigned cached_cq_tail; unsigned cq_entries; struct eventfd_ctx *cq_ev_fd; - struct wait_queue_head poll_wait; struct wait_queue_head cq_wait; unsigned cq_extra; atomic_t cq_timeouts; @@ -1300,7 +1299,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) ctx->flags = p->flags; init_waitqueue_head(&ctx->sqo_sq_wait); INIT_LIST_HEAD(&ctx->sqd_list); - init_waitqueue_head(&ctx->poll_wait); INIT_LIST_HEAD(&ctx->cq_overflow_list); init_completion(&ctx->ref_comp); xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1); @@ -1621,8 +1619,6 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx) wake_up_all(&ctx->cq_wait); if (io_should_trigger_evfd(ctx)) eventfd_signal(ctx->cq_ev_fd, 1); - if (waitqueue_active(&ctx->poll_wait)) - wake_up_interruptible(&ctx->poll_wait); } static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) @@ -1636,8 +1632,6 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) } if (io_should_trigger_evfd(ctx)) eventfd_signal(ctx->cq_ev_fd, 1); - if (waitqueue_active(&ctx->poll_wait)) - wake_up_interruptible(&ctx->poll_wait); } /* Returns true if there are no backlogged entries after the flush */ @@ -9253,7 +9247,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) struct io_ring_ctx *ctx = file->private_data; __poll_t mask = 0; - poll_wait(file, &ctx->poll_wait, wait); + poll_wait(file, &ctx->cq_wait, wait); /* * synchronizes with barrier from wq_has_sleeper call in * io_commit_cqring From 37f0e767e177af328fcaf74d3772495235e361a8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:53 +0100 Subject: [PATCH 138/255] io_uring: optimise ctx referencing by requests Currenlty, we allocate one ctx reference per request at submission time and put them at free. It's batched and not so expensive but it still bloats the kernel, adds 2 function calls for rcu and adds some overhead for request counting in io_free_batch_list(). Always keep one reference with a request, even when it's freed and in io_uring request caches. There is extra work at ring exit / quiesce paths, which now need to put all cached requests. io_ring_exit_work() is already looping, so it's not a problem. Add hybrid-busy waiting to io_ctx_quiesce() as well for now. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/99613fbe396e80777228cde39bbda1aa8938554e.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c9e6f4b68718..9ec86e35668f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1807,7 +1807,6 @@ static void io_req_complete_post(struct io_kiocb *req, long res, io_put_task(req->task, 1); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; - percpu_ref_put(&ctx->refs); } io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); @@ -1929,6 +1928,7 @@ static bool __io_alloc_req_refill(struct io_ring_ctx *ctx) ret = 1; } + percpu_ref_get_many(&ctx->refs, ret); for (i = 0; i < ret; i++) { req = reqs[i]; @@ -1986,8 +1986,6 @@ static void __io_free_req(struct io_kiocb *req) wq_list_add_head(&req->comp_list, &ctx->locked_free_list); ctx->locked_free_nr++; spin_unlock(&ctx->completion_lock); - - percpu_ref_put(&ctx->refs); } static inline void io_remove_next_linked(struct io_kiocb *req) @@ -2276,7 +2274,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, __must_hold(&ctx->uring_lock) { struct task_struct *task = NULL; - int task_refs = 0, ctx_refs = 0; + int task_refs = 0; do { struct io_kiocb *req = container_of(node, struct io_kiocb, @@ -2296,12 +2294,9 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, task_refs = 0; } task_refs++; - ctx_refs++; wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); } while (node); - if (ctx_refs) - percpu_ref_put_many(&ctx->refs, ctx_refs); if (task) io_put_task(task, task_refs); } @@ -7212,8 +7207,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) return 0; /* make sure SQ entry isn't read before tail */ nr = min3(nr, ctx->sq_entries, entries); - if (unlikely(!percpu_ref_tryget_many(&ctx->refs, nr))) - return -EAGAIN; io_get_task_refs(nr); io_submit_state_start(&ctx->submit_state, nr); @@ -7243,7 +7236,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) int unused = nr - ref_used; current->io_uring->cached_refs += unused; - percpu_ref_put_many(&ctx->refs, unused); } io_submit_state_end(ctx); @@ -9164,6 +9156,7 @@ static void io_destroy_buffers(struct io_ring_ctx *ctx) static void io_req_caches_free(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; + int nr = 0; mutex_lock(&ctx->uring_lock); io_flush_cached_locked_reqs(ctx, state); @@ -9175,7 +9168,10 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) node = wq_stack_extract(&state->free_list); req = container_of(node, struct io_kiocb, comp_list); kmem_cache_free(req_cachep, req); + nr++; } + if (nr) + percpu_ref_put_many(&ctx->refs, nr); mutex_unlock(&ctx->uring_lock); } @@ -9345,6 +9341,8 @@ static void io_ring_exit_work(struct work_struct *work) io_sq_thread_unpark(sqd); } + io_req_caches_free(ctx); + if (WARN_ON_ONCE(time_after(jiffies, timeout))) { /* there is little hope left, don't run it too often */ interval = HZ * 60; @@ -10724,10 +10722,14 @@ static int io_ctx_quiesce(struct io_ring_ctx *ctx) */ mutex_unlock(&ctx->uring_lock); do { - ret = wait_for_completion_interruptible(&ctx->ref_comp); - if (!ret) + ret = wait_for_completion_interruptible_timeout(&ctx->ref_comp, HZ); + if (ret) { + ret = min(0L, ret); break; + } + ret = io_run_task_work_sig(); + io_req_caches_free(ctx); } while (ret >= 0); mutex_lock(&ctx->uring_lock); From c072481ded14bceb9a4163ac81ce73d907a43855 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:54 +0100 Subject: [PATCH 139/255] io_uring: mark cold functions Attribute cold functions so compilers can optimise them for size. It shrinks the binary by 2.5-3% text data bss dec hex filename 90670 14002 8 104680 198e8 ./fs/io_uring.o 88053 14002 8 102063 18eaf ./fs/io_uring.o Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b53d385f91dca45170b67d7f11c7abd787e821f6.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 123 ++++++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9ec86e35668f..e404c98e846a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1230,7 +1230,7 @@ static inline void req_fail_link_node(struct io_kiocb *req, int res) req->result = res; } -static void io_ring_ctx_ref_free(struct percpu_ref *ref) +static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -1242,7 +1242,7 @@ static inline bool io_is_timeout_noseq(struct io_kiocb *req) return !req->timeout.off; } -static void io_fallback_req_func(struct work_struct *work) +static __cold void io_fallback_req_func(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work); @@ -1262,7 +1262,7 @@ static void io_fallback_req_func(struct work_struct *work) } -static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) +static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) { struct io_ring_ctx *ctx; int hash_bits; @@ -1361,7 +1361,7 @@ static inline bool io_req_ffs_set(struct io_kiocb *req) return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE); } -static void io_req_track_inflight(struct io_kiocb *req) +static inline void io_req_track_inflight(struct io_kiocb *req) { if (!(req->flags & REQ_F_INFLIGHT)) { req->flags |= REQ_F_INFLIGHT; @@ -1500,7 +1500,7 @@ static void io_kill_timeout(struct io_kiocb *req, int status) } } -static void io_queue_deferred(struct io_ring_ctx *ctx) +static __cold void io_queue_deferred(struct io_ring_ctx *ctx) { while (!list_empty(&ctx->defer_list)) { struct io_defer_entry *de = list_first_entry(&ctx->defer_list, @@ -1514,7 +1514,7 @@ static void io_queue_deferred(struct io_ring_ctx *ctx) } } -static void io_flush_timeouts(struct io_ring_ctx *ctx) +static __cold void io_flush_timeouts(struct io_ring_ctx *ctx) __must_hold(&ctx->completion_lock) { u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); @@ -1547,7 +1547,7 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) spin_unlock_irq(&ctx->timeout_lock); } -static void __io_commit_cqring_flush(struct io_ring_ctx *ctx) +static __cold void __io_commit_cqring_flush(struct io_ring_ctx *ctx) { if (ctx->off_timeout_used) io_flush_timeouts(ctx); @@ -1903,7 +1903,7 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) * Because of that, io_alloc_req() should be called only under ->uring_lock * and with extra caution to not get a request that is still worked on. */ -static bool __io_alloc_req_refill(struct io_ring_ctx *ctx) +static __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_submit_state *state = &ctx->submit_state; @@ -1975,7 +1975,7 @@ static inline void io_dismantle_req(struct io_kiocb *req) } } -static void __io_free_req(struct io_kiocb *req) +static __cold void __io_free_req(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; @@ -2467,7 +2467,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) * We can't just wait for polled events to come to us, we have to actively * find and complete them. */ -static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) +static __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx) { if (!(ctx->flags & IORING_SETUP_IOPOLL)) return; @@ -5632,8 +5632,8 @@ static bool io_poll_remove_one(struct io_kiocb *req) /* * Returns true if we found and killed one or more poll requests */ -static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk, - bool cancel_all) +static __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, + struct task_struct *tsk, bool cancel_all) { struct hlist_node *tmp; struct io_kiocb *req; @@ -6425,7 +6425,7 @@ static u32 io_get_sequence(struct io_kiocb *req) return seq; } -static void io_drain_req(struct io_kiocb *req) +static __cold void io_drain_req(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; struct io_defer_entry *de; @@ -7305,7 +7305,7 @@ static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries) return ret; } -static void io_sqd_update_thread_idle(struct io_sq_data *sqd) +static __cold void io_sqd_update_thread_idle(struct io_sq_data *sqd) { struct io_ring_ctx *ctx; unsigned sq_thread_idle = 0; @@ -7559,7 +7559,7 @@ static void io_free_page_table(void **table, size_t size) kfree(table); } -static void **io_alloc_page_table(size_t size) +static __cold void **io_alloc_page_table(size_t size) { unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE); size_t init_size = size; @@ -7588,7 +7588,7 @@ static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node) kfree(ref_node); } -static void io_rsrc_node_ref_zero(struct percpu_ref *ref) +static __cold void io_rsrc_node_ref_zero(struct percpu_ref *ref) { struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs); struct io_ring_ctx *ctx = node->rsrc_data->ctx; @@ -7665,7 +7665,8 @@ static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx) return ctx->rsrc_backup_node ? 0 : -ENOMEM; } -static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx) +static __cold int io_rsrc_ref_quiesce(struct io_rsrc_data *data, + struct io_ring_ctx *ctx) { int ret; @@ -7721,9 +7722,9 @@ static void io_rsrc_data_free(struct io_rsrc_data *data) kfree(data); } -static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, - u64 __user *utags, unsigned nr, - struct io_rsrc_data **pdata) +static __cold int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put, + u64 __user *utags, unsigned nr, + struct io_rsrc_data **pdata) { struct io_rsrc_data *data; int ret = -ENOMEM; @@ -8493,8 +8494,8 @@ static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx, return io_wq_create(concurrency, &data); } -static int io_uring_alloc_task_context(struct task_struct *task, - struct io_ring_ctx *ctx) +static __cold int io_uring_alloc_task_context(struct task_struct *task, + struct io_ring_ctx *ctx) { struct io_uring_task *tctx; int ret; @@ -8541,8 +8542,8 @@ void __io_uring_free(struct task_struct *tsk) tsk->io_uring = NULL; } -static int io_sq_offload_create(struct io_ring_ctx *ctx, - struct io_uring_params *p) +static __cold int io_sq_offload_create(struct io_ring_ctx *ctx, + struct io_uring_params *p) { int ret; @@ -9181,7 +9182,7 @@ static void io_wait_rsrc_data(struct io_rsrc_data *data) wait_for_completion(&data->done); } -static void io_ring_ctx_free(struct io_ring_ctx *ctx) +static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_sq_thread_finish(ctx); @@ -9290,7 +9291,7 @@ struct io_tctx_exit { struct io_ring_ctx *ctx; }; -static void io_tctx_exit_cb(struct callback_head *cb) +static __cold void io_tctx_exit_cb(struct callback_head *cb) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_exit *work; @@ -9305,14 +9306,14 @@ static void io_tctx_exit_cb(struct callback_head *cb) complete(&work->completion); } -static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) +static __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); return req->ctx == data; } -static void io_ring_exit_work(struct work_struct *work) +static __cold void io_ring_exit_work(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work); unsigned long timeout = jiffies + HZ * 60 * 5; @@ -9383,8 +9384,8 @@ static void io_ring_exit_work(struct work_struct *work) } /* Returns true if we found and killed one or more timeouts */ -static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, - bool cancel_all) +static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, + struct task_struct *tsk, bool cancel_all) { struct io_kiocb *req, *tmp; int canceled = 0; @@ -9406,7 +9407,7 @@ static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, return canceled != 0; } -static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) +static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) { unsigned long index; struct creds *creds; @@ -9468,8 +9469,9 @@ static bool io_cancel_task_cb(struct io_wq_work *work, void *data) return ret; } -static bool io_cancel_defer_files(struct io_ring_ctx *ctx, - struct task_struct *task, bool cancel_all) +static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx, + struct task_struct *task, + bool cancel_all) { struct io_defer_entry *de; LIST_HEAD(list); @@ -9494,7 +9496,7 @@ static bool io_cancel_defer_files(struct io_ring_ctx *ctx, return true; } -static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) +static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) { struct io_tctx_node *node; enum io_wq_cancel cret; @@ -9518,9 +9520,9 @@ static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx) return ret; } -static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, - struct task_struct *task, - bool cancel_all) +static __cold void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, + struct task_struct *task, + bool cancel_all) { struct io_task_cancel cancel = { .task = task, .all = cancel_all, }; struct io_uring_task *tctx = task ? task->io_uring : NULL; @@ -9610,7 +9612,7 @@ static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) /* * Remove this io_uring_file -> task mapping. */ -static void io_uring_del_tctx_node(unsigned long index) +static __cold void io_uring_del_tctx_node(unsigned long index) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; @@ -9633,7 +9635,7 @@ static void io_uring_del_tctx_node(unsigned long index) kfree(node); } -static void io_uring_clean_tctx(struct io_uring_task *tctx) +static __cold void io_uring_clean_tctx(struct io_uring_task *tctx) { struct io_wq *wq = tctx->io_wq; struct io_tctx_node *node; @@ -9660,7 +9662,7 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked) return percpu_counter_sum(&tctx->inflight); } -static void io_uring_drop_tctx_refs(struct task_struct *task) +static __cold void io_uring_drop_tctx_refs(struct task_struct *task) { struct io_uring_task *tctx = task->io_uring; unsigned int refs = tctx->cached_refs; @@ -9676,7 +9678,8 @@ static void io_uring_drop_tctx_refs(struct task_struct *task) * Find any io_uring ctx that this task has registered or done IO on, and cancel * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation. */ -static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd) +static __cold void io_uring_cancel_generic(bool cancel_all, + struct io_sq_data *sqd) { struct io_uring_task *tctx = current->io_uring; struct io_ring_ctx *ctx; @@ -9769,7 +9772,7 @@ static void *io_uring_validate_mmap_request(struct file *file, #ifdef CONFIG_MMU -static int io_uring_mmap(struct file *file, struct vm_area_struct *vma) +static __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) { size_t sz = vma->vm_end - vma->vm_start; unsigned long pfn; @@ -9954,7 +9957,7 @@ out_fput: } #ifdef CONFIG_PROC_FS -static int io_uring_show_cred(struct seq_file *m, unsigned int id, +static __cold int io_uring_show_cred(struct seq_file *m, unsigned int id, const struct cred *cred) { struct user_namespace *uns = seq_user_ns(m); @@ -9986,7 +9989,8 @@ static int io_uring_show_cred(struct seq_file *m, unsigned int id, return 0; } -static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) +static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, + struct seq_file *m) { struct io_sq_data *sq = NULL; struct io_overflow_cqe *ocqe; @@ -10098,7 +10102,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m) spin_unlock(&ctx->completion_lock); } -static void io_uring_show_fdinfo(struct seq_file *m, struct file *f) +static __cold void io_uring_show_fdinfo(struct seq_file *m, struct file *f) { struct io_ring_ctx *ctx = f->private_data; @@ -10122,8 +10126,8 @@ static const struct file_operations io_uring_fops = { #endif }; -static int io_allocate_scq_urings(struct io_ring_ctx *ctx, - struct io_uring_params *p) +static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, + struct io_uring_params *p) { struct io_rings *rings; size_t size, sq_array_offset; @@ -10212,8 +10216,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) return file; } -static int io_uring_create(unsigned entries, struct io_uring_params *p, - struct io_uring_params __user *params) +static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, + struct io_uring_params __user *params) { struct io_ring_ctx *ctx; struct file *file; @@ -10371,7 +10375,8 @@ SYSCALL_DEFINE2(io_uring_setup, u32, entries, return io_uring_setup(entries, params); } -static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args) +static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, + unsigned nr_args) { struct io_uring_probe *p; size_t size; @@ -10427,8 +10432,8 @@ static int io_register_personality(struct io_ring_ctx *ctx) return id; } -static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, - unsigned int nr_args) +static __cold int io_register_restrictions(struct io_ring_ctx *ctx, + void __user *arg, unsigned int nr_args) { struct io_uring_restriction *res; size_t size; @@ -10562,7 +10567,7 @@ static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg, return __io_register_rsrc_update(ctx, type, &up, up.nr); } -static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, +static __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, unsigned int size, unsigned int type) { struct io_uring_rsrc_register rr; @@ -10588,8 +10593,8 @@ static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, return -EINVAL; } -static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, - unsigned len) +static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, + void __user *arg, unsigned len) { struct io_uring_task *tctx = current->io_uring; cpumask_var_t new_mask; @@ -10615,7 +10620,7 @@ static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, return ret; } -static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) +static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) { struct io_uring_task *tctx = current->io_uring; @@ -10625,8 +10630,8 @@ static int io_unregister_iowq_aff(struct io_ring_ctx *ctx) return io_wq_cpu_affinity(tctx->io_wq, NULL); } -static int io_register_iowq_max_workers(struct io_ring_ctx *ctx, - void __user *arg) +static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, + void __user *arg) { struct io_uring_task *tctx = NULL; struct io_sq_data *sqd = NULL; @@ -10707,7 +10712,7 @@ static bool io_register_op_must_quiesce(int op) } } -static int io_ctx_quiesce(struct io_ring_ctx *ctx) +static __cold int io_ctx_quiesce(struct io_ring_ctx *ctx) { long ret; From c1e53a6988b9c83dd8bbc759414bc0f13ff1fe0c Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:55 +0100 Subject: [PATCH 140/255] io_uring: optimise io_free_batch_list() Delay reading the next node in io_free_batch_list(), allows the compiler to load the value a bit later improving register spilling in some cases. With gcc 11.1 it helped to move @task_refs variable from the stack to a register and optimises out a couple of per request instructions. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/cc9fdfb6f72a4e8bc9918a5e9f2d97869a263ae4.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e404c98e846a..1b007f77fe52 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2280,9 +2280,10 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); - node = req->comp_list.next; - if (!req_ref_put_and_test(req)) + if (!req_ref_put_and_test(req)) { + node = req->comp_list.next; continue; + } io_queue_next(req); io_dismantle_req(req); @@ -2294,6 +2295,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, task_refs = 0; } task_refs++; + node = req->comp_list.next; wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); } while (node); From d886e185a128a4f350c4df6471c0f403c2982ae8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:56 +0100 Subject: [PATCH 141/255] io_uring: control ->async_data with a REQ_F flag ->async_data is a slow path, so it won't matter much if we do the clean up inside io_clean_op(). Moreover, in many cases it's allocated together with setting one or more of IO_REQ_CLEAN_FLAGS flags, so it'd go through io_clean_op() anyway. Control ->async_data allocation with a new flag REQ_F_ASYNC_DATA, so we can do all the maintainence under io_req_needs_clean() fast check. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6892cf5883c459f36bda26f30ceb16742b20b84b.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 72 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 1b007f77fe52..aee8ecc09168 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -109,7 +109,8 @@ #define SQE_VALID_FLAGS (SQE_COMMON_FLAGS|IOSQE_BUFFER_SELECT|IOSQE_IO_DRAIN) #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \ - REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS) + REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \ + REQ_F_ASYNC_DATA) #define IO_TCTX_REFS_CACHE_NR (1U << 10) @@ -732,6 +733,7 @@ enum { REQ_F_CREDS_BIT, REQ_F_REFCOUNT_BIT, REQ_F_ARM_LTIMEOUT_BIT, + REQ_F_ASYNC_DATA_BIT, /* keep async read/write and isreg together and in order */ REQ_F_NOWAIT_READ_BIT, REQ_F_NOWAIT_WRITE_BIT, @@ -787,6 +789,8 @@ enum { REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT), /* there is a linked timeout that has to be armed */ REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT), + /* ->async_data allocated */ + REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT), }; struct async_poll { @@ -847,8 +851,6 @@ struct io_kiocb { struct io_completion compl; }; - /* opcode allocated if it needs to store data for async defer */ - void *async_data; u8 opcode; /* polled IO has completed */ u8 iopoll_completed; @@ -863,6 +865,8 @@ struct io_kiocb { u64 user_data; struct percpu_ref *fixed_rsrc_refs; + /* store used ubuf, so we can prevent reloading */ + struct io_mapped_ubuf *imu; /* used by request caches, completion batching and iopoll */ struct io_wq_work_node comp_list; @@ -872,8 +876,9 @@ struct io_kiocb { struct hlist_node hash_node; /* internal polling, see IORING_FEAT_FAST_POLL */ struct async_poll *apoll; - /* store used ubuf, so we can prevent reloading */ - struct io_mapped_ubuf *imu; + + /* opcode allocated if it needs to store data for async defer */ + void *async_data; struct io_wq_work work; /* custom credentials, valid IFF REQ_F_CREDS is set */ const struct cred *creds; @@ -1219,6 +1224,11 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task, return false; } +static inline bool req_has_async_data(struct io_kiocb *req) +{ + return req->flags & REQ_F_ASYNC_DATA; +} + static inline void req_set_fail(struct io_kiocb *req) { req->flags |= REQ_F_FAIL; @@ -1969,10 +1979,6 @@ static inline void io_dismantle_req(struct io_kiocb *req) io_put_file(req->file); if (req->fixed_rsrc_refs) percpu_ref_put(req->fixed_rsrc_refs); - if (req->async_data) { - kfree(req->async_data); - req->async_data = NULL; - } } static __cold void __io_free_req(struct io_kiocb *req) @@ -2566,7 +2572,7 @@ static bool io_resubmit_prep(struct io_kiocb *req) { struct io_async_rw *rw = req->async_data; - if (!rw) + if (!req_has_async_data(req)) return !io_req_prep_async(req); iov_iter_restore(&rw->iter, &rw->iter_state); return true; @@ -2878,7 +2884,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_async_rw *io = req->async_data; /* add previously done IO, if any */ - if (io && io->bytes_done > 0) { + if (req_has_async_data(req) && io->bytes_done > 0) { if (ret < 0) ret = io->bytes_done; else @@ -3257,7 +3263,11 @@ static inline bool io_alloc_async_data(struct io_kiocb *req) { WARN_ON_ONCE(!io_op_defs[req->opcode].async_size); req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL); - return req->async_data == NULL; + if (req->async_data) { + req->flags |= REQ_F_ASYNC_DATA; + return false; + } + return true; } static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, @@ -3266,7 +3276,7 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, { if (!force && !io_op_defs[req->opcode].needs_async_setup) return 0; - if (!req->async_data) { + if (!req_has_async_data(req)) { struct io_async_rw *iorw; if (io_alloc_async_data(req)) { @@ -3399,12 +3409,13 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; struct iov_iter __iter, *iter = &__iter; - struct io_async_rw *rw = req->async_data; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; struct iov_iter_state __state, *state; + struct io_async_rw *rw; ssize_t ret, ret2; - if (rw) { + if (req_has_async_data(req)) { + rw = req->async_data; iter = &rw->iter; state = &rw->iter_state; /* @@ -3534,12 +3545,13 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; struct kiocb *kiocb = &req->rw.kiocb; struct iov_iter __iter, *iter = &__iter; - struct io_async_rw *rw = req->async_data; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; struct iov_iter_state __state, *state; + struct io_async_rw *rw; ssize_t ret, ret2; - if (rw) { + if (req_has_async_data(req)) { + rw = req->async_data; iter = &rw->iter; state = &rw->iter_state; iov_iter_restore(iter, state); @@ -4708,8 +4720,9 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(!sock)) return -ENOTSOCK; - kmsg = req->async_data; - if (!kmsg) { + if (req_has_async_data(req)) { + kmsg = req->async_data; + } else { ret = io_sendmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -4925,8 +4938,9 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(!sock)) return -ENOTSOCK; - kmsg = req->async_data; - if (!kmsg) { + if (req_has_async_data(req)) { + kmsg = req->async_data; + } else { ret = io_recvmsg_copy_hdr(req, &iomsg); if (ret) return ret; @@ -5117,7 +5131,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags) int ret; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; - if (req->async_data) { + if (req_has_async_data(req)) { io = req->async_data; } else { ret = move_addr_to_kernel(req->connect.addr, @@ -5133,7 +5147,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_connect_file(req->file, &io->address, req->connect.addr_len, file_flags); if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { - if (req->async_data) + if (req_has_async_data(req)) return -EAGAIN; if (io_alloc_async_data(req)) { ret = -ENOMEM; @@ -5424,7 +5438,10 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake); req_ref_get(req); poll->wait.private = req; + *poll_ptr = poll; + if (req->opcode == IORING_OP_POLL_ADD) + req->flags |= REQ_F_ASYNC_DATA; } pt->nr_entries++; @@ -6086,7 +6103,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (unlikely(off && !req->ctx->off_timeout_used)) req->ctx->off_timeout_used = true; - if (!req->async_data && io_alloc_async_data(req)) + if (!req_has_async_data(req) && io_alloc_async_data(req)) return -ENOMEM; data = req->async_data; @@ -6395,7 +6412,7 @@ static int io_req_prep_async(struct io_kiocb *req) { if (!io_op_defs[req->opcode].needs_async_setup) return 0; - if (WARN_ON_ONCE(req->async_data)) + if (WARN_ON_ONCE(req_has_async_data(req))) return -EFAULT; if (io_alloc_async_data(req)) return -EAGAIN; @@ -6538,7 +6555,10 @@ static void io_clean_op(struct io_kiocb *req) } if (req->flags & REQ_F_CREDS) put_cred(req->creds); - + if (req->flags & REQ_F_ASYNC_DATA) { + kfree(req->async_data); + req->async_data = NULL; + } req->flags &= ~IO_REQ_CLEAN_FLAGS; } From d17e56eb4907c72054e63c71a2123d32b04ebd67 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:57 +0100 Subject: [PATCH 142/255] io_uring: remove struct io_completion We keep struct io_completion only as a temporal storage of cflags, Place it in io_kiocb, it's cleaner, removes extra bits and even might be used for future optimisations. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5299bd5c223204065464bd87a515d0e405316086.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index aee8ecc09168..c7dea155f43a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -684,11 +684,6 @@ struct io_hardlink { int flags; }; -struct io_completion { - struct file *file; - u32 cflags; -}; - struct io_async_connect { struct sockaddr_storage address; }; @@ -847,22 +842,20 @@ struct io_kiocb { struct io_mkdir mkdir; struct io_symlink symlink; struct io_hardlink hardlink; - /* use only after cleaning per-op data, see io_clean_op() */ - struct io_completion compl; }; u8 opcode; /* polled IO has completed */ u8 iopoll_completed; - u16 buf_index; + unsigned int flags; + + u64 user_data; u32 result; + u32 cflags; struct io_ring_ctx *ctx; - unsigned int flags; - atomic_t refs; struct task_struct *task; - u64 user_data; struct percpu_ref *fixed_rsrc_refs; /* store used ubuf, so we can prevent reloading */ @@ -870,13 +863,13 @@ struct io_kiocb { /* used by request caches, completion batching and iopoll */ struct io_wq_work_node comp_list; + atomic_t refs; struct io_kiocb *link; struct io_task_work io_task_work; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; /* internal polling, see IORING_FEAT_FAST_POLL */ struct async_poll *apoll; - /* opcode allocated if it needs to store data for async defer */ void *async_data; struct io_wq_work work; @@ -1831,11 +1824,8 @@ static inline bool io_req_needs_clean(struct io_kiocb *req) static inline void io_req_complete_state(struct io_kiocb *req, long res, unsigned int cflags) { - /* clean per-opcode space, because req->compl is aliased with it */ - if (io_req_needs_clean(req)) - io_clean_op(req); req->result = res; - req->compl.cflags = cflags; + req->cflags = cflags; req->flags |= REQ_F_COMPLETE_INLINE; } @@ -2321,7 +2311,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) comp_list); __io_cqring_fill_event(ctx, req->user_data, req->result, - req->compl.cflags); + req->cflags); } io_commit_cqring(ctx); spin_unlock(&ctx->completion_lock); From 867f8fa5aeb7fa51290bf3567ce2bbc45580a469 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:58 +0100 Subject: [PATCH 143/255] io_uring: inline io_req_needs_clean() There is only a single user of io_req_needs_clean() inline it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6111d0221ef4b439cad401e135dd6a5f990a0501.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c7dea155f43a..dc83effa8317 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1816,11 +1816,6 @@ static void io_req_complete_post(struct io_kiocb *req, long res, io_cqring_ev_posted(ctx); } -static inline bool io_req_needs_clean(struct io_kiocb *req) -{ - return req->flags & IO_REQ_CLEAN_FLAGS; -} - static inline void io_req_complete_state(struct io_kiocb *req, long res, unsigned int cflags) { @@ -1963,7 +1958,7 @@ static inline void io_dismantle_req(struct io_kiocb *req) { unsigned int flags = req->flags; - if (unlikely(io_req_needs_clean(req))) + if (unlikely(flags & IO_REQ_CLEAN_FLAGS)) io_clean_op(req); if (!(flags & REQ_F_FIXED_FILE)) io_put_file(req->file); From eb6e6f0690c846f7de46181bab3954c12c96e11e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:02:59 +0100 Subject: [PATCH 144/255] io_uring: inline io_poll_complete Inline io_poll_complete(), it's simple and doesn't have any particular purpose. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/933d7ee3e4450749a2d892235462c8f18d030293.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index dc83effa8317..815a7ec68dbc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5292,16 +5292,6 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) return !(flags & IORING_CQE_F_MORE); } -static inline bool io_poll_complete(struct io_kiocb *req, __poll_t mask) - __must_hold(&req->ctx->completion_lock) -{ - bool done; - - done = __io_poll_complete(req, mask); - io_commit_cqring(req->ctx); - return done; -} - static void io_poll_task_func(struct io_kiocb *req, bool *locked) { struct io_ring_ctx *ctx = req->ctx; @@ -5791,7 +5781,8 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) if (mask) { /* no async, we'd stolen it */ ipt.error = 0; - done = io_poll_complete(req, mask); + done = __io_poll_complete(req, mask); + io_commit_cqring(req->ctx); } spin_unlock(&ctx->completion_lock); From 54daa9b2d80ab35824464b35a99f716e1cdf2ccb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 4 Oct 2021 20:03:00 +0100 Subject: [PATCH 145/255] io_uring: correct fill events helpers types CQE result is a 32-bit integer, so the functions generating CQEs are better to accept not long but ints. Convert io_cqring_fill_event() and other helpers. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7ca6f15255e9117eae28adcac272744cae29b113.1633373302.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 815a7ec68dbc..44946711fe62 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1072,7 +1072,7 @@ static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data, - long res, unsigned int cflags); + s32 res, u32 cflags); static void io_put_req(struct io_kiocb *req); static void io_put_req_deferred(struct io_kiocb *req); static void io_dismantle_req(struct io_kiocb *req); @@ -1730,7 +1730,7 @@ static inline void io_get_task_refs(int nr) } static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, - long res, unsigned int cflags) + s32 res, u32 cflags) { struct io_overflow_cqe *ocqe; @@ -1758,7 +1758,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, } static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data, - long res, unsigned int cflags) + s32 res, u32 cflags) { struct io_uring_cqe *cqe; @@ -1781,13 +1781,13 @@ static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data /* not as hot to bloat with inlining */ static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data, - long res, unsigned int cflags) + s32 res, u32 cflags) { return __io_cqring_fill_event(ctx, user_data, res, cflags); } -static void io_req_complete_post(struct io_kiocb *req, long res, - unsigned int cflags) +static void io_req_complete_post(struct io_kiocb *req, s32 res, + u32 cflags) { struct io_ring_ctx *ctx = req->ctx; @@ -1816,8 +1816,8 @@ static void io_req_complete_post(struct io_kiocb *req, long res, io_cqring_ev_posted(ctx); } -static inline void io_req_complete_state(struct io_kiocb *req, long res, - unsigned int cflags) +static inline void io_req_complete_state(struct io_kiocb *req, s32 res, + u32 cflags) { req->result = res; req->cflags = cflags; @@ -1825,7 +1825,7 @@ static inline void io_req_complete_state(struct io_kiocb *req, long res, } static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, - long res, unsigned cflags) + s32 res, u32 cflags) { if (issue_flags & IO_URING_F_COMPLETE_DEFER) io_req_complete_state(req, res, cflags); @@ -1833,12 +1833,12 @@ static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags, io_req_complete_post(req, res, cflags); } -static inline void io_req_complete(struct io_kiocb *req, long res) +static inline void io_req_complete(struct io_kiocb *req, s32 res) { __io_req_complete(req, 0, res, 0); } -static void io_req_complete_failed(struct io_kiocb *req, long res) +static void io_req_complete_failed(struct io_kiocb *req, s32 res) { req_set_fail(req); io_req_complete_post(req, res, 0); @@ -2618,7 +2618,7 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) static void io_req_task_complete(struct io_kiocb *req, bool *locked) { unsigned int cflags = io_put_rw_kbuf(req); - long res = req->result; + int res = req->result; if (*locked) { io_req_complete_state(req, res, cflags); From 6d63416dc57eb27a3d35e7b7526e9915479d7eff Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 6 Oct 2021 16:06:46 +0100 Subject: [PATCH 146/255] io_uring: optimise plugging Plugging is only needed with requests that also need a file, so hide plugging under a ->needs_file check. Also, place ->needs_file and ->plug bits into the same byte of io_op_defs, it may matter for compilers, e.g. only with the change a tested one decided to optimise two memory testb into a more with two register testb. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1600d1287bb7d16451d4ef3343252787a5314927.1633532552.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 44946711fe62..7759fabf9ae0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -894,12 +894,12 @@ struct io_defer_entry { struct io_op_def { /* needs req->file assigned */ unsigned needs_file : 1; + /* should block plug */ + unsigned plug : 1; /* hash wq insertion if file is a regular file */ unsigned hash_reg_file : 1; /* unbound wq insertion if file is a non-regular file */ unsigned unbound_nonreg_file : 1; - /* opcode is not supported by this kernel */ - unsigned not_supported : 1; /* set if opcode supports polled "wait" */ unsigned pollin : 1; unsigned pollout : 1; @@ -907,8 +907,8 @@ struct io_op_def { unsigned buffer_select : 1; /* do prep async if is going to be punted */ unsigned needs_async_setup : 1; - /* should block plug */ - unsigned plug : 1; + /* opcode is not supported by this kernel */ + unsigned not_supported : 1; /* size of async data needed, if any */ unsigned short async_size; }; @@ -6978,7 +6978,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, const struct io_uring_sqe *sqe) __must_hold(&ctx->uring_lock) { - struct io_submit_state *state; unsigned int sqe_flags; int personality; @@ -7025,19 +7024,20 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, get_cred(req->creds); req->flags |= REQ_F_CREDS; } - state = &ctx->submit_state; - - /* - * Plug now if we have more than 1 IO left after this, and the target - * is potentially a read/write to block based storage. - */ - if (state->need_plug && io_op_defs[req->opcode].plug) { - blk_start_plug(&state->plug); - state->plug_started = true; - state->need_plug = false; - } if (io_op_defs[req->opcode].needs_file) { + struct io_submit_state *state = &ctx->submit_state; + + /* + * Plug now if we have more than 2 IO left after this, and the + * target is potentially a read/write to block based storage. + */ + if (state->need_plug && io_op_defs[req->opcode].plug) { + state->plug_started = true; + state->need_plug = false; + blk_start_plug(&state->plug); + } + req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), (sqe_flags & IOSQE_FIXED_FILE)); if (unlikely(!req->file)) From 756ab7c0ec71e5e1e8e6ea11af53324b23d5efd2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 6 Oct 2021 16:06:47 +0100 Subject: [PATCH 147/255] io_uring: safer fallback_work free Add extra wq flushing for fallback_work, that's not necessary but safer if invariants of io_fallback_req_func() change. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/24179419d6748516299600bc914f50b9e0b02275.1633532552.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7759fabf9ae0..90bef813556e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1262,7 +1262,6 @@ static __cold void io_fallback_req_func(struct work_struct *work) mutex_unlock(&ctx->uring_lock); } percpu_ref_put(&ctx->refs); - } static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) @@ -9212,6 +9211,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) if (ctx->rsrc_backup_node) io_rsrc_node_destroy(ctx->rsrc_backup_node); flush_delayed_work(&ctx->rsrc_put_work); + flush_delayed_work(&ctx->fallback_work); WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)); WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist)); From 5a158c6b0d033893cc80c28b182e1207253768a5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 6 Oct 2021 16:06:48 +0100 Subject: [PATCH 148/255] io_uring: reshuffle io_submit_state bits struct io_submit_state's ->free_list and ->link are hotter and smaller than ->plug, place them first. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6ad3c15849f50b27ad012c042c73e6e069d22df7.1633532552.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 90bef813556e..04d6e35ea0df 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -308,19 +308,15 @@ struct io_submit_link { }; struct io_submit_state { - struct blk_plug plug; + /* inline/task_work completion list, under ->uring_lock */ + struct io_wq_work_node free_list; + /* batch completion logic */ + struct io_wq_work_list compl_reqs; struct io_submit_link link; bool plug_started; bool need_plug; - - /* - * Batch completion logic - */ - struct io_wq_work_list compl_reqs; - - /* inline/task_work completion list, under ->uring_lock */ - struct io_wq_work_node free_list; + struct blk_plug plug; }; struct io_ring_ctx { From 4a04d1d14831d31f2cd0e31eb1568cc9c1be0095 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 6 Oct 2021 16:06:49 +0100 Subject: [PATCH 149/255] io_uring: optimise out req->opcode reloading Looking at the assembly, the compiler decided to reload req->opcode in io_op_defs[opcode].needs_file instead of one it had in a register, so store it in a temp variable so it can be optimised out. Also move the personality block later, it's better for spilling/etc. as it only depends on @sqe, which we're keeping anyway. By the way, zero req->opcode if it over IORING_OP_LAST, not a problem, at the moment but is safer. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6ba869f5f8b7b0f991c87fdf089f0abf87cbe06b.1633532552.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 04d6e35ea0df..7918a320104d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6975,9 +6975,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { unsigned int sqe_flags; int personality; + u8 opcode; /* req is partially pre-initialised, see io_preinit_req() */ - req->opcode = READ_ONCE(sqe->opcode); + req->opcode = opcode = READ_ONCE(sqe->opcode); /* same numerical values with corresponding REQ_F_*, safe to copy */ req->flags = sqe_flags = READ_ONCE(sqe->flags); req->user_data = READ_ONCE(sqe->user_data); @@ -6985,14 +6986,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->fixed_rsrc_refs = NULL; req->task = current; - if (unlikely(req->opcode >= IORING_OP_LAST)) + if (unlikely(opcode >= IORING_OP_LAST)) { + req->opcode = 0; return -EINVAL; + } if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) { /* enforce forwards compatibility on users */ if (sqe_flags & ~SQE_VALID_FLAGS) return -EINVAL; if ((sqe_flags & IOSQE_BUFFER_SELECT) && - !io_op_defs[req->opcode].buffer_select) + !io_op_defs[opcode].buffer_select) return -EOPNOTSUPP; if (sqe_flags & IOSQE_IO_DRAIN) io_init_req_drain(req); @@ -7011,23 +7014,14 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, } } - personality = READ_ONCE(sqe->personality); - if (personality) { - req->creds = xa_load(&ctx->personalities, personality); - if (!req->creds) - return -EINVAL; - get_cred(req->creds); - req->flags |= REQ_F_CREDS; - } - - if (io_op_defs[req->opcode].needs_file) { + if (io_op_defs[opcode].needs_file) { struct io_submit_state *state = &ctx->submit_state; /* * Plug now if we have more than 2 IO left after this, and the * target is potentially a read/write to block based storage. */ - if (state->need_plug && io_op_defs[req->opcode].plug) { + if (state->need_plug && io_op_defs[opcode].plug) { state->plug_started = true; state->need_plug = false; blk_start_plug(&state->plug); @@ -7039,6 +7033,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return -EBADF; } + personality = READ_ONCE(sqe->personality); + if (personality) { + req->creds = xa_load(&ctx->personalities, personality); + if (!req->creds) + return -EINVAL; + get_cred(req->creds); + req->flags |= REQ_F_CREDS; + } + return io_req_prep(req, sqe); } From 0cd3e3ddb4f60062c401929b2005eb7ff6399a4d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 6 Oct 2021 16:06:50 +0100 Subject: [PATCH 150/255] io_uring: remove extra io_ring_exit_work wake up task_work_add() takes care of waking up the thread, remove useless wake_up_process(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/de9a71ee255112dcaed3b5d426be24934e74722c.1633532552.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7918a320104d..383b8f61359d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -9367,7 +9367,6 @@ static __cold void io_ring_exit_work(struct work_struct *work) ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL); if (WARN_ON_ONCE(ret)) continue; - wake_up_process(node->task); mutex_unlock(&ctx->uring_lock); wait_for_completion(&exit.completion); From def77acf4396a850544b41623d7720ea3a2674a9 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 12 Oct 2021 15:02:14 +0100 Subject: [PATCH 151/255] io_uring: fix io_free_batch_list races [ 158.514382] WARNING: CPU: 5 PID: 15251 at fs/io_uring.c:1141 io_free_batch_list+0x269/0x360 [ 158.514426] RIP: 0010:io_free_batch_list+0x269/0x360 [ 158.514437] Call Trace: [ 158.514440] __io_submit_flush_completions+0xde/0x180 [ 158.514444] tctx_task_work+0x14a/0x220 [ 158.514447] task_work_run+0x64/0xa0 [ 158.514448] __do_sys_io_uring_enter+0x7c/0x970 [ 158.514450] __x64_sys_io_uring_enter+0x22/0x30 [ 158.514451] do_syscall_64+0x43/0x90 [ 158.514453] entry_SYSCALL_64_after_hwframe+0x44/0xae We should not touch request internals including req->comp_list.next after putting our ref if it's not final, e.g. we can start freeing requests from the free cache. Fixed: 62ca9cb93e7f8 ("io_uring: optimise io_free_batch_list()") Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b1f4df38fbb8f111f52911a02fd418d0283a4e6f.1634047298.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 383b8f61359d..141bfe720ce2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2266,9 +2266,10 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); - if (!req_ref_put_and_test(req)) { + if (unlikely(req->flags & REQ_F_REFCOUNT)) { node = req->comp_list.next; - continue; + if (!req_ref_put_and_test(req)) + continue; } io_queue_next(req); From a46be971edb69fe4b1dcc4359c3ddf9127629dab Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Oct 2021 23:14:40 +0100 Subject: [PATCH 152/255] io_uring: optimise io_req_set_rsrc_node() io_req_set_rsrc_node() reloads loads req->ctx, however it's already in registers in all use cases, so better to pass it as a parameter. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/67a25557b8a51e90bfd578447a6f1671911b05ae.1633817310.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 141bfe720ce2..6486ebf31866 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1174,10 +1174,9 @@ static inline void io_req_set_refcount(struct io_kiocb *req) __io_req_set_refcount(req, 1); } -static inline void io_req_set_rsrc_node(struct io_kiocb *req) +static inline void io_req_set_rsrc_node(struct io_kiocb *req, + struct io_ring_ctx *ctx) { - struct io_ring_ctx *ctx = req->ctx; - if (!req->fixed_rsrc_refs) { req->fixed_rsrc_refs = &ctx->rsrc_node->refs; percpu_ref_get(req->fixed_rsrc_refs); @@ -2828,7 +2827,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (req->opcode == IORING_OP_READ_FIXED || req->opcode == IORING_OP_WRITE_FIXED) { req->imu = NULL; - io_req_set_rsrc_node(req); + io_req_set_rsrc_node(req, ctx); } req->rw.addr = READ_ONCE(sqe->addr); @@ -6757,7 +6756,7 @@ static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, file_ptr &= ~FFS_MASK; /* mask in overlapping REQ_F and FFS bits */ req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT); - io_req_set_rsrc_node(req); + io_req_set_rsrc_node(req, ctx); return file; } From ab409402478462b5da007bfc46d165587c3adfc3 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Oct 2021 23:14:41 +0100 Subject: [PATCH 153/255] io_uring: optimise rsrc referencing Apparently, percpu_ref_put/get() are expensive enough if done per request, get them in a batch and cache on the submission side to avoid getting it over and over again. Also, if we're completing under uring_lock, return refs back into the cache instead of perfcpu_ref_put(). Pretty similar to how we do tctx->cached_refs accounting, but fall back to normal putting when we already changed a rsrc node by the time of free. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b40d8c5bc77d3c9550df8a319117a374ac85f8f4.1633817310.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6486ebf31866..6859dae218f9 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -360,6 +360,7 @@ struct io_ring_ctx { * uring_lock, and updated through io_uring_register(2) */ struct io_rsrc_node *rsrc_node; + int rsrc_cached_refs; struct io_file_table file_table; unsigned nr_user_files; unsigned nr_user_bufs; @@ -1174,12 +1175,52 @@ static inline void io_req_set_refcount(struct io_kiocb *req) __io_req_set_refcount(req, 1); } +#define IO_RSRC_REF_BATCH 100 + +static inline void io_req_put_rsrc_locked(struct io_kiocb *req, + struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + struct percpu_ref *ref = req->fixed_rsrc_refs; + + if (ref) { + if (ref == &ctx->rsrc_node->refs) + ctx->rsrc_cached_refs++; + else + percpu_ref_put(ref); + } +} + +static inline void io_req_put_rsrc(struct io_kiocb *req, struct io_ring_ctx *ctx) +{ + if (req->fixed_rsrc_refs) + percpu_ref_put(req->fixed_rsrc_refs); +} + +static __cold void io_rsrc_refs_drop(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + if (ctx->rsrc_cached_refs) { + percpu_ref_put_many(&ctx->rsrc_node->refs, ctx->rsrc_cached_refs); + ctx->rsrc_cached_refs = 0; + } +} + +static void io_rsrc_refs_refill(struct io_ring_ctx *ctx) + __must_hold(&ctx->uring_lock) +{ + ctx->rsrc_cached_refs += IO_RSRC_REF_BATCH; + percpu_ref_get_many(&ctx->rsrc_node->refs, IO_RSRC_REF_BATCH); +} + static inline void io_req_set_rsrc_node(struct io_kiocb *req, struct io_ring_ctx *ctx) { if (!req->fixed_rsrc_refs) { req->fixed_rsrc_refs = &ctx->rsrc_node->refs; - percpu_ref_get(req->fixed_rsrc_refs); + ctx->rsrc_cached_refs--; + if (unlikely(ctx->rsrc_cached_refs < 0)) + io_rsrc_refs_refill(ctx); } } @@ -1800,6 +1841,7 @@ static void io_req_complete_post(struct io_kiocb *req, s32 res, req->link = NULL; } } + io_req_put_rsrc(req, ctx); io_dismantle_req(req); io_put_task(req->task, 1); wq_list_add_head(&req->comp_list, &ctx->locked_free_list); @@ -1956,14 +1998,13 @@ static inline void io_dismantle_req(struct io_kiocb *req) io_clean_op(req); if (!(flags & REQ_F_FIXED_FILE)) io_put_file(req->file); - if (req->fixed_rsrc_refs) - percpu_ref_put(req->fixed_rsrc_refs); } static __cold void __io_free_req(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; + io_req_put_rsrc(req, ctx); io_dismantle_req(req); io_put_task(req->task, 1); @@ -2271,6 +2312,7 @@ static void io_free_batch_list(struct io_ring_ctx *ctx, continue; } + io_req_put_rsrc_locked(req, ctx); io_queue_next(req); io_dismantle_req(req); @@ -7630,10 +7672,13 @@ static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx) static void io_rsrc_node_switch(struct io_ring_ctx *ctx, struct io_rsrc_data *data_to_kill) + __must_hold(&ctx->uring_lock) { WARN_ON_ONCE(!ctx->rsrc_backup_node); WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node); + io_rsrc_refs_drop(ctx); + if (data_to_kill) { struct io_rsrc_node *rsrc_node = ctx->rsrc_node; @@ -9187,6 +9232,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) ctx->mm_account = NULL; } + io_rsrc_refs_drop(ctx); /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ io_wait_rsrc_data(ctx->buf_data); io_wait_rsrc_data(ctx->file_data); From 04f34081c5de35749274cf3bea8d8eb1c79b6ad1 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:12 +0100 Subject: [PATCH 154/255] io_uring: consistent typing for issue_flags Some of the functions keep issue_flags as int, change those to unsigned. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/04ad43797783bc9cc7567f287ab545518f8e8cf2.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 6859dae218f9..094cd5f49214 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3786,7 +3786,7 @@ static int io_mkdirat_prep(struct io_kiocb *req, return 0; } -static int io_mkdirat(struct io_kiocb *req, int issue_flags) +static int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) { struct io_mkdir *mkd = &req->mkdir; int ret; @@ -3835,7 +3835,7 @@ static int io_symlinkat_prep(struct io_kiocb *req, return 0; } -static int io_symlinkat(struct io_kiocb *req, int issue_flags) +static int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) { struct io_symlink *sl = &req->symlink; int ret; @@ -3885,7 +3885,7 @@ static int io_linkat_prep(struct io_kiocb *req, return 0; } -static int io_linkat(struct io_kiocb *req, int issue_flags) +static int io_linkat(struct io_kiocb *req, unsigned int issue_flags) { struct io_hardlink *lnk = &req->hardlink; int ret; From f80a50a632d6f5c64729563892b0deb95d563f6d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:13 +0100 Subject: [PATCH 155/255] io_uring: prioritise read success path over fails Rearrange io_read return handling so first we expect it completing successfully and only then checking for errors, which is a colder path. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/c91c7c2da11815ec8b04b5d872f60dc4cde662c5.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 094cd5f49214..86ef803db27a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3487,7 +3487,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) ret = 0; } else if (ret == -EIOCBQUEUED) { goto out_free; - } else if (ret <= 0 || ret == req->result || !force_nonblock || + } else if (ret == req->result || ret <= 0 || !force_nonblock || (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) { /* read all, failed, already did sync or don't want to retry */ goto done; From 258f3a7f84d11ce1d6cb1f60fe524bb37f7c1019 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:14 +0100 Subject: [PATCH 156/255] io_uring: optimise rw comletion handlers Don't override req->result in io_complete_rw_iopoll() when it's already of the same value, we have an if just above it, so move the assignment there. Also, add one simle unlikely() in __io_complete_rw_common(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8dfeb4f84026a20172bcf82c05010abe955874ae.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 86ef803db27a..18a91d268fb6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2640,7 +2640,7 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) { if (req->rw.kiocb.ki_flags & IOCB_WRITE) kiocb_end_write(req); - if (res != req->result) { + if (unlikely(res != req->result)) { if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) { req->flags |= REQ_F_REISSUE; @@ -2695,9 +2695,9 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) req->flags |= REQ_F_REISSUE; return; } + req->result = res; } - req->result = res; /* order with io_iopoll_complete() checking ->iopoll_completed */ smp_store_release(&req->iopoll_completed, 1); } From 538941e2681c76ba6d6a7dd28ce691a9e02a9bdf Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:15 +0100 Subject: [PATCH 157/255] io_uring: encapsulate rw state Add a new struct io_rw_state storing all iov related bits: fast iov, iterator and iterator state. Not much changes here, simply convert struct io_async_rw to use it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/e8245ffcb568b228a009ec1eb79c993c813679f1.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 18a91d268fb6..47365610de5d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -694,11 +694,15 @@ struct io_async_msghdr { struct sockaddr_storage addr; }; -struct io_async_rw { +struct io_rw_state { struct iovec fast_iov[UIO_FASTIOV]; - const struct iovec *free_iovec; struct iov_iter iter; struct iov_iter_state iter_state; +}; + +struct io_async_rw { + struct io_rw_state s; + const struct iovec *free_iovec; size_t bytes_done; struct wait_page_queue wpq; }; @@ -2596,7 +2600,7 @@ static bool io_resubmit_prep(struct io_kiocb *req) if (!req_has_async_data(req)) return !io_req_prep_async(req); - iov_iter_restore(&rw->iter, &rw->iter_state); + iov_iter_restore(&rw->s.iter, &rw->s.iter_state); return true; } @@ -3259,7 +3263,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, { struct io_async_rw *rw = req->async_data; - memcpy(&rw->iter, iter, sizeof(*iter)); + memcpy(&rw->s.iter, iter, sizeof(*iter)); rw->free_iovec = iovec; rw->bytes_done = 0; /* can only be fixed buffers, no need to do anything */ @@ -3268,13 +3272,13 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec, if (!iovec) { unsigned iov_off = 0; - rw->iter.iov = rw->fast_iov; + rw->s.iter.iov = rw->s.fast_iov; if (iter->iov != fast_iov) { iov_off = iter->iov - fast_iov; - rw->iter.iov += iov_off; + rw->s.iter.iov += iov_off; } - if (rw->fast_iov != fast_iov) - memcpy(rw->fast_iov + iov_off, fast_iov + iov_off, + if (rw->s.fast_iov != fast_iov) + memcpy(rw->s.fast_iov + iov_off, fast_iov + iov_off, sizeof(struct iovec) * iter->nr_segs); } else { req->flags |= REQ_F_NEED_CLEANUP; @@ -3309,7 +3313,7 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, io_req_map_rw(req, iovec, fast_iov, iter); iorw = req->async_data; /* we've copied and mapped the iter, ensure state is saved */ - iov_iter_save_state(&iorw->iter, &iorw->iter_state); + iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); } return 0; } @@ -3317,10 +3321,10 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, static inline int io_rw_prep_async(struct io_kiocb *req, int rw) { struct io_async_rw *iorw = req->async_data; - struct iovec *iov = iorw->fast_iov; + struct iovec *iov = iorw->s.fast_iov; int ret; - ret = io_import_iovec(rw, req, &iov, &iorw->iter, false); + ret = io_import_iovec(rw, req, &iov, &iorw->s.iter, false); if (unlikely(ret < 0)) return ret; @@ -3328,7 +3332,7 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) iorw->free_iovec = iov; if (iov) req->flags |= REQ_F_NEED_CLEANUP; - iov_iter_save_state(&iorw->iter, &iorw->iter_state); + iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); return 0; } @@ -3438,8 +3442,8 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) if (req_has_async_data(req)) { rw = req->async_data; - iter = &rw->iter; - state = &rw->iter_state; + iter = &rw->s.iter; + state = &rw->s.iter_state; /* * We come here from an earlier attempt, restore our state to * match in case it doesn't. It's cheap enough that we don't @@ -3510,9 +3514,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) * Now use our persistent iterator and state, if we aren't already. * We've restored and mapped the iter to match. */ - if (iter != &rw->iter) { - iter = &rw->iter; - state = &rw->iter_state; + if (iter != &rw->s.iter) { + iter = &rw->s.iter; + state = &rw->s.iter_state; } do { @@ -3574,8 +3578,8 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) if (req_has_async_data(req)) { rw = req->async_data; - iter = &rw->iter; - state = &rw->iter_state; + iter = &rw->s.iter; + state = &rw->s.iter_state; iov_iter_restore(iter, state); iovec = NULL; } else { From c88598a92a587109d915b4d97831bcea774c8b6f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:16 +0100 Subject: [PATCH 158/255] io_uring: optimise read/write iov state storing Currently io_read() and io_write() keep separate pointers to an iter and to struct iov_iter_state, which is not great for register spilling and requires more on-stack copies. They are both either on-stack or in req->async_data at the same time, so use struct io_rw_state and keep a pointer only to it, so having all the state with just one pointer. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5c5e7ffd7dc25fc35075c70411ba99df72f237fa.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 81 ++++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 47365610de5d..bb5cd8715f61 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -695,9 +695,9 @@ struct io_async_msghdr { }; struct io_rw_state { - struct iovec fast_iov[UIO_FASTIOV]; struct iov_iter iter; struct iov_iter_state iter_state; + struct iovec fast_iov[UIO_FASTIOV]; }; struct io_async_rw { @@ -3297,8 +3297,7 @@ static inline bool io_alloc_async_data(struct io_kiocb *req) } static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, - const struct iovec *fast_iov, - struct iov_iter *iter, bool force) + struct io_rw_state *s, bool force) { if (!force && !io_op_defs[req->opcode].needs_async_setup) return 0; @@ -3310,7 +3309,7 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, return -ENOMEM; } - io_req_map_rw(req, iovec, fast_iov, iter); + io_req_map_rw(req, iovec, s->fast_iov, &s->iter); iorw = req->async_data; /* we've copied and mapped the iter, ensure state is saved */ iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); @@ -3432,33 +3431,33 @@ static bool need_read_all(struct io_kiocb *req) static int io_read(struct io_kiocb *req, unsigned int issue_flags) { - struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; + struct io_rw_state __s, *s; + struct iovec *iovec; struct kiocb *kiocb = &req->rw.kiocb; - struct iov_iter __iter, *iter = &__iter; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; - struct iov_iter_state __state, *state; struct io_async_rw *rw; ssize_t ret, ret2; if (req_has_async_data(req)) { rw = req->async_data; - iter = &rw->s.iter; - state = &rw->s.iter_state; + s = &rw->s; /* * We come here from an earlier attempt, restore our state to * match in case it doesn't. It's cheap enough that we don't * need to make this conditional. */ - iov_iter_restore(iter, state); + iov_iter_restore(&s->iter, &s->iter_state); iovec = NULL; } else { - ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock); + s = &__s; + iovec = s->fast_iov; + ret = io_import_iovec(READ, req, &iovec, &s->iter, !force_nonblock); if (ret < 0) return ret; - state = &__state; - iov_iter_save_state(iter, state); + + iov_iter_save_state(&s->iter, &s->iter_state); } - req->result = iov_iter_count(iter); + req->result = iov_iter_count(&s->iter); /* Ensure we clear previously set non-block flag */ if (!force_nonblock) @@ -3468,7 +3467,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) /* If the file doesn't support async, just async punt */ if (force_nonblock && !io_file_supports_nowait(req, READ)) { - ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true); + ret = io_setup_async_rw(req, iovec, s, true); return ret ?: -EAGAIN; } @@ -3478,7 +3477,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) return ret; } - ret = io_iter_do_read(req, iter); + ret = io_iter_do_read(req, &s->iter); if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { req->flags &= ~REQ_F_REISSUE; @@ -3502,22 +3501,19 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) * untouched in case of error. Restore it and we'll advance it * manually if we need to. */ - iov_iter_restore(iter, state); + iov_iter_restore(&s->iter, &s->iter_state); - ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true); + ret2 = io_setup_async_rw(req, iovec, s, true); if (ret2) return ret2; iovec = NULL; rw = req->async_data; + s = &rw->s; /* * Now use our persistent iterator and state, if we aren't already. * We've restored and mapped the iter to match. */ - if (iter != &rw->s.iter) { - iter = &rw->s.iter; - state = &rw->s.iter_state; - } do { /* @@ -3525,11 +3521,11 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) * above or inside this loop. Advance the iter by the bytes * that were consumed. */ - iov_iter_advance(iter, ret); - if (!iov_iter_count(iter)) + iov_iter_advance(&s->iter, ret); + if (!iov_iter_count(&s->iter)) break; rw->bytes_done += ret; - iov_iter_save_state(iter, state); + iov_iter_save_state(&s->iter, &s->iter_state); /* if we can retry, do so with the callbacks armed */ if (!io_rw_should_retry(req)) { @@ -3543,12 +3539,12 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) * desired page gets unlocked. We can also get a partial read * here, and if we do, then just retry at the new offset. */ - ret = io_iter_do_read(req, iter); + ret = io_iter_do_read(req, &s->iter); if (ret == -EIOCBQUEUED) return 0; /* we got some bytes, but not all. retry. */ kiocb->ki_flags &= ~IOCB_WAITQ; - iov_iter_restore(iter, state); + iov_iter_restore(&s->iter, &s->iter_state); } while (ret > 0); done: kiocb_done(kiocb, ret, issue_flags); @@ -3568,28 +3564,27 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_write(struct io_kiocb *req, unsigned int issue_flags) { - struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; - struct kiocb *kiocb = &req->rw.kiocb; - struct iov_iter __iter, *iter = &__iter; - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; - struct iov_iter_state __state, *state; + struct io_rw_state __s, *s; struct io_async_rw *rw; + struct iovec *iovec; + struct kiocb *kiocb = &req->rw.kiocb; + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ssize_t ret, ret2; if (req_has_async_data(req)) { rw = req->async_data; - iter = &rw->s.iter; - state = &rw->s.iter_state; - iov_iter_restore(iter, state); + s = &rw->s; + iov_iter_restore(&s->iter, &s->iter_state); iovec = NULL; } else { - ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock); + s = &__s; + iovec = s->fast_iov; + ret = io_import_iovec(WRITE, req, &iovec, &s->iter, !force_nonblock); if (ret < 0) return ret; - state = &__state; - iov_iter_save_state(iter, state); + iov_iter_save_state(&s->iter, &s->iter_state); } - req->result = iov_iter_count(iter); + req->result = iov_iter_count(&s->iter); /* Ensure we clear previously set non-block flag */ if (!force_nonblock) @@ -3625,9 +3620,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) kiocb->ki_flags |= IOCB_WRITE; if (req->file->f_op->write_iter) - ret2 = call_write_iter(req->file, kiocb, iter); + ret2 = call_write_iter(req->file, kiocb, &s->iter); else if (req->file->f_op->write) - ret2 = loop_rw_iter(WRITE, req, iter); + ret2 = loop_rw_iter(WRITE, req, &s->iter); else ret2 = -EINVAL; @@ -3653,8 +3648,8 @@ done: kiocb_done(kiocb, ret2, issue_flags); } else { copy_iov: - iov_iter_restore(iter, state); - ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false); + iov_iter_restore(&s->iter, &s->iter_state); + ret = io_setup_async_rw(req, iovec, s, false); return ret ?: -EAGAIN; } out_free: From 51aac424aef980a6238d4907a5bb5a411fa926eb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:17 +0100 Subject: [PATCH 159/255] io_uring: optimise io_import_iovec nonblock passing First, change IO_URING_F_NONBLOCK to take sign bit of the int, so checking for it can be turned into test + sign-based-jump, makes the binary smaller and may be faster. Then, instead of passing need_lock boolean into io_import_iovec() just give it issue_flags, which is already stored somewhere. Saves some space on stack, a couple of test + cmov operations and other conversions. note: we still leave force_nonblock = issue_flags & IO_URING_F_NONBLOCK variable, but it's optimised out by the compiler into testing issue_flags directly. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/ee96547e692f6c975c229cd82fc721679571a734.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bb5cd8715f61..0a0f84b28eba 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -198,8 +198,9 @@ struct io_rings { }; enum io_uring_cmd_flags { - IO_URING_F_NONBLOCK = 1, - IO_URING_F_COMPLETE_DEFER = 2, + IO_URING_F_COMPLETE_DEFER = 1, + /* int's last bit, sign checks are usually faster than a bit test */ + IO_URING_F_NONBLOCK = INT_MIN, }; struct io_mapped_ubuf { @@ -3037,10 +3038,11 @@ static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock) } static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, - int bgid, bool needs_lock) + int bgid, unsigned int issue_flags) { struct io_buffer *kbuf = req->kbuf; struct io_buffer *head; + bool needs_lock = !(issue_flags & IO_URING_F_NONBLOCK); if (req->flags & REQ_F_BUFFER_SELECTED) return kbuf; @@ -3072,13 +3074,13 @@ static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, } static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, - bool needs_lock) + unsigned int issue_flags) { struct io_buffer *kbuf; u16 bgid; bgid = req->buf_index; - kbuf = io_buffer_select(req, len, bgid, needs_lock); + kbuf = io_buffer_select(req, len, bgid, issue_flags); if (IS_ERR(kbuf)) return kbuf; return u64_to_user_ptr(kbuf->addr); @@ -3086,7 +3088,7 @@ static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, #ifdef CONFIG_COMPAT static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, - bool needs_lock) + unsigned int issue_flags) { struct compat_iovec __user *uiov; compat_ssize_t clen; @@ -3102,7 +3104,7 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, return -EINVAL; len = clen; - buf = io_rw_buffer_select(req, &len, needs_lock); + buf = io_rw_buffer_select(req, &len, issue_flags); if (IS_ERR(buf)) return PTR_ERR(buf); iov[0].iov_base = buf; @@ -3112,7 +3114,7 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, #endif static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, - bool needs_lock) + unsigned int issue_flags) { struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr); void __user *buf; @@ -3124,7 +3126,7 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, len = iov[0].iov_len; if (len < 0) return -EINVAL; - buf = io_rw_buffer_select(req, &len, needs_lock); + buf = io_rw_buffer_select(req, &len, issue_flags); if (IS_ERR(buf)) return PTR_ERR(buf); iov[0].iov_base = buf; @@ -3133,7 +3135,7 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, } static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, - bool needs_lock) + unsigned int issue_flags) { if (req->flags & REQ_F_BUFFER_SELECTED) { struct io_buffer *kbuf = req->kbuf; @@ -3147,14 +3149,14 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, #ifdef CONFIG_COMPAT if (req->ctx->compat) - return io_compat_import(req, iov, needs_lock); + return io_compat_import(req, iov, issue_flags); #endif - return __io_iov_buffer_select(req, iov, needs_lock); + return __io_iov_buffer_select(req, iov, issue_flags); } static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, - struct iov_iter *iter, bool needs_lock) + struct iov_iter *iter, unsigned int issue_flags) { void __user *buf = u64_to_user_ptr(req->rw.addr); size_t sqe_len = req->rw.len; @@ -3172,7 +3174,7 @@ static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { if (req->flags & REQ_F_BUFFER_SELECT) { - buf = io_rw_buffer_select(req, &sqe_len, needs_lock); + buf = io_rw_buffer_select(req, &sqe_len, issue_flags); if (IS_ERR(buf)) return PTR_ERR(buf); req->rw.len = sqe_len; @@ -3184,7 +3186,7 @@ static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, } if (req->flags & REQ_F_BUFFER_SELECT) { - ret = io_iov_buffer_select(req, *iovec, needs_lock); + ret = io_iov_buffer_select(req, *iovec, issue_flags); if (!ret) iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); *iovec = NULL; @@ -3323,7 +3325,8 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) struct iovec *iov = iorw->s.fast_iov; int ret; - ret = io_import_iovec(rw, req, &iov, &iorw->s.iter, false); + /* submission path, ->uring_lock should already be taken */ + ret = io_import_iovec(rw, req, &iov, &iorw->s.iter, IO_URING_F_NONBLOCK); if (unlikely(ret < 0)) return ret; @@ -3451,7 +3454,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) } else { s = &__s; iovec = s->fast_iov; - ret = io_import_iovec(READ, req, &iovec, &s->iter, !force_nonblock); + ret = io_import_iovec(READ, req, &iovec, &s->iter, issue_flags); if (ret < 0) return ret; @@ -3579,7 +3582,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) } else { s = &__s; iovec = s->fast_iov; - ret = io_import_iovec(WRITE, req, &iovec, &s->iter, !force_nonblock); + ret = io_import_iovec(WRITE, req, &iovec, &s->iter, issue_flags); if (ret < 0) return ret; iov_iter_save_state(&s->iter, &s->iter_state); @@ -4902,11 +4905,11 @@ static int io_recvmsg_copy_hdr(struct io_kiocb *req, } static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req, - bool needs_lock) + unsigned int issue_flags) { struct io_sr_msg *sr = &req->sr_msg; - return io_buffer_select(req, &sr->len, sr->bgid, needs_lock); + return io_buffer_select(req, &sr->len, sr->bgid, issue_flags); } static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req) @@ -4969,7 +4972,7 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) } if (req->flags & REQ_F_BUFFER_SELECT) { - kbuf = io_recv_buffer_select(req, !force_nonblock); + kbuf = io_recv_buffer_select(req, issue_flags); if (IS_ERR(kbuf)) return PTR_ERR(kbuf); kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr); @@ -5021,7 +5024,7 @@ static int io_recv(struct io_kiocb *req, unsigned int issue_flags) return -ENOTSOCK; if (req->flags & REQ_F_BUFFER_SELECT) { - kbuf = io_recv_buffer_select(req, !force_nonblock); + kbuf = io_recv_buffer_select(req, issue_flags); if (IS_ERR(kbuf)) return PTR_ERR(kbuf); buf = u64_to_user_ptr(kbuf->addr); From 5e49c973fc39ffce287ab0763cb6f1e607962689 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:18 +0100 Subject: [PATCH 160/255] io_uring: clean up io_import_iovec Make io_import_iovec taking struct io_rw_state instead of an iter pointer. First it takes care of initialising iovec pointer, which can be forgotten. Even more, we can not init it if not needed, e.g. in case of IORING_OP_READ_FIXED or IORING_OP_READ. Also hide saving iter_state inside of it by splitting out an inline function of it to avoid extra ifs. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b1bbc213a95e5272d4da5867bb977d9acb6f2109.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0a0f84b28eba..cf208d713f49 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3155,9 +3155,10 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, return __io_iov_buffer_select(req, iov, issue_flags); } -static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, - struct iov_iter *iter, unsigned int issue_flags) +static int __io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, + struct io_rw_state *s, unsigned int issue_flags) { + struct iov_iter *iter = &s->iter; void __user *buf = u64_to_user_ptr(req->rw.addr); size_t sqe_len = req->rw.len; u8 opcode = req->opcode; @@ -3180,11 +3181,13 @@ static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, req->rw.len = sqe_len; } - ret = import_single_range(rw, buf, sqe_len, *iovec, iter); + ret = import_single_range(rw, buf, sqe_len, s->fast_iov, iter); *iovec = NULL; return ret; } + *iovec = s->fast_iov; + if (req->flags & REQ_F_BUFFER_SELECT) { ret = io_iov_buffer_select(req, *iovec, issue_flags); if (!ret) @@ -3197,6 +3200,19 @@ static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, req->ctx->compat); } +static inline int io_import_iovec(int rw, struct io_kiocb *req, + struct iovec **iovec, struct io_rw_state *s, + unsigned int issue_flags) +{ + int ret; + + ret = __io_import_iovec(rw, req, iovec, s, issue_flags); + if (unlikely(ret < 0)) + return ret; + iov_iter_save_state(&s->iter, &s->iter_state); + return ret; +} + static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) { return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; @@ -3322,11 +3338,11 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec, static inline int io_rw_prep_async(struct io_kiocb *req, int rw) { struct io_async_rw *iorw = req->async_data; - struct iovec *iov = iorw->s.fast_iov; + struct iovec *iov; int ret; /* submission path, ->uring_lock should already be taken */ - ret = io_import_iovec(rw, req, &iov, &iorw->s.iter, IO_URING_F_NONBLOCK); + ret = io_import_iovec(rw, req, &iov, &iorw->s, IO_URING_F_NONBLOCK); if (unlikely(ret < 0)) return ret; @@ -3334,7 +3350,6 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) iorw->free_iovec = iov; if (iov) req->flags |= REQ_F_NEED_CLEANUP; - iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); return 0; } @@ -3453,12 +3468,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) iovec = NULL; } else { s = &__s; - iovec = s->fast_iov; - ret = io_import_iovec(READ, req, &iovec, &s->iter, issue_flags); - if (ret < 0) + ret = io_import_iovec(READ, req, &iovec, s, issue_flags); + if (unlikely(ret < 0)) return ret; - - iov_iter_save_state(&s->iter, &s->iter_state); } req->result = iov_iter_count(&s->iter); @@ -3581,11 +3593,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) iovec = NULL; } else { s = &__s; - iovec = s->fast_iov; - ret = io_import_iovec(WRITE, req, &iovec, &s->iter, issue_flags); - if (ret < 0) + ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags); + if (unlikely(ret < 0)) return ret; - iov_iter_save_state(&s->iter, &s->iter_state); } req->result = iov_iter_count(&s->iter); From 607b6fb8017a684f3f1567ccf776bdc0fe6d120e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 14 Oct 2021 16:10:19 +0100 Subject: [PATCH 161/255] io_uring: rearrange io_read()/write() Combine force_nonblock branches (which is already optimised by compiler), flip branches so the most hot/common path is the first, e.g. as with non on-stack iov setup, and add extra likely/unlikely attributions for errror paths. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2c2536c5896d70994de76e387ea09a0402173a3f.1634144845.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 77 +++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index cf208d713f49..a2f989e84583 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3433,7 +3433,7 @@ static bool io_rw_should_retry(struct io_kiocb *req) static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter) { - if (req->file->f_op->read_iter) + if (likely(req->file->f_op->read_iter)) return call_read_iter(req->file, &req->rw.kiocb, iter); else if (req->file->f_op->read) return loop_rw_iter(READ, req, iter); @@ -3449,14 +3449,18 @@ static bool need_read_all(struct io_kiocb *req) static int io_read(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw_state __s, *s; + struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &req->rw.kiocb; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; struct io_async_rw *rw; ssize_t ret, ret2; - if (req_has_async_data(req)) { + if (!req_has_async_data(req)) { + ret = io_import_iovec(READ, req, &iovec, s, issue_flags); + if (unlikely(ret < 0)) + return ret; + } else { rw = req->async_data; s = &rw->s; /* @@ -3466,24 +3470,19 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) */ iov_iter_restore(&s->iter, &s->iter_state); iovec = NULL; - } else { - s = &__s; - ret = io_import_iovec(READ, req, &iovec, s, issue_flags); - if (unlikely(ret < 0)) - return ret; } req->result = iov_iter_count(&s->iter); - /* Ensure we clear previously set non-block flag */ - if (!force_nonblock) - kiocb->ki_flags &= ~IOCB_NOWAIT; - else + if (force_nonblock) { + /* If the file doesn't support async, just async punt */ + if (unlikely(!io_file_supports_nowait(req, READ))) { + ret = io_setup_async_rw(req, iovec, s, true); + return ret ?: -EAGAIN; + } kiocb->ki_flags |= IOCB_NOWAIT; - - /* If the file doesn't support async, just async punt */ - if (force_nonblock && !io_file_supports_nowait(req, READ)) { - ret = io_setup_async_rw(req, iovec, s, true); - return ret ?: -EAGAIN; + } else { + /* Ensure we clear previously set non-block flag */ + kiocb->ki_flags &= ~IOCB_NOWAIT; } ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result); @@ -3579,40 +3578,40 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_write(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw_state __s, *s; - struct io_async_rw *rw; + struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &req->rw.kiocb; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; ssize_t ret, ret2; - if (req_has_async_data(req)) { - rw = req->async_data; - s = &rw->s; - iov_iter_restore(&s->iter, &s->iter_state); - iovec = NULL; - } else { - s = &__s; + if (!req_has_async_data(req)) { ret = io_import_iovec(WRITE, req, &iovec, s, issue_flags); if (unlikely(ret < 0)) return ret; + } else { + struct io_async_rw *rw = req->async_data; + + s = &rw->s; + iov_iter_restore(&s->iter, &s->iter_state); + iovec = NULL; } req->result = iov_iter_count(&s->iter); - /* Ensure we clear previously set non-block flag */ - if (!force_nonblock) - kiocb->ki_flags &= ~IOCB_NOWAIT; - else + if (force_nonblock) { + /* If the file doesn't support async, just async punt */ + if (unlikely(!io_file_supports_nowait(req, WRITE))) + goto copy_iov; + + /* file path doesn't support NOWAIT for non-direct_IO */ + if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && + (req->flags & REQ_F_ISREG)) + goto copy_iov; + kiocb->ki_flags |= IOCB_NOWAIT; - - /* If the file doesn't support async, just async punt */ - if (force_nonblock && !io_file_supports_nowait(req, WRITE)) - goto copy_iov; - - /* file path doesn't support NOWAIT for non-direct_IO */ - if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) && - (req->flags & REQ_F_ISREG)) - goto copy_iov; + } else { + /* Ensure we clear previously set non-block flag */ + kiocb->ki_flags &= ~IOCB_NOWAIT; + } ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result); if (unlikely(ret)) From 9983028e7660a2cc5e58403d8ce29569dbf3162d Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:11 +0100 Subject: [PATCH 162/255] io_uring: optimise req->ctx reloads Don't load req->ctx in advance, it takes an extra register and the field stays valid even after opcode handlers. It also optimises out req->ctx load in io_iopoll_req_issued() once it's inlined. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/1e45ff671c44be0eb904f2e448a211734893fa0b.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index a2f989e84583..5a4d13ad1277 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6588,7 +6588,6 @@ static void io_clean_op(struct io_kiocb *req) static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) { - struct io_ring_ctx *ctx = req->ctx; const struct cred *creds = NULL; int ret; @@ -6715,7 +6714,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (ret) return ret; /* If the op doesn't have a file, we're not polling for it */ - if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) + if ((req->ctx->flags & IORING_SETUP_IOPOLL) && req->file) io_iopoll_req_issued(req); return 0; From 9882131cd9de525d484de117644e5008c6557ac7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:12 +0100 Subject: [PATCH 163/255] io_uring: kill io_wq_current_is_worker() in iopoll Don't decide about locking based on io_wq_current_is_worker(), it's not consistent with all other code and is expensive, use issue_flags. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/7546d5a58efa4360173541c6fe02ee6b8c7b4ea7.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 5a4d13ad1277..348e587524cf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2713,13 +2713,13 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) * find it from a io_do_iopoll() thread before the issuer is done * accessing the kiocb cookie. */ -static void io_iopoll_req_issued(struct io_kiocb *req) +static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - const bool in_async = io_wq_current_is_worker(); + const bool need_lock = !(issue_flags & IO_URING_F_NONBLOCK); /* workqueue context doesn't hold uring_lock, grab it now */ - if (unlikely(in_async)) + if (unlikely(need_lock)) mutex_lock(&ctx->uring_lock); /* @@ -2747,7 +2747,7 @@ static void io_iopoll_req_issued(struct io_kiocb *req) else wq_list_add_tail(&req->comp_list, &ctx->iopoll_list); - if (unlikely(in_async)) { + if (unlikely(need_lock)) { /* * If IORING_SETUP_SQPOLL is enabled, sqes are either handle * in sq thread task context or in io worker task context. If @@ -6715,7 +6715,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) return ret; /* If the op doesn't have a file, we're not polling for it */ if ((req->ctx->flags & IORING_SETUP_IOPOLL) && req->file) - io_iopoll_req_issued(req); + io_iopoll_req_issued(req, issue_flags); return 0; } From d1d681b0846af8585904be562610bdfc70bf21aa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:13 +0100 Subject: [PATCH 164/255] io_uring: optimise io_import_iovec fixed path Delay loading req->rw.{addr,len} in io_import_iovec until it's really needed, so removing extra loads for the fixed path, which doesn't use them. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/3cc48dd0c4f1a37c4ce9aab5784281a2d83ad8be.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 348e587524cf..00267e799ac2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3159,9 +3159,9 @@ static int __io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, struct io_rw_state *s, unsigned int issue_flags) { struct iov_iter *iter = &s->iter; - void __user *buf = u64_to_user_ptr(req->rw.addr); - size_t sqe_len = req->rw.len; u8 opcode = req->opcode; + void __user *buf; + size_t sqe_len; ssize_t ret; if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { @@ -3170,9 +3170,12 @@ static int __io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, } /* buffer index only valid with fixed read/write, or buffer select */ - if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT)) + if (unlikely(req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))) return -EINVAL; + buf = u64_to_user_ptr(req->rw.addr); + sqe_len = req->rw.len; + if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { if (req->flags & REQ_F_BUFFER_SELECT) { buf = io_rw_buffer_select(req, &sqe_len, issue_flags); From caa8fe6e86fd32e26d05f724e07ec826807c6ad4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:14 +0100 Subject: [PATCH 165/255] io_uring: return iovec from __io_import_iovec We pass iovec** into __io_import_iovec(), which should keep it, initialise and modify accordingly. It's expensive, return it directly from __io_import_iovec encoding errors with ERR_PTR if needed. io_import_iovec keeps the old interface, but it's inline and so everything is optimised nicely. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6230e9769982f03a8f86fa58df24666088c44d3e.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 00267e799ac2..d97545e43a16 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3155,23 +3155,25 @@ static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, return __io_iov_buffer_select(req, iov, issue_flags); } -static int __io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, - struct io_rw_state *s, unsigned int issue_flags) +static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req, + struct io_rw_state *s, + unsigned int issue_flags) { struct iov_iter *iter = &s->iter; u8 opcode = req->opcode; + struct iovec *iovec; void __user *buf; size_t sqe_len; ssize_t ret; - if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) { - *iovec = NULL; - return io_import_fixed(req, rw, iter); - } + BUILD_BUG_ON(ERR_PTR(0) != NULL); + + if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) + return ERR_PTR(io_import_fixed(req, rw, iter)); /* buffer index only valid with fixed read/write, or buffer select */ if (unlikely(req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))) - return -EINVAL; + return ERR_PTR(-EINVAL); buf = u64_to_user_ptr(req->rw.addr); sqe_len = req->rw.len; @@ -3180,40 +3182,39 @@ static int __io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, if (req->flags & REQ_F_BUFFER_SELECT) { buf = io_rw_buffer_select(req, &sqe_len, issue_flags); if (IS_ERR(buf)) - return PTR_ERR(buf); + return ERR_PTR(PTR_ERR(buf)); req->rw.len = sqe_len; } ret = import_single_range(rw, buf, sqe_len, s->fast_iov, iter); - *iovec = NULL; - return ret; + return ERR_PTR(ret); } - *iovec = s->fast_iov; - + iovec = s->fast_iov; if (req->flags & REQ_F_BUFFER_SELECT) { - ret = io_iov_buffer_select(req, *iovec, issue_flags); + ret = io_iov_buffer_select(req, iovec, issue_flags); if (!ret) - iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len); - *iovec = NULL; - return ret; + iov_iter_init(iter, rw, iovec, 1, iovec->iov_len); + return ERR_PTR(ret); } - return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter, + ret = __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, &iovec, iter, req->ctx->compat); + if (unlikely(ret < 0)) + return ERR_PTR(ret); + return iovec; } static inline int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec, struct io_rw_state *s, unsigned int issue_flags) { - int ret; + *iovec = __io_import_iovec(rw, req, s, issue_flags); + if (unlikely(IS_ERR(*iovec))) + return PTR_ERR(*iovec); - ret = __io_import_iovec(rw, req, iovec, s, issue_flags); - if (unlikely(ret < 0)) - return ret; iov_iter_save_state(&s->iter, &s->iter_state); - return ret; + return 0; } static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb) From 578c0ee234e51d9746e526a26bdb2c1b7ad81dba Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:15 +0100 Subject: [PATCH 166/255] io_uring: optimise fixed rw rsrc node setting Move fixed rw io_req_set_rsrc_node() from rw prep into io_import_fixed(), if we're using fixed buffers it will always be called during submission as we save the state in advance, Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/68c06f66d5aa9661f1e4b88d08c52d23528297ec.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d97545e43a16..014363abc2b2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2871,12 +2871,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_complete = io_complete_rw; } - if (req->opcode == IORING_OP_READ_FIXED || - req->opcode == IORING_OP_WRITE_FIXED) { - req->imu = NULL; - io_req_set_rsrc_node(req, ctx); - } - + req->imu = NULL; req->rw.addr = READ_ONCE(sqe->addr); req->rw.len = READ_ONCE(sqe->len); req->buf_index = READ_ONCE(sqe->buf_index); @@ -3005,13 +3000,15 @@ static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter) { - struct io_ring_ctx *ctx = req->ctx; struct io_mapped_ubuf *imu = req->imu; u16 index, buf_index = req->buf_index; if (likely(!imu)) { + struct io_ring_ctx *ctx = req->ctx; + if (unlikely(buf_index >= ctx->nr_user_bufs)) return -EFAULT; + io_req_set_rsrc_node(req, ctx); index = array_index_nospec(buf_index, ctx->nr_user_bufs); imu = READ_ONCE(ctx->user_bufs[index]); req->imu = imu; From 5cb03d63420bcf533111d30690141a5b28c172fa Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 15 Oct 2021 17:09:16 +0100 Subject: [PATCH 167/255] io_uring: clean io_prep_rw() We already store req->file in a variable in io_prep_rw(), just use it instead of a couple of left references to kicob->ki_filp. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/2f5889fc7ab670daefd5ccaedd99416d8355f0ad.1634314022.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 014363abc2b2..d5d06e034959 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2832,8 +2832,8 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, req->flags |= REQ_F_CUR_POS; kiocb->ki_pos = file->f_pos; } - kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp)); - kiocb->ki_flags = iocb_flags(kiocb->ki_filp); + kiocb->ki_hint = ki_hint_validate(file_write_hint(file)); + kiocb->ki_flags = iocb_flags(file); ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); if (unlikely(ret)) return ret; @@ -2858,8 +2858,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_ioprio = get_current_ioprio(); if (ctx->flags & IORING_SETUP_IOPOLL) { - if (!(kiocb->ki_flags & IOCB_DIRECT) || - !kiocb->ki_filp->f_op->iopoll) + if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) return -EOPNOTSUPP; kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE; From b10841c98c8980ee50cc1e90640b04c935a24285 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Sat, 16 Oct 2021 20:32:29 -0500 Subject: [PATCH 168/255] fs/io_uring: Prioritise checking faster conditions first in io_write This commit reorders the conditions in a branch in io_write. The reorder to check 'ret2 == -EAGAIN' first as checking '(req->ctx->flags & IORING_SETUP_IOPOLL)' will likely be more expensive due to 2x memory derefences. Signed-off-by: Noah Goldstein Link: https://lore.kernel.org/r/20211017013229.4124279-1-goldstein.w.n@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d5d06e034959..11bd0dd1425a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3654,7 +3654,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) goto done; if (!force_nonblock || ret2 != -EAGAIN) { /* IOPOLL retry should happen for io-wq threads */ - if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN) + if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) goto copy_iov; done: kiocb_done(kiocb, ret2, issue_flags); From e74ead135bc4459f7d40b1f8edab1333a28b54e8 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 Oct 2021 00:07:08 +0100 Subject: [PATCH 169/255] io_uring: arm poll for non-nowait files Don't check if we can do nowait before arming apoll, there are several reasons for that. First, we don't care much about files that don't support nowait. Second, it may be useful -- we don't want to be taking away extra workers from io-wq when it can go in some async. Even if it will go through io-wq eventually, it make difference in the numbers of workers actually used. And the last one, it's needed to clean nowait in future commits. [kernel test robot: fix unused-var] Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9d06f3cb2c8b686d970269a87986f154edb83043.1634425438.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 11bd0dd1425a..8a5b20eafccf 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5588,7 +5588,6 @@ static int io_arm_poll_handler(struct io_kiocb *req) struct async_poll *apoll; struct io_poll_table ipt; __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI; - int rw; if (!req->file || !file_can_poll(req->file)) return IO_APOLL_ABORTED; @@ -5598,7 +5597,6 @@ static int io_arm_poll_handler(struct io_kiocb *req) return IO_APOLL_ABORTED; if (def->pollin) { - rw = READ; mask |= POLLIN | POLLRDNORM; /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */ @@ -5606,14 +5604,9 @@ static int io_arm_poll_handler(struct io_kiocb *req) (req->sr_msg.msg_flags & MSG_ERRQUEUE)) mask &= ~POLLIN; } else { - rw = WRITE; mask |= POLLOUT | POLLWRNORM; } - /* if we can't nonblock try, then no point in arming a poll handler */ - if (!io_file_supports_nowait(req, rw)) - return IO_APOLL_ABORTED; - apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC); if (unlikely(!apoll)) return IO_APOLL_ABORTED; From 35645ac3c1853fbb54d8acd50fd12184f7905d5f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 Oct 2021 00:07:09 +0100 Subject: [PATCH 170/255] io_uring: combine REQ_F_NOWAIT_{READ,WRITE} flags Merge REQ_F_NOWAIT_READ and REQ_F_NOWAIT_WRITE into one flag, i.e. REQ_F_SUPPORT_NOWAIT. First it gets rid of dependence on CONFIG_64BIT but also simplifies the code. One thing to consider is when we don't have ->{read,write}_iter and go through loop_rw_iter(). Just fail it with -EAGAIN if we expect nowait behaviour but not sure whether it supports it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/f832a20e5186c2e79c6519280c238f559a1d2bbc.1634425438.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 61 ++++++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8a5b20eafccf..d7bf2002ab0c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -732,8 +732,7 @@ enum { REQ_F_ARM_LTIMEOUT_BIT, REQ_F_ASYNC_DATA_BIT, /* keep async read/write and isreg together and in order */ - REQ_F_NOWAIT_READ_BIT, - REQ_F_NOWAIT_WRITE_BIT, + REQ_F_SUPPORT_NOWAIT_BIT, REQ_F_ISREG_BIT, /* not a real bit, just to check we're not overflowing the space */ @@ -774,10 +773,8 @@ enum { REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT), /* caller should reissue async */ REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT), - /* supports async reads */ - REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT), - /* supports async writes */ - REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT), + /* supports async reads/writes */ + REQ_F_SUPPORT_NOWAIT = BIT(REQ_F_SUPPORT_NOWAIT_BIT), /* regular file */ REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), /* has creds assigned */ @@ -1390,18 +1387,13 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) return false; } -#define FFS_ASYNC_READ 0x1UL -#define FFS_ASYNC_WRITE 0x2UL -#ifdef CONFIG_64BIT -#define FFS_ISREG 0x4UL -#else -#define FFS_ISREG 0x0UL -#endif -#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG) +#define FFS_NOWAIT 0x1UL +#define FFS_ISREG 0x2UL +#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG) static inline bool io_req_ffs_set(struct io_kiocb *req) { - return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE); + return req->flags & REQ_F_FIXED_FILE; } static inline void io_req_track_inflight(struct io_kiocb *req) @@ -2772,7 +2764,7 @@ static bool io_bdev_nowait(struct block_device *bdev) * any file. For now, just ensure that anything potentially problematic is done * inline. */ -static bool __io_file_supports_nowait(struct file *file, int rw) +static bool __io_file_supports_nowait(struct file *file) { umode_t mode = file_inode(file)->i_mode; @@ -2795,24 +2787,14 @@ static bool __io_file_supports_nowait(struct file *file, int rw) /* any ->read/write should understand O_NONBLOCK */ if (file->f_flags & O_NONBLOCK) return true; - - if (!(file->f_mode & FMODE_NOWAIT)) - return false; - - if (rw == READ) - return file->f_op->read_iter != NULL; - - return file->f_op->write_iter != NULL; + return file->f_mode & FMODE_NOWAIT; } -static bool io_file_supports_nowait(struct io_kiocb *req, int rw) +static inline bool io_file_supports_nowait(struct io_kiocb *req) { - if (rw == READ && (req->flags & REQ_F_NOWAIT_READ)) + if (likely(req->flags & REQ_F_SUPPORT_NOWAIT)) return true; - else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE)) - return true; - - return __io_file_supports_nowait(req->file, rw); + return __io_file_supports_nowait(req->file); } static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, @@ -2844,7 +2826,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, * reliably. If not, or it IOCB_NOWAIT is set, don't retry. */ if ((kiocb->ki_flags & IOCB_NOWAIT) || - ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw))) + ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) req->flags |= REQ_F_NOWAIT; ioprio = READ_ONCE(sqe->ioprio); @@ -3235,7 +3217,8 @@ static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter) */ if (kiocb->ki_flags & IOCB_HIPRI) return -EOPNOTSUPP; - if (kiocb->ki_flags & IOCB_NOWAIT) + if ((kiocb->ki_flags & IOCB_NOWAIT) && + !(kiocb->ki_filp->f_flags & O_NONBLOCK)) return -EAGAIN; while (iov_iter_count(iter)) { @@ -3475,7 +3458,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) if (force_nonblock) { /* If the file doesn't support async, just async punt */ - if (unlikely(!io_file_supports_nowait(req, READ))) { + if (unlikely(!io_file_supports_nowait(req))) { ret = io_setup_async_rw(req, iovec, s, true); return ret ?: -EAGAIN; } @@ -3599,7 +3582,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) if (force_nonblock) { /* If the file doesn't support async, just async punt */ - if (unlikely(!io_file_supports_nowait(req, WRITE))) + if (unlikely(!io_file_supports_nowait(req))) goto copy_iov; /* file path doesn't support NOWAIT for non-direct_IO */ @@ -3631,7 +3614,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) } kiocb->ki_flags |= IOCB_WRITE; - if (req->file->f_op->write_iter) + if (likely(req->file->f_op->write_iter)) ret2 = call_write_iter(req->file, kiocb, &s->iter); else if (req->file->f_op->write) ret2 = loop_rw_iter(WRITE, req, &s->iter); @@ -6778,10 +6761,8 @@ static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file { unsigned long file_ptr = (unsigned long) file; - if (__io_file_supports_nowait(file, READ)) - file_ptr |= FFS_ASYNC_READ; - if (__io_file_supports_nowait(file, WRITE)) - file_ptr |= FFS_ASYNC_WRITE; + if (__io_file_supports_nowait(file)) + file_ptr |= FFS_NOWAIT; if (S_ISREG(file_inode(file)->i_mode)) file_ptr |= FFS_ISREG; file_slot->file_ptr = file_ptr; @@ -6800,7 +6781,7 @@ static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx, file = (struct file *) (file_ptr & FFS_MASK); file_ptr &= ~FFS_MASK; /* mask in overlapping REQ_F and FFS bits */ - req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT); + req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT); io_req_set_rsrc_node(req, ctx); return file; } From 88459b50b42a4bd58e528006663afabd0b8652f2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 Oct 2021 00:07:10 +0100 Subject: [PATCH 171/255] io_uring: simplify io_file_supports_nowait() Make sure that REQ_F_SUPPORT_NOWAIT is always set io_prep_rw(), and so we can stop caring about setting it down the line simplifying io_file_supports_nowait(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/60c8f1f5e2cb45e00f4897b2cec10c5b3669da91.1634425438.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d7bf2002ab0c..b834cf2778f6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2764,10 +2764,8 @@ static bool io_bdev_nowait(struct block_device *bdev) * any file. For now, just ensure that anything potentially problematic is done * inline. */ -static bool __io_file_supports_nowait(struct file *file) +static bool __io_file_supports_nowait(struct file *file, umode_t mode) { - umode_t mode = file_inode(file)->i_mode; - if (S_ISBLK(mode)) { if (IS_ENABLED(CONFIG_BLOCK) && io_bdev_nowait(I_BDEV(file->f_mapping->host))) @@ -2790,11 +2788,26 @@ static bool __io_file_supports_nowait(struct file *file) return file->f_mode & FMODE_NOWAIT; } +/* + * If we tracked the file through the SCM inflight mechanism, we could support + * any file. For now, just ensure that anything potentially problematic is done + * inline. + */ +static unsigned int io_file_get_flags(struct file *file) +{ + umode_t mode = file_inode(file)->i_mode; + unsigned int res = 0; + + if (S_ISREG(mode)) + res |= FFS_ISREG; + if (__io_file_supports_nowait(file, mode)) + res |= FFS_NOWAIT; + return res; +} + static inline bool io_file_supports_nowait(struct io_kiocb *req) { - if (likely(req->flags & REQ_F_SUPPORT_NOWAIT)) - return true; - return __io_file_supports_nowait(req->file); + return req->flags & REQ_F_SUPPORT_NOWAIT; } static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, @@ -2806,8 +2819,8 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, unsigned ioprio; int ret; - if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode)) - req->flags |= REQ_F_ISREG; + if (!io_req_ffs_set(req)) + req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT; kiocb->ki_pos = READ_ONCE(sqe->off); if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) { @@ -6761,10 +6774,7 @@ static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file { unsigned long file_ptr = (unsigned long) file; - if (__io_file_supports_nowait(file)) - file_ptr |= FFS_NOWAIT; - if (S_ISREG(file_inode(file)->i_mode)) - file_ptr |= FFS_ISREG; + file_ptr |= io_file_get_flags(file); file_slot->file_ptr = file_ptr; } From 5ca7a8b3f698b111729ed8e133b14e30f961de0f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 6 Oct 2021 11:01:42 -0600 Subject: [PATCH 172/255] io_uring: inform block layer of how many requests we are submitting The block layer can use this knowledge to make smarter decisions on how to handle the request, if it knows that N more may be coming. Switch to using blk_start_plug_nr_ios() to pass in that information. Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index b834cf2778f6..240222da2989 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -317,6 +317,7 @@ struct io_submit_state { bool plug_started; bool need_plug; + unsigned short submit_nr; struct blk_plug plug; }; @@ -7060,7 +7061,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (state->need_plug && io_op_defs[opcode].plug) { state->plug_started = true; state->need_plug = false; - blk_start_plug(&state->plug); + blk_start_plug_nr_ios(&state->plug, state->submit_nr); } req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd), @@ -7181,6 +7182,7 @@ static void io_submit_state_start(struct io_submit_state *state, { state->plug_started = false; state->need_plug = max_ios > 2; + state->submit_nr = max_ios; /* set only head, no need to init link_last in advance */ state->link.head = NULL; } From 86d46fdaa12ae5befc16b8d73fc85a3ca0399ea6 Mon Sep 17 00:00:00 2001 From: Michael Schmitz Date: Tue, 19 Oct 2021 19:13:21 +1300 Subject: [PATCH 173/255] block: ataflop: fix breakage introduced at blk-mq refactoring Refactoring of the Atari floppy driver when converting to blk-mq has broken the state machine in not-so-subtle ways: finish_fdc() must be called when operations on the floppy device have completed. This is crucial in order to relase the ST-DMA lock, which protects against concurrent access to the ST-DMA controller by other drivers (some DMA related, most just related to device register access - broken beyond compare, I know). When rewriting the driver's old do_request() function, the fact that finish_fdc() was called only when all queued requests had completed appears to have been overlooked. Instead, the new request function calls finish_fdc() immediately after the last request has been queued. finish_fdc() executes a dummy seek after most requests, and this overwrites the state machine's interrupt hander that was set up to wait for completion of the read/write request just prior. To make matters worse, finish_fdc() is called before device interrupts are re-enabled, making certain that the read/write interupt is missed. Shifting the finish_fdc() call into the read/write request completion handler ensures the driver waits for the request to actually complete. With a queue depth of 2, we won't see long request sequences, so calling finish_fdc() unconditionally just adds a little overhead for the dummy seeks, and keeps the code simple. While we're at it, kill ataflop_commit_rqs() which does nothing but run finish_fdc() unconditionally, again likely wiping out an in-flight request. Signed-off-by: Michael Schmitz Fixes: 6ec3938cff95 ("ataflop: convert to blk-mq") CC: linux-block@vger.kernel.org CC: Tetsuo Handa Link: https://lore.kernel.org/r/20211019061321.26425-1-schmitzmic@gmail.com Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 9bc5cce6b29a..3d217f1d7440 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -655,9 +655,6 @@ static inline void copy_buffer(void *from, void *to) *p2++ = *p1++; } - - - /* General Interrupt Handling */ static void (*FloppyIRQHandler)( int status ) = NULL; @@ -1230,6 +1227,7 @@ static void fd_rwsec_done1(int status) } else { /* all sectors finished */ + finish_fdc(); fd_end_request_cur(BLK_STS_OK); } return; @@ -1477,15 +1475,6 @@ static void setup_req_params( int drive ) ReqTrack, ReqSector, (unsigned long)ReqData )); } -static void ataflop_commit_rqs(struct blk_mq_hw_ctx *hctx) -{ - spin_lock_irq(&ataflop_lock); - atari_disable_irq(IRQ_MFP_FDC); - finish_fdc(); - atari_enable_irq(IRQ_MFP_FDC); - spin_unlock_irq(&ataflop_lock); -} - static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -1493,6 +1482,8 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, int drive = floppy - unit; int type = floppy->type; + DPRINT(("Queue request: drive %d type %d last %d\n", drive, type, bd->last)); + spin_lock_irq(&ataflop_lock); if (fd_request) { spin_unlock_irq(&ataflop_lock); @@ -1552,8 +1543,6 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, setup_req_params( drive ); do_fd_action( drive ); - if (bd->last) - finish_fdc(); atari_enable_irq( IRQ_MFP_FDC ); out: @@ -1964,7 +1953,6 @@ static const struct block_device_operations floppy_fops = { static const struct blk_mq_ops ataflop_mq_ops = { .queue_rq = ataflop_queue_rq, - .commit_rqs = ataflop_commit_rqs, }; static int ataflop_alloc_disk(unsigned int drive, unsigned int type) From 946e99373037be4841e8b42dcd136e03093c9fd5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Oct 2021 08:20:22 +0200 Subject: [PATCH 174/255] block/ioctl: use bdev_nr_sectors and bdev_nr_bytes Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211019062024.2171074-2-hch@lst.de Signed-off-by: Jens Axboe --- block/ioctl.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/block/ioctl.c b/block/ioctl.c index 77b1b2453f39..d6af0ac97e57 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -132,7 +132,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, if (len & 511) return -EINVAL; - if (start + len > i_size_read(bdev->bd_inode)) + if (start + len > bdev_nr_bytes(bdev)) return -EINVAL; err = truncate_bdev_range(bdev, mode, start, start + len - 1); @@ -164,7 +164,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, fmode_t mode, return -EINVAL; if (len & 511) return -EINVAL; - if (end >= (uint64_t)i_size_read(bdev->bd_inode)) + if (end >= (uint64_t)bdev_nr_bytes(bdev)) return -EINVAL; if (end < start) return -EINVAL; @@ -543,7 +543,6 @@ long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) struct block_device *bdev = I_BDEV(file->f_mapping->host); void __user *argp = (void __user *)arg; fmode_t mode = file->f_mode; - loff_t size; int ret; /* @@ -570,10 +569,9 @@ long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) return put_long(argp, (bdev->bd_disk->bdi->ra_pages * PAGE_SIZE) / 512); case BLKGETSIZE: - size = i_size_read(bdev->bd_inode); - if ((size >> 9) > ~0UL) + if (bdev_nr_sectors(bdev) > ~0UL) return -EFBIG; - return put_ulong(argp, size >> 9); + return put_ulong(argp, bdev_nr_sectors(bdev)); /* The data is compatible, but the command number is different */ case BLKBSZGET: /* get block device soft block size (cf. BLKSSZGET) */ @@ -581,7 +579,7 @@ long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) case BLKBSZSET: return blkdev_bszset(bdev, mode, argp); case BLKGETSIZE64: - return put_u64(argp, i_size_read(bdev->bd_inode)); + return put_u64(argp, bdev_nr_bytes(bdev)); /* Incompatible alignment on i386 */ case BLKTRACESETUP: @@ -615,7 +613,6 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) struct block_device *bdev = I_BDEV(file->f_mapping->host); struct gendisk *disk = bdev->bd_disk; fmode_t mode = file->f_mode; - loff_t size; /* * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have @@ -641,10 +638,9 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) return compat_put_long(argp, (bdev->bd_disk->bdi->ra_pages * PAGE_SIZE) / 512); case BLKGETSIZE: - size = i_size_read(bdev->bd_inode); - if ((size >> 9) > ~0UL) + if (bdev_nr_sectors(bdev) > ~0UL) return -EFBIG; - return compat_put_ulong(argp, size >> 9); + return compat_put_ulong(argp, bdev_nr_sectors(bdev)); /* The data is compatible, but the command number is different */ case BLKBSZGET_32: /* get the logical block size (cf. BLKSSZGET) */ @@ -652,7 +648,7 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) case BLKBSZSET_32: return blkdev_bszset(bdev, mode, argp); case BLKGETSIZE64_32: - return put_u64(argp, i_size_read(bdev->bd_inode)); + return put_u64(argp, bdev_nr_bytes(bdev)); /* Incompatible alignment on i386 */ case BLKTRACESETUP32: From f9831b885709978ed9d16833ceeb3a2ec174a2d2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Oct 2021 08:20:23 +0200 Subject: [PATCH 175/255] partitions/efi: use bdev_nr_bytes instead of open coding it Use the proper helper to read the block device size. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211019062024.2171074-3-hch@lst.de Signed-off-by: Jens Axboe --- block/partitions/efi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/partitions/efi.c b/block/partitions/efi.c index 7ca5c4c374d4..5e9be13a56a8 100644 --- a/block/partitions/efi.c +++ b/block/partitions/efi.c @@ -133,7 +133,7 @@ efi_crc32(const void *buf, unsigned long len) */ static u64 last_lba(struct gendisk *disk) { - return div_u64(disk->part0->bd_inode->i_size, + return div_u64(bdev_nr_bytes(disk->part0), queue_logical_block_size(disk->queue)) - 1ULL; } From 97eeb5fc14cc4b2091df8b841a07a1ac69f2d762 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Oct 2021 08:20:24 +0200 Subject: [PATCH 176/255] partitions/ibm: use bdev_nr_sectors instead of open coding it Use the proper helper to read the block device size and switch various places to pass the size in terms of sectors which is more practical. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211019062024.2171074-4-hch@lst.de [axboe: fix comment typo] Signed-off-by: Jens Axboe --- block/partitions/ibm.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/block/partitions/ibm.c b/block/partitions/ibm.c index 9bca396aef4a..403756dbd50d 100644 --- a/block/partitions/ibm.c +++ b/block/partitions/ibm.c @@ -198,7 +198,7 @@ static int find_lnx1_partitions(struct parsed_partitions *state, char name[], union label_t *label, sector_t labelsect, - loff_t i_size, + sector_t nr_sectors, dasd_information2_t *info) { loff_t offset, geo_size, size; @@ -213,14 +213,14 @@ static int find_lnx1_partitions(struct parsed_partitions *state, } else { /* * Formated w/o large volume support. If the sanity check - * 'size based on geo == size based on i_size' is true, then + * 'size based on geo == size based on nr_sectors' is true, then * we can safely assume that we know the formatted size of * the disk, otherwise we need additional information * that we can only get from a real DASD device. */ geo_size = geo->cylinders * geo->heads * geo->sectors * secperblk; - size = i_size >> 9; + size = nr_sectors; if (size != geo_size) { if (!info) { strlcat(state->pp_buf, "\n", PAGE_SIZE); @@ -229,7 +229,7 @@ static int find_lnx1_partitions(struct parsed_partitions *state, if (!strcmp(info->type, "ECKD")) if (geo_size < size) size = geo_size; - /* else keep size based on i_size */ + /* else keep size based on nr_sectors */ } } /* first and only partition starts in the first block after the label */ @@ -293,7 +293,8 @@ int ibm_partition(struct parsed_partitions *state) struct gendisk *disk = state->disk; struct block_device *bdev = disk->part0; int blocksize, res; - loff_t i_size, offset, size; + loff_t offset, size; + sector_t nr_sectors; dasd_information2_t *info; struct hd_geometry *geo; char type[5] = {0,}; @@ -308,8 +309,8 @@ int ibm_partition(struct parsed_partitions *state) blocksize = bdev_logical_block_size(bdev); if (blocksize <= 0) goto out_symbol; - i_size = i_size_read(bdev->bd_inode); - if (i_size == 0) + nr_sectors = bdev_nr_sectors(bdev); + if (nr_sectors == 0) goto out_symbol; info = kmalloc(sizeof(dasd_information2_t), GFP_KERNEL); if (info == NULL) @@ -336,7 +337,7 @@ int ibm_partition(struct parsed_partitions *state) label); } else if (!strncmp(type, "LNX1", 4)) { res = find_lnx1_partitions(state, geo, blocksize, name, - label, labelsect, i_size, + label, labelsect, nr_sectors, info); } else if (!strncmp(type, "CMS1", 4)) { res = find_cms1_partitions(state, geo, blocksize, name, @@ -353,7 +354,7 @@ int ibm_partition(struct parsed_partitions *state) res = 1; if (info->format == DASD_FORMAT_LDL) { strlcat(state->pp_buf, "(nonl)", PAGE_SIZE); - size = i_size >> 9; + size = nr_sectors; offset = (info->label_block + 1) * (blocksize >> 9); put_partition(state, 1, offset, size-offset); strlcat(state->pp_buf, "\n", PAGE_SIZE); From 00169246e6981752e53266c62d0ab0c827493634 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 19 Oct 2021 17:34:53 +0200 Subject: [PATCH 177/255] io_uring: warning about unused-but-set parameter When enabling -Wunused warnings by building with W=1, I get an instance of the -Wunused-but-set-parameter warning in the io_uring code: fs/io_uring.c: In function 'io_queue_async_work': fs/io_uring.c:1445:61: error: parameter 'locked' set but not used [-Werror=unused-but-set-parameter] 1445 | static void io_queue_async_work(struct io_kiocb *req, bool *locked) | ~~~~~~^~~~~~ There are very few warnings of this type, so it would be nice to enable this by default and fix all the existing instances. As the assignment serves no purpose by itself other than to prevent developers from using the variable, an easy workaround is to remove the assignment and just rename the argument to "dont_use". Fixes: f237c30a5610 ("io_uring: batch task work locking") Link: https://lore.kernel.org/lkml/20210920121352.93063-1-arnd@kernel.org/ Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20211019153507.348480-1-arnd@kernel.org Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 240222da2989..df07ce2fa6bc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1487,15 +1487,12 @@ static inline void io_req_add_compl_list(struct io_kiocb *req) wq_list_add_tail(&req->comp_list, &state->compl_reqs); } -static void io_queue_async_work(struct io_kiocb *req, bool *locked) +static void io_queue_async_work(struct io_kiocb *req, bool *dont_use) { struct io_ring_ctx *ctx = req->ctx; struct io_kiocb *link = io_prep_linked_timeout(req); struct io_uring_task *tctx = req->task->io_uring; - /* must not take the lock, NULL it as a precaution */ - locked = NULL; - BUG_ON(!tctx); BUG_ON(!tctx->io_wq); From 9c3d29296fe4c297447d2055e7a9535c981a8370 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 18 Oct 2021 06:45:06 -0600 Subject: [PATCH 178/255] nvme: move command clear into the various setup helpers We don't have to worry about doing extra memsets by moving it outside the protection of RQF_DONTPREP, as nvme doesn't do partial completions. This is in preparation for making the read/write fast path not do a full memset of the command. Reviewed-by: Keith Busch Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 10 +++++++--- drivers/nvme/host/zns.c | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 3109bdf137e4..824790bed2f5 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -834,6 +834,7 @@ static void nvme_assign_write_stream(struct nvme_ctrl *ctrl, static inline void nvme_setup_flush(struct nvme_ns *ns, struct nvme_command *cmnd) { + memset(cmnd, 0, sizeof(*cmnd)); cmnd->common.opcode = nvme_cmd_flush; cmnd->common.nsid = cpu_to_le32(ns->head->ns_id); } @@ -885,6 +886,7 @@ static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req, return BLK_STS_IOERR; } + memset(cmnd, 0, sizeof(*cmnd)); cmnd->dsm.opcode = nvme_cmd_dsm; cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id); cmnd->dsm.nr = cpu_to_le32(segments - 1); @@ -901,6 +903,8 @@ static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req, static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns, struct request *req, struct nvme_command *cmnd) { + memset(cmnd, 0, sizeof(*cmnd)); + if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) return nvme_setup_discard(ns, req, cmnd); @@ -925,6 +929,8 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, u16 control = 0; u32 dsmgmt = 0; + memset(cmnd, 0, sizeof(*cmnd)); + if (req->cmd_flags & REQ_FUA) control |= NVME_RW_FUA; if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) @@ -993,10 +999,8 @@ blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req) struct nvme_ctrl *ctrl = nvme_req(req)->ctrl; blk_status_t ret = BLK_STS_OK; - if (!(req->rq_flags & RQF_DONTPREP)) { + if (!(req->rq_flags & RQF_DONTPREP)) nvme_clear_nvme_request(req); - memset(cmd, 0, sizeof(*cmd)); - } switch (req_op(req)) { case REQ_OP_DRV_IN: diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index d95010481fce..bfc259e0d7b8 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -233,6 +233,8 @@ out_free: blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req, struct nvme_command *c, enum nvme_zone_mgmt_action action) { + memset(c, 0, sizeof(*c)); + c->zms.opcode = nvme_cmd_zone_mgmt_send; c->zms.nsid = cpu_to_le32(ns->head->ns_id); c->zms.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); From a9a7e30fd918588bc312ba782426e3a1282df359 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 18 Oct 2021 06:47:18 -0600 Subject: [PATCH 179/255] nvme: don't memset() the normal read/write command This memset in the fast path costs a lot of cycles on my setup. Here's a top-of-profile of doing ~6.7M IOPS: + 5.90% io_uring [nvme] [k] nvme_queue_rq + 5.32% io_uring [nvme_core] [k] nvme_setup_cmd + 5.17% io_uring [kernel.vmlinux] [k] io_submit_sqes + 4.97% io_uring [kernel.vmlinux] [k] blkdev_direct_IO and a perf diff with this patch: 0.92% +4.40% [nvme_core] [k] nvme_setup_cmd reducing it from 5.3% to only 0.9%. This takes it from the 2nd most cycle consumer to something that's mostly irrelevant. Reviewed-by: Chaitanya Kulkarni Reviewed-by: Keith Busch Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 824790bed2f5..c415c3faf420 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -929,8 +929,6 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, u16 control = 0; u32 dsmgmt = 0; - memset(cmnd, 0, sizeof(*cmnd)); - if (req->cmd_flags & REQ_FUA) control |= NVME_RW_FUA; if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD)) @@ -940,9 +938,15 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; cmnd->rw.opcode = op; + cmnd->rw.flags = 0; cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); + cmnd->rw.rsvd2 = 0; + cmnd->rw.metadata = 0; cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); + cmnd->rw.reftag = 0; + cmnd->rw.apptag = 0; + cmnd->rw.appmask = 0; if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams) nvme_assign_write_stream(ctrl, req, &control, &dsmgmt); From 3b44b3712c5b19b0af11c25cd978abdc3680d5e7 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Mon, 18 Oct 2021 21:34:31 +0800 Subject: [PATCH 180/255] io_uring: split logic of force_nonblock Currently force_nonblock stands for three meanings: - nowait or not - in an io-worker or not(hold uring_lock or not) Let's split the logic to two flags, IO_URING_F_NONBLOCK and IO_URING_F_UNLOCKED for convenience of the next patch. Suggested-by: Pavel Begunkov Signed-off-by: Hao Xu Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20211018133431.103298-1-haoxu@linux.alibaba.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index df07ce2fa6bc..c87931d8b503 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -199,6 +199,7 @@ struct io_rings { enum io_uring_cmd_flags { IO_URING_F_COMPLETE_DEFER = 1, + IO_URING_F_UNLOCKED = 2, /* int's last bit, sign checks are usually faster than a bit test */ IO_URING_F_NONBLOCK = INT_MIN, }; @@ -2706,10 +2707,10 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - const bool need_lock = !(issue_flags & IO_URING_F_NONBLOCK); + const bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; /* workqueue context doesn't hold uring_lock, grab it now */ - if (unlikely(need_lock)) + if (unlikely(needs_lock)) mutex_lock(&ctx->uring_lock); /* @@ -2737,7 +2738,7 @@ static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) else wq_list_add_tail(&req->comp_list, &ctx->iopoll_list); - if (unlikely(need_lock)) { + if (unlikely(needs_lock)) { /* * If IORING_SETUP_SQPOLL is enabled, sqes are either handle * in sq thread task context or in io worker task context. If @@ -2921,7 +2922,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_ring_ctx *ctx = req->ctx; req_set_fail(req); - if (!(issue_flags & IO_URING_F_NONBLOCK)) { + if (issue_flags & IO_URING_F_UNLOCKED) { mutex_lock(&ctx->uring_lock); __io_req_complete(req, issue_flags, ret, cflags); mutex_unlock(&ctx->uring_lock); @@ -3031,7 +3032,7 @@ static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len, { struct io_buffer *kbuf = req->kbuf; struct io_buffer *head; - bool needs_lock = !(issue_flags & IO_URING_F_NONBLOCK); + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; if (req->flags & REQ_F_BUFFER_SELECTED) return kbuf; @@ -3336,7 +3337,7 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw) int ret; /* submission path, ->uring_lock should already be taken */ - ret = io_import_iovec(rw, req, &iov, &iorw->s, IO_URING_F_NONBLOCK); + ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); if (unlikely(ret < 0)) return ret; @@ -4309,9 +4310,9 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head; int ret = 0; - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; - io_ring_submit_lock(ctx, !force_nonblock); + io_ring_submit_lock(ctx, needs_lock); lockdep_assert_held(&ctx->uring_lock); @@ -4324,7 +4325,7 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) /* complete before unlock, IOPOLL may need the lock */ __io_req_complete(req, issue_flags, ret, 0); - io_ring_submit_unlock(ctx, !force_nonblock); + io_ring_submit_unlock(ctx, needs_lock); return 0; } @@ -4396,9 +4397,9 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) struct io_ring_ctx *ctx = req->ctx; struct io_buffer *head, *list; int ret = 0; - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; - io_ring_submit_lock(ctx, !force_nonblock); + io_ring_submit_lock(ctx, needs_lock); lockdep_assert_held(&ctx->uring_lock); @@ -4414,7 +4415,7 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) req_set_fail(req); /* complete before unlock, IOPOLL may need the lock */ __io_req_complete(req, issue_flags, ret, 0); - io_ring_submit_unlock(ctx, !force_nonblock); + io_ring_submit_unlock(ctx, needs_lock); return 0; } @@ -6271,6 +6272,7 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; u64 sqe_addr = req->cancel.addr; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; struct io_tctx_node *node; int ret; @@ -6279,7 +6281,7 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) goto done; /* slow path, try all io-wq's */ - io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_lock(ctx, needs_lock); ret = -ENOENT; list_for_each_entry(node, &ctx->tctx_list, ctx_node) { struct io_uring_task *tctx = node->task->io_uring; @@ -6288,7 +6290,7 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) if (ret != -ENOENT) break; } - io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_unlock(ctx, needs_lock); done: if (ret < 0) req_set_fail(req); @@ -6315,6 +6317,7 @@ static int io_rsrc_update_prep(struct io_kiocb *req, static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; struct io_uring_rsrc_update2 up; int ret; @@ -6324,10 +6327,10 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) up.tags = 0; up.resv = 0; - io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_lock(ctx, needs_lock); ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, req->rsrc_update.nr_args); - io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_unlock(ctx, needs_lock); if (ret < 0) req_set_fail(req); @@ -6737,7 +6740,7 @@ static void io_wq_submit_work(struct io_wq_work *work) if (!ret) { do { - ret = io_issue_sqe(req, 0); + ret = io_issue_sqe(req, IO_URING_F_UNLOCKED); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -8326,12 +8329,12 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, unsigned int issue_flags, u32 slot_index) { struct io_ring_ctx *ctx = req->ctx; - bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; bool needs_switch = false; struct io_fixed_file *file_slot; int ret = -EBADF; - io_ring_submit_lock(ctx, !force_nonblock); + io_ring_submit_lock(ctx, needs_lock); if (file->f_op == &io_uring_fops) goto err; ret = -ENXIO; @@ -8372,7 +8375,7 @@ static int io_install_fixed_file(struct io_kiocb *req, struct file *file, err: if (needs_switch) io_rsrc_node_switch(ctx, ctx->file_data); - io_ring_submit_unlock(ctx, !force_nonblock); + io_ring_submit_unlock(ctx, needs_lock); if (ret) fput(file); return ret; @@ -8382,11 +8385,12 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) { unsigned int offset = req->close.file_slot - 1; struct io_ring_ctx *ctx = req->ctx; + bool needs_lock = issue_flags & IO_URING_F_UNLOCKED; struct io_fixed_file *file_slot; struct file *file; int ret, i; - io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_lock(ctx, needs_lock); ret = -ENXIO; if (unlikely(!ctx->file_data)) goto out; @@ -8412,7 +8416,7 @@ static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) io_rsrc_node_switch(ctx, ctx->file_data); ret = 0; out: - io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK)); + io_ring_submit_unlock(ctx, needs_lock); return ret; } From 898df2447b9ee8d759e85d33087505d3905bf2f0 Mon Sep 17 00:00:00 2001 From: Changcheng Deng Date: Wed, 20 Oct 2021 08:49:48 +0000 Subject: [PATCH 181/255] io_uring: Use ERR_CAST() instead of ERR_PTR(PTR_ERR()) Use ERR_CAST() instead of ERR_PTR(PTR_ERR()). This makes it more readable and also fix this warning detected by err_cast.cocci: ./fs/io_uring.c: WARNING: 3208: 11-18: ERR_CAST can be used with buf Reported-by: Zeal Robot Signed-off-by: Changcheng Deng Link: https://lore.kernel.org/r/20211020084948.1038420-1-deng.changcheng@zte.com.cn Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c87931d8b503..88c5ee4dc242 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3172,7 +3172,7 @@ static struct iovec *__io_import_iovec(int rw, struct io_kiocb *req, if (req->flags & REQ_F_BUFFER_SELECT) { buf = io_rw_buffer_select(req, &sqe_len, issue_flags); if (IS_ERR(buf)) - return ERR_PTR(PTR_ERR(buf)); + return ERR_CAST(buf); req->rw.len = sqe_len; } From 0c98057be9efa32de78dbc4685fc73da9d71faa1 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Wed, 20 Oct 2021 15:39:59 +0800 Subject: [PATCH 182/255] nbd: Fix use-after-free in pid_show I got issue as follows: [ 263.886511] BUG: KASAN: use-after-free in pid_show+0x11f/0x13f [ 263.888359] Read of size 4 at addr ffff8880bf0648c0 by task cat/746 [ 263.890479] CPU: 0 PID: 746 Comm: cat Not tainted 4.19.90-dirty #140 [ 263.893162] Call Trace: [ 263.893509] dump_stack+0x108/0x15f [ 263.893999] print_address_description+0xa5/0x372 [ 263.894641] kasan_report.cold+0x236/0x2a8 [ 263.895696] __asan_report_load4_noabort+0x25/0x30 [ 263.896365] pid_show+0x11f/0x13f [ 263.897422] dev_attr_show+0x48/0x90 [ 263.898361] sysfs_kf_seq_show+0x24d/0x4b0 [ 263.899479] kernfs_seq_show+0x14e/0x1b0 [ 263.900029] seq_read+0x43f/0x1150 [ 263.900499] kernfs_fop_read+0xc7/0x5a0 [ 263.903764] vfs_read+0x113/0x350 [ 263.904231] ksys_read+0x103/0x270 [ 263.905230] __x64_sys_read+0x77/0xc0 [ 263.906284] do_syscall_64+0x106/0x360 [ 263.906797] entry_SYSCALL_64_after_hwframe+0x44/0xa9 Reproduce this issue as follows: 1. nbd-server 8000 /tmp/disk 2. nbd-client localhost 8000 /dev/nbd1 3. cat /sys/block/nbd1/pid Then trigger use-after-free in pid_show. Reason is after do step '2', nbd-client progress is already exit. So it's task_struct already freed. To solve this issue, revert part of 6521d39a64b3's modify and remove useless 'recv_task' member of nbd_device. Fixes: 6521d39a64b3 ("nbd: Remove variable 'pid'") Signed-off-by: Ye Bin Reviewed-by: Josef Bacik Link: https://lore.kernel.org/r/20211020073959.2679255-1-yebin10@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 0d064fab6186..0ee104fbb628 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -122,10 +122,10 @@ struct nbd_device { struct work_struct remove_work; struct list_head list; - struct task_struct *task_recv; struct task_struct *task_setup; unsigned long flags; + pid_t pid; /* pid of nbd-client, if attached */ char *backend; }; @@ -223,7 +223,7 @@ static ssize_t pid_show(struct device *dev, struct gendisk *disk = dev_to_disk(dev); struct nbd_device *nbd = (struct nbd_device *)disk->private_data; - return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv)); + return sprintf(buf, "%d\n", nbd->pid); } static const struct device_attribute pid_attr = { @@ -335,7 +335,7 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, nbd->config->bytesize = bytesize; nbd->config->blksize_bits = __ffs(blksize); - if (!nbd->task_recv) + if (!nbd->pid) return 0; if (nbd->config->flags & NBD_FLAG_SEND_TRIM) { @@ -1300,7 +1300,7 @@ static void nbd_config_put(struct nbd_device *nbd) if (test_and_clear_bit(NBD_RT_HAS_PID_FILE, &config->runtime_flags)) device_remove_file(disk_to_dev(nbd->disk), &pid_attr); - nbd->task_recv = NULL; + nbd->pid = 0; if (test_and_clear_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags)) { device_remove_file(disk_to_dev(nbd->disk), &backend_attr); @@ -1341,7 +1341,7 @@ static int nbd_start_device(struct nbd_device *nbd) int num_connections = config->num_connections; int error = 0, i; - if (nbd->task_recv) + if (nbd->pid) return -EBUSY; if (!config->socks) return -EINVAL; @@ -1360,7 +1360,7 @@ static int nbd_start_device(struct nbd_device *nbd) } blk_mq_update_nr_hw_queues(&nbd->tag_set, config->num_connections); - nbd->task_recv = current; + nbd->pid = task_pid_nr(current); nbd_parse_flags(nbd); @@ -1616,8 +1616,8 @@ static int nbd_dbg_tasks_show(struct seq_file *s, void *unused) { struct nbd_device *nbd = s->private; - if (nbd->task_recv) - seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv)); + if (nbd->pid) + seq_printf(s, "recv: %d\n", nbd->pid); return 0; } @@ -2198,7 +2198,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) mutex_lock(&nbd->config_lock); config = nbd->config; if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) || - !nbd->task_recv) { + !nbd->pid) { dev_err(nbd_to_dev(nbd), "not configured, cannot reconfigure\n"); ret = -EINVAL; From 169bbdacaa473a2f3abd3ab8170e1c7795931560 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 20 Oct 2021 13:51:18 +0200 Subject: [PATCH 183/255] s390/dasd: handle request magic consistently as unsigned int MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get rid of the rather odd casts to character pointer of the dasd_ccw_req magic member and simply use the unsigned int value unmodified everywhere. Acked-by: Jan Höppner Signed-off-by: Heiko Carstens Signed-off-by: Stefan Haberland Link: https://lore.kernel.org/r/20211020115124.1735254-2-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_3990_erp.c | 6 +++--- drivers/s390/block/dasd_erp.c | 8 ++++---- drivers/s390/block/dasd_int.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 4691a3c35d72..299001ad9a32 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -201,7 +201,7 @@ dasd_3990_erp_DCTL(struct dasd_ccw_req * erp, char modifier) struct ccw1 *ccw; struct dasd_ccw_req *dctl_cqr; - dctl_cqr = dasd_alloc_erp_request((char *) &erp->magic, 1, + dctl_cqr = dasd_alloc_erp_request(erp->magic, 1, sizeof(struct DCTL_data), device); if (IS_ERR(dctl_cqr)) { @@ -1652,7 +1652,7 @@ dasd_3990_erp_action_1B_32(struct dasd_ccw_req * default_erp, char *sense) } /* Build new ERP request including DE/LO */ - erp = dasd_alloc_erp_request((char *) &cqr->magic, + erp = dasd_alloc_erp_request(cqr->magic, 2 + 1,/* DE/LO + TIC */ sizeof(struct DE_eckd_data) + sizeof(struct LO_eckd_data), device); @@ -2388,7 +2388,7 @@ static struct dasd_ccw_req *dasd_3990_erp_add_erp(struct dasd_ccw_req *cqr) } /* allocate additional request block */ - erp = dasd_alloc_erp_request((char *) &cqr->magic, + erp = dasd_alloc_erp_request(cqr->magic, cplength, datasize, device); if (IS_ERR(erp)) { if (cqr->retries <= 0) { diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index ba4fa372d02d..c07e6e713518 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -24,7 +24,7 @@ #include "dasd_int.h" struct dasd_ccw_req * -dasd_alloc_erp_request(char *magic, int cplength, int datasize, +dasd_alloc_erp_request(unsigned int magic, int cplength, int datasize, struct dasd_device * device) { unsigned long flags; @@ -33,8 +33,8 @@ dasd_alloc_erp_request(char *magic, int cplength, int datasize, int size; /* Sanity checks */ - BUG_ON( magic == NULL || datasize > PAGE_SIZE || - (cplength*sizeof(struct ccw1)) > PAGE_SIZE); + BUG_ON(datasize > PAGE_SIZE || + (cplength*sizeof(struct ccw1)) > PAGE_SIZE); size = (sizeof(struct dasd_ccw_req) + 7L) & -8L; if (cplength > 0) @@ -62,7 +62,7 @@ dasd_alloc_erp_request(char *magic, int cplength, int datasize, cqr->data = data; memset(cqr->data, 0, datasize); } - strncpy((char *) &cqr->magic, magic, 4); + cqr->magic = magic; ASCEBC((char *) &cqr->magic, 4); set_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags); dasd_get_device(device); diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 155428bfed8a..15d8731c1eee 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -887,7 +887,7 @@ void dasd_proc_exit(void); /* externals in dasd_erp.c */ struct dasd_ccw_req *dasd_default_erp_action(struct dasd_ccw_req *); struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *); -struct dasd_ccw_req *dasd_alloc_erp_request(char *, int, int, +struct dasd_ccw_req *dasd_alloc_erp_request(unsigned int, int, int, struct dasd_device *); void dasd_free_erp_request(struct dasd_ccw_req *, struct dasd_device *); void dasd_log_sense(struct dasd_ccw_req *, struct irb *); From 10c78e53eea3d9359d4e2a0a4a17a73aacd85497 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 20 Oct 2021 13:51:19 +0200 Subject: [PATCH 184/255] s390/dasd: fix kernel doc comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix this: drivers/s390/block/dasd_ioctl.c:666: warning: Function parameter or member 'disk' not described in 'dasd_biodasdinfo' drivers/s390/block/dasd_ioctl.c:666: warning: Function parameter or member 'info' not described in 'dasd_biodasdinfo' Acked-by: Jan Höppner Signed-off-by: Heiko Carstens Signed-off-by: Stefan Haberland Link: https://lore.kernel.org/r/20211020115124.1735254-3-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_ioctl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 468cbeb539ff..95349f95758c 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -650,8 +650,8 @@ int dasd_ioctl(struct block_device *bdev, fmode_t mode, /** * dasd_biodasdinfo() - fill out the dasd information structure - * @disk [in]: pointer to gendisk structure that references a DASD - * @info [out]: pointer to the dasd_information2_t structure + * @disk: [in] pointer to gendisk structure that references a DASD + * @info: [out] pointer to the dasd_information2_t structure * * Provide access to DASD specific information. * The gendisk structure is checked if it belongs to the DASD driver by From 23596961b43752be871ea3a5756c7267f8140cff Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 20 Oct 2021 13:51:20 +0200 Subject: [PATCH 185/255] s390/dasd: split up dasd_eckd_read_conf Move the cabling check out of dasd_eckd_read_conf and split it up into separate functions to improve readability and re-use functions. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20211020115124.1735254-4-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 130 +++++++++++++-------------------- 1 file changed, 51 insertions(+), 79 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 460e0f1cca53..f6ff26472936 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1071,15 +1071,55 @@ static void dasd_eckd_read_fc_security(struct dasd_device *device) } } +static void dasd_eckd_get_uid_string(struct dasd_eckd_private *private, + char *print_uid) +{ + struct dasd_uid *uid; + + uid = &private->uid; + if (strlen(uid->vduit) > 0) + snprintf(print_uid, sizeof(*print_uid), + "%s.%s.%04x.%02x.%s", + uid->vendor, uid->serial, uid->ssid, + uid->real_unit_addr, uid->vduit); + else + snprintf(print_uid, sizeof(*print_uid), + "%s.%s.%04x.%02x", + uid->vendor, uid->serial, uid->ssid, + uid->real_unit_addr); +} + +static int dasd_eckd_check_cabling(struct dasd_device *device, + void *conf_data, __u8 lpm) +{ + struct dasd_eckd_private *private = device->private; + char print_path_uid[60], print_device_uid[60]; + struct dasd_eckd_private path_private; + + path_private.conf_data = conf_data; + path_private.conf_len = DASD_ECKD_RCD_DATA_SIZE; + if (dasd_eckd_identify_conf_parts(&path_private)) + return 1; + + if (dasd_eckd_compare_path_uid(device, &path_private)) { + dasd_eckd_get_uid_string(&path_private, print_path_uid); + dasd_eckd_get_uid_string(private, print_device_uid); + dev_err(&device->cdev->dev, + "Not all channel paths lead to the same device, path %02X leads to device %s instead of %s\n", + lpm, print_path_uid, print_device_uid); + return 1; + } + + return 0; +} + static int dasd_eckd_read_conf(struct dasd_device *device) { void *conf_data; int conf_len, conf_data_saved; int rc, path_err, pos; __u8 lpm, opm; - struct dasd_eckd_private *private, path_private; - struct dasd_uid *uid; - char print_path_uid[60], print_device_uid[60]; + struct dasd_eckd_private *private; private = device->private; opm = ccw_device_get_path_mask(device->cdev); @@ -1123,59 +1163,11 @@ static int dasd_eckd_read_conf(struct dasd_device *device) */ dasd_eckd_generate_uid(device); conf_data_saved++; - } else { - path_private.conf_data = conf_data; - path_private.conf_len = DASD_ECKD_RCD_DATA_SIZE; - if (dasd_eckd_identify_conf_parts( - &path_private)) { - path_private.conf_data = NULL; - path_private.conf_len = 0; - kfree(conf_data); - continue; - } - if (dasd_eckd_compare_path_uid( - device, &path_private)) { - uid = &path_private.uid; - if (strlen(uid->vduit) > 0) - snprintf(print_path_uid, - sizeof(print_path_uid), - "%s.%s.%04x.%02x.%s", - uid->vendor, uid->serial, - uid->ssid, uid->real_unit_addr, - uid->vduit); - else - snprintf(print_path_uid, - sizeof(print_path_uid), - "%s.%s.%04x.%02x", - uid->vendor, uid->serial, - uid->ssid, - uid->real_unit_addr); - uid = &private->uid; - if (strlen(uid->vduit) > 0) - snprintf(print_device_uid, - sizeof(print_device_uid), - "%s.%s.%04x.%02x.%s", - uid->vendor, uid->serial, - uid->ssid, uid->real_unit_addr, - uid->vduit); - else - snprintf(print_device_uid, - sizeof(print_device_uid), - "%s.%s.%04x.%02x", - uid->vendor, uid->serial, - uid->ssid, - uid->real_unit_addr); - dev_err(&device->cdev->dev, - "Not all channel paths lead to " - "the same device, path %02X leads to " - "device %s instead of %s\n", lpm, - print_path_uid, print_device_uid); - path_err = -EINVAL; - dasd_path_add_cablepm(device, lpm); - continue; - } - path_private.conf_data = NULL; - path_private.conf_len = 0; + } else if (dasd_eckd_check_cabling(device, conf_data, lpm)) { + dasd_path_add_cablepm(device, lpm); + path_err = -EINVAL; + kfree(conf_data); + continue; } pos = pathmask_to_pos(lpm); @@ -1300,7 +1292,6 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, struct pe_handler_work_data *data) { struct dasd_eckd_private path_private; - struct dasd_uid *uid; __u8 path_rcd_buf[DASD_ECKD_RCD_DATA_SIZE]; __u8 lpm, opm, npm, ppm, epm, hpfpm, cablepm; struct dasd_conf_data *conf_data; @@ -1397,19 +1388,8 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, if (rebuild_device_uid(device, data) || dasd_eckd_compare_path_uid( device, &path_private)) { - uid = &path_private.uid; - if (strlen(uid->vduit) > 0) - snprintf(print_uid, sizeof(print_uid), - "%s.%s.%04x.%02x.%s", - uid->vendor, uid->serial, - uid->ssid, uid->real_unit_addr, - uid->vduit); - else - snprintf(print_uid, sizeof(print_uid), - "%s.%s.%04x.%02x", - uid->vendor, uid->serial, - uid->ssid, - uid->real_unit_addr); + dasd_eckd_get_uid_string(&path_private, + print_uid); dev_err(&device->cdev->dev, "The newly added channel path %02X " "will not be used because it leads " @@ -5820,15 +5800,7 @@ static int dasd_eckd_reload_device(struct dasd_device *device) dasd_eckd_get_uid(device, &uid); if (old_base != uid.base_unit_addr) { - if (strlen(uid.vduit) > 0) - snprintf(print_uid, sizeof(print_uid), - "%s.%s.%04x.%02x.%s", uid.vendor, uid.serial, - uid.ssid, uid.base_unit_addr, uid.vduit); - else - snprintf(print_uid, sizeof(print_uid), - "%s.%s.%04x.%02x", uid.vendor, uid.serial, - uid.ssid, uid.base_unit_addr); - + dasd_eckd_get_uid_string(private, print_uid); dev_info(&device->cdev->dev, "An Alias device was reassigned to a new base device " "with UID: %s\n", print_uid); From 74e2f2110258d5cb5f3bcbf3f9813d523eb049b9 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 20 Oct 2021 13:51:21 +0200 Subject: [PATCH 186/255] s390/dasd: move dasd_eckd_read_fc_security dasd_eckd_read_conf is called multiple times during device setup but the fc_security feature needs to be read only once. So move it into the calling function. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20211020115124.1735254-5-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index f6ff26472936..248006291c29 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1189,8 +1189,6 @@ static int dasd_eckd_read_conf(struct dasd_device *device) } } - dasd_eckd_read_fc_security(device); - return path_err; } @@ -2101,6 +2099,7 @@ dasd_eckd_check_characteristics(struct dasd_device *device) if (rc) goto out_err3; + dasd_eckd_read_fc_security(device); dasd_path_create_kobjects(device); /* Read Feature Codes */ @@ -5788,6 +5787,8 @@ static int dasd_eckd_reload_device(struct dasd_device *device) if (rc) goto out_err; + dasd_eckd_read_fc_security(device); + rc = dasd_eckd_generate_uid(device); if (rc) goto out_err; From 542e30ce8e6e1104e99c78a520a821b05b6ea98b Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 20 Oct 2021 13:51:22 +0200 Subject: [PATCH 187/255] s390/dasd: summarize dasd configuration data in a separate structure Summarize the dasd configuration data in a separate structure so that functions that need temporary config data do not need to allocate the whole eckd_private structure. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20211020115124.1735254-6-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 182 ++++++++++++++++----------------- drivers/s390/block/dasd_eckd.h | 13 ++- 2 files changed, 98 insertions(+), 97 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 248006291c29..6cfaaddb4fbb 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -560,8 +560,8 @@ static int prefix_LRE(struct ccw1 *ccw, struct PFX_eckd_data *pfxdata, return -EINVAL; } pfxdata->format = format; - pfxdata->base_address = basepriv->ned->unit_addr; - pfxdata->base_lss = basepriv->ned->ID; + pfxdata->base_address = basepriv->conf.ned->unit_addr; + pfxdata->base_lss = basepriv->conf.ned->ID; pfxdata->validity.define_extent = 1; /* private uid is kept up to date, conf_data may be outdated */ @@ -736,32 +736,30 @@ dasd_eckd_cdl_reclen(int recid) return LABEL_SIZE; } /* create unique id from private structure. */ -static void create_uid(struct dasd_eckd_private *private) +static void create_uid(struct dasd_conf *conf, struct dasd_uid *uid) { int count; - struct dasd_uid *uid; - uid = &private->uid; memset(uid, 0, sizeof(struct dasd_uid)); - memcpy(uid->vendor, private->ned->HDA_manufacturer, + memcpy(uid->vendor, conf->ned->HDA_manufacturer, sizeof(uid->vendor) - 1); EBCASC(uid->vendor, sizeof(uid->vendor) - 1); - memcpy(uid->serial, &private->ned->serial, + memcpy(uid->serial, &conf->ned->serial, sizeof(uid->serial) - 1); EBCASC(uid->serial, sizeof(uid->serial) - 1); - uid->ssid = private->gneq->subsystemID; - uid->real_unit_addr = private->ned->unit_addr; - if (private->sneq) { - uid->type = private->sneq->sua_flags; + uid->ssid = conf->gneq->subsystemID; + uid->real_unit_addr = conf->ned->unit_addr; + if (conf->sneq) { + uid->type = conf->sneq->sua_flags; if (uid->type == UA_BASE_PAV_ALIAS) - uid->base_unit_addr = private->sneq->base_unit_addr; + uid->base_unit_addr = conf->sneq->base_unit_addr; } else { uid->type = UA_BASE_DEVICE; } - if (private->vdsneq) { + if (conf->vdsneq) { for (count = 0; count < 16; count++) { sprintf(uid->vduit+2*count, "%02x", - private->vdsneq->uit[count]); + conf->vdsneq->uit[count]); } } } @@ -776,10 +774,10 @@ static int dasd_eckd_generate_uid(struct dasd_device *device) if (!private) return -ENODEV; - if (!private->ned || !private->gneq) + if (!private->conf.ned || !private->conf.gneq) return -ENODEV; spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - create_uid(private); + create_uid(&private->conf, &private->uid); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); return 0; } @@ -803,14 +801,15 @@ static int dasd_eckd_get_uid(struct dasd_device *device, struct dasd_uid *uid) * return 0 for match */ static int dasd_eckd_compare_path_uid(struct dasd_device *device, - struct dasd_eckd_private *private) + struct dasd_conf *path_conf) { struct dasd_uid device_uid; + struct dasd_uid path_uid; - create_uid(private); + create_uid(path_conf, &path_uid); dasd_eckd_get_uid(device, &device_uid); - return memcmp(&device_uid, &private->uid, sizeof(struct dasd_uid)); + return memcmp(&device_uid, &path_uid, sizeof(struct dasd_uid)); } static void dasd_eckd_fill_rcd_cqr(struct dasd_device *device, @@ -946,34 +945,34 @@ out_error: return ret; } -static int dasd_eckd_identify_conf_parts(struct dasd_eckd_private *private) +static int dasd_eckd_identify_conf_parts(struct dasd_conf *conf) { struct dasd_sneq *sneq; int i, count; - private->ned = NULL; - private->sneq = NULL; - private->vdsneq = NULL; - private->gneq = NULL; - count = private->conf_len / sizeof(struct dasd_sneq); - sneq = (struct dasd_sneq *)private->conf_data; + conf->ned = NULL; + conf->sneq = NULL; + conf->vdsneq = NULL; + conf->gneq = NULL; + count = conf->len / sizeof(struct dasd_sneq); + sneq = (struct dasd_sneq *)conf->data; for (i = 0; i < count; ++i) { if (sneq->flags.identifier == 1 && sneq->format == 1) - private->sneq = sneq; + conf->sneq = sneq; else if (sneq->flags.identifier == 1 && sneq->format == 4) - private->vdsneq = (struct vd_sneq *)sneq; + conf->vdsneq = (struct vd_sneq *)sneq; else if (sneq->flags.identifier == 2) - private->gneq = (struct dasd_gneq *)sneq; + conf->gneq = (struct dasd_gneq *)sneq; else if (sneq->flags.identifier == 3 && sneq->res1 == 1) - private->ned = (struct dasd_ned *)sneq; + conf->ned = (struct dasd_ned *)sneq; sneq++; } - if (!private->ned || !private->gneq) { - private->ned = NULL; - private->sneq = NULL; - private->vdsneq = NULL; - private->gneq = NULL; + if (!conf->ned || !conf->gneq) { + conf->ned = NULL; + conf->sneq = NULL; + conf->vdsneq = NULL; + conf->gneq = NULL; return -EINVAL; } return 0; @@ -1016,9 +1015,9 @@ static void dasd_eckd_store_conf_data(struct dasd_device *device, * with the new one if this points to the same data */ cdp = device->path[chp].conf_data; - if (private->conf_data == cdp) { - private->conf_data = (void *)conf_data; - dasd_eckd_identify_conf_parts(private); + if (private->conf.data == cdp) { + private->conf.data = (void *)conf_data; + dasd_eckd_identify_conf_parts(&private->conf); } ccw_device_get_schid(device->cdev, &sch_id); device->path[chp].conf_data = conf_data; @@ -1036,8 +1035,8 @@ static void dasd_eckd_clear_conf_data(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int i; - private->conf_data = NULL; - private->conf_len = 0; + private->conf.data = NULL; + private->conf.len = 0; for (i = 0; i < 8; i++) { kfree(device->path[i].conf_data); device->path[i].conf_data = NULL; @@ -1071,22 +1070,22 @@ static void dasd_eckd_read_fc_security(struct dasd_device *device) } } -static void dasd_eckd_get_uid_string(struct dasd_eckd_private *private, +static void dasd_eckd_get_uid_string(struct dasd_conf *conf, char *print_uid) { - struct dasd_uid *uid; + struct dasd_uid uid; - uid = &private->uid; - if (strlen(uid->vduit) > 0) + create_uid(conf, &uid); + if (strlen(uid.vduit) > 0) snprintf(print_uid, sizeof(*print_uid), "%s.%s.%04x.%02x.%s", - uid->vendor, uid->serial, uid->ssid, - uid->real_unit_addr, uid->vduit); + uid.vendor, uid.serial, uid.ssid, + uid.real_unit_addr, uid.vduit); else snprintf(print_uid, sizeof(*print_uid), "%s.%s.%04x.%02x", - uid->vendor, uid->serial, uid->ssid, - uid->real_unit_addr); + uid.vendor, uid.serial, uid.ssid, + uid.real_unit_addr); } static int dasd_eckd_check_cabling(struct dasd_device *device, @@ -1094,16 +1093,16 @@ static int dasd_eckd_check_cabling(struct dasd_device *device, { struct dasd_eckd_private *private = device->private; char print_path_uid[60], print_device_uid[60]; - struct dasd_eckd_private path_private; + struct dasd_conf path_conf; - path_private.conf_data = conf_data; - path_private.conf_len = DASD_ECKD_RCD_DATA_SIZE; - if (dasd_eckd_identify_conf_parts(&path_private)) + path_conf.data = conf_data; + path_conf.len = DASD_ECKD_RCD_DATA_SIZE; + if (dasd_eckd_identify_conf_parts(&path_conf)) return 1; - if (dasd_eckd_compare_path_uid(device, &path_private)) { - dasd_eckd_get_uid_string(&path_private, print_path_uid); - dasd_eckd_get_uid_string(private, print_device_uid); + if (dasd_eckd_compare_path_uid(device, &path_conf)) { + dasd_eckd_get_uid_string(&path_conf, print_path_uid); + dasd_eckd_get_uid_string(&private->conf, print_device_uid); dev_err(&device->cdev->dev, "Not all channel paths lead to the same device, path %02X leads to device %s instead of %s\n", lpm, print_path_uid, print_device_uid); @@ -1149,11 +1148,11 @@ static int dasd_eckd_read_conf(struct dasd_device *device) if (!conf_data_saved) { /* initially clear previously stored conf_data */ dasd_eckd_clear_conf_data(device); - private->conf_data = conf_data; - private->conf_len = conf_len; - if (dasd_eckd_identify_conf_parts(private)) { - private->conf_data = NULL; - private->conf_len = 0; + private->conf.data = conf_data; + private->conf.len = conf_len; + if (dasd_eckd_identify_conf_parts(&private->conf)) { + private->conf.data = NULL; + private->conf.len = 0; kfree(conf_data); continue; } @@ -1203,7 +1202,7 @@ static u32 get_fcx_max_data(struct dasd_device *device) return 0; /* is transport mode supported? */ fcx_in_css = css_general_characteristics.fcx; - fcx_in_gneq = private->gneq->reserved2[7] & 0x04; + fcx_in_gneq = private->conf.gneq->reserved2[7] & 0x04; fcx_in_features = private->features.feature[40] & 0x80; tpm = fcx_in_css && fcx_in_gneq && fcx_in_features; @@ -1272,9 +1271,9 @@ static int rebuild_device_uid(struct dasd_device *device, "returned error %d", rc); break; } - memcpy(private->conf_data, data->rcd_buffer, + memcpy(private->conf.data, data->rcd_buffer, DASD_ECKD_RCD_DATA_SIZE); - if (dasd_eckd_identify_conf_parts(private)) { + if (dasd_eckd_identify_conf_parts(&private->conf)) { rc = -ENODEV; } else /* first valid path is enough */ break; @@ -1289,10 +1288,10 @@ static int rebuild_device_uid(struct dasd_device *device, static void dasd_eckd_path_available_action(struct dasd_device *device, struct pe_handler_work_data *data) { - struct dasd_eckd_private path_private; __u8 path_rcd_buf[DASD_ECKD_RCD_DATA_SIZE]; __u8 lpm, opm, npm, ppm, epm, hpfpm, cablepm; struct dasd_conf_data *conf_data; + struct dasd_conf path_conf; unsigned long flags; char print_uid[60]; int rc, pos; @@ -1356,11 +1355,11 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, */ memcpy(&path_rcd_buf, data->rcd_buffer, DASD_ECKD_RCD_DATA_SIZE); - path_private.conf_data = (void *) &path_rcd_buf; - path_private.conf_len = DASD_ECKD_RCD_DATA_SIZE; - if (dasd_eckd_identify_conf_parts(&path_private)) { - path_private.conf_data = NULL; - path_private.conf_len = 0; + path_conf.data = (void *)&path_rcd_buf; + path_conf.len = DASD_ECKD_RCD_DATA_SIZE; + if (dasd_eckd_identify_conf_parts(&path_conf)) { + path_conf.data = NULL; + path_conf.len = 0; continue; } @@ -1371,7 +1370,7 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, * the first working path UID will be used as device UID */ if (dasd_path_get_opm(device) && - dasd_eckd_compare_path_uid(device, &path_private)) { + dasd_eckd_compare_path_uid(device, &path_conf)) { /* * the comparison was not successful * rebuild the device UID with at least one @@ -1385,9 +1384,8 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, */ if (rebuild_device_uid(device, data) || dasd_eckd_compare_path_uid( - device, &path_private)) { - dasd_eckd_get_uid_string(&path_private, - print_uid); + device, &path_conf)) { + dasd_eckd_get_uid_string(&path_conf, print_uid); dev_err(&device->cdev->dev, "The newly added channel path %02X " "will not be used because it leads " @@ -1603,8 +1601,8 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device) prssdp = cqr->data; prssdp->order = PSF_ORDER_PRSSD; prssdp->suborder = PSF_SUBORDER_VSQ; /* Volume Storage Query */ - prssdp->lss = private->ned->ID; - prssdp->volume = private->ned->unit_addr; + prssdp->lss = private->conf.ned->ID; + prssdp->volume = private->conf.ned->unit_addr; ccw = cqr->cpaddr; ccw->cmd_code = DASD_ECKD_CCW_PSF; @@ -2063,11 +2061,11 @@ dasd_eckd_check_characteristics(struct dasd_device *device) device->path_thrhld = DASD_ECKD_PATH_THRHLD; device->path_interval = DASD_ECKD_PATH_INTERVAL; - if (private->gneq) { + if (private->conf.gneq) { value = 1; - for (i = 0; i < private->gneq->timeout.value; i++) + for (i = 0; i < private->conf.gneq->timeout.value; i++) value = 10 * value; - value = value * private->gneq->timeout.number; + value = value * private->conf.gneq->timeout.number; /* do not accept useless values */ if (value != 0 && value <= DASD_EXPIRES_MAX) device->default_expires = value; @@ -2174,10 +2172,10 @@ static void dasd_eckd_uncheck_device(struct dasd_device *device) return; dasd_alias_disconnect_device_from_lcu(device); - private->ned = NULL; - private->sneq = NULL; - private->vdsneq = NULL; - private->gneq = NULL; + private->conf.ned = NULL; + private->conf.sneq = NULL; + private->conf.vdsneq = NULL; + private->conf.gneq = NULL; dasd_eckd_clear_conf_data(device); dasd_path_remove_kobjects(device); } @@ -3729,8 +3727,8 @@ dasd_eckd_dso_ras(struct dasd_device *device, struct dasd_block *block, * subset. */ ras_data->op_flags.guarantee_init = !!(features->feature[56] & 0x01); - ras_data->lss = private->ned->ID; - ras_data->dev_addr = private->ned->unit_addr; + ras_data->lss = private->conf.ned->ID; + ras_data->dev_addr = private->conf.ned->unit_addr; ras_data->nr_exts = nr_exts; if (by_extent) { @@ -4272,8 +4270,8 @@ static int prepare_itcw(struct itcw *itcw, memset(&pfxdata, 0, sizeof(pfxdata)); pfxdata.format = 1; /* PFX with LRE */ - pfxdata.base_address = basepriv->ned->unit_addr; - pfxdata.base_lss = basepriv->ned->ID; + pfxdata.base_address = basepriv->conf.ned->unit_addr; + pfxdata.base_lss = basepriv->conf.ned->ID; pfxdata.validity.define_extent = 1; /* private uid is kept up to date, conf_data may be outdated */ @@ -4942,9 +4940,9 @@ dasd_eckd_fill_info(struct dasd_device * device, info->characteristics_size = sizeof(private->rdc_data); memcpy(info->characteristics, &private->rdc_data, sizeof(private->rdc_data)); - info->confdata_size = min((unsigned long)private->conf_len, - sizeof(info->configuration_data)); - memcpy(info->configuration_data, private->conf_data, + info->confdata_size = min_t(unsigned long, private->conf.len, + sizeof(info->configuration_data)); + memcpy(info->configuration_data, private->conf.data, info->confdata_size); return 0; } @@ -5801,7 +5799,7 @@ static int dasd_eckd_reload_device(struct dasd_device *device) dasd_eckd_get_uid(device, &uid); if (old_base != uid.base_unit_addr) { - dasd_eckd_get_uid_string(private, print_uid); + dasd_eckd_get_uid_string(&private->conf, print_uid); dev_info(&device->cdev->dev, "An Alias device was reassigned to a new base device " "with UID: %s\n", print_uid); @@ -5939,8 +5937,8 @@ static int dasd_eckd_query_host_access(struct dasd_device *device, prssdp->order = PSF_ORDER_PRSSD; prssdp->suborder = PSF_SUBORDER_QHA; /* query host access */ /* LSS and Volume that will be queried */ - prssdp->lss = private->ned->ID; - prssdp->volume = private->ned->unit_addr; + prssdp->lss = private->conf.ned->ID; + prssdp->volume = private->conf.ned->unit_addr; /* all other bytes of prssdp must be zero */ ccw = cqr->cpaddr; diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 65e4630ad2ae..a91b265441cc 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -658,16 +658,19 @@ struct dasd_conf_data { struct dasd_gneq gneq; } __packed; -struct dasd_eckd_private { - struct dasd_eckd_characteristics rdc_data; - u8 *conf_data; - int conf_len; - +struct dasd_conf { + u8 *data; + int len; /* pointers to specific parts in the conf_data */ struct dasd_ned *ned; struct dasd_sneq *sneq; struct vd_sneq *vdsneq; struct dasd_gneq *gneq; +}; + +struct dasd_eckd_private { + struct dasd_eckd_characteristics rdc_data; + struct dasd_conf conf; struct eckd_count count_area[5]; int init_cqr_status; From 9dffede0115e96d0ff0a07e4382569a9c6dba735 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 20 Oct 2021 13:51:23 +0200 Subject: [PATCH 188/255] s390/dasd: fix missing path conf_data after failed allocation dasd_eckd_path_available_action() does a memory allocation to store the per path configuration data permanently. In the unlikely case that this allocation fails there is no conf_data stored for the corresponding path. This is OK since this is not necessary for an operational path but some features like control unit initiated reconfiguration (CUIR) do not work. To fix this add the path to the 'to be verified pathmask' again and schedule the handler again. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20211020115124.1735254-7-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 6cfaaddb4fbb..8410a25a65c1 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1403,6 +1403,14 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, if (conf_data) { memcpy(conf_data, data->rcd_buffer, DASD_ECKD_RCD_DATA_SIZE); + } else { + /* + * path is operational but path config data could not + * be stored due to low mem condition + * add it to the error path mask and schedule a path + * verification later that this could be added again + */ + epm |= lpm; } pos = pathmask_to_pos(lpm); dasd_eckd_store_conf_data(device, conf_data, pos); @@ -1423,7 +1431,10 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, } dasd_path_add_nppm(device, npm); dasd_path_add_ppm(device, ppm); - dasd_path_add_tbvpm(device, epm); + if (epm) { + dasd_path_add_tbvpm(device, epm); + dasd_device_set_timer(device, 50); + } dasd_path_add_cablepm(device, cablepm); dasd_path_add_nohpfpm(device, hpfpm); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); From a8e5d491dfc184c6b78cbb7f44107b01229c9df2 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 20 Oct 2021 13:51:24 +0200 Subject: [PATCH 189/255] s390/dasd: fix possibly missed path verification __dasd_device_check_path_events() calls the discipline path event handler. This handler can leave the 'to be verified pathmask' populated for an additional verification. There is a race window where the worker has finished before dasd_path_clear_all_verify() is called which resets the tbvpm. Due to this there could be outstanding path verifications missed. Fix by clearing the pathmasks before calling the handler and add them again in case of an error. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20211020115124.1735254-8-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 9 ++++++--- drivers/s390/block/dasd_int.h | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index e34c6cc61983..8e87a31e329d 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -2077,12 +2077,15 @@ static void __dasd_device_check_path_events(struct dasd_device *device) if (device->stopped & ~(DASD_STOPPED_DC_WAIT)) return; + + dasd_path_clear_all_verify(device); + dasd_path_clear_all_fcsec(device); + rc = device->discipline->pe_handler(device, tbvpm, fcsecpm); if (rc) { + dasd_path_add_tbvpm(device, tbvpm); + dasd_path_add_fcsecpm(device, fcsecpm); dasd_device_set_timer(device, 50); - } else { - dasd_path_clear_all_verify(device); - dasd_path_clear_all_fcsec(device); } }; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 15d8731c1eee..8b458010f88a 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -1305,6 +1305,15 @@ static inline void dasd_path_add_ppm(struct dasd_device *device, __u8 pm) dasd_path_preferred(device, chp); } +static inline void dasd_path_add_fcsecpm(struct dasd_device *device, __u8 pm) +{ + int chp; + + for (chp = 0; chp < 8; chp++) + if (pm & (0x80 >> chp)) + dasd_path_fcsec(device, chp); +} + /* * set functions for path masks * the existing path mask will be replaced by the given path mask From a307e2abfc22880a3026bc2f2a997402b7c2d833 Mon Sep 17 00:00:00 2001 From: Ding Senjie Date: Wed, 20 Oct 2021 22:38:05 +0800 Subject: [PATCH 190/255] md: bcache: Fix spelling of 'acquire' acqurie -> acquire Signed-off-by: Ding Senjie Reviewed-by: Hannes Reinecke Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-2-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index f2874c77ff79..330d6c167265 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -2750,7 +2750,7 @@ static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x) * The reason bch_register_lock is not held to call * bch_cache_set_stop() and bcache_device_stop() is to * avoid potential deadlock during reboot, because cache - * set or bcache device stopping process will acqurie + * set or bcache device stopping process will acquire * bch_register_lock too. * * We are safe here because bcache_is_reboot sets to From 0a2b3e363566c4cc8792d37c5e73b9d9295e075c Mon Sep 17 00:00:00 2001 From: Coly Li Date: Wed, 20 Oct 2021 22:38:06 +0800 Subject: [PATCH 191/255] bcache: reserve never used bits from bkey.high There sre 3 bits in member high of struct bkey are never used, and no plan to support them in future, - HEADER_SIZE, start at bit 58, length 2 bits - KEY_PINNED, start at bit 55, length 1 bit No any kernel code, or user space tool references or accesses the three bits. Therefore it is possible and feasible to reserve the valuable bits from bkey.high. They can be used in future for other purpose. Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-3-colyli@suse.de Signed-off-by: Jens Axboe --- include/uapi/linux/bcache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/bcache.h b/include/uapi/linux/bcache.h index cf7399f03b71..97413586195b 100644 --- a/include/uapi/linux/bcache.h +++ b/include/uapi/linux/bcache.h @@ -43,9 +43,9 @@ static inline void SET_##name(struct bkey *k, unsigned int i, __u64 v) \ #define KEY_MAX_U64S 8 KEY_FIELD(KEY_PTRS, high, 60, 3) -KEY_FIELD(HEADER_SIZE, high, 58, 2) +KEY_FIELD(__PAD0, high, 58, 2) KEY_FIELD(KEY_CSUM, high, 56, 2) -KEY_FIELD(KEY_PINNED, high, 55, 1) +KEY_FIELD(__PAD1, high, 55, 1) KEY_FIELD(KEY_DIRTY, high, 36, 1) KEY_FIELD(KEY_SIZE, high, 20, KEY_SIZE_BITS) From d55f7cb2e5c053010d2b527494da9bbb722a78ba Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Wed, 20 Oct 2021 22:38:07 +0800 Subject: [PATCH 192/255] bcache: fix error info in register_bcache() In register_bcache(), there are several cases we didn't set correct error info (return value and/or error message): - if kzalloc() fails, it needs to return ENOMEM and print "cannot allocate memory"; - if register_cache() fails, it's better to propagate its return value rather than using default EINVAL. Signed-off-by: Chao Yu Reviewed-by: Hannes Reinecke Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-4-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 330d6c167265..62b0140b0a73 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -2617,8 +2617,11 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, if (SB_IS_BDEV(sb)) { struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL); - if (!dc) + if (!dc) { + ret = -ENOMEM; + err = "cannot allocate memory"; goto out_put_sb_page; + } mutex_lock(&bch_register_lock); ret = register_bdev(sb, sb_disk, bdev, dc); @@ -2629,11 +2632,15 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, } else { struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL); - if (!ca) + if (!ca) { + ret = -ENOMEM; + err = "cannot allocate memory"; goto out_put_sb_page; + } /* blkdev_put() will be called in bch_cache_release() */ - if (register_cache(sb, sb_disk, bdev, ca) != 0) + ret = register_cache(sb, sb_disk, bdev, ca); + if (ret) goto out_free_sb; } From 0259d4498ba48454749ecfb9c81e892cdb8d1a32 Mon Sep 17 00:00:00 2001 From: Lin Feng Date: Wed, 20 Oct 2021 22:38:08 +0800 Subject: [PATCH 193/255] bcache: move calc_cached_dev_sectors to proper place on backing device detach Calculation of cache_set's cached sectors is done by travelling cached_devs list as shown below: static void calc_cached_dev_sectors(struct cache_set *c) { ... list_for_each_entry(dc, &c->cached_devs, list) sectors += bdev_sectors(dc->bdev); c->cached_dev_sectors = sectors; } But cached_dev won't be unlinked from c->cached_devs list until we call following list_move(&dc->list, &uncached_devices), so previous fix in 'commit 46010141da6677b81cc77f9b47f8ac62bd1cbfd3 ("bcache: recal cached_dev_sectors on detach")' is wrong, now we move it to its right place. Signed-off-by: Lin Feng Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-5-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 62b0140b0a73..dced2ea17431 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1154,9 +1154,9 @@ static void cached_dev_detach_finish(struct work_struct *w) mutex_lock(&bch_register_lock); - calc_cached_dev_sectors(dc->disk.c); bcache_device_detach(&dc->disk); list_move(&dc->list, &uncached_devices); + calc_cached_dev_sectors(dc->disk.c); clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags); clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags); From 7e84c2150731faec088ebfe33459f61d118b2497 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 20 Oct 2021 22:38:09 +0800 Subject: [PATCH 194/255] bcache: remove the cache_dev_name field from struct cache Just use the %pg format specifier to print the name directly. Signed-off-by: Christoph Hellwig Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-6-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/bcache.h | 2 -- drivers/md/bcache/io.c | 8 ++++---- drivers/md/bcache/super.c | 7 +++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 5fc989a6d452..47ff9ecea2e2 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -470,8 +470,6 @@ struct cache { atomic_long_t meta_sectors_written; atomic_long_t btree_sectors_written; atomic_long_t sectors_written; - - char cache_dev_name[BDEVNAME_SIZE]; }; struct gc_stat { diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index e4388fe3ab7e..564357de7640 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c @@ -123,13 +123,13 @@ void bch_count_io_errors(struct cache *ca, errors >>= IO_ERROR_SHIFT; if (errors < ca->set->error_limit) - pr_err("%s: IO error on %s%s\n", - ca->cache_dev_name, m, + pr_err("%pg: IO error on %s%s\n", + ca->bdev, m, is_read ? ", recovering." : "."); else bch_cache_set_error(ca->set, - "%s: too many IO errors %s\n", - ca->cache_dev_name, m); + "%pg: too many IO errors %s\n", + ca->bdev, m); } } diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index dced2ea17431..88cdce218f5c 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -2338,7 +2338,7 @@ err_btree_alloc: err_free: module_put(THIS_MODULE); if (err) - pr_notice("error %s: %s\n", ca->cache_dev_name, err); + pr_notice("error %pg: %s\n", ca->bdev, err); return ret; } @@ -2348,7 +2348,6 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk, const char *err = NULL; /* must be set for any error case */ int ret = 0; - bdevname(bdev, ca->cache_dev_name); memcpy(&ca->sb, sb, sizeof(struct cache_sb)); ca->bdev = bdev; ca->bdev->bd_holder = ca; @@ -2390,14 +2389,14 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk, goto out; } - pr_info("registered cache device %s\n", ca->cache_dev_name); + pr_info("registered cache device %pg\n", ca->bdev); out: kobject_put(&ca->kobj); err: if (err) - pr_notice("error %s: %s\n", ca->cache_dev_name, err); + pr_notice("error %pg: %s\n", ca->bdev, err); return ret; } From 0f5cd7815f7f4bb1dd340a9aeb9b9d6a7c7eec22 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 20 Oct 2021 22:38:10 +0800 Subject: [PATCH 195/255] bcache: remove the backing_dev_name field from struct cached_dev Just use the %pg format specifier to print the name directly. Signed-off-by: Christoph Hellwig Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-7-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/bcache.h | 2 -- drivers/md/bcache/debug.c | 4 ++-- drivers/md/bcache/io.c | 8 +++---- drivers/md/bcache/request.c | 4 ++-- drivers/md/bcache/super.c | 48 ++++++++++++++++--------------------- drivers/md/bcache/sysfs.c | 2 +- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 47ff9ecea2e2..941685409c68 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -395,8 +395,6 @@ struct cached_dev { atomic_t io_errors; unsigned int error_limit; unsigned int offline_seconds; - - char backing_dev_name[BDEVNAME_SIZE]; }; enum alloc_reserve { diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index 116edda845c3..e803cad864be 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -137,8 +137,8 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) p2 + bv.bv_offset, bv.bv_len), dc->disk.c, - "verify failed at dev %s sector %llu", - dc->backing_dev_name, + "verify failed at dev %pg sector %llu", + dc->bdev, (uint64_t) bio->bi_iter.bi_sector); kunmap_atomic(p1); diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index 564357de7640..9c6f9ec55b72 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c @@ -65,15 +65,15 @@ void bch_count_backing_io_errors(struct cached_dev *dc, struct bio *bio) * we shouldn't count failed REQ_RAHEAD bio to dc->io_errors. */ if (bio->bi_opf & REQ_RAHEAD) { - pr_warn_ratelimited("%s: Read-ahead I/O failed on backing device, ignore\n", - dc->backing_dev_name); + pr_warn_ratelimited("%pg: Read-ahead I/O failed on backing device, ignore\n", + dc->bdev); return; } errors = atomic_add_return(1, &dc->io_errors); if (errors < dc->error_limit) - pr_err("%s: IO error on backing device, unrecoverable\n", - dc->backing_dev_name); + pr_err("%pg: IO error on backing device, unrecoverable\n", + dc->bdev); else bch_cached_dev_error(dc); } diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 23b28edae90f..f86909a66ac6 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -651,8 +651,8 @@ static void backing_request_endio(struct bio *bio) */ if (unlikely(s->iop.writeback && bio->bi_opf & REQ_PREFLUSH)) { - pr_err("Can't flush %s: returned bi_status %i\n", - dc->backing_dev_name, bio->bi_status); + pr_err("Can't flush %pg: returned bi_status %i\n", + dc->bdev, bio->bi_status); } else { /* set to orig_bio->bi_status in bio_complete() */ s->iop.status = bio->bi_status; diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 88cdce218f5c..dc35f6e1d8d3 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1026,8 +1026,8 @@ static int cached_dev_status_update(void *arg) dc->offline_seconds = 0; if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) { - pr_err("%s: device offline for %d seconds\n", - dc->backing_dev_name, + pr_err("%pg: device offline for %d seconds\n", + dc->bdev, BACKING_DEV_OFFLINE_TIMEOUT); pr_err("%s: disable I/O request due to backing device offline\n", dc->disk.name); @@ -1058,15 +1058,13 @@ int bch_cached_dev_run(struct cached_dev *dc) }; if (dc->io_disable) { - pr_err("I/O disabled on cached dev %s\n", - dc->backing_dev_name); + pr_err("I/O disabled on cached dev %pg\n", dc->bdev); ret = -EIO; goto out; } if (atomic_xchg(&dc->running, 1)) { - pr_info("cached dev %s is running already\n", - dc->backing_dev_name); + pr_info("cached dev %pg is running already\n", dc->bdev); ret = -EBUSY; goto out; } @@ -1163,7 +1161,7 @@ static void cached_dev_detach_finish(struct work_struct *w) mutex_unlock(&bch_register_lock); - pr_info("Caching disabled for %s\n", dc->backing_dev_name); + pr_info("Caching disabled for %pg\n", dc->bdev); /* Drop ref we took in cached_dev_detach() */ closure_put(&dc->disk.cl); @@ -1203,29 +1201,27 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, return -ENOENT; if (dc->disk.c) { - pr_err("Can't attach %s: already attached\n", - dc->backing_dev_name); + pr_err("Can't attach %pg: already attached\n", dc->bdev); return -EINVAL; } if (test_bit(CACHE_SET_STOPPING, &c->flags)) { - pr_err("Can't attach %s: shutting down\n", - dc->backing_dev_name); + pr_err("Can't attach %pg: shutting down\n", dc->bdev); return -EINVAL; } if (dc->sb.block_size < c->cache->sb.block_size) { /* Will die */ - pr_err("Couldn't attach %s: block size less than set's block size\n", - dc->backing_dev_name); + pr_err("Couldn't attach %pg: block size less than set's block size\n", + dc->bdev); return -EINVAL; } /* Check whether already attached */ list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) { if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) { - pr_err("Tried to attach %s but duplicate UUID already attached\n", - dc->backing_dev_name); + pr_err("Tried to attach %pg but duplicate UUID already attached\n", + dc->bdev); return -EINVAL; } @@ -1243,15 +1239,13 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, if (!u) { if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { - pr_err("Couldn't find uuid for %s in set\n", - dc->backing_dev_name); + pr_err("Couldn't find uuid for %pg in set\n", dc->bdev); return -ENOENT; } u = uuid_find_empty(c); if (!u) { - pr_err("Not caching %s, no room for UUID\n", - dc->backing_dev_name); + pr_err("Not caching %pg, no room for UUID\n", dc->bdev); return -EINVAL; } } @@ -1319,8 +1313,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, */ kthread_stop(dc->writeback_thread); cancel_writeback_rate_update_dwork(dc); - pr_err("Couldn't run cached device %s\n", - dc->backing_dev_name); + pr_err("Couldn't run cached device %pg\n", dc->bdev); return ret; } @@ -1336,8 +1329,8 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, /* Allow the writeback thread to proceed */ up_write(&dc->writeback_lock); - pr_info("Caching %s as %s on set %pU\n", - dc->backing_dev_name, + pr_info("Caching %pg as %s on set %pU\n", + dc->bdev, dc->disk.disk->disk_name, dc->disk.c->set_uuid); return 0; @@ -1461,7 +1454,6 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, struct cache_set *c; int ret = -ENOMEM; - bdevname(bdev, dc->backing_dev_name); memcpy(&dc->sb, sb, sizeof(struct cache_sb)); dc->bdev = bdev; dc->bdev->bd_holder = dc; @@ -1476,7 +1468,7 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj)) goto err; - pr_info("registered backing device %s\n", dc->backing_dev_name); + pr_info("registered backing device %pg\n", dc->bdev); list_add(&dc->list, &uncached_devices); /* attach to a matched cache set if it exists */ @@ -1493,7 +1485,7 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, return 0; err: - pr_notice("error %s: %s\n", dc->backing_dev_name, err); + pr_notice("error %pg: %s\n", dc->bdev, err); bcache_device_stop(&dc->disk); return ret; } @@ -1621,8 +1613,8 @@ bool bch_cached_dev_error(struct cached_dev *dc) /* make others know io_disable is true earlier */ smp_mb(); - pr_err("stop %s: too many IO errors on backing device %s\n", - dc->disk.disk->disk_name, dc->backing_dev_name); + pr_err("stop %s: too many IO errors on backing device %pg\n", + dc->disk.disk->disk_name, dc->bdev); bcache_device_stop(&dc->disk); return true; diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 05ac1d6fbbf3..1f0dce30fa75 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -271,7 +271,7 @@ SHOW(__bch_cached_dev) } if (attr == &sysfs_backing_dev_name) { - snprintf(buf, BDEVNAME_SIZE + 1, "%s", dc->backing_dev_name); + snprintf(buf, BDEVNAME_SIZE + 1, "%pg", dc->bdev); strcat(buf, "\n"); return strlen(buf); } From 00387bd21dac98f9e793294c895768d9e5441f82 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 20 Oct 2021 22:38:11 +0800 Subject: [PATCH 196/255] bcache: use bvec_kmap_local in bch_data_verify Using local kmaps slightly reduces the chances to stray writes, and the bvec interface cleans up the code a little bit. Also switch from page_address to bvec_kmap_local for cbv to be on the safe side and to avoid pointlessly poking into bvec internals. Signed-off-by: Christoph Hellwig Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-8-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/debug.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index e803cad864be..6230dfdd9286 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -127,21 +127,20 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) citer.bi_size = UINT_MAX; bio_for_each_segment(bv, bio, iter) { - void *p1 = kmap_atomic(bv.bv_page); + void *p1 = bvec_kmap_local(&bv); void *p2; cbv = bio_iter_iovec(check, citer); - p2 = page_address(cbv.bv_page); + p2 = bvec_kmap_local(&cbv); - cache_set_err_on(memcmp(p1 + bv.bv_offset, - p2 + bv.bv_offset, - bv.bv_len), + cache_set_err_on(memcmp(p1, p2, bv.bv_len), dc->disk.c, "verify failed at dev %pg sector %llu", dc->bdev, (uint64_t) bio->bi_iter.bi_sector); - kunmap_atomic(p1); + kunmap_local(p2); + kunmap_local(p1); bio_advance_iter(check, &citer, bv.bv_len); } From 39fa7a95552cc851029267b97c1317f1dea61cad Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 20 Oct 2021 22:38:12 +0800 Subject: [PATCH 197/255] bcache: remove bch_crc64_update bch_crc64_update is an entirely pointless wrapper around crc64_be. Signed-off-by: Christoph Hellwig Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211020143812.6403-9-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/btree.c | 2 +- drivers/md/bcache/request.c | 2 +- drivers/md/bcache/util.h | 8 -------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 0595559de174..93b67b8d31c3 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -141,7 +141,7 @@ static uint64_t btree_csum_set(struct btree *b, struct bset *i) uint64_t crc = b->key.ptr[0]; void *data = (void *) i + 8, *end = bset_bkey_last(i); - crc = bch_crc64_update(crc, data, end - data); + crc = crc64_be(crc, data, end - data); return crc ^ 0xffffffffffffffffULL; } diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index f86909a66ac6..d15aae6c51c1 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -46,7 +46,7 @@ static void bio_csum(struct bio *bio, struct bkey *k) bio_for_each_segment(bv, bio, iter) { void *d = kmap(bv.bv_page) + bv.bv_offset; - csum = bch_crc64_update(csum, d, bv.bv_len); + csum = crc64_be(csum, d, bv.bv_len); kunmap(bv.bv_page); } diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h index b64460a76267..6274d6a17e5e 100644 --- a/drivers/md/bcache/util.h +++ b/drivers/md/bcache/util.h @@ -548,14 +548,6 @@ static inline uint64_t bch_crc64(const void *p, size_t len) return crc ^ 0xffffffffffffffffULL; } -static inline uint64_t bch_crc64_update(uint64_t crc, - const void *p, - size_t len) -{ - crc = crc64_be(crc, p, len); - return crc; -} - /* * A stepwise-linear pseudo-exponential. This returns 1 << (x >> * frac_bits), with the less-significant bits filled in by linear From f6f09c15a767fad1206068bc09ec62d3cf273460 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 6 Oct 2021 11:13:13 +0200 Subject: [PATCH 198/255] nvme: generate uevent once a multipath namespace is operational again When fast_io_fail_tmo is set I/O will be aborted while recovery is still ongoing. This causes MD to set the namespace to failed, and no futher I/O will be submitted to that namespace. However, once the recovery succeeds and the namespace becomes operational again the NVMe subsystem doesn't send a notification, so MD cannot automatically reinstate operation and requires manual interaction. This patch will send a KOBJ_CHANGE uevent per multipathed namespace once the underlying controller transitions to LIVE, allowing an automatic MD reassembly with these udev rules: /etc/udev/rules.d/65-md-auto-re-add.rules: SUBSYSTEM!="block", GOTO="md_end" ACTION!="change", GOTO="md_end" ENV{ID_FS_TYPE}!="linux_raid_member", GOTO="md_end" PROGRAM="/sbin/md_raid_auto_readd.sh $devnode" LABEL="md_end" /sbin/md_raid_auto_readd.sh: MDADM=/sbin/mdadm DEVNAME=$1 export $(${MDADM} --examine --export ${DEVNAME}) if [ -z "${MD_UUID}" ]; then exit 1 fi UUID_LINK=$(readlink /dev/disk/by-id/md-uuid-${MD_UUID}) MD_DEVNAME=${UUID_LINK##*/} export $(${MDADM} --detail --export /dev/${MD_DEVNAME}) if [ -z "${MD_METADATA}" ] ; then exit 1 fi if [ $(cat /sys/block/${MD_DEVNAME}/md/degraded) != 1 ]; then echo "${MD_DEVNAME}: array not degraded, nothing to do" exit 0 fi MD_STATE=$(cat /sys/block/${MD_DEVNAME}/md/array_state) if [ ${MD_STATE} != "clean" ] ; then echo "${MD_DEVNAME}: array state ${MD_STATE}, cannot re-add" exit 1 fi MD_VARNAME="MD_DEVICE_dev_${DEVNAME##*/}_ROLE" if [ ${!MD_VARNAME} = "spare" ] ; then ${MDADM} --manage /dev/${MD_DEVNAME} --re-add ${DEVNAME} fi Changes to v2: - Add udev rules example to description Changes to v1: - use disk_uevent() as suggested by hch Signed-off-by: Hannes Reinecke Signed-off-by: Christoph Hellwig --- drivers/nvme/host/multipath.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 11440c86881e..22c6b64f4d93 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -105,8 +105,11 @@ void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) down_read(&ctrl->namespaces_rwsem); list_for_each_entry(ns, &ctrl->namespaces, list) { - if (ns->head->disk) - kblockd_schedule_work(&ns->head->requeue_work); + if (!ns->head->disk) + continue; + kblockd_schedule_work(&ns->head->requeue_work); + if (ctrl->state == NVME_CTRL_LIVE) + disk_uevent(ns->head->disk, KOBJ_CHANGE); } up_read(&ctrl->namespaces_rwsem); } From 01d838164b4c305c1cafb0c3f71fb0027d99358b Mon Sep 17 00:00:00 2001 From: Saurav Kashyap Date: Mon, 23 Aug 2021 05:56:48 -0700 Subject: [PATCH 199/255] nvme-fc: add support for ->map_queues NVMe FC don't have support for map queues, unlike the PCI, RDMA and TCP transports. Add a ->map_queues callout for the LLDDs to provide such functionality. Signed-off-by: Saurav Kashyap Signed-off-by: Nilesh Javali Signed-off-by: Christoph Hellwig --- drivers/nvme/host/fc.c | 24 ++++++++++++++++++++++++ include/linux/nvme-fc-driver.h | 7 +++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index aa14ad963d91..34f7a7c236bf 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -16,6 +16,7 @@ #include #include "fc.h" #include +#include /* *************************** Data Structures/Defines ****************** */ @@ -2841,6 +2842,28 @@ nvme_fc_complete_rq(struct request *rq) nvme_fc_ctrl_put(ctrl); } +static int nvme_fc_map_queues(struct blk_mq_tag_set *set) +{ + struct nvme_fc_ctrl *ctrl = set->driver_data; + int i; + + for (i = 0; i < set->nr_maps; i++) { + struct blk_mq_queue_map *map = &set->map[i]; + + if (!map->nr_queues) { + WARN_ON(i == HCTX_TYPE_DEFAULT); + continue; + } + + /* Call LLDD map queue functionality if defined */ + if (ctrl->lport->ops->map_queues) + ctrl->lport->ops->map_queues(&ctrl->lport->localport, + map); + else + blk_mq_map_queues(map); + } + return 0; +} static const struct blk_mq_ops nvme_fc_mq_ops = { .queue_rq = nvme_fc_queue_rq, @@ -2849,6 +2872,7 @@ static const struct blk_mq_ops nvme_fc_mq_ops = { .exit_request = nvme_fc_exit_request, .init_hctx = nvme_fc_init_hctx, .timeout = nvme_fc_timeout, + .map_queues = nvme_fc_map_queues, }; static int diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h index 2a38f2b477a5..cb909edb76c4 100644 --- a/include/linux/nvme-fc-driver.h +++ b/include/linux/nvme-fc-driver.h @@ -7,6 +7,7 @@ #define _NVME_FC_DRIVER_H 1 #include +#include /* @@ -497,6 +498,8 @@ struct nvme_fc_port_template { int (*xmt_ls_rsp)(struct nvme_fc_local_port *localport, struct nvme_fc_remote_port *rport, struct nvmefc_ls_rsp *ls_rsp); + void (*map_queues)(struct nvme_fc_local_port *localport, + struct blk_mq_queue_map *map); u32 max_hw_queues; u16 max_sgl_segments; @@ -779,6 +782,10 @@ struct nvmet_fc_target_port { * LS received. * Entrypoint is Mandatory. * + * @map_queues: This functions lets the driver expose the queue mapping + * to the block layer. + * Entrypoint is Optional. + * * @fcp_op: Called to perform a data transfer or transmit a response. * The nvmefc_tgt_fcp_req structure is the same LLDD-supplied * exchange structure specified in the nvmet_fc_rcv_fcp_req() call From 2b2af50ae8367b0655e12ccfe6252d5c1d7ae7bf Mon Sep 17 00:00:00 2001 From: Saurav Kashyap Date: Mon, 23 Aug 2021 05:56:49 -0700 Subject: [PATCH 200/255] qla2xxx: add ->map_queues support for nvme Implement ->map queues and use the block layer blk_mq_pci_map_queues helper for mapping queues to CPUs. With this mapping minimum 10%+ increase in performance is noticed. Signed-off-by: Saurav Kashyap Signed-off-by: Nilesh Javali Signed-off-by: Christoph Hellwig --- drivers/scsi/qla2xxx/qla_nvme.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 1c5da2dbd6f9..253055cf9daf 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include static struct nvme_fc_port_template qla_nvme_fc_transport; @@ -642,6 +644,18 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport, return rval; } +static void qla_nvme_map_queues(struct nvme_fc_local_port *lport, + struct blk_mq_queue_map *map) +{ + struct scsi_qla_host *vha = lport->private; + int rc; + + rc = blk_mq_pci_map_queues(map, vha->hw->pdev, vha->irq_offset); + if (rc) + ql_log(ql_log_warn, vha, 0x21de, + "pci map queue failed 0x%x", rc); +} + static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport) { struct scsi_qla_host *vha = lport->private; @@ -676,6 +690,7 @@ static struct nvme_fc_port_template qla_nvme_fc_transport = { .ls_abort = qla_nvme_ls_abort, .fcp_io = qla_nvme_post_cmd, .fcp_abort = qla_nvme_fcp_abort, + .map_queues = qla_nvme_map_queues, .max_hw_queues = 8, .max_sgl_segments = 1024, .max_dif_sgl_segments = 64, From e3e19dcc4c416d65f99f13d55be2b787f8d0050e Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Wed, 6 Oct 2021 08:09:43 +0000 Subject: [PATCH 201/255] nvmet: fix use-after-free when a port is removed When a port is removed through configfs, any connected controllers are starting teardown flow asynchronously and can still send commands. This causes a use-after-free bug for any command that dereferences req->port (like in nvmet_parse_io_cmd). To fix this, wait for all the teardown scheduled works to complete (like release_work at rdma/tcp drivers). This ensures there are no active controllers when the port is eventually removed. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index be5d82421e3a..496d775c6770 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1553,6 +1553,8 @@ static void nvmet_port_release(struct config_item *item) { struct nvmet_port *port = to_nvmet_port(item); + /* Let inflight controllers teardown complete */ + flush_scheduled_work(); list_del(&port->global_entry); kfree(port->ana_state); From fcf73a804c7d6bbf0ea63531c6122aa363852e04 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Wed, 6 Oct 2021 08:09:44 +0000 Subject: [PATCH 202/255] nvmet-rdma: fix use-after-free when a port is removed When removing a port, all its controllers are being removed, but there are queues on the port that doesn't belong to any controller (during connection time). This causes a use-after-free bug for any command that dereferences req->port (like in nvmet_alloc_ctrl). Those queues should be destroyed before freeing the port via configfs. Destroy the remaining queues after the RDMA-CM was destroyed guarantees that no new queue will be created. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/rdma.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 38d1f292ecc2..6f75e6be2ce3 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -1819,12 +1819,36 @@ restart: mutex_unlock(&nvmet_rdma_queue_mutex); } +static void nvmet_rdma_destroy_port_queues(struct nvmet_rdma_port *port) +{ + struct nvmet_rdma_queue *queue, *tmp; + struct nvmet_port *nport = port->nport; + + mutex_lock(&nvmet_rdma_queue_mutex); + list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list, + queue_list) { + if (queue->port != nport) + continue; + + list_del_init(&queue->queue_list); + __nvmet_rdma_queue_disconnect(queue); + } + mutex_unlock(&nvmet_rdma_queue_mutex); +} + static void nvmet_rdma_disable_port(struct nvmet_rdma_port *port) { struct rdma_cm_id *cm_id = xchg(&port->cm_id, NULL); if (cm_id) rdma_destroy_id(cm_id); + + /* + * Destroy the remaining queues, which are not belong to any + * controller yet. Do it here after the RDMA-CM was destroyed + * guarantees that no new queue will be created. + */ + nvmet_rdma_destroy_port_queues(port); } static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port) From 2351ead99ce9164fb42555aee3f96af84c4839e9 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Wed, 6 Oct 2021 08:09:45 +0000 Subject: [PATCH 203/255] nvmet-tcp: fix use-after-free when a port is removed When removing a port, all its controllers are being removed, but there are queues on the port that doesn't belong to any controller (during connection time). This causes a use-after-free bug for any command that dereferences req->port (like in nvmet_alloc_ctrl). Those queues should be destroyed before freeing the port via configfs. Destroy the remaining queues after the accept_work was cancelled guarantees that no new queue will be created. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/tcp.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 07ee347ea3f3..6eb0b3153477 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1737,6 +1737,17 @@ err_port: return ret; } +static void nvmet_tcp_destroy_port_queues(struct nvmet_tcp_port *port) +{ + struct nvmet_tcp_queue *queue; + + mutex_lock(&nvmet_tcp_queue_mutex); + list_for_each_entry(queue, &nvmet_tcp_queue_list, queue_list) + if (queue->port == port) + kernel_sock_shutdown(queue->sock, SHUT_RDWR); + mutex_unlock(&nvmet_tcp_queue_mutex); +} + static void nvmet_tcp_remove_port(struct nvmet_port *nport) { struct nvmet_tcp_port *port = nport->priv; @@ -1746,6 +1757,11 @@ static void nvmet_tcp_remove_port(struct nvmet_port *nport) port->sock->sk->sk_user_data = NULL; write_unlock_bh(&port->sock->sk->sk_callback_lock); cancel_work_sync(&port->accept_work); + /* + * Destroy the remaining queues, which are not belong to any + * controller yet. + */ + nvmet_tcp_destroy_port_queues(port); sock_release(port->sock); kfree(port); From 44c3c6257e99c6284f312206de73783575fc8906 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 23 Sep 2021 00:55:35 +0300 Subject: [PATCH 204/255] nvme-rdma: limit the maximal queue size for RDMA controllers Corrent limit of 1024 isn't valid for some of the RDMA based ctrls. In case the target expose a cap of larger amount of entries (e.g. 1024), the initiator may fail to create a QP with this size. Thus limit to a value that works for all RDMA adapters. Future general solution should use RDMA/core API to calculate this size according to device capabilities and number of WRs needed per NVMe IO request. Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/host/rdma.c | 7 +++++++ include/linux/nvme-rdma.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 1624da3702d4..027ee57cbdb0 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1112,6 +1112,13 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) ctrl->ctrl.opts->queue_size, ctrl->ctrl.sqsize + 1); } + if (ctrl->ctrl.sqsize + 1 > NVME_RDMA_MAX_QUEUE_SIZE) { + dev_warn(ctrl->ctrl.device, + "ctrl sqsize %u > max queue size %u, clamping down\n", + ctrl->ctrl.sqsize + 1, NVME_RDMA_MAX_QUEUE_SIZE); + ctrl->ctrl.sqsize = NVME_RDMA_MAX_QUEUE_SIZE - 1; + } + if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) { dev_warn(ctrl->ctrl.device, "sqsize %u > ctrl maxcmd %u, clamping down\n", diff --git a/include/linux/nvme-rdma.h b/include/linux/nvme-rdma.h index 3ec8e50efa16..4dd7e6fe92fb 100644 --- a/include/linux/nvme-rdma.h +++ b/include/linux/nvme-rdma.h @@ -6,6 +6,8 @@ #ifndef _LINUX_NVME_RDMA_H #define _LINUX_NVME_RDMA_H +#define NVME_RDMA_MAX_QUEUE_SIZE 128 + enum nvme_rdma_cm_fmt { NVME_RDMA_CM_FMT_1_0 = 0x0, }; From 6d1555cc41c088d738b4968009b32aaeda8542a3 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 23 Sep 2021 00:55:36 +0300 Subject: [PATCH 205/255] nvmet: add get_max_queue_size op for controllers Some transports, such as RDMA, would like to set the queue size according to device/port/ctrl characteristics. Add a new nvmet transport op that is called during ctrl initialization. This will not effect transports that don't implement this option. Reviewed-by: Chaitanya Kulkarni Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 8 +++++--- drivers/nvme/target/nvmet.h | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index b8425fa34300..93107af3310d 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1205,7 +1205,10 @@ static void nvmet_init_cap(struct nvmet_ctrl *ctrl) /* CC.EN timeout in 500msec units: */ ctrl->cap |= (15ULL << 24); /* maximum queue entries supported: */ - ctrl->cap |= NVMET_QUEUE_SIZE - 1; + if (ctrl->ops->get_max_queue_size) + ctrl->cap |= ctrl->ops->get_max_queue_size(ctrl) - 1; + else + ctrl->cap |= NVMET_QUEUE_SIZE - 1; if (nvmet_is_passthru_subsys(ctrl->subsys)) nvmet_passthrough_override_cap(ctrl); @@ -1367,6 +1370,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, mutex_init(&ctrl->lock); ctrl->port = req->port; + ctrl->ops = req->ops; INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); INIT_LIST_HEAD(&ctrl->async_events); @@ -1405,8 +1409,6 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, } ctrl->cntlid = ret; - ctrl->ops = req->ops; - /* * Discovery controllers may use some arbitrary high value * in order to cleanup stale discovery sessions diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 7143c7fa7464..f8e0ee131dc6 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -309,6 +309,7 @@ struct nvmet_fabrics_ops { u16 (*install_queue)(struct nvmet_sq *nvme_sq); void (*discovery_chg)(struct nvmet_port *port); u8 (*get_mdts)(const struct nvmet_ctrl *ctrl); + u16 (*get_max_queue_size)(const struct nvmet_ctrl *ctrl); }; #define NVMET_MAX_INLINE_BIOVEC 8 From c7d792f9b8b0502c807ecda57aeb5eac70cc7ab9 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 23 Sep 2021 00:55:37 +0300 Subject: [PATCH 206/255] nvmet-rdma: implement get_max_queue_size controller op Limit the maximal queue size for RDMA controllers. Today, the target reports a limit of 1024 and this limit isn't valid for some of the RDMA based controllers. For now, limit RDMA transport to 128 entries (the max queue depth configured for Linux NVMe/RDMA host). Future general solution should use RDMA/core API to calculate this size according to device capabilities and number of WRs needed per NVMe IO request. Reported-by: Mark Ruijter Reviewed-by: Chaitanya Kulkarni Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/rdma.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 6f75e6be2ce3..1deb4043e242 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -2000,6 +2000,11 @@ static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl) return NVMET_RDMA_MAX_MDTS; } +static u16 nvmet_rdma_get_max_queue_size(const struct nvmet_ctrl *ctrl) +{ + return NVME_RDMA_MAX_QUEUE_SIZE; +} + static const struct nvmet_fabrics_ops nvmet_rdma_ops = { .owner = THIS_MODULE, .type = NVMF_TRTYPE_RDMA, @@ -2011,6 +2016,7 @@ static const struct nvmet_fabrics_ops nvmet_rdma_ops = { .delete_ctrl = nvmet_rdma_delete_ctrl, .disc_traddr = nvmet_rdma_disc_port_addr, .get_mdts = nvmet_rdma_get_mdts, + .get_max_queue_size = nvmet_rdma_get_max_queue_size, }; static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data) From 626851e9225df93eda00f6b256cb74ad0860e418 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:19 +0200 Subject: [PATCH 207/255] nvmet: make discovery NQN configurable TPAR8013 allows for unique discovery NQNs, so make the discovery controller NQN configurable by exposing a subsys attribute 'discovery_nqn'. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 39 ++++++++++++++++++++++++++++++++++ drivers/nvme/target/core.c | 3 ++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 496d775c6770..091a0ca16361 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1233,6 +1233,44 @@ static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, } CONFIGFS_ATTR(nvmet_subsys_, attr_model); +static ssize_t nvmet_subsys_attr_discovery_nqn_show(struct config_item *item, + char *page) +{ + return snprintf(page, PAGE_SIZE, "%s\n", + nvmet_disc_subsys->subsysnqn); +} + +static ssize_t nvmet_subsys_attr_discovery_nqn_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_subsys *subsys = to_subsys(item); + char *subsysnqn; + int len; + + len = strcspn(page, "\n"); + if (!len) + return -EINVAL; + + subsysnqn = kmemdup_nul(page, len, GFP_KERNEL); + if (!subsysnqn) + return -ENOMEM; + + /* + * The discovery NQN must be different from subsystem NQN. + */ + if (!strcmp(subsysnqn, subsys->subsysnqn)) { + kfree(subsysnqn); + return -EBUSY; + } + down_write(&nvmet_config_sem); + kfree(nvmet_disc_subsys->subsysnqn); + nvmet_disc_subsys->subsysnqn = subsysnqn; + up_write(&nvmet_config_sem); + + return count; +} +CONFIGFS_ATTR(nvmet_subsys_, attr_discovery_nqn); + #ifdef CONFIG_BLK_DEV_INTEGRITY static ssize_t nvmet_subsys_attr_pi_enable_show(struct config_item *item, char *page) @@ -1262,6 +1300,7 @@ static struct configfs_attribute *nvmet_subsys_attrs[] = { &nvmet_subsys_attr_attr_cntlid_min, &nvmet_subsys_attr_attr_cntlid_max, &nvmet_subsys_attr_attr_model, + &nvmet_subsys_attr_attr_discovery_nqn, #ifdef CONFIG_BLK_DEV_INTEGRITY &nvmet_subsys_attr_attr_pi_enable, #endif diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 93107af3310d..e1f41436ea2c 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1493,7 +1493,8 @@ static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, if (!port) return NULL; - if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn)) { + if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn) || + !strcmp(nvmet_disc_subsys->subsysnqn, subsysnqn)) { if (!kref_get_unless_zero(&nvmet_disc_subsys->ref)) return NULL; return nvmet_disc_subsys; From e15a8a9755659ff5972f30de4dd64867c97f242d Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:20 +0200 Subject: [PATCH 208/255] nvme: add CNTRLTYPE definitions for 'identify controller' Update the 'identify controller' structure to define the newly added CNTRLTYPE field. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig --- include/linux/nvme.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index b7c4c4130b65..ed2428918bca 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -31,6 +31,12 @@ enum nvme_subsys_type { NVME_NQN_NVME = 2, /* NVME type target subsystem */ }; +enum nvme_ctrl_type { + NVME_CTRL_IO = 1, /* I/O controller */ + NVME_CTRL_DISC = 2, /* Discovery controller */ + NVME_CTRL_ADMIN = 3, /* Administrative controller */ +}; + /* Address Family codes for Discovery Log Page entry ADRFAM field */ enum { NVMF_ADDR_FAMILY_PCI = 0, /* PCIe */ @@ -244,7 +250,9 @@ struct nvme_id_ctrl { __le32 rtd3e; __le32 oaes; __le32 ctratt; - __u8 rsvd100[28]; + __u8 rsvd100[11]; + __u8 cntrltype; + __u8 fguid[16]; __le16 crdt1; __le16 crdt2; __le16 crdt3; From a294711ed5123f757ed8ed2f103c851b8ee416c9 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:21 +0200 Subject: [PATCH 209/255] nvmet: add nvmet_is_disc_subsys() helper Add a helper function to determine if a given subsystem is a discovery subsystem. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 2 +- drivers/nvme/target/core.c | 6 +++--- drivers/nvme/target/nvmet.h | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index aa6d84d8848e..b653ea4244fe 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -1008,7 +1008,7 @@ u16 nvmet_parse_admin_cmd(struct nvmet_req *req) if (nvme_is_fabrics(cmd)) return nvmet_parse_fabrics_cmd(req); - if (nvmet_req_subsys(req)->type == NVME_NQN_DISC) + if (nvmet_is_disc_subsys(nvmet_req_subsys(req))) return nvmet_parse_discovery_cmd(req); ret = nvmet_check_ctrl_status(req); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index e1f41436ea2c..d844fb9ef2eb 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1140,7 +1140,7 @@ static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl) * should verify iosqes,iocqes are zeroed, however that * would break backwards compatibility, so don't enforce it. */ - if (ctrl->subsys->type != NVME_NQN_DISC && + if (!nvmet_is_disc_subsys(ctrl->subsys) && (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES || nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES)) { ctrl->csts = NVME_CSTS_CFS; @@ -1281,7 +1281,7 @@ bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn) if (subsys->allow_any_host) return true; - if (subsys->type == NVME_NQN_DISC) /* allow all access to disc subsys */ + if (nvmet_is_disc_subsys(subsys)) /* allow all access to disc subsys */ return true; list_for_each_entry(p, &subsys->hosts, entry) { @@ -1413,7 +1413,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, * Discovery controllers may use some arbitrary high value * in order to cleanup stale discovery sessions */ - if ((ctrl->subsys->type == NVME_NQN_DISC) && !kato) + if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) kato = NVMET_DISC_KATO_MS; /* keep-alive timeout in seconds */ diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index f8e0ee131dc6..f31dcc4fb1a2 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -577,6 +577,11 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) return req->sq->ctrl->subsys; } +static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) +{ + return subsys->type == NVME_NQN_DISC; +} + #ifdef CONFIG_NVME_TARGET_PASSTHRU void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys); int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys); From d3aef70124e7f69975eabeb340866bc91672532d Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:22 +0200 Subject: [PATCH 210/255] nvmet: set 'CNTRLTYPE' in the identify controller data Set the correct 'CNTRLTYPE' field in the identify controller data. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 5 +++++ drivers/nvme/target/discovery.c | 2 ++ drivers/nvme/target/fabrics-cmd.c | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index b653ea4244fe..3616a06f7836 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -374,6 +374,11 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) id->rab = 6; + if (nvmet_is_disc_subsys(ctrl->subsys)) + id->cntrltype = NVME_CTRL_DISC; + else + id->cntrltype = NVME_CTRL_IO; + /* * XXX: figure out how we can assign a IEEE OUI, but until then * the safest is to leave it as zeroes. diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index 7aa62bc6ae84..7b360f8d07e9 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -268,6 +268,8 @@ static void nvmet_execute_disc_identify(struct nvmet_req *req) memcpy_and_pad(id->fr, sizeof(id->fr), UTS_RELEASE, strlen(UTS_RELEASE), ' '); + id->cntrltype = NVME_CTRL_DISC; + /* no limit on data transfer sizes for now */ id->mdts = 0; id->cntlid = cpu_to_le16(ctrl->cntlid); diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 7d0454cee920..70fb587e9413 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -221,7 +221,8 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) goto out; } - pr_info("creating controller %d for subsystem %s for NQN %s%s.\n", + pr_info("creating %s controller %d for subsystem %s for NQN %s%s.\n", + nvmet_is_disc_subsys(ctrl->subsys) ? "discovery" : "nvm", ctrl->cntlid, ctrl->subsys->subsysnqn, ctrl->hostnqn, ctrl->pi_support ? " T10-PI is enabled" : ""); req->cqe->result.u16 = cpu_to_le16(ctrl->cntlid); From 954ae16681f6bdf684f016ca626329302a38e177 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:23 +0200 Subject: [PATCH 211/255] nvme: expose subsystem type in sysfs attribute 'subsystype' With unique discovery controller NQNs we cannot distinguish the subsystem type by the NQN alone, but need to check the subsystem type, too. So expose the subsystem type in a new sysfs attribute 'subsystype'. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 27 +++++++++++++++++++++++++++ drivers/nvme/host/nvme.h | 1 + 2 files changed, 28 insertions(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c415c3faf420..0f0f64db1a91 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2620,6 +2620,24 @@ static ssize_t nvme_subsys_show_nqn(struct device *dev, } static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn); +static ssize_t nvme_subsys_show_type(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct nvme_subsystem *subsys = + container_of(dev, struct nvme_subsystem, dev); + + switch (subsys->subtype) { + case NVME_NQN_DISC: + return sysfs_emit(buf, "discovery\n"); + case NVME_NQN_NVME: + return sysfs_emit(buf, "nvm\n"); + default: + return sysfs_emit(buf, "reserved\n"); + } +} +static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type); + #define nvme_subsys_show_str_function(field) \ static ssize_t subsys_##field##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ @@ -2640,6 +2658,7 @@ static struct attribute *nvme_subsys_attrs[] = { &subsys_attr_serial.attr, &subsys_attr_firmware_rev.attr, &subsys_attr_subsysnqn.attr, + &subsys_attr_subsystype.attr, #ifdef CONFIG_NVME_MULTIPATH &subsys_attr_iopolicy.attr, #endif @@ -2710,6 +2729,14 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev)); subsys->vendor_id = le16_to_cpu(id->vid); subsys->cmic = id->cmic; + + /* Versions prior to 1.4 don't necessarily report a valid type */ + if (id->cntrltype == NVME_CTRL_DISC || + !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME)) + subsys->subtype = NVME_NQN_DISC; + else + subsys->subtype = NVME_NQN_NVME; + subsys->awupf = le16_to_cpu(id->awupf); #ifdef CONFIG_NVME_MULTIPATH subsys->iopolicy = NVME_IOPOLICY_NUMA; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ef2467b93adb..ba3edd4d02f0 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -372,6 +372,7 @@ struct nvme_subsystem { char model[40]; char firmware_rev[8]; u8 cmic; + enum nvme_subsys_type subtype; u16 vendor_id; u16 awupf; /* 0's based awupf value. */ struct ida ns_ida; From 20e8b689c9088027b7495ffd6f80812c11ecc872 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:24 +0200 Subject: [PATCH 212/255] nvme: Add connect option 'discovery' Add a connect option 'discovery' to specify that the connection should be made to a discovery controller, not a normal I/O controller. With discovery controllers supporting unique subsystem NQNs we cannot easily distinguish by the subsystem NQN if this should be a discovery connection, but we need this information to blank out options not supported by discovery controllers. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 7 +++++++ drivers/nvme/host/fabrics.c | 6 +++++- drivers/nvme/host/fabrics.h | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0f0f64db1a91..fd856eec0912 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2737,6 +2737,13 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) else subsys->subtype = NVME_NQN_NVME; + if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) { + dev_err(ctrl->device, + "Subsystem %s is not a discovery controller", + subsys->subnqn); + kfree(subsys); + return -EINVAL; + } subsys->awupf = le16_to_cpu(id->awupf); #ifdef CONFIG_NVME_MULTIPATH subsys->iopolicy = NVME_IOPOLICY_NUMA; diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 668c6bb7a567..c5a2b71c5268 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -548,6 +548,7 @@ static const match_table_t opt_tokens = { { NVMF_OPT_NR_POLL_QUEUES, "nr_poll_queues=%d" }, { NVMF_OPT_TOS, "tos=%d" }, { NVMF_OPT_FAIL_FAST_TMO, "fast_io_fail_tmo=%d" }, + { NVMF_OPT_DISCOVERY, "discovery" }, { NVMF_OPT_ERR, NULL } }; @@ -823,6 +824,9 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } opts->tos = token; break; + case NVMF_OPT_DISCOVERY: + opts->discovery_nqn = true; + break; default: pr_warn("unknown parameter or missing value '%s' in ctrl creation request\n", p); @@ -949,7 +953,7 @@ EXPORT_SYMBOL_GPL(nvmf_free_options); #define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\ - NVMF_OPT_DISABLE_SQFLOW |\ + NVMF_OPT_DISABLE_SQFLOW | NVMF_OPT_DISCOVERY |\ NVMF_OPT_FAIL_FAST_TMO) static struct nvme_ctrl * diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index a146cb903869..b61b666e10ec 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -67,6 +67,7 @@ enum { NVMF_OPT_TOS = 1 << 19, NVMF_OPT_FAIL_FAST_TMO = 1 << 20, NVMF_OPT_HOST_IFACE = 1 << 21, + NVMF_OPT_DISCOVERY = 1 << 22, }; /** From e5ea42faa773c6a6bb5d9e9f5c2cc808940b5a55 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 22 Sep 2021 08:35:25 +0200 Subject: [PATCH 213/255] nvme: display correct subsystem NQN With discovery controllers supporting unique subsystem NQNs the actual subsystem NQN might be different from that one passed in via the connect args. So add a helper to display the resulting subsystem NQN. Signed-off-by: Hannes Reinecke Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 2 +- drivers/nvme/host/fabrics.h | 7 +++++++ drivers/nvme/host/fc.c | 2 +- drivers/nvme/host/rdma.c | 2 +- drivers/nvme/host/tcp.c | 2 +- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index fd856eec0912..3825b596ca16 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -222,7 +222,7 @@ int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl) static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl) { dev_info(ctrl->device, - "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn); + "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl)); flush_work(&ctrl->reset_work); nvme_stop_ctrl(ctrl); diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index b61b666e10ec..c3203ff1c654 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -179,6 +179,13 @@ nvmf_ctlr_matches_baseopts(struct nvme_ctrl *ctrl, return true; } +static inline char *nvmf_ctrl_subsysnqn(struct nvme_ctrl *ctrl) +{ + if (!ctrl->subsys) + return ctrl->opts->subsysnqn; + return ctrl->subsys->subnqn; +} + int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val); int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val); int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 34f7a7c236bf..be9892894849 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -3596,7 +3596,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, dev_info(ctrl->ctrl.device, "NVME-FC{%d}: new ctrl: NQN \"%s\"\n", - ctrl->cnum, ctrl->ctrl.opts->subsysnqn); + ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl)); return &ctrl->ctrl; diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 027ee57cbdb0..5ba386f00c3f 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -2393,7 +2393,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev, goto out_uninit_ctrl; dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n", - ctrl->ctrl.opts->subsysnqn, &ctrl->addr); + nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); mutex_lock(&nvme_rdma_ctrl_mutex); list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 9ce3458ee1dd..07156ea9d1a8 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2582,7 +2582,7 @@ static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, goto out_uninit_ctrl; dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", - ctrl->ctrl.opts->subsysnqn, &ctrl->addr); + nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); mutex_lock(&nvme_tcp_ctrl_mutex); list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); From 571b5444d1eee112e82eebd0cf877dc510b8b5a5 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 23 Sep 2021 13:17:43 +0300 Subject: [PATCH 214/255] nvmet: use macro definition for setting nmic value This makes the code more readable. Signed-off-by: Max Gurtovoy Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3616a06f7836..dce53030b375 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -541,7 +541,7 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req) * Our namespace might always be shared. Not just with other * controllers, but also with any other user of the block device. */ - id->nmic = (1 << 0); + id->nmic = NVME_NS_NMIC_SHARED; id->anagrpid = cpu_to_le32(req->ns->anagrpid); memcpy(&id->nguid, &req->ns->nguid, sizeof(id->nguid)); From d56ae18f063e38eba47550c632519c9fe3f76b19 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 23 Sep 2021 13:17:44 +0300 Subject: [PATCH 215/255] nvmet: use macro definitions for setting cmic value This makes the code more readable. Signed-off-by: Max Gurtovoy Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 3 ++- include/linux/nvme.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index dce53030b375..6e75890bc8d9 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -385,7 +385,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) */ /* we support multiple ports, multiples hosts and ANA: */ - id->cmic = (1 << 0) | (1 << 1) | (1 << 3); + id->cmic = NVME_CTRL_CMIC_MULTI_PORT | NVME_CTRL_CMIC_MULTI_CTRL | + NVME_CTRL_CMIC_ANA; /* Limit MDTS according to transport capability */ if (ctrl->ops->get_mdts) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index ed2428918bca..357482dedb59 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -320,6 +320,7 @@ struct nvme_id_ctrl { }; enum { + NVME_CTRL_CMIC_MULTI_PORT = 1 << 0, NVME_CTRL_CMIC_MULTI_CTRL = 1 << 1, NVME_CTRL_CMIC_ANA = 1 << 3, NVME_CTRL_ONCS_COMPARE = 1 << 0, From 11384580e3322b41dbaa68dbcd949af875b8aa22 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:52:08 -0700 Subject: [PATCH 216/255] nvme-multipath: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Since we now can tell for sure when a disk was added, move setting the bit NVME_NSHEAD_DISK_LIVE only when we did add the disk successfully. Nothing to do here as the cleanup is done elsewhere. We take care and use test_and_set_bit() because it is protects against two nvme paths simultaneously calling device_add_disk() on the same namespace head. Signed-off-by: Luis Chamberlain Reviewed-by: Keith Busch Signed-off-by: Christoph Hellwig --- drivers/nvme/host/multipath.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 22c6b64f4d93..bc1e7d8a2c0f 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -509,13 +509,23 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) static void nvme_mpath_set_live(struct nvme_ns *ns) { struct nvme_ns_head *head = ns->head; + int rc; if (!head->disk) return; + /* + * test_and_set_bit() is used because it is protecting against two nvme + * paths simultaneously calling device_add_disk() on the same namespace + * head. + */ if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { - device_add_disk(&head->subsys->dev, head->disk, - nvme_ns_id_attr_groups); + rc = device_add_disk(&head->subsys->dev, head->disk, + nvme_ns_id_attr_groups); + if (rc) { + clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags); + return; + } nvme_add_ns_head_cdev(head); } From 09748122009aed7bfaa7acc33c10c083a4758322 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Sun, 17 Oct 2021 11:58:16 +0300 Subject: [PATCH 217/255] nvme-rdma: fix error code in nvme_rdma_setup_ctrl In case that icdoff is not zero or mandatory keyed sgls are not supported by the NVMe/RDMA target, we'll go to error flow but we'll return 0 to the caller. Fix it by returning an appropriate error code. Fixes: c66e2998c8ca ("nvme-rdma: centralize controller setup sequence") Signed-off-by: Max Gurtovoy Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/host/rdma.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 5ba386f00c3f..9317f26e51e0 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1096,11 +1096,13 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new) return ret; if (ctrl->ctrl.icdoff) { + ret = -EOPNOTSUPP; dev_err(ctrl->ctrl.device, "icdoff is not supported!\n"); goto destroy_admin; } if (!(ctrl->ctrl.sgls & (1 << 2))) { + ret = -EOPNOTSUPP; dev_err(ctrl->ctrl.device, "Mandatory keyed sgls are not supported!\n"); goto destroy_admin; From 58847f12fe7823c56f844218abcca6920901097d Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 14 Oct 2021 09:45:42 -0700 Subject: [PATCH 218/255] nvme-pci: clear shadow doorbell memory on resets The host memory doorbell and event buffers need to be initialized on each reset so the driver doesn't observe stale values from the previous instantiation. Signed-off-by: Keith Busch Tested-by: John Levon Signed-off-by: Christoph Hellwig --- drivers/nvme/host/pci.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index ed684874842f..6e05cfb4879f 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -245,8 +245,15 @@ static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) { unsigned int mem_size = nvme_dbbuf_size(dev); - if (dev->dbbuf_dbs) + if (dev->dbbuf_dbs) { + /* + * Clear the dbbuf memory so the driver doesn't observe stale + * values from the previous instantiation. + */ + memset(dev->dbbuf_dbs, 0, mem_size); + memset(dev->dbbuf_eis, 0, mem_size); return 0; + } dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size, &dev->dbbuf_dbs_dma_addr, From 2b81a5f015199f3d585ce710190a9e87714d3c1e Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Wed, 20 Oct 2021 07:59:10 +0200 Subject: [PATCH 219/255] nvme: drop scan_lock and always kick requeue list when removing namespaces When reading the partition table on initial scan hits an I/O error the I/O will hang with the scan_mutex held: [<0>] do_read_cache_page+0x49b/0x790 [<0>] read_part_sector+0x39/0xe0 [<0>] read_lba+0xf9/0x1d0 [<0>] efi_partition+0xf1/0x7f0 [<0>] bdev_disk_changed+0x1ee/0x550 [<0>] blkdev_get_whole+0x81/0x90 [<0>] blkdev_get_by_dev+0x128/0x2e0 [<0>] device_add_disk+0x377/0x3c0 [<0>] nvme_mpath_set_live+0x130/0x1b0 [nvme_core] [<0>] nvme_mpath_add_disk+0x150/0x160 [nvme_core] [<0>] nvme_alloc_ns+0x417/0x950 [nvme_core] [<0>] nvme_validate_or_alloc_ns+0xe9/0x1e0 [nvme_core] [<0>] nvme_scan_work+0x168/0x310 [nvme_core] [<0>] process_one_work+0x231/0x420 and trying to delete the controller will deadlock as it tries to grab the scan mutex: [<0>] nvme_mpath_clear_ctrl_paths+0x25/0x80 [nvme_core] [<0>] nvme_remove_namespaces+0x31/0xf0 [nvme_core] [<0>] nvme_do_delete_ctrl+0x4b/0x80 [nvme_core] As we're now properly ordering the namespace list there is no need to hold the scan_mutex in nvme_mpath_clear_ctrl_paths() anymore. And we always need to kick the requeue list as the path will be marked as unusable and I/O will be requeued _without_ a current path. Signed-off-by: Hannes Reinecke Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/host/multipath.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index bc1e7d8a2c0f..954e84df6eb7 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -146,13 +146,12 @@ void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) { struct nvme_ns *ns; - mutex_lock(&ctrl->scan_lock); down_read(&ctrl->namespaces_rwsem); - list_for_each_entry(ns, &ctrl->namespaces, list) - if (nvme_mpath_clear_current_path(ns)) - kblockd_schedule_work(&ns->head->requeue_work); + list_for_each_entry(ns, &ctrl->namespaces, list) { + nvme_mpath_clear_current_path(ns); + kblockd_schedule_work(&ns->head->requeue_work); + } up_read(&ctrl->namespaces_rwsem); - mutex_unlock(&ctrl->scan_lock); } void nvme_mpath_revalidate_paths(struct nvme_ns *ns) From 117d5b6d00ee02f73d7065fe906e2ef1af74bb68 Mon Sep 17 00:00:00 2001 From: Len Baker Date: Sun, 17 Oct 2021 11:56:50 +0200 Subject: [PATCH 220/255] nvmet: use struct_size over open coded arithmetic As noted in the "Deprecated Interfaces, Language Features, Attributes, and Conventions" documentation [1], size calculations (especially multiplication) should not be performed in memory allocator (or similar) function arguments due to the risk of them overflowing. This could lead to values wrapping around and a smaller allocation being made than the caller was expecting. Using those allocations could lead to linear overflows of heap memory and other misbehaviors. In this case this is not actually dynamic size: all the operands involved in the calculation are constant values. However it is better to refactor this anyway, just to keep the open-coded math idiom out of code. So, use the struct_size() helper to do the arithmetic instead of the argument "size + count * size" in the kmalloc() function. This code was detected with the help of Coccinelle and audited and fixed manually. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments Signed-off-by: Len Baker Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 6e75890bc8d9..403de678fd06 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -278,8 +278,8 @@ static void nvmet_execute_get_log_page_ana(struct nvmet_req *req) u16 status; status = NVME_SC_INTERNAL; - desc = kmalloc(sizeof(struct nvme_ana_group_desc) + - NVMET_MAX_NAMESPACES * sizeof(__le32), GFP_KERNEL); + desc = kmalloc(struct_size(desc, nsids, NVMET_MAX_NAMESPACES), + GFP_KERNEL); if (!desc) goto out; From bbc3925cf696422492ebdaba386e61450fa2294c Mon Sep 17 00:00:00 2001 From: luo penghao Date: Thu, 21 Oct 2021 08:46:21 +0100 Subject: [PATCH 221/255] cdrom: Remove redundant variable and its assignment Variable is not used in functions, and its assignment is redundant too. So it should be deleted. Also the inner-most set of parentheses is no longer needed. The clang_analyzer complains as follows: drivers/cdrom/cdrom.c:877: warning: Although the value stored to 'ret' is used in the enclosing expression, the value is never actually read from 'ret'. Reported-by: Zeal Robot Signed-off-by: luo penghao Link: https://lore.kernel.org/all/20211020024229.1036219-1-luo.penghao@zte.com.cn Signed-off-by: Phillip Potter Link: https://lore.kernel.org/r/20211021074621.901-1-phil@philpotter.co.uk Signed-off-by: Jens Axboe --- drivers/cdrom/cdrom.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 89a68457820a..9877e413fce3 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -871,7 +871,7 @@ static void cdrom_mmc3_profile(struct cdrom_device_info *cdi) { struct packet_command cgc; char buffer[32]; - int ret, mmc3_profile; + int mmc3_profile; init_cdrom_command(&cgc, buffer, sizeof(buffer), CGC_DATA_READ); @@ -881,7 +881,7 @@ static void cdrom_mmc3_profile(struct cdrom_device_info *cdi) cgc.cmd[8] = sizeof(buffer); /* Allocation Length */ cgc.quiet = 1; - if ((ret = cdi->ops->generic_packet(cdi, &cgc))) + if (cdi->ops->generic_packet(cdi, &cgc)) mmc3_profile = 0xffff; else mmc3_profile = (buffer[6] << 8) | buffer[7]; From ff06ed7e815ccd3f80cfd4d5dd30687bc722e871 Mon Sep 17 00:00:00 2001 From: Ye Guojin Date: Thu, 21 Oct 2021 06:49:31 +0000 Subject: [PATCH 222/255] block: aoe: fixup coccinelle warnings coccicheck complains about the use of snprintf() in sysfs show functions: WARNING use scnprintf or sprintf Use sysfs_emit instead of scnprintf or sprintf makes more sense. Reported-by: Zeal Robot Signed-off-by: Ye Guojin Link: https://lore.kernel.org/r/20211021064931.1047687-1-ye.guojin@zte.com.cn Signed-off-by: Jens Axboe --- drivers/block/aoe/aoeblk.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index e436b0e8eff5..52484bcdedb9 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -37,8 +37,7 @@ static ssize_t aoedisk_show_state(struct device *dev, struct gendisk *disk = dev_to_disk(dev); struct aoedev *d = disk->private_data; - return snprintf(page, PAGE_SIZE, - "%s%s\n", + return sysfs_emit(page, "%s%s\n", (d->flags & DEVFL_UP) ? "up" : "down", (d->flags & DEVFL_KICKME) ? ",kickme" : (d->nopen && !(d->flags & DEVFL_UP)) ? ",closewait" : ""); @@ -52,8 +51,8 @@ static ssize_t aoedisk_show_mac(struct device *dev, struct aoetgt *t = d->targets[0]; if (t == NULL) - return snprintf(page, PAGE_SIZE, "none\n"); - return snprintf(page, PAGE_SIZE, "%pm\n", t->addr); + return sysfs_emit(page, "none\n"); + return sysfs_emit(page, "%pm\n", t->addr); } static ssize_t aoedisk_show_netif(struct device *dev, struct device_attribute *attr, char *page) @@ -85,7 +84,7 @@ static ssize_t aoedisk_show_netif(struct device *dev, ne = nd; nd = nds; if (*nd == NULL) - return snprintf(page, PAGE_SIZE, "none\n"); + return sysfs_emit(page, "none\n"); for (p = page; nd < ne; nd++) p += scnprintf(p, PAGE_SIZE - (p-page), "%s%s", p == page ? "" : ",", (*nd)->name); @@ -99,7 +98,7 @@ static ssize_t aoedisk_show_fwver(struct device *dev, struct gendisk *disk = dev_to_disk(dev); struct aoedev *d = disk->private_data; - return snprintf(page, PAGE_SIZE, "0x%04x\n", (unsigned int) d->fw_ver); + return sysfs_emit(page, "0x%04x\n", (unsigned int) d->fw_ver); } static ssize_t aoedisk_show_payload(struct device *dev, struct device_attribute *attr, char *page) @@ -107,7 +106,7 @@ static ssize_t aoedisk_show_payload(struct device *dev, struct gendisk *disk = dev_to_disk(dev); struct aoedev *d = disk->private_data; - return snprintf(page, PAGE_SIZE, "%lu\n", d->maxbcnt); + return sysfs_emit(page, "%lu\n", d->maxbcnt); } static int aoedisk_debugfs_show(struct seq_file *s, void *ignored) From e7089f65dd51afeda5eb760506b5950d95f9ec29 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:22 -0700 Subject: [PATCH 223/255] dm: add add_disk() error handling We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. There are two calls to dm_setup_md_queue() which can fail then, one on dm_early_create() and we can easily see that the error path there calls dm_destroy in the error path. The other use case is on the ioctl table_load case. If that fails userspace needs to call the DM_DEV_REMOVE_CMD to cleanup the state - similar to any other failure. Reviewed-by: Hannes Reinecke Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-4-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/md/dm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 7870e6460633..79d4ac4aab05 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2078,7 +2078,9 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) if (r) return r; - add_disk(md->disk); + r = add_disk(md->disk); + if (r) + return r; r = dm_sysfs_init(md); if (r) { From 2961c3bbcaec0ed7fb7b9a465b3796f37f2294e5 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:23 -0700 Subject: [PATCH 224/255] bcache: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. This driver doesn't do any unwinding with blk_cleanup_disk() even on errors after add_disk() and so we follow that tradition. Acked-by: Coly Li Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-5-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index dc35f6e1d8d3..84a48eed8e24 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1080,7 +1080,9 @@ int bch_cached_dev_run(struct cached_dev *dc) closure_sync(&cl); } - add_disk(d->disk); + ret = add_disk(d->disk); + if (ret) + goto out; bd_link_disk_holder(dc->bdev, dc->disk.disk); /* * won't show up in the uevent file, use udevadm monitor -e instead @@ -1526,10 +1528,11 @@ static void flash_dev_flush(struct closure *cl) static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) { + int err = -ENOMEM; struct bcache_device *d = kzalloc(sizeof(struct bcache_device), GFP_KERNEL); if (!d) - return -ENOMEM; + goto err_ret; closure_init(&d->cl, NULL); set_closure_fn(&d->cl, flash_dev_flush, system_wq); @@ -1543,9 +1546,12 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) bcache_device_attach(d, c, u - c->uuids); bch_sectors_dirty_init(d); bch_flash_dev_request_init(d); - add_disk(d->disk); + err = add_disk(d->disk); + if (err) + goto err; - if (kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache")) + err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache"); + if (err) goto err; bcache_device_link(d, c, "volume"); @@ -1559,7 +1565,8 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) return 0; err: kobject_put(&d->kobj); - return -ENOMEM; +err_ret: + return err; } static int flash_devs_run(struct cache_set *c) From 293a7c528803321479593d42d0898bb5a9769db1 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:24 -0700 Subject: [PATCH 225/255] xen-blkfront: add error handling support for add_disk() We never checked for errors on device_add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. The function xlvbd_alloc_gendisk() typically does the unwinding on error on allocating the disk and creating the tag, but since all that error handling was stuffed inside xlvbd_alloc_gendisk() we must repeat the tag free'ing as well. We set the info->rq to NULL to ensure blkif_free() doesn't crash on blk_mq_stop_hw_queues() on device_add_disk() error as the queue will be long gone by then. Reviewed-by: Juergen Gross Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-6-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/xen-blkfront.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index df0deb927760..8e3983e456f3 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2386,7 +2386,13 @@ static void blkfront_connect(struct blkfront_info *info) for_each_rinfo(info, rinfo, i) kick_pending_request_queues(rinfo); - device_add_disk(&info->xbdev->dev, info->gd, NULL); + err = device_add_disk(&info->xbdev->dev, info->gd, NULL); + if (err) { + blk_cleanup_disk(info->gd); + blk_mq_free_tag_set(&info->tag_set); + info->rq = NULL; + goto fail; + } info->is_ready = 1; return; From 21fd880d3da7564bab68979417cab7408e4f9642 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:25 -0700 Subject: [PATCH 226/255] m68k/emu/nfblock: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Reviewed-by: Geert Uytterhoeven Acked-by: Geert Uytterhoeven Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-7-mcgrof@kernel.org Signed-off-by: Jens Axboe --- arch/m68k/emu/nfblock.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 4ef457ba5220..9c57b245dc12 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -99,6 +99,7 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) { struct nfhd_device *dev; int dev_id = id - NFHD_DEV_OFFSET; + int err = -ENOMEM; pr_info("nfhd%u: found device with %u blocks (%u bytes)\n", dev_id, blocks, bsize); @@ -129,16 +130,20 @@ static int __init nfhd_init_one(int id, u32 blocks, u32 bsize) sprintf(dev->disk->disk_name, "nfhd%u", dev_id); set_capacity(dev->disk, (sector_t)blocks * (bsize / 512)); blk_queue_logical_block_size(dev->disk->queue, bsize); - add_disk(dev->disk); + err = add_disk(dev->disk); + if (err) + goto out_cleanup_disk; list_add_tail(&dev->list, &nfhd_list); return 0; +out_cleanup_disk: + blk_cleanup_disk(dev->disk); free_dev: kfree(dev); out: - return -ENOMEM; + return err; } static int __init nfhd_init(void) From 66638f163a2b5c5b462ca38525129b14a20117eb Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:26 -0700 Subject: [PATCH 227/255] um/drivers/ubd_kern: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. ubd_disk_register() never returned an error, so just fix that now and let the caller handle the error condition. Reviewed-by: Gabriel Krisman Bertazi Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-8-mcgrof@kernel.org Signed-off-by: Jens Axboe --- arch/um/drivers/ubd_kern.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index fefd343412c7..69d2d0049a61 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -855,8 +855,8 @@ static const struct attribute_group *ubd_attr_groups[] = { NULL, }; -static void ubd_disk_register(int major, u64 size, int unit, - struct gendisk *disk) +static int ubd_disk_register(int major, u64 size, int unit, + struct gendisk *disk) { disk->major = major; disk->first_minor = unit << UBD_SHIFT; @@ -873,7 +873,7 @@ static void ubd_disk_register(int major, u64 size, int unit, disk->private_data = &ubd_devs[unit]; disk->queue = ubd_devs[unit].queue; - device_add_disk(&ubd_devs[unit].pdev.dev, disk, ubd_attr_groups); + return device_add_disk(&ubd_devs[unit].pdev.dev, disk, ubd_attr_groups); } #define ROUND_BLOCK(n) ((n + (SECTOR_SIZE - 1)) & (-SECTOR_SIZE)) @@ -920,10 +920,15 @@ static int ubd_add(int n, char **error_out) blk_queue_write_cache(ubd_dev->queue, true, false); blk_queue_max_segments(ubd_dev->queue, MAX_SG); blk_queue_segment_boundary(ubd_dev->queue, PAGE_SIZE - 1); - ubd_disk_register(UBD_MAJOR, ubd_dev->size, n, disk); + err = ubd_disk_register(UBD_MAJOR, ubd_dev->size, n, disk); + if (err) + goto out_cleanup_disk; + ubd_gendisk[n] = disk; return 0; +out_cleanup_disk: + blk_cleanup_disk(disk); out_cleanup_tags: blk_mq_free_tag_set(&ubd_dev->tag_set); out: From 2e9e31bea01997450397d64da43b6675e0adb9e3 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:27 -0700 Subject: [PATCH 228/255] rnbd: add error handling support for add_disk() We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Acked-by: Jack Wang Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-9-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/block/rnbd/rnbd-clt.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 0ec0191d4196..2df0657cdf00 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -1384,8 +1384,10 @@ static void setup_request_queue(struct rnbd_clt_dev *dev) blk_queue_write_cache(dev->queue, dev->wc, dev->fua); } -static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) +static int rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) { + int err; + dev->gd->major = rnbd_client_major; dev->gd->first_minor = idx << RNBD_PART_BITS; dev->gd->minors = 1 << RNBD_PART_BITS; @@ -1410,7 +1412,11 @@ static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) if (!dev->rotational) blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue); - add_disk(dev->gd); + err = add_disk(dev->gd); + if (err) + blk_cleanup_disk(dev->gd); + + return err; } static int rnbd_client_setup_device(struct rnbd_clt_dev *dev) @@ -1426,8 +1432,7 @@ static int rnbd_client_setup_device(struct rnbd_clt_dev *dev) rnbd_init_mq_hw_queues(dev); setup_request_queue(dev); - rnbd_clt_setup_gen_disk(dev, idx); - return 0; + return rnbd_clt_setup_gen_disk(dev, idx); } static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, From 83b863f4a3f0de4ece7802d9121fed0c3e64145f Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Oct 2021 16:30:28 -0700 Subject: [PATCH 229/255] mtd: add add_disk() error handling We never checked for errors on add_disk() as this function returned void. Now that this is fixed, use the shiny new error handling. Acked-by: Miquel Raynal Signed-off-by: Luis Chamberlain Link: https://lore.kernel.org/r/20211015233028.2167651-10-mcgrof@kernel.org Signed-off-by: Jens Axboe --- drivers/mtd/mtd_blkdevs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c index b8ae1ec14e17..4eaba6f4ec68 100644 --- a/drivers/mtd/mtd_blkdevs.c +++ b/drivers/mtd/mtd_blkdevs.c @@ -384,7 +384,9 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) if (new->readonly) set_disk_ro(gd, 1); - device_add_disk(&new->mtd->dev, gd, NULL); + ret = device_add_disk(&new->mtd->dev, gd, NULL); + if (ret) + goto out_cleanup_disk; if (new->disk_attributes) { ret = sysfs_create_group(&disk_to_dev(gd)->kobj, @@ -393,6 +395,8 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) } return 0; +out_cleanup_disk: + blk_cleanup_disk(new->disk); out_free_tag_set: blk_mq_free_tag_set(new->tag_set); out_kfree_tag_set: From 47e9624616c80c9879feda536c48c6a3a0ed9835 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Oct 2021 09:56:39 +0200 Subject: [PATCH 230/255] block: remove support for cryptoloop and the xor transfer Support for cyrptoloop has been officially marked broken and deprecated in favor of dm-crypt (which supports the same broken algorithms if needed) in Linux 2.6.4 (released in March 2004), and support for it has been entirely removed from losetup in util-linux 2.23 (released in April 2013). The XOR transfer has never been more than a toy to demonstrate the transfer in the bad old times of crypto export restrictions. Remove them as they have some nasty interactions with loop device life times due to the iteration over all loop devices in loop_unregister_transfer. Suggested-by: Milan Broz Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211019075639.2333969-1-hch@lst.de Signed-off-by: Jens Axboe --- drivers/block/Kconfig | 23 --- drivers/block/Makefile | 1 - drivers/block/cryptoloop.c | 206 -------------------- drivers/block/loop.c | 376 +++---------------------------------- drivers/block/loop.h | 30 --- 5 files changed, 26 insertions(+), 610 deletions(-) delete mode 100644 drivers/block/cryptoloop.c diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index ab3e37aa1830..f7f32eeaee63 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -180,14 +180,6 @@ config BLK_DEV_LOOP bits of, say, a sound file). This is also safe if the file resides on a remote file server. - There are several ways of encrypting disks. Some of these require - kernel patches. The vanilla kernel offers the cryptoloop option - and a Device Mapper target (which is superior, as it supports all - file systems). If you want to use the cryptoloop, say Y to both - LOOP and CRYPTOLOOP, and make sure you have a recent (version 2.12 - or later) version of util-linux. Additionally, be aware that - the cryptoloop is not safe for storing journaled filesystems. - Note that this loop device has nothing to do with the loopback device used for network connections from the machine to itself. @@ -211,21 +203,6 @@ config BLK_DEV_LOOP_MIN_COUNT is used, it can be set to 0, since needed loop devices can be dynamically allocated with the /dev/loop-control interface. -config BLK_DEV_CRYPTOLOOP - tristate "Cryptoloop Support (DEPRECATED)" - select CRYPTO - select CRYPTO_CBC - depends on BLK_DEV_LOOP - help - Say Y here if you want to be able to use the ciphers that are - provided by the CryptoAPI as loop transformation. This might be - used as hard disk encryption. - - WARNING: This device is not safe for journaled file systems like - ext3 or Reiserfs. Please use the Device Mapper crypto module - instead, which can be configured to be on-disk compatible with the - cryptoloop device. cryptoloop support will be removed in Linux 5.16. - source "drivers/block/drbd/Kconfig" config BLK_DEV_NBD diff --git a/drivers/block/Makefile b/drivers/block/Makefile index bc68817ef496..11a74f17c9ad 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -24,7 +24,6 @@ obj-$(CONFIG_CDROM_PKTCDVD) += pktcdvd.o obj-$(CONFIG_SUNVDC) += sunvdc.o obj-$(CONFIG_BLK_DEV_NBD) += nbd.o -obj-$(CONFIG_BLK_DEV_CRYPTOLOOP) += cryptoloop.o obj-$(CONFIG_VIRTIO_BLK) += virtio_blk.o obj-$(CONFIG_BLK_DEV_SX8) += sx8.o diff --git a/drivers/block/cryptoloop.c b/drivers/block/cryptoloop.c deleted file mode 100644 index f0a91faa43a8..000000000000 --- a/drivers/block/cryptoloop.c +++ /dev/null @@ -1,206 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - Linux loop encryption enabling module - - Copyright (C) 2002 Herbert Valerio Riedel - Copyright (C) 2003 Fruhwirth Clemens - - */ - -#include - -#include -#include -#include -#include -#include -#include -#include "loop.h" - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("loop blockdevice transferfunction adaptor / CryptoAPI"); -MODULE_AUTHOR("Herbert Valerio Riedel "); - -#define LOOP_IV_SECTOR_BITS 9 -#define LOOP_IV_SECTOR_SIZE (1 << LOOP_IV_SECTOR_BITS) - -static int -cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info) -{ - int err = -EINVAL; - int cipher_len; - int mode_len; - char cms[LO_NAME_SIZE]; /* cipher-mode string */ - char *mode; - char *cmsp = cms; /* c-m string pointer */ - struct crypto_sync_skcipher *tfm; - - /* encryption breaks for non sector aligned offsets */ - - if (info->lo_offset % LOOP_IV_SECTOR_SIZE) - goto out; - - strncpy(cms, info->lo_crypt_name, LO_NAME_SIZE); - cms[LO_NAME_SIZE - 1] = 0; - - cipher_len = strcspn(cmsp, "-"); - - mode = cmsp + cipher_len; - mode_len = 0; - if (*mode) { - mode++; - mode_len = strcspn(mode, "-"); - } - - if (!mode_len) { - mode = "cbc"; - mode_len = 3; - } - - if (cipher_len + mode_len + 3 > LO_NAME_SIZE) - return -EINVAL; - - memmove(cms, mode, mode_len); - cmsp = cms + mode_len; - *cmsp++ = '('; - memcpy(cmsp, info->lo_crypt_name, cipher_len); - cmsp += cipher_len; - *cmsp++ = ')'; - *cmsp = 0; - - tfm = crypto_alloc_sync_skcipher(cms, 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - err = crypto_sync_skcipher_setkey(tfm, info->lo_encrypt_key, - info->lo_encrypt_key_size); - - if (err != 0) - goto out_free_tfm; - - lo->key_data = tfm; - return 0; - - out_free_tfm: - crypto_free_sync_skcipher(tfm); - - out: - return err; -} - - -typedef int (*encdec_cbc_t)(struct skcipher_request *req); - -static int -cryptoloop_transfer(struct loop_device *lo, int cmd, - struct page *raw_page, unsigned raw_off, - struct page *loop_page, unsigned loop_off, - int size, sector_t IV) -{ - struct crypto_sync_skcipher *tfm = lo->key_data; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); - struct scatterlist sg_out; - struct scatterlist sg_in; - - encdec_cbc_t encdecfunc; - struct page *in_page, *out_page; - unsigned in_offs, out_offs; - int err; - - skcipher_request_set_sync_tfm(req, tfm); - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, - NULL, NULL); - - sg_init_table(&sg_out, 1); - sg_init_table(&sg_in, 1); - - if (cmd == READ) { - in_page = raw_page; - in_offs = raw_off; - out_page = loop_page; - out_offs = loop_off; - encdecfunc = crypto_skcipher_decrypt; - } else { - in_page = loop_page; - in_offs = loop_off; - out_page = raw_page; - out_offs = raw_off; - encdecfunc = crypto_skcipher_encrypt; - } - - while (size > 0) { - const int sz = min(size, LOOP_IV_SECTOR_SIZE); - u32 iv[4] = { 0, }; - iv[0] = cpu_to_le32(IV & 0xffffffff); - - sg_set_page(&sg_in, in_page, sz, in_offs); - sg_set_page(&sg_out, out_page, sz, out_offs); - - skcipher_request_set_crypt(req, &sg_in, &sg_out, sz, iv); - err = encdecfunc(req); - if (err) - goto out; - - IV++; - size -= sz; - in_offs += sz; - out_offs += sz; - } - - err = 0; - -out: - skcipher_request_zero(req); - return err; -} - -static int -cryptoloop_ioctl(struct loop_device *lo, int cmd, unsigned long arg) -{ - return -EINVAL; -} - -static int -cryptoloop_release(struct loop_device *lo) -{ - struct crypto_sync_skcipher *tfm = lo->key_data; - if (tfm != NULL) { - crypto_free_sync_skcipher(tfm); - lo->key_data = NULL; - return 0; - } - printk(KERN_ERR "cryptoloop_release(): tfm == NULL?\n"); - return -EINVAL; -} - -static struct loop_func_table cryptoloop_funcs = { - .number = LO_CRYPT_CRYPTOAPI, - .init = cryptoloop_init, - .ioctl = cryptoloop_ioctl, - .transfer = cryptoloop_transfer, - .release = cryptoloop_release, - .owner = THIS_MODULE -}; - -static int __init -init_cryptoloop(void) -{ - int rc = loop_register_transfer(&cryptoloop_funcs); - - if (rc) - printk(KERN_ERR "cryptoloop: loop_register_transfer failed\n"); - else - pr_warn("the cryptoloop driver has been deprecated and will be removed in in Linux 5.16\n"); - return rc; -} - -static void __exit -cleanup_cryptoloop(void) -{ - if (loop_unregister_transfer(LO_CRYPT_CRYPTOAPI)) - printk(KERN_ERR - "cryptoloop: loop_unregister_transfer failed\n"); -} - -module_init(init_cryptoloop); -module_exit(cleanup_cryptoloop); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 00ee365ed5e0..302ac8f4f8ac 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -133,58 +133,6 @@ static void loop_global_unlock(struct loop_device *lo, bool global) static int max_part; static int part_shift; -static int transfer_xor(struct loop_device *lo, int cmd, - struct page *raw_page, unsigned raw_off, - struct page *loop_page, unsigned loop_off, - int size, sector_t real_block) -{ - char *raw_buf = kmap_atomic(raw_page) + raw_off; - char *loop_buf = kmap_atomic(loop_page) + loop_off; - char *in, *out, *key; - int i, keysize; - - if (cmd == READ) { - in = raw_buf; - out = loop_buf; - } else { - in = loop_buf; - out = raw_buf; - } - - key = lo->lo_encrypt_key; - keysize = lo->lo_encrypt_key_size; - for (i = 0; i < size; i++) - *out++ = *in++ ^ key[(i & 511) % keysize]; - - kunmap_atomic(loop_buf); - kunmap_atomic(raw_buf); - cond_resched(); - return 0; -} - -static int xor_init(struct loop_device *lo, const struct loop_info64 *info) -{ - if (unlikely(info->lo_encrypt_key_size <= 0)) - return -EINVAL; - return 0; -} - -static struct loop_func_table none_funcs = { - .number = LO_CRYPT_NONE, -}; - -static struct loop_func_table xor_funcs = { - .number = LO_CRYPT_XOR, - .transfer = transfer_xor, - .init = xor_init -}; - -/* xfer_funcs[0] is special - its release function is never called */ -static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = { - &none_funcs, - &xor_funcs -}; - static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) { loff_t loopsize; @@ -228,8 +176,7 @@ static void __loop_update_dio(struct loop_device *lo, bool dio) /* * We support direct I/O only if lo_offset is aligned with the * logical I/O size of backing device, and the logical block - * size of loop is bigger than the backing device's and the loop - * needn't transform transfer. + * size of loop is bigger than the backing device's. * * TODO: the above condition may be loosed in the future, and * direct I/O may be switched runtime at that time because most @@ -238,8 +185,7 @@ static void __loop_update_dio(struct loop_device *lo, bool dio) if (dio) { if (queue_logical_block_size(lo->lo_queue) >= sb_bsize && !(lo->lo_offset & dio_align) && - mapping->a_ops->direct_IO && - !lo->transfer) + mapping->a_ops->direct_IO) use_dio = true; else use_dio = false; @@ -299,24 +245,6 @@ static void loop_set_size(struct loop_device *lo, loff_t size) kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); } -static inline int -lo_do_transfer(struct loop_device *lo, int cmd, - struct page *rpage, unsigned roffs, - struct page *lpage, unsigned loffs, - int size, sector_t rblock) -{ - int ret; - - ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); - if (likely(!ret)) - return 0; - - printk_ratelimited(KERN_ERR - "loop: Transfer error at byte offset %llu, length %i.\n", - (unsigned long long)rblock << 9, size); - return ret; -} - static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) { struct iov_iter i; @@ -356,41 +284,6 @@ static int lo_write_simple(struct loop_device *lo, struct request *rq, return ret; } -/* - * This is the slow, transforming version that needs to double buffer the - * data as it cannot do the transformations in place without having direct - * access to the destination pages of the backing file. - */ -static int lo_write_transfer(struct loop_device *lo, struct request *rq, - loff_t pos) -{ - struct bio_vec bvec, b; - struct req_iterator iter; - struct page *page; - int ret = 0; - - page = alloc_page(GFP_NOIO); - if (unlikely(!page)) - return -ENOMEM; - - rq_for_each_segment(bvec, rq, iter) { - ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page, - bvec.bv_offset, bvec.bv_len, pos >> 9); - if (unlikely(ret)) - break; - - b.bv_page = page; - b.bv_offset = 0; - b.bv_len = bvec.bv_len; - ret = lo_write_bvec(lo->lo_backing_file, &b, &pos); - if (ret < 0) - break; - } - - __free_page(page); - return ret; -} - static int lo_read_simple(struct loop_device *lo, struct request *rq, loff_t pos) { @@ -420,64 +313,12 @@ static int lo_read_simple(struct loop_device *lo, struct request *rq, return 0; } -static int lo_read_transfer(struct loop_device *lo, struct request *rq, - loff_t pos) -{ - struct bio_vec bvec, b; - struct req_iterator iter; - struct iov_iter i; - struct page *page; - ssize_t len; - int ret = 0; - - page = alloc_page(GFP_NOIO); - if (unlikely(!page)) - return -ENOMEM; - - rq_for_each_segment(bvec, rq, iter) { - loff_t offset = pos; - - b.bv_page = page; - b.bv_offset = 0; - b.bv_len = bvec.bv_len; - - iov_iter_bvec(&i, READ, &b, 1, b.bv_len); - len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); - if (len < 0) { - ret = len; - goto out_free_page; - } - - ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page, - bvec.bv_offset, len, offset >> 9); - if (ret) - goto out_free_page; - - flush_dcache_page(bvec.bv_page); - - if (len != bvec.bv_len) { - struct bio *bio; - - __rq_for_each_bio(bio, rq) - zero_fill_bio(bio); - break; - } - } - - ret = 0; -out_free_page: - __free_page(page); - return ret; -} - static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos, int mode) { /* * We use fallocate to manipulate the space mappings used by the image - * a.k.a. discard/zerorange. However we do not support this if - * encryption is enabled, because it may give an attacker useful - * information. + * a.k.a. discard/zerorange. */ struct file *file = lo->lo_backing_file; struct request_queue *q = lo->lo_queue; @@ -660,16 +501,12 @@ static int do_req_filebacked(struct loop_device *lo, struct request *rq) case REQ_OP_DISCARD: return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE); case REQ_OP_WRITE: - if (lo->transfer) - return lo_write_transfer(lo, rq, pos); - else if (cmd->use_aio) + if (cmd->use_aio) return lo_rw_aio(lo, cmd, pos, WRITE); else return lo_write_simple(lo, rq, pos); case REQ_OP_READ: - if (lo->transfer) - return lo_read_transfer(lo, rq, pos); - else if (cmd->use_aio) + if (cmd->use_aio) return lo_rw_aio(lo, cmd, pos, READ); else return lo_read_simple(lo, rq, pos); @@ -934,7 +771,7 @@ static void loop_config_discard(struct loop_device *lo) * not blkdev_issue_discard(). This maintains consistent behavior with * file-backed loop devices: discarded regions read back as zero. */ - if (S_ISBLK(inode->i_mode) && !lo->lo_encrypt_key_size) { + if (S_ISBLK(inode->i_mode)) { struct request_queue *backingq = bdev_get_queue(I_BDEV(inode)); max_discard_sectors = backingq->limits.max_write_zeroes_sectors; @@ -943,11 +780,9 @@ static void loop_config_discard(struct loop_device *lo) /* * We use punch hole to reclaim the free space used by the - * image a.k.a. discard. However we do not support discard if - * encryption is enabled, because it may give an attacker - * useful information. + * image a.k.a. discard. */ - } else if (!file->f_op->fallocate || lo->lo_encrypt_key_size) { + } else if (!file->f_op->fallocate) { max_discard_sectors = 0; granularity = 0; @@ -1084,43 +919,6 @@ static void loop_update_rotational(struct loop_device *lo) blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); } -static int -loop_release_xfer(struct loop_device *lo) -{ - int err = 0; - struct loop_func_table *xfer = lo->lo_encryption; - - if (xfer) { - if (xfer->release) - err = xfer->release(lo); - lo->transfer = NULL; - lo->lo_encryption = NULL; - module_put(xfer->owner); - } - return err; -} - -static int -loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, - const struct loop_info64 *i) -{ - int err = 0; - - if (xfer) { - struct module *owner = xfer->owner; - - if (!try_module_get(owner)) - return -EINVAL; - if (xfer->init) - err = xfer->init(lo, i); - if (err) - module_put(owner); - else - lo->lo_encryption = xfer; - } - return err; -} - /** * loop_set_status_from_info - configure device from loop_info * @lo: struct loop_device to configure @@ -1133,55 +931,27 @@ static int loop_set_status_from_info(struct loop_device *lo, const struct loop_info64 *info) { - int err; - struct loop_func_table *xfer; - kuid_t uid = current_uid(); - if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) return -EINVAL; - err = loop_release_xfer(lo); - if (err) - return err; - - if (info->lo_encrypt_type) { - unsigned int type = info->lo_encrypt_type; - - if (type >= MAX_LO_CRYPT) - return -EINVAL; - xfer = xfer_funcs[type]; - if (xfer == NULL) - return -EINVAL; - } else - xfer = NULL; - - err = loop_init_xfer(lo, xfer, info); - if (err) - return err; + switch (info->lo_encrypt_type) { + case LO_CRYPT_NONE: + break; + case LO_CRYPT_XOR: + pr_warn("support for the xor transformation has been removed.\n"); + return -EINVAL; + case LO_CRYPT_CRYPTOAPI: + pr_warn("support for cryptoloop has been removed. Use dm-crypt instead.\n"); + return -EINVAL; + default: + return -EINVAL; + } lo->lo_offset = info->lo_offset; lo->lo_sizelimit = info->lo_sizelimit; memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); - memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); lo->lo_file_name[LO_NAME_SIZE-1] = 0; - lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; - - if (!xfer) - xfer = &none_funcs; - lo->transfer = xfer->transfer; - lo->ioctl = xfer->ioctl; - lo->lo_flags = info->lo_flags; - - lo->lo_encrypt_key_size = info->lo_encrypt_key_size; - lo->lo_init[0] = info->lo_init[0]; - lo->lo_init[1] = info->lo_init[1]; - if (info->lo_encrypt_key_size) { - memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, - info->lo_encrypt_key_size); - lo->lo_key_owner = uid; - } - return 0; } @@ -1381,16 +1151,9 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) lo->lo_backing_file = NULL; spin_unlock_irq(&lo->lo_lock); - loop_release_xfer(lo); - lo->transfer = NULL; - lo->ioctl = NULL; lo->lo_device = NULL; - lo->lo_encryption = NULL; lo->lo_offset = 0; lo->lo_sizelimit = 0; - lo->lo_encrypt_key_size = 0; - memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); - memset(lo->lo_crypt_name, 0, LO_NAME_SIZE); memset(lo->lo_file_name, 0, LO_NAME_SIZE); blk_queue_logical_block_size(lo->lo_queue, 512); blk_queue_physical_block_size(lo->lo_queue, 512); @@ -1498,7 +1261,6 @@ static int loop_set_status(struct loop_device *lo, const struct loop_info64 *info) { int err; - kuid_t uid = current_uid(); int prev_lo_flags; bool partscan = false; bool size_changed = false; @@ -1506,12 +1268,6 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) err = mutex_lock_killable(&lo->lo_mutex); if (err) return err; - if (lo->lo_encrypt_key_size && - !uid_eq(lo->lo_key_owner, uid) && - !capable(CAP_SYS_ADMIN)) { - err = -EPERM; - goto out_unlock; - } if (lo->lo_state != Lo_bound) { err = -ENXIO; goto out_unlock; @@ -1597,14 +1353,6 @@ loop_get_status(struct loop_device *lo, struct loop_info64 *info) info->lo_sizelimit = lo->lo_sizelimit; info->lo_flags = lo->lo_flags; memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); - memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE); - info->lo_encrypt_type = - lo->lo_encryption ? lo->lo_encryption->number : 0; - if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) { - info->lo_encrypt_key_size = lo->lo_encrypt_key_size; - memcpy(info->lo_encrypt_key, lo->lo_encrypt_key, - lo->lo_encrypt_key_size); - } /* Drop lo_mutex while we call into the filesystem. */ path = lo->lo_backing_file->f_path; @@ -1630,16 +1378,8 @@ loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) info64->lo_rdevice = info->lo_rdevice; info64->lo_offset = info->lo_offset; info64->lo_sizelimit = 0; - info64->lo_encrypt_type = info->lo_encrypt_type; - info64->lo_encrypt_key_size = info->lo_encrypt_key_size; info64->lo_flags = info->lo_flags; - info64->lo_init[0] = info->lo_init[0]; - info64->lo_init[1] = info->lo_init[1]; - if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) - memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE); - else - memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); - memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE); + memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); } static int @@ -1651,16 +1391,8 @@ loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) info->lo_inode = info64->lo_inode; info->lo_rdevice = info64->lo_rdevice; info->lo_offset = info64->lo_offset; - info->lo_encrypt_type = info64->lo_encrypt_type; - info->lo_encrypt_key_size = info64->lo_encrypt_key_size; info->lo_flags = info64->lo_flags; - info->lo_init[0] = info64->lo_init[0]; - info->lo_init[1] = info64->lo_init[1]; - if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) - memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE); - else - memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); - memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); + memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); /* error in case values were truncated */ if (info->lo_device != info64->lo_device || @@ -1809,7 +1541,7 @@ static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, err = loop_set_block_size(lo, arg); break; default: - err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL; + err = -EINVAL; } mutex_unlock(&lo->lo_mutex); return err; @@ -1885,7 +1617,6 @@ struct compat_loop_info { compat_ulong_t lo_inode; /* ioctl r/o */ compat_dev_t lo_rdevice; /* ioctl r/o */ compat_int_t lo_offset; - compat_int_t lo_encrypt_type; compat_int_t lo_encrypt_key_size; /* ioctl w/o */ compat_int_t lo_flags; /* ioctl r/o */ char lo_name[LO_NAME_SIZE]; @@ -1914,16 +1645,8 @@ loop_info64_from_compat(const struct compat_loop_info __user *arg, info64->lo_rdevice = info.lo_rdevice; info64->lo_offset = info.lo_offset; info64->lo_sizelimit = 0; - info64->lo_encrypt_type = info.lo_encrypt_type; - info64->lo_encrypt_key_size = info.lo_encrypt_key_size; info64->lo_flags = info.lo_flags; - info64->lo_init[0] = info.lo_init[0]; - info64->lo_init[1] = info.lo_init[1]; - if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) - memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE); - else - memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); - memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE); + memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); return 0; } @@ -1943,24 +1666,14 @@ loop_info64_to_compat(const struct loop_info64 *info64, info.lo_inode = info64->lo_inode; info.lo_rdevice = info64->lo_rdevice; info.lo_offset = info64->lo_offset; - info.lo_encrypt_type = info64->lo_encrypt_type; - info.lo_encrypt_key_size = info64->lo_encrypt_key_size; info.lo_flags = info64->lo_flags; - info.lo_init[0] = info64->lo_init[0]; - info.lo_init[1] = info64->lo_init[1]; - if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) - memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE); - else - memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); - memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); + memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); /* error in case values were truncated */ if (info.lo_device != info64->lo_device || info.lo_rdevice != info64->lo_rdevice || info.lo_inode != info64->lo_inode || - info.lo_offset != info64->lo_offset || - info.lo_init[0] != info64->lo_init[0] || - info.lo_init[1] != info64->lo_init[1]) + info.lo_offset != info64->lo_offset) return -EOVERFLOW; if (copy_to_user(arg, &info, sizeof(info))) @@ -2101,43 +1814,6 @@ MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); MODULE_LICENSE("GPL"); MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); -int loop_register_transfer(struct loop_func_table *funcs) -{ - unsigned int n = funcs->number; - - if (n >= MAX_LO_CRYPT || xfer_funcs[n]) - return -EINVAL; - xfer_funcs[n] = funcs; - return 0; -} - -int loop_unregister_transfer(int number) -{ - unsigned int n = number; - struct loop_func_table *xfer; - - if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL) - return -EINVAL; - /* - * This function is called from only cleanup_cryptoloop(). - * Given that each loop device that has a transfer enabled holds a - * reference to the module implementing it we should never get here - * with a transfer that is set (unless forced module unloading is - * requested). Thus, check module's refcount and warn if this is - * not a clean unloading. - */ -#ifdef CONFIG_MODULE_UNLOAD - if (xfer->owner && module_refcount(xfer->owner) != -1) - pr_err("Danger! Unregistering an in use transfer function.\n"); -#endif - - xfer_funcs[n] = NULL; - return 0; -} - -EXPORT_SYMBOL(loop_register_transfer); -EXPORT_SYMBOL(loop_unregister_transfer); - static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { diff --git a/drivers/block/loop.h b/drivers/block/loop.h index 04c88dd6eabd..082d4b6bfc6a 100644 --- a/drivers/block/loop.h +++ b/drivers/block/loop.h @@ -32,23 +32,10 @@ struct loop_device { loff_t lo_offset; loff_t lo_sizelimit; int lo_flags; - int (*transfer)(struct loop_device *, int cmd, - struct page *raw_page, unsigned raw_off, - struct page *loop_page, unsigned loop_off, - int size, sector_t real_block); char lo_file_name[LO_NAME_SIZE]; - char lo_crypt_name[LO_NAME_SIZE]; - char lo_encrypt_key[LO_KEY_SIZE]; - int lo_encrypt_key_size; - struct loop_func_table *lo_encryption; - __u32 lo_init[2]; - kuid_t lo_key_owner; /* Who set the key */ - int (*ioctl)(struct loop_device *, int cmd, - unsigned long arg); struct file * lo_backing_file; struct block_device *lo_device; - void *key_data; gfp_t old_gfp_mask; @@ -82,21 +69,4 @@ struct loop_cmd { struct cgroup_subsys_state *memcg_css; }; -/* Support for loadable transfer modules */ -struct loop_func_table { - int number; /* filter type */ - int (*transfer)(struct loop_device *lo, int cmd, - struct page *raw_page, unsigned raw_off, - struct page *loop_page, unsigned loop_off, - int size, sector_t real_block); - int (*init)(struct loop_device *, const struct loop_info64 *); - /* release is called from loop_unregister_transfer or clr_fd */ - int (*release)(struct loop_device *); - int (*ioctl)(struct loop_device *, int cmd, unsigned long arg); - struct module *owner; -}; - -int loop_register_transfer(struct loop_func_table *funcs); -int loop_unregister_transfer(int number); - #endif From 90fa02883f063b971ebfd9f5b2184b38b83b7ee3 Mon Sep 17 00:00:00 2001 From: Hao Xu Date: Mon, 18 Oct 2021 21:34:45 +0800 Subject: [PATCH 231/255] io_uring: implement async hybrid mode for pollable requests The current logic of requests with IOSQE_ASYNC is first queueing it to io-worker, then execute it in a synchronous way. For unbound works like pollable requests(e.g. read/write a socketfd), the io-worker may stuck there waiting for events for a long time. And thus other works wait in the list for a long time too. Let's introduce a new way for unbound works (currently pollable requests), with this a request will first be queued to io-worker, then executed in a nonblock try rather than a synchronous way. Failure of that leads it to arm poll stuff and then the worker can begin to handle other works. The detail process of this kind of requests is: step1: original context: queue it to io-worker step2: io-worker context: nonblock try(the old logic is a synchronous try here) | |--fail--> arm poll | |--(fail/ready)-->synchronous issue | |--(succeed)-->worker finish it's job, tw take over the req This works much better than the old IOSQE_ASYNC logic in cases where unbound max_worker is relatively small. In this case, number of io-worker eazily increments to max_worker, new worker cannot be created and running workers stuck there handling old works in IOSQE_ASYNC mode. In my 64-core machine, set unbound max_worker to 20, run echo-server, turns out: (arguments: register_file, connetion number is 1000, message size is 12 Byte) original IOSQE_ASYNC: 76664.151 tps after this patch: 166934.985 tps Suggested-by: Jens Axboe Signed-off-by: Hao Xu Link: https://lore.kernel.org/r/20211018133445.103438-1-haoxu@linux.alibaba.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 88c5ee4dc242..736d456e7913 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6739,8 +6739,18 @@ static void io_wq_submit_work(struct io_wq_work *work) ret = -ECANCELED; if (!ret) { + bool needs_poll = false; + unsigned int issue_flags = IO_URING_F_UNLOCKED; + + if (req->flags & REQ_F_FORCE_ASYNC) { + needs_poll = req->file && file_can_poll(req->file); + if (needs_poll) + issue_flags |= IO_URING_F_NONBLOCK; + } + do { - ret = io_issue_sqe(req, IO_URING_F_UNLOCKED); +issue_sqe: + ret = io_issue_sqe(req, issue_flags); /* * We can get EAGAIN for polled IO even though we're * forcing a sync submission from here, since we can't @@ -6748,6 +6758,30 @@ static void io_wq_submit_work(struct io_wq_work *work) */ if (ret != -EAGAIN) break; + if (needs_poll) { + bool armed = false; + + ret = 0; + needs_poll = false; + issue_flags &= ~IO_URING_F_NONBLOCK; + + switch (io_arm_poll_handler(req)) { + case IO_APOLL_READY: + goto issue_sqe; + case IO_APOLL_ABORTED: + /* + * somehow we failed to arm the poll infra, + * fallback it to a normal async worker try. + */ + break; + case IO_APOLL_OK: + armed = true; + break; + } + + if (armed) + break; + } cond_resched(); } while (1); } From c907e52c72dee710f3bf7c751822df8371924a84 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:13:55 +0100 Subject: [PATCH 232/255] io-wq: use helper for worker refcounting Use io_worker_release() instead of hand coding it in io_worker_exit(). Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6f95f09d2cdbafcbb2e22ad0d1a2bc4d3962bf65.1634987320.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 811299ac9684..0c283bb18fb2 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -178,8 +178,7 @@ static void io_worker_exit(struct io_worker *worker) { struct io_wqe *wqe = worker->wqe; - if (refcount_dec_and_test(&worker->ref)) - complete(&worker->ref_done); + io_worker_release(worker); wait_for_completion(&worker->ref_done); raw_spin_lock(&wqe->lock); From 255657d237042fd51673aef6f22463f662f9933f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:13:56 +0100 Subject: [PATCH 233/255] io_uring: clean io_wq_submit_work()'s main loop Do a bit of cleaning for the main loop of io_wq_submit_work(). Get rid of switch, just replace it with a single if as we're retrying in both other cases. Kill issue_sqe label, Get rid of needs_poll nesting and disambiguate a bit the comment. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/ed12ce0c64e051f9a6b8a37a24f8ea554d299c29.1634987320.git.asml.silence@gmail.com Reviewed-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 40 ++++++++++++---------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 736d456e7913..7f92523c1282 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6749,40 +6749,24 @@ static void io_wq_submit_work(struct io_wq_work *work) } do { -issue_sqe: ret = io_issue_sqe(req, issue_flags); + if (ret != -EAGAIN) + break; /* - * We can get EAGAIN for polled IO even though we're + * We can get EAGAIN for iopolled IO even though we're * forcing a sync submission from here, since we can't * wait for request slots on the block side. */ - if (ret != -EAGAIN) - break; - if (needs_poll) { - bool armed = false; - - ret = 0; - needs_poll = false; - issue_flags &= ~IO_URING_F_NONBLOCK; - - switch (io_arm_poll_handler(req)) { - case IO_APOLL_READY: - goto issue_sqe; - case IO_APOLL_ABORTED: - /* - * somehow we failed to arm the poll infra, - * fallback it to a normal async worker try. - */ - break; - case IO_APOLL_OK: - armed = true; - break; - } - - if (armed) - break; + if (!needs_poll) { + cond_resched(); + continue; } - cond_resched(); + + if (io_arm_poll_handler(req) == IO_APOLL_OK) + return; + /* aborted or ready, in either case retry blocking */ + needs_poll = false; + issue_flags &= ~IO_URING_F_NONBLOCK; } while (1); } From d01905db14eb6223dd1c375001f4daa26cb15c1f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:13:57 +0100 Subject: [PATCH 234/255] io_uring: clean iowq submit work cancellation If we've got IO_WQ_WORK_CANCEL in io_wq_submit_work(), handle the error on the same lines as the check instead of having a weird code flow. The main loop doesn't change but goes one indention left. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/ff4a09cf41f7a22bbb294b6f1faea721e21fe615.1634987320.git.asml.silence@gmail.com Reviewed-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 59 +++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7f92523c1282..58cb3a14d58e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6721,6 +6721,8 @@ static struct io_wq_work *io_wq_free_work(struct io_wq_work *work) static void io_wq_submit_work(struct io_wq_work *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); + unsigned int issue_flags = IO_URING_F_UNLOCKED; + bool needs_poll = false; struct io_kiocb *timeout; int ret = 0; @@ -6735,40 +6737,37 @@ static void io_wq_submit_work(struct io_wq_work *work) io_queue_linked_timeout(timeout); /* either cancelled or io-wq is dying, so don't touch tctx->iowq */ - if (work->flags & IO_WQ_WORK_CANCEL) - ret = -ECANCELED; + if (work->flags & IO_WQ_WORK_CANCEL) { + io_req_task_queue_fail(req, -ECANCELED); + return; + } - if (!ret) { - bool needs_poll = false; - unsigned int issue_flags = IO_URING_F_UNLOCKED; + if (req->flags & REQ_F_FORCE_ASYNC) { + needs_poll = req->file && file_can_poll(req->file); + if (needs_poll) + issue_flags |= IO_URING_F_NONBLOCK; + } - if (req->flags & REQ_F_FORCE_ASYNC) { - needs_poll = req->file && file_can_poll(req->file); - if (needs_poll) - issue_flags |= IO_URING_F_NONBLOCK; + do { + ret = io_issue_sqe(req, issue_flags); + if (ret != -EAGAIN) + break; + /* + * We can get EAGAIN for iopolled IO even though we're + * forcing a sync submission from here, since we can't + * wait for request slots on the block side. + */ + if (!needs_poll) { + cond_resched(); + continue; } - do { - ret = io_issue_sqe(req, issue_flags); - if (ret != -EAGAIN) - break; - /* - * We can get EAGAIN for iopolled IO even though we're - * forcing a sync submission from here, since we can't - * wait for request slots on the block side. - */ - if (!needs_poll) { - cond_resched(); - continue; - } - - if (io_arm_poll_handler(req) == IO_APOLL_OK) - return; - /* aborted or ready, in either case retry blocking */ - needs_poll = false; - issue_flags &= ~IO_URING_F_NONBLOCK; - } while (1); - } + if (io_arm_poll_handler(req) == IO_APOLL_OK) + return; + /* aborted or ready, in either case retry blocking */ + needs_poll = false; + issue_flags &= ~IO_URING_F_NONBLOCK; + } while (1); /* avoid locking problems by failing it from a clean context */ if (ret) From 658d0a401637ffbc733b0f03723e9e1255289429 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:13:58 +0100 Subject: [PATCH 235/255] io_uring: check if opcode needs poll first on arming ->pollout or ->pollin are set only for opcodes that need a file, so if io_arm_poll_handler() tests them first we can be sure that the request has file set and the ->file check can be removed. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/9adfe4f543d984875e516fce6da35348aab48668.1634987320.git.asml.silence@gmail.com Reviewed-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 58cb3a14d58e..bff911f951ed 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5584,12 +5584,10 @@ static int io_arm_poll_handler(struct io_kiocb *req) struct io_poll_table ipt; __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI; - if (!req->file || !file_can_poll(req->file)) - return IO_APOLL_ABORTED; - if (req->flags & REQ_F_POLLED) - return IO_APOLL_ABORTED; if (!def->pollin && !def->pollout) return IO_APOLL_ABORTED; + if (!file_can_poll(req->file) || (req->flags & REQ_F_POLLED)) + return IO_APOLL_ABORTED; if (def->pollin) { mask |= POLLIN | POLLRDNORM; From afb7f56fc624fb6ade8fde70a67eda4d831b4ed0 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:13:59 +0100 Subject: [PATCH 236/255] io_uring: don't try io-wq polling if not supported If an opcode doesn't support polling, just let it be executed synchronously in iowq, otherwise it will do a nonblock attempt just to fail in io_arm_poll_handler() and return back to blocking execution. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/6401256db01b88f448f15fcd241439cb76f5b940.1634987320.git.asml.silence@gmail.com Reviewed-by: Hao Xu Signed-off-by: Jens Axboe --- fs/io_uring.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bff911f951ed..c6f32fcf387b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6741,9 +6741,13 @@ static void io_wq_submit_work(struct io_wq_work *work) } if (req->flags & REQ_F_FORCE_ASYNC) { - needs_poll = req->file && file_can_poll(req->file); - if (needs_poll) + const struct io_op_def *def = &io_op_defs[req->opcode]; + bool opcode_poll = def->pollin || def->pollout; + + if (opcode_poll && file_can_poll(req->file)) { + needs_poll = true; issue_flags |= IO_URING_F_NONBLOCK; + } } do { From d6a644a795451d5fd063a5c08d6bb3a91d021887 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:14:00 +0100 Subject: [PATCH 237/255] io_uring: clean up timeout async_data allocation opcode prep functions are one of the first things that are called, we can't have ->async_data allocated at this point and it's certainly a bug. Reflect this assumption in io_timeout_prep() and add a WARN_ONCE just in case. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/75a28ca7dbcc5af8b6cd9092819e8384c24dedd4.1634987320.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c6f32fcf387b..e775529a36d8 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6113,7 +6113,9 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (unlikely(off && !req->ctx->off_timeout_used)) req->ctx->off_timeout_used = true; - if (!req_has_async_data(req) && io_alloc_async_data(req)) + if (WARN_ON_ONCE(req_has_async_data(req))) + return -EFAULT; + if (io_alloc_async_data(req)) return -ENOMEM; data = req->async_data; From b9a6b8f92f6feebf40609e4f5f22a3c0404afb60 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:14:01 +0100 Subject: [PATCH 238/255] io_uring: kill unused param from io_file_supports_nowait io_file_supports_nowait() doesn't use rw argument anymore, remove it. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/4bd6709fc573d70c866ea656cb7a7dbe94be8026.1634987320.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index e775529a36d8..7042ed870b52 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2809,8 +2809,7 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) return req->flags & REQ_F_SUPPORT_NOWAIT; } -static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, - int rw) +static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_ring_ctx *ctx = req->ctx; struct kiocb *kiocb = &req->rw.kiocb; @@ -3352,7 +3351,7 @@ static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (unlikely(!(req->file->f_mode & FMODE_READ))) return -EBADF; - return io_prep_rw(req, sqe, READ); + return io_prep_rw(req, sqe); } /* @@ -3568,7 +3567,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (unlikely(!(req->file->f_mode & FMODE_WRITE))) return -EBADF; - return io_prep_rw(req, sqe, WRITE); + return io_prep_rw(req, sqe); } static int io_write(struct io_kiocb *req, unsigned int issue_flags) From fb27274a90eac5a687fe73229775ad36df737d8b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 23 Oct 2021 12:14:02 +0100 Subject: [PATCH 239/255] io_uring: clusterise ki_flags access in rw_prep ioprio setup doesn't depend on other fields that are modified in io_prep_rw() and we can move it down in the function without worrying about performance. It's useful as it makes iocb->ki_flags accesses/modifications closer together, so it's more likely the compiler will cache it in a register and avoid extra reloads. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/8ee98779c06f1b59f6039b1e292db4332efd664b.1634987320.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7042ed870b52..bba2f77ae7e7 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2840,16 +2840,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) req->flags |= REQ_F_NOWAIT; - ioprio = READ_ONCE(sqe->ioprio); - if (ioprio) { - ret = ioprio_check_cap(ioprio); - if (ret) - return ret; - - kiocb->ki_ioprio = ioprio; - } else - kiocb->ki_ioprio = get_current_ioprio(); - if (ctx->flags & IORING_SETUP_IOPOLL) { if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) return -EOPNOTSUPP; @@ -2863,6 +2853,17 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) kiocb->ki_complete = io_complete_rw; } + ioprio = READ_ONCE(sqe->ioprio); + if (ioprio) { + ret = ioprio_check_cap(ioprio); + if (ret) + return ret; + + kiocb->ki_ioprio = ioprio; + } else { + kiocb->ki_ioprio = get_current_ioprio(); + } + req->imu = NULL; req->rw.addr = READ_ONCE(sqe->addr); req->rw.len = READ_ONCE(sqe->len); From d28e4dff085c5a87025c9a0a85fb798bd8e9ca17 Mon Sep 17 00:00:00 2001 From: Michael Schmitz Date: Sun, 24 Oct 2021 13:20:13 +1300 Subject: [PATCH 240/255] block: ataflop: more blk-mq refactoring fixes As it turns out, my earlier patch in commit 86d46fdaa12a (block: ataflop: fix breakage introduced at blk-mq refactoring) was incomplete. This patch fixes any remaining issues found during more testing and code review. Requests exceeding 4 k are handled in 4k segments but __blk_mq_end_request() is never called on these (still sectors outstanding on the request). With redo_fd_request() removed, there is no provision to kick off processing of the next segment, causing requests exceeding 4k to hang. (By setting /sys/block/fd0/queue/max_sectors_k <= 4 as workaround, this behaviour can be avoided). Instead of reintroducing redo_fd_request(), requeue the remainder of the request by calling blk_mq_requeue_request() on incomplete requests (i.e. when blk_update_request() still returns true), and rely on the block layer to queue the residual as new request. Both error handling and formatting needs to release the ST-DMA lock, so call finish_fdc() on these (this was previously handled by redo_fd_request()). finish_fdc() may be called legitimately without the ST-DMA lock held - make sure we only release the lock if we actually held it. In a similar way, early exit due to errors in ataflop_queue_rq() must release the lock. After minor errors, fd_error sets up to recalibrate the drive but never re-runs the current operation (another task handled by redo_fd_request() before). Call do_fd_action() to get the next steps (seek, retry read/write) underway. Signed-off-by: Michael Schmitz Fixes: 6ec3938cff95f (ataflop: convert to blk-mq) CC: linux-block@vger.kernel.org Link: https://lore.kernel.org/r/20211024002013.9332-1-schmitzmic@gmail.com Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 45 +++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 3d217f1d7440..2622803ef71a 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -458,10 +458,20 @@ static DEFINE_TIMER(fd_timer, check_change); static void fd_end_request_cur(blk_status_t err) { + DPRINT(("fd_end_request_cur(), bytes %d of %d\n", + blk_rq_cur_bytes(fd_request), + blk_rq_bytes(fd_request))); + if (!blk_update_request(fd_request, err, blk_rq_cur_bytes(fd_request))) { + DPRINT(("calling __blk_mq_end_request()\n")); __blk_mq_end_request(fd_request, err); fd_request = NULL; + } else { + /* requeue rest of request */ + DPRINT(("calling blk_mq_requeue_request()\n")); + blk_mq_requeue_request(fd_request, true); + fd_request = NULL; } } @@ -699,12 +709,21 @@ static void fd_error( void ) if (fd_request->error_count >= MAX_ERRORS) { printk(KERN_ERR "fd%d: too many errors.\n", SelectedDrive ); fd_end_request_cur(BLK_STS_IOERR); + finish_fdc(); + return; } else if (fd_request->error_count == RECALIBRATE_ERRORS) { printk(KERN_WARNING "fd%d: recalibrating\n", SelectedDrive ); if (SelectedDrive != -1) SUD.track = -1; } + /* need to re-run request to recalibrate */ + atari_disable_irq( IRQ_MFP_FDC ); + + setup_req_params( SelectedDrive ); + do_fd_action( SelectedDrive ); + + atari_enable_irq( IRQ_MFP_FDC ); } @@ -731,8 +750,10 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) if (type) { type--; if (type >= NUM_DISK_MINORS || - minor2disktype[type].drive_types > DriveType) + minor2disktype[type].drive_types > DriveType) { + finish_fdc(); return -EINVAL; + } } q = unit[drive].disk[type]->queue; @@ -750,6 +771,7 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) } if (!UDT || desc->track >= UDT->blocks/UDT->spt/2 || desc->head >= 2) { + finish_fdc(); ret = -EINVAL; goto out; } @@ -790,6 +812,7 @@ static int do_format(int drive, int type, struct atari_format_descr *desc) wait_for_completion(&format_wait); + finish_fdc(); ret = FormatError ? -EIO : 0; out: blk_mq_unquiesce_queue(q); @@ -824,6 +847,7 @@ static void do_fd_action( int drive ) else { /* all sectors finished */ fd_end_request_cur(BLK_STS_OK); + finish_fdc(); return; } } @@ -1227,8 +1251,8 @@ static void fd_rwsec_done1(int status) } else { /* all sectors finished */ - finish_fdc(); fd_end_request_cur(BLK_STS_OK); + finish_fdc(); } return; @@ -1350,7 +1374,7 @@ static void fd_times_out(struct timer_list *unused) static void finish_fdc( void ) { - if (!NeedSeek) { + if (!NeedSeek || !stdma_is_locked_by(floppy_irq)) { finish_fdc_done( 0 ); } else { @@ -1385,7 +1409,8 @@ static void finish_fdc_done( int dummy ) start_motor_off_timer(); local_irq_save(flags); - stdma_release(); + if (stdma_is_locked_by(floppy_irq)) + stdma_release(); local_irq_restore(flags); DPRINT(("finish_fdc() finished\n")); @@ -1482,7 +1507,9 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, int drive = floppy - unit; int type = floppy->type; - DPRINT(("Queue request: drive %d type %d last %d\n", drive, type, bd->last)); + DPRINT(("Queue request: drive %d type %d sectors %d of %d last %d\n", + drive, type, blk_rq_cur_sectors(bd->rq), + blk_rq_sectors(bd->rq), bd->last)); spin_lock_irq(&ataflop_lock); if (fd_request) { @@ -1504,6 +1531,7 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, /* drive not connected */ printk(KERN_ERR "Unknown Device: fd%d\n", drive ); fd_end_request_cur(BLK_STS_IOERR); + stdma_release(); goto out; } @@ -1520,11 +1548,13 @@ static blk_status_t ataflop_queue_rq(struct blk_mq_hw_ctx *hctx, if (--type >= NUM_DISK_MINORS) { printk(KERN_WARNING "fd%d: invalid disk format", drive ); fd_end_request_cur(BLK_STS_IOERR); + stdma_release(); goto out; } if (minor2disktype[type].drive_types > DriveType) { printk(KERN_WARNING "fd%d: unsupported disk format", drive ); fd_end_request_cur(BLK_STS_IOERR); + stdma_release(); goto out; } type = minor2disktype[type].index; @@ -1625,6 +1655,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, /* what if type > 0 here? Overwrite specified entry ? */ if (type) { /* refuse to re-set a predefined type for now */ + finish_fdc(); return -EINVAL; } @@ -1692,8 +1723,10 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, /* sanity check */ if (setprm.track != dtp->blocks/dtp->spt/2 || - setprm.head != 2) + setprm.head != 2) { + finish_fdc(); return -EINVAL; + } UDT = dtp; set_capacity(disk, UDT->blocks); From 3884b83dff245e41def99ceacca8ed2056baf0a8 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 25 Oct 2021 13:45:12 -0600 Subject: [PATCH 241/255] io_uring: don't assign write hint in the read path Move this out of the generic read/write prep path, and place it in the write specific kiocb setup instead. Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index bba2f77ae7e7..d001cd7a6c51 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2825,7 +2825,6 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |= REQ_F_CUR_POS; kiocb->ki_pos = file->f_pos; } - kiocb->ki_hint = ki_hint_validate(file_write_hint(file)); kiocb->ki_flags = iocb_flags(file); ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); if (unlikely(ret)) @@ -3568,6 +3567,7 @@ static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (unlikely(!(req->file->f_mode & FMODE_WRITE))) return -EBADF; + req->rw.kiocb.ki_hint = ki_hint_validate(file_write_hint(req->file)); return io_prep_rw(req, sqe); } From e815d36548f01797ce381be8f0b74f4ba9befd15 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 27 Oct 2021 11:22:20 +0900 Subject: [PATCH 242/255] scsi: sd: add concurrent positioning ranges support Add the sd_read_cpr() function to the sd scsi disk driver to discover if a device has multiple concurrent positioning ranges (i.e. multiple actuators on an HDD). The existence of VPD page B9h indicates if a device has multiple concurrent positioning ranges. The page content describes each range supported by the device. sd_read_cpr() is called from sd_revalidate_disk() and uses the block layer functions disk_alloc_independent_access_ranges() and disk_set_independent_access_ranges() to represent the set of actuators of the device as independent access ranges. The format of the Concurrent Positioning Ranges VPD page B9h is defined in section 6.6.6 of SBC-5. Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke Reviewed-by: Martin K. Petersen Reviewed-by: Keith Busch Link: https://lore.kernel.org/r/20211027022223.183838-3-damien.lemoal@wdc.com Signed-off-by: Jens Axboe --- drivers/scsi/sd.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/scsi/sd.h | 1 + 2 files changed, 82 insertions(+) diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index d8f6add416c0..55c0d951a446 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3088,6 +3088,86 @@ static void sd_read_security(struct scsi_disk *sdkp, unsigned char *buffer) sdkp->security = 1; } +static inline sector_t sd64_to_sectors(struct scsi_disk *sdkp, u8 *buf) +{ + return logical_to_sectors(sdkp->device, get_unaligned_be64(buf)); +} + +/** + * sd_read_cpr - Query concurrent positioning ranges + * @sdkp: disk to query + */ +static void sd_read_cpr(struct scsi_disk *sdkp) +{ + struct blk_independent_access_ranges *iars = NULL; + unsigned char *buffer = NULL; + unsigned int nr_cpr = 0; + int i, vpd_len, buf_len = SD_BUF_SIZE; + u8 *desc; + + /* + * We need to have the capacity set first for the block layer to be + * able to check the ranges. + */ + if (sdkp->first_scan) + return; + + if (!sdkp->capacity) + goto out; + + /* + * Concurrent Positioning Ranges VPD: there can be at most 256 ranges, + * leading to a maximum page size of 64 + 256*32 bytes. + */ + buf_len = 64 + 256*32; + buffer = kmalloc(buf_len, GFP_KERNEL); + if (!buffer || scsi_get_vpd_page(sdkp->device, 0xb9, buffer, buf_len)) + goto out; + + /* We must have at least a 64B header and one 32B range descriptor */ + vpd_len = get_unaligned_be16(&buffer[2]) + 3; + if (vpd_len > buf_len || vpd_len < 64 + 32 || (vpd_len & 31)) { + sd_printk(KERN_ERR, sdkp, + "Invalid Concurrent Positioning Ranges VPD page\n"); + goto out; + } + + nr_cpr = (vpd_len - 64) / 32; + if (nr_cpr == 1) { + nr_cpr = 0; + goto out; + } + + iars = disk_alloc_independent_access_ranges(sdkp->disk, nr_cpr); + if (!iars) { + nr_cpr = 0; + goto out; + } + + desc = &buffer[64]; + for (i = 0; i < nr_cpr; i++, desc += 32) { + if (desc[0] != i) { + sd_printk(KERN_ERR, sdkp, + "Invalid Concurrent Positioning Range number\n"); + nr_cpr = 0; + break; + } + + iars->ia_range[i].sector = sd64_to_sectors(sdkp, desc + 8); + iars->ia_range[i].nr_sectors = sd64_to_sectors(sdkp, desc + 16); + } + +out: + disk_set_independent_access_ranges(sdkp->disk, iars); + if (nr_cpr && sdkp->nr_actuators != nr_cpr) { + sd_printk(KERN_NOTICE, sdkp, + "%u concurrent positioning ranges\n", nr_cpr); + sdkp->nr_actuators = nr_cpr; + } + + kfree(buffer); +} + /* * Determine the device's preferred I/O size for reads and writes * unless the reported value is unreasonably small, large, not a @@ -3203,6 +3283,7 @@ static int sd_revalidate_disk(struct gendisk *disk) sd_read_app_tag_own(sdkp, buffer); sd_read_write_same(sdkp, buffer); sd_read_security(sdkp, buffer); + sd_read_cpr(sdkp); } /* diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index b59136c4125b..2e5932bde43d 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -106,6 +106,7 @@ struct scsi_disk { u8 protection_type;/* Data Integrity Field */ u8 provisioning_mode; u8 zeroing_mode; + u8 nr_actuators; /* Number of actuators */ unsigned ATO : 1; /* state of disk ATO bit */ unsigned cache_override : 1; /* temp override of WCE,RCD */ unsigned WCE : 1; /* state of disk WCE bit */ From fe22e1c2f705676a705d821301fc52eecc2fe055 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 27 Oct 2021 11:22:21 +0900 Subject: [PATCH 243/255] libata: support concurrent positioning ranges log Add support to discover if an ATA device supports the Concurrent Positioning Ranges data log (address 0x47), indicating that the device is capable of seeking to multiple different locations in parallel using multiple actuators serving different LBA ranges. Also add support to translate the concurrent positioning ranges log into its equivalent Concurrent Positioning Ranges VPD page B9h in libata-scsi.c. The format of the Concurrent Positioning Ranges Log is defined in ACS-5 r9. Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Keith Busch Link: https://lore.kernel.org/r/20211027022223.183838-4-damien.lemoal@wdc.com Signed-off-by: Jens Axboe --- drivers/ata/libata-core.c | 57 +++++++++++++++++++++++++++++++++++++-- drivers/ata/libata-scsi.c | 48 ++++++++++++++++++++++++++------- include/linux/ata.h | 1 + include/linux/libata.h | 15 +++++++++++ 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index eed65311b5d1..75f1a6cd6621 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2459,18 +2459,70 @@ static void ata_dev_config_devslp(struct ata_device *dev) } } +static void ata_dev_config_cpr(struct ata_device *dev) +{ + unsigned int err_mask; + size_t buf_len; + int i, nr_cpr = 0; + struct ata_cpr_log *cpr_log = NULL; + u8 *desc, *buf = NULL; + + if (!ata_identify_page_supported(dev, + ATA_LOG_CONCURRENT_POSITIONING_RANGES)) + goto out; + + /* + * Read IDENTIFY DEVICE data log, page 0x47 + * (concurrent positioning ranges). We can have at most 255 32B range + * descriptors plus a 64B header. + */ + buf_len = (64 + 255 * 32 + 511) & ~511; + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) + goto out; + + err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE, + ATA_LOG_CONCURRENT_POSITIONING_RANGES, + buf, buf_len >> 9); + if (err_mask) + goto out; + + nr_cpr = buf[0]; + if (!nr_cpr) + goto out; + + cpr_log = kzalloc(struct_size(cpr_log, cpr, nr_cpr), GFP_KERNEL); + if (!cpr_log) + goto out; + + cpr_log->nr_cpr = nr_cpr; + desc = &buf[64]; + for (i = 0; i < nr_cpr; i++, desc += 32) { + cpr_log->cpr[i].num = desc[0]; + cpr_log->cpr[i].num_storage_elements = desc[1]; + cpr_log->cpr[i].start_lba = get_unaligned_le64(&desc[8]); + cpr_log->cpr[i].num_lbas = get_unaligned_le64(&desc[16]); + } + +out: + swap(dev->cpr_log, cpr_log); + kfree(cpr_log); + kfree(buf); +} + static void ata_dev_print_features(struct ata_device *dev) { if (!(dev->flags & ATA_DFLAG_FEATURES_MASK)) return; ata_dev_info(dev, - "Features:%s%s%s%s%s\n", + "Features:%s%s%s%s%s%s\n", dev->flags & ATA_DFLAG_TRUSTED ? " Trust" : "", dev->flags & ATA_DFLAG_DA ? " Dev-Attention" : "", dev->flags & ATA_DFLAG_DEVSLP ? " Dev-Sleep" : "", dev->flags & ATA_DFLAG_NCQ_SEND_RECV ? " NCQ-sndrcv" : "", - dev->flags & ATA_DFLAG_NCQ_PRIO ? " NCQ-prio" : ""); + dev->flags & ATA_DFLAG_NCQ_PRIO ? " NCQ-prio" : "", + dev->cpr_log ? " CPR" : ""); } /** @@ -2634,6 +2686,7 @@ int ata_dev_configure(struct ata_device *dev) ata_dev_config_sense_reporting(dev); ata_dev_config_zac(dev); ata_dev_config_trusted(dev); + ata_dev_config_cpr(dev); dev->cdb_len = 32; if (ata_msg_drv(ap) && print_info) diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 1fb4611f7eeb..15a279f773c7 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1895,7 +1895,7 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) */ static unsigned int ata_scsiop_inq_00(struct ata_scsi_args *args, u8 *rbuf) { - int num_pages; + int i, num_pages = 0; static const u8 pages[] = { 0x00, /* page 0x00, this page */ 0x80, /* page 0x80, unit serial no page */ @@ -1905,13 +1905,17 @@ static unsigned int ata_scsiop_inq_00(struct ata_scsi_args *args, u8 *rbuf) 0xb1, /* page 0xb1, block device characteristics page */ 0xb2, /* page 0xb2, thin provisioning page */ 0xb6, /* page 0xb6, zoned block device characteristics */ + 0xb9, /* page 0xb9, concurrent positioning ranges */ }; - num_pages = sizeof(pages); - if (!(args->dev->flags & ATA_DFLAG_ZAC)) - num_pages--; + for (i = 0; i < sizeof(pages); i++) { + if (pages[i] == 0xb6 && + !(args->dev->flags & ATA_DFLAG_ZAC)) + continue; + rbuf[num_pages + 4] = pages[i]; + num_pages++; + } rbuf[3] = num_pages; /* number of supported VPD pages */ - memcpy(rbuf + 4, pages, num_pages); return 0; } @@ -2121,6 +2125,26 @@ static unsigned int ata_scsiop_inq_b6(struct ata_scsi_args *args, u8 *rbuf) return 0; } +static unsigned int ata_scsiop_inq_b9(struct ata_scsi_args *args, u8 *rbuf) +{ + struct ata_cpr_log *cpr_log = args->dev->cpr_log; + u8 *desc = &rbuf[64]; + int i; + + /* SCSI Concurrent Positioning Ranges VPD page: SBC-5 rev 1 or later */ + rbuf[1] = 0xb9; + put_unaligned_be16(64 + (int)cpr_log->nr_cpr * 32 - 4, &rbuf[3]); + + for (i = 0; i < cpr_log->nr_cpr; i++, desc += 32) { + desc[0] = cpr_log->cpr[i].num; + desc[1] = cpr_log->cpr[i].num_storage_elements; + put_unaligned_be64(cpr_log->cpr[i].start_lba, &desc[8]); + put_unaligned_be64(cpr_log->cpr[i].num_lbas, &desc[16]); + } + + return 0; +} + /** * modecpy - Prepare response for MODE SENSE * @dest: output buffer @@ -4120,11 +4144,17 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd) ata_scsi_rbuf_fill(&args, ata_scsiop_inq_b2); break; case 0xb6: - if (dev->flags & ATA_DFLAG_ZAC) { + if (dev->flags & ATA_DFLAG_ZAC) ata_scsi_rbuf_fill(&args, ata_scsiop_inq_b6); - break; - } - fallthrough; + else + ata_scsi_set_invalid_field(dev, cmd, 2, 0xff); + break; + case 0xb9: + if (dev->cpr_log) + ata_scsi_rbuf_fill(&args, ata_scsiop_inq_b9); + else + ata_scsi_set_invalid_field(dev, cmd, 2, 0xff); + break; default: ata_scsi_set_invalid_field(dev, cmd, 2, 0xff); break; diff --git a/include/linux/ata.h b/include/linux/ata.h index 1b44f40c7700..199e47e97d64 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -329,6 +329,7 @@ enum { ATA_LOG_SECURITY = 0x06, ATA_LOG_SATA_SETTINGS = 0x08, ATA_LOG_ZONED_INFORMATION = 0x09, + ATA_LOG_CONCURRENT_POSITIONING_RANGES = 0x47, /* Identify device SATA settings log:*/ ATA_LOG_DEVSLP_OFFSET = 0x30, diff --git a/include/linux/libata.h b/include/linux/libata.h index c0c64f03e107..236ec689056a 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -676,6 +676,18 @@ struct ata_ering { struct ata_ering_entry ring[ATA_ERING_SIZE]; }; +struct ata_cpr { + u8 num; + u8 num_storage_elements; + u64 start_lba; + u64 num_lbas; +}; + +struct ata_cpr_log { + u8 nr_cpr; + struct ata_cpr cpr[]; +}; + struct ata_device { struct ata_link *link; unsigned int devno; /* 0 or 1 */ @@ -735,6 +747,9 @@ struct ata_device { u32 zac_zones_optimal_nonseq; u32 zac_zones_max_open; + /* Concurrent positioning ranges */ + struct ata_cpr_log *cpr_log; + /* error history */ int spdn_cnt; /* ering is CLEAR_END, read comment above CLEAR_END */ From 6b3bae2324d2ecaa404ceab869018011b7ef6a90 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 27 Oct 2021 11:22:22 +0900 Subject: [PATCH 244/255] doc: document sysfs queue/independent_access_ranges attributes Update the file Documentation/block/queue-sysfs.rst to add a description of a device queue sysfs entries related to independent access ranges (e.g. concurrent positioning ranges for multi-actuator hard-disks). Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke Reviewed-by: Martin K. Petersen Reviewed-by: Keith Busch Link: https://lore.kernel.org/r/20211027022223.183838-5-damien.lemoal@wdc.com Signed-off-by: Jens Axboe --- Documentation/block/queue-sysfs.rst | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/Documentation/block/queue-sysfs.rst b/Documentation/block/queue-sysfs.rst index 4dc7f0d499a8..b6e8983d8eda 100644 --- a/Documentation/block/queue-sysfs.rst +++ b/Documentation/block/queue-sysfs.rst @@ -286,4 +286,35 @@ sequential zones of zoned block devices (devices with a zoned attributed that reports "host-managed" or "host-aware"). This value is always 0 for regular block devices. +independent_access_ranges (RO) +------------------------------ + +The presence of this sub-directory of the /sys/block/xxx/queue/ directory +indicates that the device is capable of executing requests targeting +different sector ranges in parallel. For instance, single LUN multi-actuator +hard-disks will have an independent_access_ranges directory if the device +correctly advertizes the sector ranges of its actuators. + +The independent_access_ranges directory contains one directory per access +range, with each range described using the sector (RO) attribute file to +indicate the first sector of the range and the nr_sectors (RO) attribute file +to indicate the total number of sectors in the range starting from the first +sector of the range. For example, a dual-actuator hard-disk will have the +following independent_access_ranges entries.:: + + $ tree /sys/block//queue/independent_access_ranges/ + /sys/block//queue/independent_access_ranges/ + |-- 0 + | |-- nr_sectors + | `-- sector + `-- 1 + |-- nr_sectors + `-- sector + +The sector and nr_sectors attributes use 512B sector unit, regardless of +the actual block size of the device. Independent access ranges do not +overlap and include all sectors within the device capacity. The access +ranges are numbered in increasing order of the range start sector, +that is, the sector attribute of range 0 always has the value 0. + Jens Axboe , February 2009 From 9d824642889823c464847342d6ff530b9eee3241 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Wed, 27 Oct 2021 11:22:23 +0900 Subject: [PATCH 245/255] doc: Fix typo in request queue sysfs documentation Fix a typo (are -> as) in the introduction paragraph of Documentation/block/queue-sysfs.rst. Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke Reviewed-by: Martin K. Petersen Reviewed-by: Keith Busch Link: https://lore.kernel.org/r/20211027022223.183838-6-damien.lemoal@wdc.com Signed-off-by: Jens Axboe --- Documentation/block/queue-sysfs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/block/queue-sysfs.rst b/Documentation/block/queue-sysfs.rst index b6e8983d8eda..e8c74306f70a 100644 --- a/Documentation/block/queue-sysfs.rst +++ b/Documentation/block/queue-sysfs.rst @@ -4,7 +4,7 @@ Queue sysfs files This text file will detail the queue files that are located in the sysfs tree for each block device. Note that stacked devices typically do not export -any settings, since their queue merely functions are a remapping target. +any settings, since their queue merely functions as a remapping target. These files are the ones found in the /sys/block/xxx/queue/ directory. Files denoted with a RO postfix are readonly and the RW postfix means From 785d584c30ffc1224027536fe55bdc15ee509f14 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Mon, 18 Oct 2021 17:21:37 +0200 Subject: [PATCH 246/255] nvme: add new discovery log page entry definitions TP8014 adds a new SUBTYPE value and a new field EFLAGS for the discovery log page entry. Signed-off-by: Hannes Reinecke Signed-off-by: Christoph Hellwig --- include/linux/nvme.h | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 357482dedb59..855dd9b3e84b 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -27,8 +27,14 @@ #define NVME_NSID_ALL 0xffffffff enum nvme_subsys_type { - NVME_NQN_DISC = 1, /* Discovery type target subsystem */ - NVME_NQN_NVME = 2, /* NVME type target subsystem */ + /* Referral to another discovery type target subsystem */ + NVME_NQN_DISC = 1, + + /* NVME type target subsystem */ + NVME_NQN_NVME = 2, + + /* Current discovery type target subsystem */ + NVME_NQN_CURR = 3, }; enum nvme_ctrl_type { @@ -1312,6 +1318,12 @@ struct nvmf_common_command { #define MAX_DISC_LOGS 255 +/* Discovery log page entry flags (EFLAGS): */ +enum { + NVME_DISC_EFLAGS_EPCSD = (1 << 1), + NVME_DISC_EFLAGS_DUPRETINFO = (1 << 0), +}; + /* Discovery log page entry */ struct nvmf_disc_rsp_page_entry { __u8 trtype; @@ -1321,7 +1333,8 @@ struct nvmf_disc_rsp_page_entry { __le16 portid; __le16 cntlid; __le16 asqsz; - __u8 resv8[22]; + __le16 eflags; + __u8 resv10[20]; char trsvcid[NVMF_TRSVCID_SIZE]; __u8 resv64[192]; char subnqn[NVMF_NQN_FIELD_LEN]; From 598e75934c38e2e8af6be92374053e59df8071ad Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Mon, 18 Oct 2021 17:21:36 +0200 Subject: [PATCH 247/255] nvmet: switch check for subsystem type Invert the check for discovery subsystem type to allow for additional discovery subsystem types. Signed-off-by: Hannes Reinecke Signed-off-by: Christoph Hellwig --- drivers/nvme/target/nvmet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index f31dcc4fb1a2..af193423c10b 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -579,7 +579,7 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) { - return subsys->type == NVME_NQN_DISC; + return subsys->type != NVME_NQN_NVME; } #ifdef CONFIG_NVME_TARGET_PASSTHRU From 2953b30b1d9feb1bc555682e64e6479d197b9231 Mon Sep 17 00:00:00 2001 From: Hannes Reinecke Date: Mon, 18 Oct 2021 17:21:38 +0200 Subject: [PATCH 248/255] nvmet: register discovery subsystem as 'current' Register the discovery subsystem as the 'current' discovery subsystem, and add a new discovery log page entry for it. Signed-off-by: Hannes Reinecke Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 1 + drivers/nvme/target/discovery.c | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index d844fb9ef2eb..5119c687de68 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1541,6 +1541,7 @@ struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, subsys->max_qid = NVMET_NR_QUEUES; break; case NVME_NQN_DISC: + case NVME_NQN_CURR: subsys->max_qid = 0; break; default: diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index 7b360f8d07e9..c2162eef8ce1 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -146,7 +146,7 @@ static size_t discovery_log_entries(struct nvmet_req *req) struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_subsys_link *p; struct nvmet_port *r; - size_t entries = 0; + size_t entries = 1; list_for_each_entry(p, &req->port->subsystems, entry) { if (!nvmet_host_allowed(p->subsys, ctrl->hostnqn)) @@ -171,6 +171,7 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) u32 numrec = 0; u16 status = 0; void *buffer; + char traddr[NVMF_TRADDR_SIZE]; if (!nvmet_check_transfer_len(req, data_len)) return; @@ -203,15 +204,19 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) status = NVME_SC_INTERNAL; goto out; } - hdr = buffer; - list_for_each_entry(p, &req->port->subsystems, entry) { - char traddr[NVMF_TRADDR_SIZE]; + nvmet_set_disc_traddr(req, req->port, traddr); + + nvmet_format_discovery_entry(hdr, req->port, + nvmet_disc_subsys->subsysnqn, + traddr, NVME_NQN_CURR, numrec); + numrec++; + + list_for_each_entry(p, &req->port->subsystems, entry) { if (!nvmet_host_allowed(p->subsys, ctrl->hostnqn)) continue; - nvmet_set_disc_traddr(req, req->port, traddr); nvmet_format_discovery_entry(hdr, req->port, p->subsys->subsysnqn, traddr, NVME_NQN_NVME, numrec); @@ -389,7 +394,7 @@ u16 nvmet_parse_discovery_cmd(struct nvmet_req *req) int __init nvmet_init_discovery(void) { nvmet_disc_subsys = - nvmet_subsys_alloc(NVME_DISC_SUBSYS_NAME, NVME_NQN_DISC); + nvmet_subsys_alloc(NVME_DISC_SUBSYS_NAME, NVME_NQN_CURR); return PTR_ERR_OR_ZERO(nvmet_disc_subsys); } From d156cfcafbd0eae4224ea007d95ebda467eb0c46 Mon Sep 17 00:00:00 2001 From: Len Baker Date: Sun, 24 Oct 2021 19:29:21 +0200 Subject: [PATCH 249/255] nvmet: use flex_array_size and struct_size In an effort to avoid open-coded arithmetic in the kernel [1], use the flex_array_size() and struct_size() helpers instead of an open-coded calculation. [1] https://github.com/KSPP/linux/issues/160 Signed-off-by: Len Baker Reviewed-by: Sagi Grimberg Reviewed-by: Gustavo A. R. Silva Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/multipath.c | 2 +- drivers/nvme/target/admin-cmd.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 954e84df6eb7..7f2071f2460c 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -562,7 +562,7 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, return -EINVAL; nr_nsids = le32_to_cpu(desc->nnsids); - nsid_buf_size = nr_nsids * sizeof(__le32); + nsid_buf_size = flex_array_size(desc, nsids, nr_nsids); if (WARN_ON_ONCE(desc->grpid == 0)) return -EINVAL; diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 403de678fd06..6fb24746de06 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -264,7 +264,7 @@ static u32 nvmet_format_ana_group(struct nvmet_req *req, u32 grpid, desc->chgcnt = cpu_to_le64(nvmet_ana_chgcnt); desc->state = req->port->ana_state[grpid]; memset(desc->rsvd17, 0, sizeof(desc->rsvd17)); - return sizeof(struct nvme_ana_group_desc) + count * sizeof(__le32); + return struct_size(desc, nsids, count); } static void nvmet_execute_get_log_page_ana(struct nvmet_req *req) From cf2197ca4b8c199d188593ca6800ea1827c42171 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Fri, 29 Oct 2021 14:09:29 +0800 Subject: [PATCH 250/255] bcache: move uapi header bcache.h to bcache code directory The header file include/uapi/linux/bcache.h is not really a user space API heaer. This file defines the ondisk format of bcache internal meta data but no one includes it from user space, bcache-tools has its own copy of this header with minor modification. Therefore, this patch moves include/uapi/linux/bcache.h to bcache code directory as drivers/md/bcache/bcache_ondisk.h. Suggested-by: Arnd Bergmann Suggested-by: Christoph Hellwig Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211029060930.119923-2-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/bcache.h | 2 +- .../uapi/linux/bcache.h => drivers/md/bcache/bcache_ondisk.h | 0 drivers/md/bcache/bset.h | 2 +- drivers/md/bcache/features.c | 2 +- drivers/md/bcache/features.h | 3 ++- 5 files changed, 5 insertions(+), 4 deletions(-) rename include/uapi/linux/bcache.h => drivers/md/bcache/bcache_ondisk.h (100%) diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 941685409c68..9ed9c955add7 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -178,7 +178,6 @@ #define pr_fmt(fmt) "bcache: %s() " fmt, __func__ -#include #include #include #include @@ -190,6 +189,7 @@ #include #include +#include "bcache_ondisk.h" #include "bset.h" #include "util.h" #include "closure.h" diff --git a/include/uapi/linux/bcache.h b/drivers/md/bcache/bcache_ondisk.h similarity index 100% rename from include/uapi/linux/bcache.h rename to drivers/md/bcache/bcache_ondisk.h diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h index a50dcfda656f..d795c84246b0 100644 --- a/drivers/md/bcache/bset.h +++ b/drivers/md/bcache/bset.h @@ -2,10 +2,10 @@ #ifndef _BCACHE_BSET_H #define _BCACHE_BSET_H -#include #include #include +#include "bcache_ondisk.h" #include "util.h" /* for time_stats */ /* diff --git a/drivers/md/bcache/features.c b/drivers/md/bcache/features.c index 6d2b7b84a7b7..634922c5601d 100644 --- a/drivers/md/bcache/features.c +++ b/drivers/md/bcache/features.c @@ -6,7 +6,7 @@ * Copyright 2020 Coly Li * */ -#include +#include "bcache_ondisk.h" #include "bcache.h" #include "features.h" diff --git a/drivers/md/bcache/features.h b/drivers/md/bcache/features.h index d1c8fd3977fc..09161b89c63e 100644 --- a/drivers/md/bcache/features.h +++ b/drivers/md/bcache/features.h @@ -2,10 +2,11 @@ #ifndef _BCACHE_FEATURES_H #define _BCACHE_FEATURES_H -#include #include #include +#include "bcache_ondisk.h" + #define BCH_FEATURE_COMPAT 0 #define BCH_FEATURE_RO_COMPAT 1 #define BCH_FEATURE_INCOMPAT 2 From 1b86db5f4e025840e0bf7cef2b10e84531954386 Mon Sep 17 00:00:00 2001 From: Qing Wang Date: Fri, 29 Oct 2021 14:09:30 +0800 Subject: [PATCH 251/255] bcache: replace snprintf in show functions with sysfs_emit coccicheck complains about the use of snprintf() in sysfs show functions. Fix the following coccicheck warning: drivers/md/bcache/sysfs.h:54:12-20: WARNING: use scnprintf or sprintf. Implement sysfs_print() by sysfs_emit() and remove snprint() since no one uses it any more. Suggested-by: Coly Li Signed-off-by: Qing Wang Signed-off-by: Coly Li Link: https://lore.kernel.org/r/20211029060930.119923-3-colyli@suse.de Signed-off-by: Jens Axboe --- drivers/md/bcache/sysfs.h | 18 ++++++++++++++++-- drivers/md/bcache/util.h | 17 ----------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/drivers/md/bcache/sysfs.h b/drivers/md/bcache/sysfs.h index 215df32f567b..c1752ba2e05b 100644 --- a/drivers/md/bcache/sysfs.h +++ b/drivers/md/bcache/sysfs.h @@ -51,13 +51,27 @@ STORE(fn) \ #define sysfs_printf(file, fmt, ...) \ do { \ if (attr == &sysfs_ ## file) \ - return snprintf(buf, PAGE_SIZE, fmt "\n", __VA_ARGS__); \ + return sysfs_emit(buf, fmt "\n", __VA_ARGS__); \ } while (0) #define sysfs_print(file, var) \ do { \ if (attr == &sysfs_ ## file) \ - return snprint(buf, PAGE_SIZE, var); \ + return sysfs_emit(buf, \ + __builtin_types_compatible_p(typeof(var), int) \ + ? "%i\n" : \ + __builtin_types_compatible_p(typeof(var), unsigned int) \ + ? "%u\n" : \ + __builtin_types_compatible_p(typeof(var), long) \ + ? "%li\n" : \ + __builtin_types_compatible_p(typeof(var), unsigned long)\ + ? "%lu\n" : \ + __builtin_types_compatible_p(typeof(var), int64_t) \ + ? "%lli\n" : \ + __builtin_types_compatible_p(typeof(var), uint64_t) \ + ? "%llu\n" : \ + __builtin_types_compatible_p(typeof(var), const char *) \ + ? "%s\n" : "%i\n", var); \ } while (0) #define sysfs_hprint(file, val) \ diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h index 6274d6a17e5e..cdb165517d0b 100644 --- a/drivers/md/bcache/util.h +++ b/drivers/md/bcache/util.h @@ -340,23 +340,6 @@ static inline int bch_strtoul_h(const char *cp, long *res) _r; \ }) -#define snprint(buf, size, var) \ - snprintf(buf, size, \ - __builtin_types_compatible_p(typeof(var), int) \ - ? "%i\n" : \ - __builtin_types_compatible_p(typeof(var), unsigned int) \ - ? "%u\n" : \ - __builtin_types_compatible_p(typeof(var), long) \ - ? "%li\n" : \ - __builtin_types_compatible_p(typeof(var), unsigned long)\ - ? "%lu\n" : \ - __builtin_types_compatible_p(typeof(var), int64_t) \ - ? "%lli\n" : \ - __builtin_types_compatible_p(typeof(var), uint64_t) \ - ? "%llu\n" : \ - __builtin_types_compatible_p(typeof(var), const char *) \ - ? "%s\n" : "%i\n", var) - ssize_t bch_hprint(char *buf, int64_t v); bool bch_is_zero(const char *p, size_t n); From df75db1fc1e5608271397de37cab43371bb838d2 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Fri, 29 Oct 2021 17:50:29 +0800 Subject: [PATCH 252/255] block: ataflop: Fix warning comparing pointer to 0 Fix the following coccicheck warning: ./drivers/block/ataflop.c:1464:20-21: WARNING comparing pointer to 0. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Link: https://lore.kernel.org/r/1635501029-81391-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Jens Axboe --- drivers/block/ataflop.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 2622803ef71a..d14bdc3589b2 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c @@ -1460,8 +1460,7 @@ static int floppy_revalidate(struct gendisk *disk) unsigned int drive = p - unit; if (test_bit(drive, &changed_floppies) || - test_bit(drive, &fake_change) || - p->disktype == 0) { + test_bit(drive, &fake_change) || !p->disktype) { if (UD.flags & FTD_MSG) printk(KERN_ERR "floppy: clear format %p!\n", UDT); BufferDrive = -1; From 15dfc662ef31a20b59097d59b0792b06770255fa Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Fri, 29 Oct 2021 19:39:26 +0900 Subject: [PATCH 253/255] null_blk: Fix handling of submit_queues and poll_queues attributes Commit 0a593fbbc245 ("null_blk: poll queue support") introduced the poll queue feature to null_blk. After this change, null_blk device has both submit queues and poll queues, and null_map_queues() callback maps the both queues for corresponding hardware contexts. The commit also added the device configuration attribute 'poll_queues' in same manner as the existing attribute 'submit_queues'. These attributes allow to modify the numbers of queues. However, when the new values are stored to these attributes, the values are just handled only for the corresponding queue. When number of submit_queue is updated, number of poll_queue is not counted, or vice versa. This caused inconsistent number of queues and queue mapping and resulted in null-ptr-dereference. This failure was observed in blktests block/029 and block/030. To avoid the inconsistency, fix the attribute updates to care both submit_queues and poll_queues. Introduce the helper function nullb_update_nr_hw_queues() to handle stores to the both two attributes. Add poll_queues field to the struct nullb_device to track the number in same manner as submit_queues. Add two more fields prev_submit_queues and prev_poll_queues to keep the previous values before change. In case the block layer failed to update the nr_hw_queues, refer the previous values in null_map_queues() to map queues in same manner as before change. Also add poll_queues value checks in nullb_update_nr_hw_queues() and null_validate_conf(). They ensure the poll_queues value of each device is within the range from 1 to module parameter value of poll_queues. Fixes: 0a593fbbc245 ("null_blk: poll queue support") Reported-by: Yi Zhang Signed-off-by: Shin'ichiro Kawasaki Link: https://lore.kernel.org/r/20211029103926.845635-1-shinichiro.kawasaki@wdc.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 106 ++++++++++++++++++++++++------ drivers/block/null_blk/null_blk.h | 2 + 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f4af95c2f9a9..323af5c9c802 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -328,30 +328,69 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ } \ CONFIGFS_ATTR(nullb_device_, NAME); -static int nullb_apply_submit_queues(struct nullb_device *dev, - unsigned int submit_queues) -{ - struct nullb *nullb = dev->nullb; - struct blk_mq_tag_set *set; +static int nullb_update_nr_hw_queues(struct nullb_device *dev, + unsigned int submit_queues, + unsigned int poll_queues) - if (!nullb) +{ + struct blk_mq_tag_set *set; + int ret, nr_hw_queues; + + if (!dev->nullb) return 0; + /* + * Make sure at least one queue exists for each of submit and poll. + */ + if (!submit_queues || !poll_queues) + return -EINVAL; + /* * Make sure that null_init_hctx() does not access nullb->queues[] past * the end of that array. */ - if (submit_queues > nr_cpu_ids) + if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues) return -EINVAL; - set = nullb->tag_set; - blk_mq_update_nr_hw_queues(set, submit_queues); - return set->nr_hw_queues == submit_queues ? 0 : -ENOMEM; + + /* + * Keep previous and new queue numbers in nullb_device for reference in + * the call back function null_map_queues(). + */ + dev->prev_submit_queues = dev->submit_queues; + dev->prev_poll_queues = dev->poll_queues; + dev->submit_queues = submit_queues; + dev->poll_queues = poll_queues; + + set = dev->nullb->tag_set; + nr_hw_queues = submit_queues + poll_queues; + blk_mq_update_nr_hw_queues(set, nr_hw_queues); + ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM; + + if (ret) { + /* on error, revert the queue numbers */ + dev->submit_queues = dev->prev_submit_queues; + dev->poll_queues = dev->prev_poll_queues; + } + + return ret; +} + +static int nullb_apply_submit_queues(struct nullb_device *dev, + unsigned int submit_queues) +{ + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); +} + +static int nullb_apply_poll_queues(struct nullb_device *dev, + unsigned int poll_queues) +{ + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL); NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues); -NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_submit_queues); +NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues); NULLB_DEVICE_ATTR(home_node, uint, NULL); NULLB_DEVICE_ATTR(queue_mode, uint, NULL); NULLB_DEVICE_ATTR(blocksize, uint, NULL); @@ -599,7 +638,9 @@ static struct nullb_device *null_alloc_dev(void) dev->size = g_gb * 1024; dev->completion_nsec = g_completion_nsec; dev->submit_queues = g_submit_queues; + dev->prev_submit_queues = g_submit_queues; dev->poll_queues = g_poll_queues; + dev->prev_poll_queues = g_poll_queues; dev->home_node = g_home_node; dev->queue_mode = g_queue_mode; dev->blocksize = g_bs; @@ -1465,25 +1506,45 @@ static int null_map_queues(struct blk_mq_tag_set *set) { struct nullb *nullb = set->driver_data; int i, qoff; + unsigned int submit_queues = g_submit_queues; + unsigned int poll_queues = g_poll_queues; + + if (nullb) { + struct nullb_device *dev = nullb->dev; + + /* + * Refer nr_hw_queues of the tag set to check if the expected + * number of hardware queues are prepared. If block layer failed + * to prepare them, use previous numbers of submit queues and + * poll queues to map queues. + */ + if (set->nr_hw_queues == + dev->submit_queues + dev->poll_queues) { + submit_queues = dev->submit_queues; + poll_queues = dev->poll_queues; + } else if (set->nr_hw_queues == + dev->prev_submit_queues + dev->prev_poll_queues) { + submit_queues = dev->prev_submit_queues; + poll_queues = dev->prev_poll_queues; + } else { + pr_warn("tag set has unexpected nr_hw_queues: %d\n", + set->nr_hw_queues); + return -EINVAL; + } + } for (i = 0, qoff = 0; i < set->nr_maps; i++) { struct blk_mq_queue_map *map = &set->map[i]; switch (i) { case HCTX_TYPE_DEFAULT: - if (nullb) - map->nr_queues = nullb->dev->submit_queues; - else - map->nr_queues = g_submit_queues; + map->nr_queues = submit_queues; break; case HCTX_TYPE_READ: map->nr_queues = 0; continue; case HCTX_TYPE_POLL: - if (nullb) - map->nr_queues = nullb->dev->poll_queues; - else - map->nr_queues = g_poll_queues; + map->nr_queues = poll_queues; break; } map->queue_offset = qoff; @@ -1853,6 +1914,13 @@ static int null_validate_conf(struct nullb_device *dev) dev->submit_queues = nr_cpu_ids; else if (dev->submit_queues == 0) dev->submit_queues = 1; + dev->prev_submit_queues = dev->submit_queues; + + if (dev->poll_queues > g_poll_queues) + dev->poll_queues = g_poll_queues; + else if (dev->poll_queues == 0) + dev->poll_queues = 1; + dev->prev_poll_queues = dev->poll_queues; dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ); dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER); diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h index 90c3eefb3ca3..78eb56b0ca55 100644 --- a/drivers/block/null_blk/null_blk.h +++ b/drivers/block/null_blk/null_blk.h @@ -86,7 +86,9 @@ struct nullb_device { unsigned int zone_max_open; /* max number of open zones */ unsigned int zone_max_active; /* max number of active zones */ unsigned int submit_queues; /* number of submission queues */ + unsigned int prev_submit_queues; /* number of submission queues before change */ unsigned int poll_queues; /* number of IOPOLL submission queues */ + unsigned int prev_poll_queues; /* number of IOPOLL submission queues before change */ unsigned int home_node; /* home node for the device */ unsigned int queue_mode; /* block interface */ unsigned int blocksize; /* block size */ From f75d118349be055d47407b4ba4ceb98e6437e472 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 29 Oct 2021 06:36:45 -0600 Subject: [PATCH 254/255] io_uring: harder fdinfo sq/cq ring iterating The ring iteration is racy, which isn't necessarily a problem except it can cause us to iterate the whole thing. That isn't desired or ideal, and it can lead to excessive runtimes of reading fdinfo. Cap the iteration at tail - head OR the ring size. While in there, clean up the ring masking and just dump the raw values along with the masks. That provides more useful debug info. Fixes: 83f84356bc8f ("io_uring: add more uring info to fdinfo for debug") Reported-by: Eric Dumazet Signed-off-by: Jens Axboe --- fs/io_uring.c | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d001cd7a6c51..c887e4e19e9e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10056,12 +10056,11 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct io_overflow_cqe *ocqe; struct io_rings *r = ctx->rings; unsigned int sq_mask = ctx->sq_entries - 1, cq_mask = ctx->cq_entries - 1; - unsigned int cached_sq_head = ctx->cached_sq_head; - unsigned int cached_cq_tail = ctx->cached_cq_tail; unsigned int sq_head = READ_ONCE(r->sq.head); unsigned int sq_tail = READ_ONCE(r->sq.tail); unsigned int cq_head = READ_ONCE(r->cq.head); unsigned int cq_tail = READ_ONCE(r->cq.tail); + unsigned int sq_entries, cq_entries; bool has_lock; unsigned int i; @@ -10071,29 +10070,37 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, * and sq_tail and cq_head are changed by userspace. But it's ok since * we usually use these info when it is stuck. */ - seq_printf(m, "SqHead:\t%u\n", sq_head & sq_mask); - seq_printf(m, "SqTail:\t%u\n", sq_tail & sq_mask); - seq_printf(m, "CachedSqHead:\t%u\n", cached_sq_head & sq_mask); - seq_printf(m, "CqHead:\t%u\n", cq_head & cq_mask); - seq_printf(m, "CqTail:\t%u\n", cq_tail & cq_mask); - seq_printf(m, "CachedCqTail:\t%u\n", cached_cq_tail & cq_mask); - seq_printf(m, "SQEs:\t%u\n", sq_tail - cached_sq_head); - for (i = cached_sq_head; i < sq_tail; i++) { - unsigned int sq_idx = READ_ONCE(ctx->sq_array[i & sq_mask]); + seq_printf(m, "SqMask:\t\t0x%x\n", sq_mask); + seq_printf(m, "SqHead:\t%u\n", sq_head); + seq_printf(m, "SqTail:\t%u\n", sq_tail); + seq_printf(m, "CachedSqHead:\t%u\n", ctx->cached_sq_head); + seq_printf(m, "CqMask:\t0x%x\n", cq_mask); + seq_printf(m, "CqHead:\t%u\n", cq_head); + seq_printf(m, "CqTail:\t%u\n", cq_tail); + seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail); + seq_printf(m, "SQEs:\t%u\n", sq_tail - ctx->cached_sq_head); + sq_entries = min(sq_tail - sq_head, ctx->sq_entries); + for (i = 0; i < sq_entries; i++) { + unsigned int entry = i + sq_head; + unsigned int sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]); + struct io_uring_sqe *sqe = &ctx->sq_sqes[sq_idx]; - if (likely(sq_idx <= sq_mask)) { - struct io_uring_sqe *sqe = &ctx->sq_sqes[sq_idx]; - - seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n", - sq_idx, sqe->opcode, sqe->fd, sqe->flags, sqe->user_data); - } + if (sq_idx > sq_mask) + continue; + sqe = &ctx->sq_sqes[sq_idx]; + seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n", + sq_idx, sqe->opcode, sqe->fd, sqe->flags, + sqe->user_data); } - seq_printf(m, "CQEs:\t%u\n", cached_cq_tail - cq_head); - for (i = cq_head; i < cached_cq_tail; i++) { - struct io_uring_cqe *cqe = &r->cqes[i & cq_mask]; + seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); + cq_entries = min(cq_tail - cq_head, ctx->cq_entries); + for (i = 0; i < cq_entries; i++) { + unsigned int entry = i + cq_head; + struct io_uring_cqe *cqe = &r->cqes[entry & cq_mask]; seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n", - i & cq_mask, cqe->user_data, cqe->res, cqe->flags); + entry & cq_mask, cqe->user_data, cqe->res, + cqe->flags); } /* From 1d5f5ea7cb7d15b9fb1cc82673ebb054f02cd7d2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Fri, 29 Oct 2021 13:11:33 +0100 Subject: [PATCH 255/255] io-wq: remove worker to owner tw dependency INFO: task iou-wrk-6609:6612 blocked for more than 143 seconds. Not tainted 5.15.0-rc5-syzkaller #0 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. task:iou-wrk-6609 state:D stack:27944 pid: 6612 ppid: 6526 flags:0x00004006 Call Trace: context_switch kernel/sched/core.c:4940 [inline] __schedule+0xb44/0x5960 kernel/sched/core.c:6287 schedule+0xd3/0x270 kernel/sched/core.c:6366 schedule_timeout+0x1db/0x2a0 kernel/time/timer.c:1857 do_wait_for_common kernel/sched/completion.c:85 [inline] __wait_for_common kernel/sched/completion.c:106 [inline] wait_for_common kernel/sched/completion.c:117 [inline] wait_for_completion+0x176/0x280 kernel/sched/completion.c:138 io_worker_exit fs/io-wq.c:183 [inline] io_wqe_worker+0x66d/0xc40 fs/io-wq.c:597 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 io-wq worker may submit a task_work to the master task and upon io_worker_exit() wait for the tw to get executed. The problem appears when the master task is waiting in coredump.c: 468 freezer_do_not_count(); 469 wait_for_completion(&core_state->startup); 470 freezer_count(); Apparently having some dependency on children threads getting everything stuck. Workaround it by cancelling the taks_work callback that causes it before going into io_worker_exit() waiting. p.s. probably a better option is to not submit tw elevating the refcount in the first place, but let's leave this excercise for the future. Cc: stable@vger.kernel.org Reported-and-tested-by: syzbot+27d62ee6f256b186883e@syzkaller.appspotmail.com Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/142a716f4ed936feae868959059154362bfa8c19.1635509451.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- fs/io-wq.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 0c283bb18fb2..fe6b2abcaa49 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -140,6 +140,7 @@ static void io_wqe_dec_running(struct io_worker *worker); static bool io_acct_cancel_pending_work(struct io_wqe *wqe, struct io_wqe_acct *acct, struct io_cb_cancel_data *match); +static void create_worker_cb(struct callback_head *cb); static bool io_worker_get(struct io_worker *worker) { @@ -174,9 +175,44 @@ static void io_worker_ref_put(struct io_wq *wq) complete(&wq->worker_done); } +static void io_worker_cancel_cb(struct io_worker *worker) +{ + struct io_wqe_acct *acct = io_wqe_get_acct(worker); + struct io_wqe *wqe = worker->wqe; + struct io_wq *wq = wqe->wq; + + atomic_dec(&acct->nr_running); + raw_spin_lock(&worker->wqe->lock); + acct->nr_workers--; + raw_spin_unlock(&worker->wqe->lock); + io_worker_ref_put(wq); + clear_bit_unlock(0, &worker->create_state); + io_worker_release(worker); +} + +static bool io_task_worker_match(struct callback_head *cb, void *data) +{ + struct io_worker *worker; + + if (cb->func != create_worker_cb) + return false; + worker = container_of(cb, struct io_worker, create_work); + return worker == data; +} + static void io_worker_exit(struct io_worker *worker) { struct io_wqe *wqe = worker->wqe; + struct io_wq *wq = wqe->wq; + + while (1) { + struct callback_head *cb = task_work_cancel_match(wq->task, + io_task_worker_match, worker); + + if (!cb) + break; + io_worker_cancel_cb(worker); + } io_worker_release(worker); wait_for_completion(&worker->ref_done); @@ -1148,17 +1184,9 @@ static void io_wq_exit_workers(struct io_wq *wq) while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) { struct io_worker *worker; - struct io_wqe_acct *acct; worker = container_of(cb, struct io_worker, create_work); - acct = io_wqe_get_acct(worker); - atomic_dec(&acct->nr_running); - raw_spin_lock(&worker->wqe->lock); - acct->nr_workers--; - raw_spin_unlock(&worker->wqe->lock); - io_worker_ref_put(wq); - clear_bit_unlock(0, &worker->create_state); - io_worker_release(worker); + io_worker_cancel_cb(worker); } rcu_read_lock();