ANDROID: block: Support configuring limits below the page size

Allow block drivers to configure the following:
* Maximum number of hardware sectors values smaller than
  PAGE_SIZE >> SECTOR_SHIFT. For PAGE_SIZE = 4096 this means that values
  below 8 become supported.
* A maximum segment size below the page size. This is most useful
  for page sizes above 4096 bytes.

The blk_sub_page_segments static branch will be used in later patches to
prevent that performance of block drivers that support segments >=
PAGE_SIZE and max_hw_sectors >= PAGE_SIZE >> SECTOR_SHIFT would be affected.

This patch may change the behavior of existing block drivers from not
working into working. An attempt to
configure a limit below what is supported by the block layer causes the
block layer to select a larger value. If that value is not supported by
the block driver, this may cause other data to be transferred than
requested, a kernel crash or other undesirable behavior.

Cc: Christoph Hellwig <hch@lst.de>
Cc: Ming Lei <ming.lei@redhat.com>
Cc: Keith Busch <kbusch@kernel.org>
Cc: Luis Chamberlain <mcgrof@kernel.org>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Sandeep Dhavale <dhavale@google.com>

Bug: 346870006

Link: https://lore.kernel.org/all/20230612203314.17820-4-bvanassche@acm.org/

[dhavale: current patch is based on the FROMLIST patch send to kernel mailing list.
As the queue config functions are removed, the logic has been adapted in
analogus function blk_validate_limits().

Block maintainers have rejected all our previous attempt to land patches
which support sub page segment size. But we have decided that these
patches are necessary to have 16KB page size kernel work with hardware
which supports maximum 4KB segment size.
]

Change-Id: I3faa20be1e83d1501d0f25f549b40301443d0df4

block/blk-core.c

@@ -263,6 +263,7 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head)
 static void blk_free_queue(struct request_queue *q)
 {
 	blk_free_queue_stats(q->stats);
+	blk_disable_sub_page_limits(&q->limits);
 	if (queue_is_mq(q))
 		blk_mq_release(q);
 

block/blk-settings.c

@@ -19,6 +19,12 @@
 #include "blk-rq-qos.h"
 #include "blk-wbt.h"
 
+
+/* Protects blk_nr_sub_page_limit_queues and blk_sub_page_limits changes. */
+static DEFINE_MUTEX(blk_sub_page_limit_lock);
+static uint32_t blk_nr_sub_page_limit_queues;
+DEFINE_STATIC_KEY_FALSE(blk_sub_page_limits);
+
 void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
 {
 	q->rq_timeout = timeout;
@@ -219,6 +225,50 @@ unsupported:
 	lim->atomic_write_unit_max = 0;
 }
 
+/**
+ * blk_enable_sub_page_limits - enable support for limits below the page size
+ * @lim: request queue limits for which to enable support of these features.
+ *
+ * Enable support for max_segment_size values smaller than PAGE_SIZE and for
+ * max_hw_sectors values below PAGE_SIZE >> SECTOR_SHIFT. Support for these
+ * features is not enabled all the time because of the runtime overhead of these
+ * features.
+ */
+static void blk_enable_sub_page_limits(struct queue_limits *lim)
+{
+	if (lim->sub_page_limits)
+		return;
+
+	lim->sub_page_limits = true;
+
+	mutex_lock(&blk_sub_page_limit_lock);
+	if (++blk_nr_sub_page_limit_queues == 1)
+		static_branch_enable(&blk_sub_page_limits);
+	mutex_unlock(&blk_sub_page_limit_lock);
+}
+
+/**
+ * blk_disable_sub_page_limits - disable support for limits below the page size
+ * @lim: request queue limits for which to enable support of these features.
+ *
+ * max_segment_size values smaller than PAGE_SIZE and for max_hw_sectors values
+ * below PAGE_SIZE >> SECTOR_SHIFT. Support for these features is not enabled
+ * all the time because of the runtime overhead of these features.
+ */
+void blk_disable_sub_page_limits(struct queue_limits *lim)
+{
+	if (!lim->sub_page_limits)
+		return;
+
+	lim->sub_page_limits = false;
+
+	mutex_lock(&blk_sub_page_limit_lock);
+	WARN_ON_ONCE(blk_nr_sub_page_limit_queues <= 0);
+	if (--blk_nr_sub_page_limit_queues == 0)
+		static_branch_disable(&blk_sub_page_limits);
+	mutex_unlock(&blk_sub_page_limit_lock);
+}
+
 /*
  * Check that the limits in lim are valid, initialize defaults for unset
  * values, and cap values based on others where needed.
@@ -262,13 +312,13 @@ static int blk_validate_limits(struct queue_limits *lim)
 	 * value.
 	 *
 	 * The block layer relies on the fact that every driver can
-	 * handle at lest a page worth of data per I/O, and needs the value
-	 * aligned to the logical block size.
+	 * handle at least a logical_block_size worth of data per I/O,
+	 * and needs the value aligned to the logical block size.
 	 */
 	if (!lim->max_hw_sectors)
 		lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS;
-	if (WARN_ON_ONCE(lim->max_hw_sectors < PAGE_SECTORS))
-		return -EINVAL;
+	if (lim->max_hw_sectors < PAGE_SECTORS)
+		blk_enable_sub_page_limits(lim);
 	logical_block_sectors = lim->logical_block_size >> SECTOR_SHIFT;
 	if (WARN_ON_ONCE(logical_block_sectors > lim->max_hw_sectors))
 		return -EINVAL;
@@ -343,7 +393,9 @@ static int blk_validate_limits(struct queue_limits *lim)
 		 */
 		if (!lim->max_segment_size)
 			lim->max_segment_size = BLK_MAX_SEGMENT_SIZE;
-		if (WARN_ON_ONCE(lim->max_segment_size < PAGE_SIZE))
+		if (lim->max_segment_size < PAGE_SIZE)
+			blk_enable_sub_page_limits(lim);
+		if (WARN_ON_ONCE(lim->max_segment_size < SECTOR_SIZE))
 			return -EINVAL;
 	}
 
@@ -387,6 +439,8 @@ int blk_set_default_limits(struct queue_limits *lim)
 	 * initialization to the max value here.
 	 */
 	lim->max_user_discard_sectors = UINT_MAX;
+	/* Set sub_page_limits to false and let validate set it if required */
+	lim->sub_page_limits = false;
 	return blk_validate_limits(lim);
 }
 

block/blk.h

@@ -17,6 +17,7 @@ struct elevator_type;
 #define BLK_MAX_TIMEOUT		(5 * HZ)
 
 extern struct dentry *blk_debugfs_root;
+DECLARE_STATIC_KEY_FALSE(blk_sub_page_limits);
 
 struct blk_flush_queue {
 	spinlock_t		mq_flush_lock;
@@ -44,6 +45,12 @@ int __bio_queue_enter(struct request_queue *q, struct bio *bio);
 void submit_bio_noacct_nocheck(struct bio *bio);
 void bio_await_chain(struct bio *bio);
 
+static inline bool blk_queue_sub_page_limits(const struct queue_limits *lim)
+{
+	return static_branch_unlikely(&blk_sub_page_limits) &&
+		lim->sub_page_limits;
+}
+void blk_disable_sub_page_limits(struct queue_limits *q);
 static inline bool blk_try_enter_queue(struct request_queue *q, bool pm)
 {
 	rcu_read_lock();

include/linux/blkdev.h

@@ -402,6 +402,7 @@ struct queue_limits {
 	unsigned int		dma_pad_mask;
 
 	struct blk_integrity	integrity;
+	bool			sub_page_limits;
 };
 
 typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,