tmakin/ack-tegra
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

io_uring/zcrx: add shared-memory notification statistics

Browse Source 
Add support for an optional stats struct embedded in the refill queue
region, allowing userspace to monitor copy-fallback in real-time.

Userspace queries the stats struct size and alignment via
IO_URING_QUERY_ZCRX_NOTIF (notif_stats_size / notif_stats_alignment),
then provides a stats_offset in zcrx_notification_desc pointing to a
location within the refill queue region.

The kernel updates the stats counters in-place on every copy-fallback
event.

Signed-off-by: Clément Léger <cleger@meta.com>
[pavel: rename io_uring_zcrx_notif_stats]
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/f6af5a21015efea4b733b9d77aba22c637788fe4.1779189667.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Clément Léger authored and Jens Axboe committed 2026-05-26 10:42:01 -06:00
1 parent 255180f703
commit 6935f63146
5 files changed
+94 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/uapi/linux/io_uring/query.h
+12
View File
@@ -23,6 +23,7 @@ enum {
IO_URING_QUERY_OPCODES = 0,
IO_URING_QUERY_ZCRX = 1,
IO_URING_QUERY_SCQ = 2,
IO_URING_QUERY_ZCRX_NOTIF = 3,
__IO_URING_QUERY_MAX,
};
@@ -62,6 +63,17 @@ struct io_uring_query_zcrx {
__u64 __resv2;
};
struct io_uring_query_zcrx_notif {
/* Bitmask of supported ZCRX_NOTIF_* flags */
__u32 notif_flags;
/* Size of io_uring_zcrx_notif_stats */
__u32 notif_stats_size;
/* Required alignment for the stats struct within the region (ie stats_offset) */
__u32 notif_stats_off_alignment;
__u32 __resv1;
__u64 __resv2[4];
};
struct io_uring_query_scq {
/* The SQ/CQ rings header size */
__u64 hdr_size;
include/uapi/linux/io_uring/zcrx.h
+13 -2
View File
@@ -75,11 +75,22 @@ enum zcrx_notification_type {
__ZCRX_NOTIF_TYPE_LAST,
};
enum zcrx_notification_desc_flags {
/* If set, stats_offset holds a valid offset to a notif_stats struct */
ZCRX_NOTIF_DESC_FLAG_STATS = 1 << 0,
};
struct zcrx_notif_stats {
__u64 copy_count; /* cumulative copy-fallback CQEs */
__u64 copy_bytes; /* cumulative bytes copied */
};
struct zcrx_notification_desc {
__u64 user_data;
__u32 type_mask;
__u32 __resv1;
__u64 __resv2[10];
__u32 flags; /* see enum zcrx_notification_desc_flags */
__u64 stats_offset; /* offset from the beginning of refill ring region for stats */
__u64 __resv2[9];
};
/*
io_uring/query.c
+16
View File
@@ -9,6 +9,7 @@
union io_query_data {
struct io_uring_query_opcode opcodes;
struct io_uring_query_zcrx zcrx;
struct io_uring_query_zcrx_notif zcrx_notif;
struct io_uring_query_scq scq;
};
@@ -44,6 +45,18 @@ static ssize_t io_query_zcrx(union io_query_data *data)
return sizeof(*e);
}
static ssize_t io_query_zcrx_notif(union io_query_data *data)
{
struct io_uring_query_zcrx_notif *e = &data->zcrx_notif;
e->notif_flags = ZCRX_NOTIF_TYPE_MASK;
e->notif_stats_size = sizeof(struct zcrx_notif_stats);
e->notif_stats_off_alignment = __alignof__(struct zcrx_notif_stats);
e->__resv1 = 0;
memset(&e->__resv2, 0, sizeof(e->__resv2));
return sizeof(*e);
}
static ssize_t io_query_scq(union io_query_data *data)
{
struct io_uring_query_scq *e = &data->scq;
@@ -83,6 +96,9 @@ static int io_handle_query_entry(union io_query_data *data, void __user *uhdr,
case IO_URING_QUERY_ZCRX:
ret = io_query_zcrx(data);
break;
case IO_URING_QUERY_ZCRX_NOTIF:
ret = io_query_zcrx_notif(data);
break;
case IO_URING_QUERY_SCQ:
ret = io_query_scq(data);
break;
io_uring/zcrx.c
+52 -2
View File
@@ -415,6 +415,7 @@ static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq)
io_free_region(ifq->user, &ifq->rq_region);
ifq->rq.ring = IO_URING_PTR_POISON;
ifq->rq.rqes = IO_URING_PTR_POISON;
ifq->notif_stats = IO_URING_PTR_POISON;
}
static void io_zcrx_free_area(struct io_zcrx_ifq *ifq,
@@ -853,6 +854,33 @@ netdev_put_unlock:
return ret;
}
static int zcrx_validate_notif_stats(struct io_zcrx_ifq *ifq,
const struct io_uring_zcrx_ifq_reg *reg,
const struct zcrx_notification_desc *notif)
{
size_t stats_off = notif->stats_offset;
size_t used, end;
used = reg->offsets.rqes +
sizeof(struct io_uring_zcrx_rqe) * reg->rq_entries;
if (!IS_ALIGNED(stats_off, __alignof__(struct zcrx_notif_stats)))
return -EINVAL;
if (stats_off < used)
return -ERANGE;
if (check_add_overflow(stats_off,
sizeof(struct zcrx_notif_stats),
&end))
return -ERANGE;
if (end > io_region_size(&ifq->rq_region))
return -ERANGE;
ifq->notif_stats = io_region_get_ptr(&ifq->rq_region) + stats_off;
memset(ifq->notif_stats, 0, sizeof(*ifq->notif_stats));
return 0;
}
int io_register_zcrx(struct io_ring_ctx *ctx,
struct io_uring_zcrx_ifq_reg __user *arg)
{
@@ -906,7 +934,13 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
return -EFAULT;
if (notif.type_mask & ~ZCRX_NOTIF_TYPE_MASK)
return -EINVAL;
if (notif.__resv1 || !mem_is_zero(&notif.__resv2, sizeof(notif.__resv2)))
if (notif.flags & ~ZCRX_NOTIF_DESC_FLAG_STATS)
return -EINVAL;
if (!(notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS)) {
if (notif.stats_offset)
return -EINVAL;
}
if (!mem_is_zero(&notif.__resv2, sizeof(notif.__resv2)))
return -EINVAL;
ifq = io_zcrx_ifq_alloc(ctx);
@@ -937,6 +971,12 @@ int io_register_zcrx(struct io_ring_ctx *ctx,
if (ret)
goto err;
if (notif.flags & ZCRX_NOTIF_DESC_FLAG_STATS) {
ret = zcrx_validate_notif_stats(ifq, &reg, &notif);
if (ret)
goto err;
}
ifq->kern_readable = !(area.flags & IORING_ZCRX_AREA_DMABUF);
if (!(reg.flags & ZCRX_REG_NODEV)) {
@@ -1152,6 +1192,11 @@ static void zcrx_notif_tw(struct io_tw_req tw_req, io_tw_token_t tw)
kmem_cache_free(req_cachep, req);
}
static void zcrx_stat_add(__u64 *p, s64 v)
{
WRITE_ONCE(*p, READ_ONCE(*p) + v);
}
static void zcrx_send_notif(struct io_zcrx_ifq *ifq, unsigned type)
{
gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN | __GFP_ZERO;
@@ -1535,8 +1580,13 @@ static int io_zcrx_copy_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
int ret;
ret = io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len);
if (ret > 0)
if (ret > 0) {
if (ifq->notif_stats) {
zcrx_stat_add(&ifq->notif_stats->copy_count, 1);
zcrx_stat_add(&ifq->notif_stats->copy_bytes, ret);
}
zcrx_send_notif(ifq, ZCRX_NOTIF_COPY);
}
return ret;
}
io_uring/zcrx.h
+1
View File
@@ -80,6 +80,7 @@ struct io_zcrx_ifq {
u32 allowed_notif_mask;
u32 fired_notifs;
u64 notif_data;
struct zcrx_notif_stats *notif_stats;
};
#if defined(CONFIG_IO_URING_ZCRX)