From 959fa5c188bf095558c417554e4772ac1fda3531 Mon Sep 17 00:00:00 2001 From: Mina Almasry Date: Mon, 8 Apr 2024 08:29:56 -0700 Subject: [PATCH 01/10] net: make napi_frag_unref reuse skb_page_unref The implementations of these 2 functions are almost identical. Remove the implementation of napi_frag_unref, and make it a call into skb_page_unref so we don't duplicate the implementation. Signed-off-by: Mina Almasry Reviewed-by: Eric Dumazet Reviewed-by: Jacob Keller Link: https://lore.kernel.org/r/20240408153000.2152844-2-almasrymina@google.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 12 +++--------- net/ipv4/esp4.c | 2 +- net/ipv6/esp6.c | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 7dfb906d92f7..c0ff85bb087a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3522,10 +3522,10 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb, bool napi_pp_put_page(struct page *page); static inline void -skb_page_unref(const struct sk_buff *skb, struct page *page) +skb_page_unref(struct page *page, bool recycle) { #ifdef CONFIG_PAGE_POOL - if (skb->pp_recycle && napi_pp_put_page(page)) + if (recycle && napi_pp_put_page(page)) return; #endif put_page(page); @@ -3534,13 +3534,7 @@ skb_page_unref(const struct sk_buff *skb, struct page *page) static inline void napi_frag_unref(skb_frag_t *frag, bool recycle) { - struct page *page = skb_frag_page(frag); - -#ifdef CONFIG_PAGE_POOL - if (recycle && napi_pp_put_page(page)) - return; -#endif - put_page(page); + skb_page_unref(skb_frag_page(frag), recycle); } /** diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 3d647c9a7a21..40330253f076 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -114,7 +114,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb) */ if (req->src != req->dst) for (sg = sg_next(req->src); sg; sg = sg_next(sg)) - skb_page_unref(skb, sg_page(sg)); + skb_page_unref(sg_page(sg), skb->pp_recycle); } #ifdef CONFIG_INET_ESPINTCP diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index fe8d53f5a5ee..fb431d0a3475 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -131,7 +131,7 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb) */ if (req->src != req->dst) for (sg = sg_next(req->src); sg; sg = sg_next(sg)) - skb_page_unref(skb, sg_page(sg)); + skb_page_unref(sg_page(sg), skb->pp_recycle); } #ifdef CONFIG_INET6_ESPINTCP From f58f3c9563409e618e591d0d540316286cb0665f Mon Sep 17 00:00:00 2001 From: Mina Almasry Date: Mon, 8 Apr 2024 08:29:58 -0700 Subject: [PATCH 02/10] net: remove napi_frag_unref With the changes in the last patches, napi_frag_unref() is now reduandant. Remove it and use skb_page_unref directly. Signed-off-by: Mina Almasry Reviewed-by: Dragos Tatulea Reviewed-by: Eric Dumazet Reviewed-by: Jacob Keller Link: https://lore.kernel.org/r/20240408153000.2152844-4-almasrymina@google.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 8 +------- net/core/skbuff.c | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c0ff85bb087a..7135a3e94afd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3531,12 +3531,6 @@ skb_page_unref(struct page *page, bool recycle) put_page(page); } -static inline void -napi_frag_unref(skb_frag_t *frag, bool recycle) -{ - skb_page_unref(skb_frag_page(frag), recycle); -} - /** * __skb_frag_unref - release a reference on a paged fragment. * @frag: the paged fragment @@ -3547,7 +3541,7 @@ napi_frag_unref(skb_frag_t *frag, bool recycle) */ static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) { - napi_frag_unref(frag, recycle); + skb_page_unref(skb_frag_page(frag), recycle); } /** diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 21cd01641f4c..888874ef8566 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1101,7 +1101,7 @@ static void skb_release_data(struct sk_buff *skb, enum skb_drop_reason reason) } for (i = 0; i < shinfo->nr_frags; i++) - napi_frag_unref(&shinfo->frags[i], skb->pp_recycle); + __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); free_head: if (shinfo->frag_list) From 2ecd487b670fcbb1ad4893fff1af4aafdecb6023 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 8 Apr 2024 16:48:17 +0300 Subject: [PATCH 03/10] net: sched: cls_api: fix slab-use-after-free in fl_dump_key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filter counter is updated under the protection of cb_lock in the cited commit. While waiting for the lock, it's possible the filter is being deleted by other thread, and thus causes UAF when dump it. Fix this issue by moving tcf_block_filter_cnt_update() after tfilter_put(). ================================================================== BUG: KASAN: slab-use-after-free in fl_dump_key+0x1d3e/0x20d0 [cls_flower] Read of size 4 at addr ffff88814f864000 by task tc/2973 CPU: 7 PID: 2973 Comm: tc Not tainted 6.9.0-rc2_for_upstream_debug_2024_04_02_12_41 #1 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014 Call Trace: dump_stack_lvl+0x7e/0xc0 print_report+0xc1/0x600 ? __virt_addr_valid+0x1cf/0x390 ? fl_dump_key+0x1d3e/0x20d0 [cls_flower] ? fl_dump_key+0x1d3e/0x20d0 [cls_flower] kasan_report+0xb9/0xf0 ? fl_dump_key+0x1d3e/0x20d0 [cls_flower] fl_dump_key+0x1d3e/0x20d0 [cls_flower] ? lock_acquire+0x1c2/0x530 ? fl_dump+0x172/0x5c0 [cls_flower] ? lockdep_hardirqs_on_prepare+0x400/0x400 ? fl_dump_key_options.part.0+0x10f0/0x10f0 [cls_flower] ? do_raw_spin_lock+0x12d/0x270 ? spin_bug+0x1d0/0x1d0 fl_dump+0x21d/0x5c0 [cls_flower] ? fl_tmplt_dump+0x1f0/0x1f0 [cls_flower] ? nla_put+0x15f/0x1c0 tcf_fill_node+0x51b/0x9a0 ? tc_skb_ext_tc_enable+0x150/0x150 ? __alloc_skb+0x17b/0x310 ? __build_skb_around+0x340/0x340 ? down_write+0x1b0/0x1e0 tfilter_notify+0x1a5/0x390 ? fl_terse_dump+0x400/0x400 [cls_flower] tc_new_tfilter+0x963/0x2170 ? tc_del_tfilter+0x1490/0x1490 ? print_usage_bug.part.0+0x670/0x670 ? lock_downgrade+0x680/0x680 ? security_capable+0x51/0x90 ? tc_del_tfilter+0x1490/0x1490 rtnetlink_rcv_msg+0x75e/0xac0 ? if_nlmsg_stats_size+0x4c0/0x4c0 ? lockdep_set_lock_cmp_fn+0x190/0x190 ? __netlink_lookup+0x35e/0x6e0 netlink_rcv_skb+0x12c/0x360 ? if_nlmsg_stats_size+0x4c0/0x4c0 ? netlink_ack+0x15e0/0x15e0 ? lockdep_hardirqs_on_prepare+0x400/0x400 ? netlink_deliver_tap+0xcd/0xa60 ? netlink_deliver_tap+0xcd/0xa60 ? netlink_deliver_tap+0x1c9/0xa60 netlink_unicast+0x43e/0x700 ? netlink_attachskb+0x750/0x750 ? lock_acquire+0x1c2/0x530 ? __might_fault+0xbb/0x170 netlink_sendmsg+0x749/0xc10 ? netlink_unicast+0x700/0x700 ? __might_fault+0xbb/0x170 ? netlink_unicast+0x700/0x700 __sock_sendmsg+0xc5/0x190 ____sys_sendmsg+0x534/0x6b0 ? import_iovec+0x7/0x10 ? kernel_sendmsg+0x30/0x30 ? __copy_msghdr+0x3c0/0x3c0 ? entry_SYSCALL_64_after_hwframe+0x46/0x4e ? lock_acquire+0x1c2/0x530 ? __virt_addr_valid+0x116/0x390 ___sys_sendmsg+0xeb/0x170 ? __virt_addr_valid+0x1ca/0x390 ? copy_msghdr_from_user+0x110/0x110 ? __delete_object+0xb8/0x100 ? __virt_addr_valid+0x1cf/0x390 ? do_sys_openat2+0x102/0x150 ? lockdep_hardirqs_on_prepare+0x284/0x400 ? do_sys_openat2+0x102/0x150 ? __fget_light+0x53/0x1d0 ? sockfd_lookup_light+0x1a/0x150 __sys_sendmsg+0xb5/0x140 ? __sys_sendmsg_sock+0x20/0x20 ? lock_downgrade+0x680/0x680 do_syscall_64+0x70/0x140 entry_SYSCALL_64_after_hwframe+0x46/0x4e RIP: 0033:0x7f98e3713367 Code: 0e 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b9 0f 1f 00 f3 0f 1e fa 64 8b 04 25 18 00 00 00 85 c0 75 10 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 51 c3 48 83 ec 28 89 54 24 1c 48 89 74 24 10 RSP: 002b:00007ffc74a64608 EFLAGS: 00000246 ORIG_RAX: 000000000000002e RAX: ffffffffffffffda RBX: 000000000047eae0 RCX: 00007f98e3713367 RDX: 0000000000000000 RSI: 00007ffc74a64670 RDI: 0000000000000003 RBP: 0000000000000008 R08: 0000000000000000 R09: 0000000000000000 R10: 00007f98e360c5e8 R11: 0000000000000246 R12: 00007ffc74a6a508 R13: 00000000660d518d R14: 0000000000484a80 R15: 00007ffc74a6a50b Allocated by task 2973: kasan_save_stack+0x20/0x40 kasan_save_track+0x10/0x30 __kasan_kmalloc+0x77/0x90 fl_change+0x27a6/0x4540 [cls_flower] tc_new_tfilter+0x879/0x2170 rtnetlink_rcv_msg+0x75e/0xac0 netlink_rcv_skb+0x12c/0x360 netlink_unicast+0x43e/0x700 netlink_sendmsg+0x749/0xc10 __sock_sendmsg+0xc5/0x190 ____sys_sendmsg+0x534/0x6b0 ___sys_sendmsg+0xeb/0x170 __sys_sendmsg+0xb5/0x140 do_syscall_64+0x70/0x140 entry_SYSCALL_64_after_hwframe+0x46/0x4e Freed by task 283: kasan_save_stack+0x20/0x40 kasan_save_track+0x10/0x30 kasan_save_free_info+0x37/0x50 poison_slab_object+0x105/0x190 __kasan_slab_free+0x11/0x30 kfree+0x111/0x340 process_one_work+0x787/0x1490 worker_thread+0x586/0xd30 kthread+0x2df/0x3b0 ret_from_fork+0x2d/0x70 ret_from_fork_asm+0x11/0x20 Last potentially related work creation: kasan_save_stack+0x20/0x40 __kasan_record_aux_stack+0x9b/0xb0 insert_work+0x25/0x1b0 __queue_work+0x640/0xc90 rcu_work_rcufn+0x42/0x70 rcu_core+0x6a9/0x1850 __do_softirq+0x264/0x88f Second to last potentially related work creation: kasan_save_stack+0x20/0x40 __kasan_record_aux_stack+0x9b/0xb0 __call_rcu_common.constprop.0+0x6f/0xac0 queue_rcu_work+0x56/0x70 fl_mask_put+0x20d/0x270 [cls_flower] __fl_delete+0x352/0x6b0 [cls_flower] fl_delete+0x97/0x160 [cls_flower] tc_del_tfilter+0x7d1/0x1490 rtnetlink_rcv_msg+0x75e/0xac0 netlink_rcv_skb+0x12c/0x360 netlink_unicast+0x43e/0x700 netlink_sendmsg+0x749/0xc10 __sock_sendmsg+0xc5/0x190 ____sys_sendmsg+0x534/0x6b0 ___sys_sendmsg+0xeb/0x170 __sys_sendmsg+0xb5/0x140 do_syscall_64+0x70/0x140 entry_SYSCALL_64_after_hwframe+0x46/0x4e Fixes: 2081fd3445fe ("net: sched: cls_api: add filter counter") Signed-off-by: Jianbo Liu Reviewed-by: Cosmin Ratiu Tested-by: Asbjørn Sloth Tønnesen Signed-off-by: David S. Miller --- net/sched/cls_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index db0653993632..17d97bbe890f 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2400,10 +2400,10 @@ replay: err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, flags, extack); if (err == 0) { - tcf_block_filter_cnt_update(block, &tp->counted, true); tfilter_notify(net, skb, n, tp, block, q, parent, fh, RTM_NEWTFILTER, false, rtnl_held, extack); tfilter_put(tp, fh); + tcf_block_filter_cnt_update(block, &tp->counted, true); /* q pointer is NULL for shared blocks */ if (q) q->flags &= ~TCQ_F_CAN_BYPASS; From e59f0e93e92e0ddfd17e3373d586218cf638571e Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 8 Apr 2024 20:15:46 -0700 Subject: [PATCH 04/10] selftests: move bpf-offload test from bpf to net We're building more python tests on the netdev side, and some of the classes from the venerable BPF offload tests can be reused. Acked-by: Stanislav Fomichev Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/r/20240409031549.3531084-2-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/testing/selftests/bpf/Makefile | 1 - tools/testing/selftests/net/Makefile | 8 +++++++- .../selftests/{bpf/test_offload.py => net/bpf_offload.py} | 0 .../progs/sample_map_ret0.c => net/sample_map_ret0.bpf.c} | 0 .../{bpf/progs/sample_ret0.c => net/sample_ret0.bpf.c} | 0 5 files changed, 7 insertions(+), 2 deletions(-) rename tools/testing/selftests/{bpf/test_offload.py => net/bpf_offload.py} (100%) rename tools/testing/selftests/{bpf/progs/sample_map_ret0.c => net/sample_map_ret0.bpf.c} (100%) rename tools/testing/selftests/{bpf/progs/sample_ret0.c => net/sample_ret0.bpf.c} (100%) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 3b9eb40d6343..b0be07f29dde 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -102,7 +102,6 @@ TEST_PROGS := test_kmod.sh \ test_xdp_redirect_multi.sh \ test_xdp_meta.sh \ test_xdp_veth.sh \ - test_offload.py \ test_sock_addr.sh \ test_tunnel.sh \ test_lwt_seg6local.sh \ diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 5e34c93aa51b..e8bfa715aa49 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -84,6 +84,8 @@ TEST_GEN_FILES += sctp_hello TEST_GEN_FILES += csum TEST_GEN_FILES += nat6to4.o TEST_GEN_FILES += xdp_dummy.o +TEST_GEN_FILES += sample_ret0.bpf.o +TEST_GEN_FILES += sample_map_ret0.bpf.o TEST_GEN_FILES += ip_local_port_range TEST_GEN_FILES += bind_wildcard TEST_PROGS += test_vxlan_mdb.sh @@ -93,6 +95,7 @@ TEST_PROGS += test_bridge_backup_port.sh TEST_PROGS += fdb_flush.sh TEST_PROGS += fq_band_pktlimit.sh TEST_PROGS += vlan_hw_filter.sh +TEST_PROGS += bpf_offload.py TEST_FILES := settings TEST_FILES += in_netns.sh lib.sh net_helper.sh setup_loopback.sh setup_veth.sh @@ -142,7 +145,10 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) -$(OUTPUT)/nat6to4.o $(OUTPUT)/xdp_dummy.o: $(OUTPUT)/%.o : %.c $(BPFOBJ) | $(MAKE_DIRS) +BPF_PROG_OBJS := $(OUTPUT)/nat6to4.o $(OUTPUT)/xdp_dummy.o \ + $(OUTPUT)/sample_map_ret0.bpf.o $(OUTPUT)/sample_ret0.bpf.o + +$(BPF_PROG_OBJS): $(OUTPUT)/%.o : %.c $(BPFOBJ) | $(MAKE_DIRS) $(CLANG) -O2 --target=bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/net/bpf_offload.py similarity index 100% rename from tools/testing/selftests/bpf/test_offload.py rename to tools/testing/selftests/net/bpf_offload.py diff --git a/tools/testing/selftests/bpf/progs/sample_map_ret0.c b/tools/testing/selftests/net/sample_map_ret0.bpf.c similarity index 100% rename from tools/testing/selftests/bpf/progs/sample_map_ret0.c rename to tools/testing/selftests/net/sample_map_ret0.bpf.c diff --git a/tools/testing/selftests/bpf/progs/sample_ret0.c b/tools/testing/selftests/net/sample_ret0.bpf.c similarity index 100% rename from tools/testing/selftests/bpf/progs/sample_ret0.c rename to tools/testing/selftests/net/sample_ret0.bpf.c From fc50c698c28bcf307dfb14ba9d0b3cacb091c1cb Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 8 Apr 2024 20:15:47 -0700 Subject: [PATCH 05/10] selftests: net: bpf_offload: wait for maps Maps are removed asynchronously. Either there's a bigger delay now or the test has always been flaky. Retry waiting in the loop. Acked-by: Stanislav Fomichev Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/r/20240409031549.3531084-3-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/testing/selftests/net/bpf_offload.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/net/bpf_offload.py b/tools/testing/selftests/net/bpf_offload.py index 6157f884d091..174dba1a48d3 100755 --- a/tools/testing/selftests/net/bpf_offload.py +++ b/tools/testing/selftests/net/bpf_offload.py @@ -201,11 +201,11 @@ def bpftool_prog_list_wait(expected=0, n_retry=20): time.sleep(0.05) raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs)) -def bpftool_map_list_wait(expected=0, n_retry=20): +def bpftool_map_list_wait(expected=0, n_retry=20, ns=""): for i in range(n_retry): - nmaps = len(bpftool_map_list()) - if nmaps == expected: - return + maps = bpftool_map_list(ns=ns) + if len(maps) == expected: + return maps time.sleep(0.05) raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps)) @@ -605,7 +605,7 @@ def pin_prog(file_name, idx=0): return file_name, bpf_pinned(file_name) def pin_map(file_name, idx=0, expected=1): - maps = bpftool_map_list(expected=expected) + maps = bpftool_map_list_wait(expected=expected) m = maps[idx] bpftool("map pin id %d %s" % (m["id"], file_name)) files.append(file_name) @@ -618,7 +618,7 @@ def check_dev_info_removed(prog_file=None, map_file=None): ret, err = bpftool("prog show pin %s" % (prog_file), fail=False) fail(ret != 0, "failed to show prog with removed device") - bpftool_map_list(expected=0) + bpftool_map_list_wait(expected=0) ret, err = bpftool("map show pin %s" % (map_file), fail=False) fail(ret == 0, "Showing map with removed device did not fail") fail(err["error"].find("No such device") == -1, @@ -642,7 +642,7 @@ def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False): else: fail("ifname" in dev.keys(), "Ifname is reported for other ns") - maps = bpftool_map_list(expected=2, ns=ns) + maps = bpftool_map_list_wait(expected=2, ns=ns) for m in maps: fail("dev" not in m.keys(), "Device parameters not reported") fail(dev != m["dev"], "Map's device different than program's") @@ -1206,7 +1206,7 @@ try: start_test("Test map update (no flags)...") sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON - maps = bpftool_map_list(expected=2) + maps = bpftool_map_list_wait(expected=2) array = maps[0] if maps[0]["type"] == "array" else maps[1] htab = maps[0] if maps[0]["type"] == "hash" else maps[1] for m in maps: From b1c2ce11d42886d08cfa28e38ee07f2b606ced0b Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 8 Apr 2024 20:15:48 -0700 Subject: [PATCH 06/10] selftests: net: declare section names for bpf_offload Non-ancient ip (iproute2-5.15.0, libbpf 0.7.0) refuses to load the sample with maps because we don't generate BTF: libbpf: BTF is required, but is missing or corrupted. ERROR: opening BPF object file failed Enable BTF by adding -g to clang flags. With that done neither of the programs load: libbpf: prog 'func': error relocating .BTF.ext function info: -22 libbpf: prog 'func': failed to relocate calls: -22 libbpf: failed to load object 'ksft-net-drv/net/sample_ret0.bpf.o' Andrii explains that this is because we don't specify section names for the code. Add the section names, too. Acked-by: Alexei Starovoitov Acked-by: Stanislav Fomichev Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/r/20240409031549.3531084-4-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/testing/selftests/net/Makefile | 3 ++- tools/testing/selftests/net/bpf_offload.py | 2 +- tools/testing/selftests/net/sample_map_ret0.bpf.c | 2 +- tools/testing/selftests/net/sample_ret0.bpf.c | 3 +++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index e8bfa715aa49..a3c781cb8367 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -149,7 +149,8 @@ BPF_PROG_OBJS := $(OUTPUT)/nat6to4.o $(OUTPUT)/xdp_dummy.o \ $(OUTPUT)/sample_map_ret0.bpf.o $(OUTPUT)/sample_ret0.bpf.o $(BPF_PROG_OBJS): $(OUTPUT)/%.o : %.c $(BPFOBJ) | $(MAKE_DIRS) - $(CLANG) -O2 --target=bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@ + $(CLANG) -O2 -g --target=bpf $(CCINCLUDE) $(CLANG_SYS_INCLUDES) \ + -c $< -o $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ $(APIDIR)/linux/bpf.h \ diff --git a/tools/testing/selftests/net/bpf_offload.py b/tools/testing/selftests/net/bpf_offload.py index 174dba1a48d3..76b53ac2c8c6 100755 --- a/tools/testing/selftests/net/bpf_offload.py +++ b/tools/testing/selftests/net/bpf_offload.py @@ -237,7 +237,7 @@ def tc(args, JSON=True, ns="", fail=True, include_stderr=False): def ethtool(dev, opt, args, fail=True): return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail) -def bpf_obj(name, sec=".text", path=bpf_test_dir,): +def bpf_obj(name, sec="xdp", path=bpf_test_dir,): return "obj %s sec %s" % (os.path.join(path, name), sec) def bpf_pinned(name): diff --git a/tools/testing/selftests/net/sample_map_ret0.bpf.c b/tools/testing/selftests/net/sample_map_ret0.bpf.c index 495990d355ef..43ca92594926 100644 --- a/tools/testing/selftests/net/sample_map_ret0.bpf.c +++ b/tools/testing/selftests/net/sample_map_ret0.bpf.c @@ -17,7 +17,7 @@ struct { } array SEC(".maps"); /* Sample program which should always load for testing control paths. */ -SEC(".text") int func() +SEC("xdp") int func() { __u64 key64 = 0; __u32 key = 0; diff --git a/tools/testing/selftests/net/sample_ret0.bpf.c b/tools/testing/selftests/net/sample_ret0.bpf.c index fec99750d6ea..1df5ca98bb65 100644 --- a/tools/testing/selftests/net/sample_ret0.bpf.c +++ b/tools/testing/selftests/net/sample_ret0.bpf.c @@ -1,6 +1,9 @@ /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +#define SEC(name) __attribute__((section(name), used)) + /* Sample program which should always load for testing control paths. */ +SEC("xdp") int func() { return 0; From 6ce2b689932ba8288ceef9a82c1caf029b0b23f9 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 8 Apr 2024 20:15:49 -0700 Subject: [PATCH 07/10] selftests: net: reuse common code in bpf_offload net/lib/py/nsim.py already contains the most useful parts of the netdevsim wrapper classes. Reuse them. Acked-by: Stanislav Fomichev Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/r/20240409031549.3531084-5-kuba@kernel.org Signed-off-by: Jakub Kicinski --- tools/testing/selftests/net/bpf_offload.py | 120 +++++---------------- tools/testing/selftests/net/lib/py/nsim.py | 9 +- 2 files changed, 35 insertions(+), 94 deletions(-) diff --git a/tools/testing/selftests/net/bpf_offload.py b/tools/testing/selftests/net/bpf_offload.py index 76b53ac2c8c6..3efe44f6e92a 100755 --- a/tools/testing/selftests/net/bpf_offload.py +++ b/tools/testing/selftests/net/bpf_offload.py @@ -29,6 +29,9 @@ import subprocess import time import traceback +from lib.py import NetdevSim, NetdevSimDev + + logfile = None log_level = 1 skip_extack = False @@ -145,8 +148,10 @@ def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False): if JSON: params += "%s " % (flags["json"]) - if ns != "": + if ns: ns = "ip netns exec %s " % (ns) + elif ns is None: + ns = "" if include_stderr: ret, stdout, stderr = cmd(ns + name + " " + params + args, @@ -334,72 +339,16 @@ class DebugfsDir: return dfs -class NetdevSimDev: +class BpfNetdevSimDev(NetdevSimDev): """ Class for netdevsim bus device and its attributes. """ - @staticmethod - def ctrl_write(path, val): - fullpath = os.path.join("/sys/bus/netdevsim/", path) - try: - with open(fullpath, "w") as f: - f.write(val) - except OSError as e: - log("WRITE %s: %r" % (fullpath, val), -e.errno) - raise e - log("WRITE %s: %r" % (fullpath, val), 0) - - def __init__(self, port_count=1): - addr = 0 - while True: - try: - self.ctrl_write("new_device", "%u %u" % (addr, port_count)) - except OSError as e: - if e.errno == errno.ENOSPC: - addr += 1 - continue - raise e - break - self.addr = addr - - # As probe of netdevsim device might happen from a workqueue, - # so wait here until all netdevs appear. - self.wait_for_netdevs(port_count) - - ret, out = cmd("udevadm settle", fail=False) - if ret: - raise Exception("udevadm settle failed") - ifnames = self.get_ifnames() - + def __init__(self, port_count=1, ns=None): + super().__init__(port_count, ns=ns) devs.append(self) - self.dfs_dir = "/sys/kernel/debug/netdevsim/netdevsim%u/" % addr - self.nsims = [] - for port_index in range(port_count): - self.nsims.append(NetdevSim(self, port_index, ifnames[port_index])) - - def get_ifnames(self): - ifnames = [] - listdir = os.listdir("/sys/bus/netdevsim/devices/netdevsim%u/net/" % self.addr) - for ifname in listdir: - ifnames.append(ifname) - ifnames.sort() - return ifnames - - def wait_for_netdevs(self, port_count): - timeout = 5 - timeout_start = time.time() - - while True: - try: - ifnames = self.get_ifnames() - except FileNotFoundError as e: - ifnames = [] - if len(ifnames) == port_count: - break - if time.time() < timeout_start + timeout: - continue - raise Exception("netdevices did not appear within timeout") + def _make_port(self, port_index, ifname): + return BpfNetdevSim(self, port_index, ifname, self.ns) def dfs_num_bound_progs(self): path = os.path.join(self.dfs_dir, "bpf_bound_progs") @@ -415,33 +364,20 @@ class NetdevSimDev: return progs def remove(self): - self.ctrl_write("del_device", "%u" % (self.addr, )) + super().remove() devs.remove(self) - def remove_nsim(self, nsim): - self.nsims.remove(nsim) - self.ctrl_write("devices/netdevsim%u/del_port" % (self.addr, ), - "%u" % (nsim.port_index, )) -class NetdevSim: +class BpfNetdevSim(NetdevSim): """ Class for netdevsim netdevice and its attributes. """ - def __init__(self, nsimdev, port_index, ifname): - # In case udev renamed the netdev to according to new schema, - # check if the name matches the port_index. - nsimnamere = re.compile("eni\d+np(\d+)") - match = nsimnamere.match(ifname) - if match and int(match.groups()[0]) != port_index + 1: - raise Exception("netdevice name mismatches the expected one") + def __init__(self, nsimdev, port_index, ifname, ns=None): + super().__init__(nsimdev, port_index, ifname, ns=ns) - self.nsimdev = nsimdev - self.port_index = port_index - self.ns = "" self.dfs_dir = "%s/ports/%u/" % (nsimdev.dfs_dir, port_index) self.dfs_refresh() - _, [self.dev] = ip("link show dev %s" % ifname) def __getitem__(self, key): return self.dev[key] @@ -468,7 +404,7 @@ class NetdevSim: raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs)) def set_ns(self, ns): - name = "1" if ns == "" else ns + name = ns if ns else "1" ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns) self.ns = ns @@ -744,7 +680,7 @@ def test_multi_prog(simdev, sim, obj, modename, modeid): start_test("Test multi-attachment XDP - device remove...") simdev.remove() - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.set_ethtool_tc_offloads(True) return [simdev, sim] @@ -809,13 +745,13 @@ try: bytecode = bpf_bytecode("1,6 0 0 4294967295,") start_test("Test destruction of generic XDP...") - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.set_xdp(obj, "generic") simdev.remove() bpftool_prog_list_wait(expected=0) - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.tc_add_ingress() @@ -967,7 +903,7 @@ try: simdev.remove() bpftool_prog_list_wait(expected=0) - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.set_ethtool_tc_offloads(True) @@ -976,7 +912,7 @@ try: simdev.remove() bpftool_prog_list_wait(expected=0) - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.set_ethtool_tc_offloads(True) @@ -1080,7 +1016,7 @@ try: bpftool_prog_list_wait(expected=0) start_test("Test attempt to use a program for a wrong device...") - simdev2 = NetdevSimDev() + simdev2 = BpfNetdevSimDev() sim2, = simdev2.nsims sim2.set_xdp(obj, "offload") pin_file, pinned = pin_prog("/sys/fs/bpf/tmp") @@ -1169,7 +1105,7 @@ try: clean_up() bpftool_prog_list_wait(expected=0) - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims map_obj = bpf_obj("sample_map_ret0.bpf.o") start_test("Test loading program with maps...") @@ -1201,7 +1137,7 @@ try: clean_up() bpftool_prog_list_wait(expected=0) - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims start_test("Test map update (no flags)...") @@ -1285,14 +1221,14 @@ try: bpftool_map_list_wait(expected=0) simdev.remove() - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON simdev.remove() bpftool_map_list_wait(expected=0) start_test("Test map creation fail path...") - simdev = NetdevSimDev() + simdev = BpfNetdevSimDev() sim, = simdev.nsims sim.dfs["bpf_map_accept"] = "N" ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False) @@ -1302,9 +1238,9 @@ try: simdev.remove() start_test("Test multi-dev ASIC program reuse...") - simdevA = NetdevSimDev() + simdevA = BpfNetdevSimDev() simA, = simdevA.nsims - simdevB = NetdevSimDev(3) + simdevB = BpfNetdevSimDev(3) simB1, simB2, simB3 = simdevB.nsims sims = (simA, simB1, simB2, simB3) simB = (simB1, simB2, simB3) diff --git a/tools/testing/selftests/net/lib/py/nsim.py b/tools/testing/selftests/net/lib/py/nsim.py index b2d696e12805..97457aca7e08 100644 --- a/tools/testing/selftests/net/lib/py/nsim.py +++ b/tools/testing/selftests/net/lib/py/nsim.py @@ -21,8 +21,11 @@ class NetdevSim: if match and int(match.groups()[0]) != port_index + 1: raise Exception("netdevice name mismatches the expected one") + self.ifname = ifname self.nsimdev = nsimdev self.port_index = port_index + self.ns = ns + self.dfs_dir = "%s/ports/%u/" % (nsimdev.dfs_dir, port_index) ret = ip("-j link show dev %s" % ifname, ns=ns) self.dev = json.loads(ret.stdout)[0] @@ -79,8 +82,10 @@ class NetdevSimDev: self.nsims = [] for port_index in range(port_count): - self.nsims.append(NetdevSim(self, port_index, ifnames[port_index], - ns=ns)) + self.nsims.append(self._make_port(port_index, ifnames[port_index])) + + def _make_port(self, port_index, ifname): + return NetdevSim(self, port_index, ifname, self.ns) def get_ifnames(self): ifnames = [] From 9b9fd45869e744bea7d32a94793736e3d9fd7d26 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 9 Apr 2024 14:09:14 +0000 Subject: [PATCH 08/10] tcp: tweak tcp_sock_write_txrx size assertion I forgot 32bit arches might have 64bit alignment for u64 fields. tcp_sock_write_txrx group does not contain pointers, but two u64 fields. It is possible that on 32bit kernel, a 32bit hole is before tp->tcp_clock_cache. I will try to remember a group can be bigger on 32bit kernels in the future. With help from Vladimir Oltean. Fixes: d2c3a7eb1afa ("tcp: more struct tcp_sock adjustments") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404082207.HCEdQhUO-lkp@intel.com/ Signed-off-by: Eric Dumazet Tested-by: Vladimir Oltean Link: https://lore.kernel.org/r/20240409140914.4105429-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/ipv4/tcp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index b07aa71b24ec..e1bf468e0d22 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4673,7 +4673,11 @@ static void __init tcp_struct_check(void) CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd); CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt); - CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92); + + /* 32bit arches with 8byte alignment on u64 fields might need padding + * before tcp_clock_cache. + */ + CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92 + 4); /* RX read-write hotpath cache lines */ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received); From 7cb31c46b9cc37a5e564667fe46daf9a35dbafdd Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Apr 2024 02:28:09 +0100 Subject: [PATCH 09/10] net: cache for same cpu skb_attempt_defer_free Optimise skb_attempt_defer_free() when run by the same CPU the skb was allocated on. Instead of __kfree_skb() -> kmem_cache_free() we can disable softirqs and put the buffer into cpu local caches. CPU bound TCP ping pong style benchmarking (i.e. netbench) showed a 1% throughput increase (392.2 -> 396.4 Krps). Cross checking with profiles, the total CPU share of skb_attempt_defer_free() dropped by 0.6%. Note, I'd expect the win doubled with rx only benchmarks, as the optimisation is for the receive path, but the test spends >55% of CPU doing writes. Signed-off-by: Pavel Begunkov Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/a887463fb219d973ec5ad275e31194812571f1f5.1712711977.git.asml.silence@gmail.com Signed-off-by: Jakub Kicinski --- net/core/skbuff.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 888874ef8566..18612f255ef5 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -6974,6 +6974,19 @@ free_now: EXPORT_SYMBOL(__skb_ext_put); #endif /* CONFIG_SKB_EXTENSIONS */ +static void kfree_skb_napi_cache(struct sk_buff *skb) +{ + /* if SKB is a clone, don't handle this case */ + if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { + __kfree_skb(skb); + return; + } + + local_bh_disable(); + __napi_kfree_skb(skb, SKB_DROP_REASON_NOT_SPECIFIED); + local_bh_enable(); +} + /** * skb_attempt_defer_free - queue skb for remote freeing * @skb: buffer @@ -6992,7 +7005,7 @@ void skb_attempt_defer_free(struct sk_buff *skb) if (WARN_ON_ONCE(cpu >= nr_cpu_ids) || !cpu_online(cpu) || cpu == raw_smp_processor_id()) { -nodefer: __kfree_skb(skb); +nodefer: kfree_skb_napi_cache(skb); return; } From d8415a165c43905ccf9ad7a4c7eaf6006b91a00b Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Wed, 10 Apr 2024 02:28:10 +0100 Subject: [PATCH 10/10] net: use SKB_CONSUMED in skb_attempt_defer_free() skb_attempt_defer_free() is used to free already processed skbs, so pass SKB_CONSUMED as the reason in kfree_skb_napi_cache(). Suggested-by: Jason Xing Suggested-by: Eric Dumazet Signed-off-by: Pavel Begunkov Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/bcf5dbdda79688b074ab7ae2238535840a6d3fc2.1712711977.git.asml.silence@gmail.com Signed-off-by: Jakub Kicinski --- net/core/skbuff.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 18612f255ef5..ab970ded8a7b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -6983,7 +6983,7 @@ static void kfree_skb_napi_cache(struct sk_buff *skb) } local_bh_disable(); - __napi_kfree_skb(skb, SKB_DROP_REASON_NOT_SPECIFIED); + __napi_kfree_skb(skb, SKB_CONSUMED); local_bh_enable(); }