From c147f663b6a5813b9860f3917cc473fb2c462d8d Mon Sep 17 00:00:00 2001 From: Javier Carrasco Date: Tue, 30 Jul 2024 07:54:37 +0200 Subject: [PATCH 001/111] clang-format: Update with v6.11-rc1's `for_each` macro list Re-run the shell fragment that generated the original list. Signed-off-by: Javier Carrasco Link: https://lore.kernel.org/r/20240730-clang-format-for-each-macro-update-v2-1-254fca862c97@gmail.com Signed-off-by: Miguel Ojeda --- .clang-format | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/.clang-format b/.clang-format index 252820d9c80a..fe1aa1a30d40 100644 --- a/.clang-format +++ b/.clang-format @@ -141,11 +141,13 @@ ForEachMacros: - 'damon_for_each_target_safe' - 'damos_for_each_filter' - 'damos_for_each_filter_safe' + - 'damos_for_each_quota_goal' + - 'damos_for_each_quota_goal_safe' - 'data__for_each_file' - 'data__for_each_file_new' - 'data__for_each_file_start' - 'device_for_each_child_node' - - 'displayid_iter_for_each' + - 'device_for_each_child_node_scoped' - 'dma_fence_array_for_each' - 'dma_fence_chain_for_each' - 'dma_fence_unwrap_for_each' @@ -172,11 +174,14 @@ ForEachMacros: - 'drm_for_each_plane' - 'drm_for_each_plane_mask' - 'drm_for_each_privobj' - - 'drm_gem_for_each_gpuva' - - 'drm_gem_for_each_gpuva_safe' + - 'drm_gem_for_each_gpuvm_bo' + - 'drm_gem_for_each_gpuvm_bo_safe' - 'drm_gpuva_for_each_op' - 'drm_gpuva_for_each_op_from_reverse' + - 'drm_gpuva_for_each_op_reverse' - 'drm_gpuva_for_each_op_safe' + - 'drm_gpuvm_bo_for_each_va' + - 'drm_gpuvm_bo_for_each_va_safe' - 'drm_gpuvm_for_each_va' - 'drm_gpuvm_for_each_va_range' - 'drm_gpuvm_for_each_va_range_safe' @@ -192,11 +197,11 @@ ForEachMacros: - 'dsa_switch_for_each_port_continue_reverse' - 'dsa_switch_for_each_port_safe' - 'dsa_switch_for_each_user_port' + - 'dsa_switch_for_each_user_port_continue_reverse' - 'dsa_tree_for_each_cpu_port' - 'dsa_tree_for_each_user_port' - 'dsa_tree_for_each_user_port_continue_reverse' - 'dso__for_each_symbol' - - 'dsos__for_each_with_build_id' - 'elf_hash_for_each_possible' - 'elf_symtab__for_each_symbol' - 'evlist__for_each_cpu' @@ -216,6 +221,7 @@ ForEachMacros: - 'for_each_and_bit' - 'for_each_andnot_bit' - 'for_each_available_child_of_node' + - 'for_each_available_child_of_node_scoped' - 'for_each_bench' - 'for_each_bio' - 'for_each_board_func_rsrc' @@ -234,6 +240,7 @@ ForEachMacros: - 'for_each_card_widgets_safe' - 'for_each_cgroup_storage_type' - 'for_each_child_of_node' + - 'for_each_child_of_node_scoped' - 'for_each_clear_bit' - 'for_each_clear_bit_from' - 'for_each_clear_bitrange' @@ -251,6 +258,7 @@ ForEachMacros: - 'for_each_cpu' - 'for_each_cpu_and' - 'for_each_cpu_andnot' + - 'for_each_cpu_from' - 'for_each_cpu_or' - 'for_each_cpu_wrap' - 'for_each_dapm_widgets' @@ -269,13 +277,14 @@ ForEachMacros: - 'for_each_element' - 'for_each_element_extid' - 'for_each_element_id' + - 'for_each_enabled_cpu' - 'for_each_endpoint_of_node' - 'for_each_event' - 'for_each_event_tps' - 'for_each_evictable_lru' - 'for_each_fib6_node_rt_rcu' - 'for_each_fib6_walker_rt' - - 'for_each_free_mem_pfn_range_in_zone' + - 'for_each_file_lock' - 'for_each_free_mem_pfn_range_in_zone_from' - 'for_each_free_mem_range' - 'for_each_free_mem_range_reverse' @@ -286,15 +295,18 @@ ForEachMacros: - 'for_each_group_member' - 'for_each_group_member_head' - 'for_each_hstate' + - 'for_each_hwgpio' - 'for_each_if' - 'for_each_inject_fn' - 'for_each_insn' + - 'for_each_insn_op_loc' - 'for_each_insn_prefix' - 'for_each_intid' - 'for_each_iommu' - 'for_each_ip_tunnel_rcu' - 'for_each_irq_nr' - 'for_each_lang' + - 'for_each_link_ch_maps' - 'for_each_link_codecs' - 'for_each_link_cpus' - 'for_each_link_platforms' @@ -332,6 +344,9 @@ ForEachMacros: - 'for_each_new_plane_in_state_reverse' - 'for_each_new_private_obj_in_state' - 'for_each_new_reg' + - 'for_each_nhlt_endpoint' + - 'for_each_nhlt_endpoint_fmtcfg' + - 'for_each_nhlt_fmtcfg' - 'for_each_node' - 'for_each_node_by_name' - 'for_each_node_by_type' @@ -387,12 +402,15 @@ ForEachMacros: - 'for_each_reloc_from' - 'for_each_requested_gpio' - 'for_each_requested_gpio_in_range' + - 'for_each_reserved_child_of_node' - 'for_each_reserved_mem_range' - 'for_each_reserved_mem_region' + - 'for_each_rtd_ch_maps' - 'for_each_rtd_codec_dais' - 'for_each_rtd_components' - 'for_each_rtd_cpu_dais' - 'for_each_rtd_dais' + - 'for_each_rtd_dais_reverse' - 'for_each_sband_iftype_data' - 'for_each_script' - 'for_each_sec' @@ -533,8 +551,6 @@ ForEachMacros: - 'lwq_for_each_safe' - 'map__for_each_symbol' - 'map__for_each_symbol_by_name' - - 'maps__for_each_entry' - - 'maps__for_each_entry_safe' - 'mas_for_each' - 'mci_for_each_dimm' - 'media_device_for_each_entity' @@ -560,7 +576,9 @@ ForEachMacros: - 'netdev_hw_addr_list_for_each' - 'nft_rule_for_each_expr' - 'nla_for_each_attr' + - 'nla_for_each_attr_type' - 'nla_for_each_nested' + - 'nla_for_each_nested_type' - 'nlmsg_for_each_attr' - 'nlmsg_for_each_msg' - 'nr_neigh_for_each' @@ -579,6 +597,7 @@ ForEachMacros: - 'perf_config_sections__for_each_entry' - 'perf_config_set__for_each_entry' - 'perf_cpu_map__for_each_cpu' + - 'perf_cpu_map__for_each_cpu_skip_any' - 'perf_cpu_map__for_each_idx' - 'perf_evlist__for_each_entry' - 'perf_evlist__for_each_entry_reverse' @@ -639,7 +658,6 @@ ForEachMacros: - 'shost_for_each_device' - 'sk_for_each' - 'sk_for_each_bound' - - 'sk_for_each_bound_bhash2' - 'sk_for_each_entry_offset_rcu' - 'sk_for_each_from' - 'sk_for_each_rcu' @@ -653,6 +671,7 @@ ForEachMacros: - 'snd_soc_dapm_widget_for_each_path_safe' - 'snd_soc_dapm_widget_for_each_sink_path' - 'snd_soc_dapm_widget_for_each_source_path' + - 'sparsebit_for_each_set_range' - 'strlist__for_each_entry' - 'strlist__for_each_entry_safe' - 'sym_for_each_insn' @@ -662,7 +681,6 @@ ForEachMacros: - 'tcf_act_for_each_action' - 'tcf_exts_for_each_action' - 'ttm_resource_manager_for_each_res' - - 'twsk_for_each_bound_bhash2' - 'udp_portaddr_for_each_entry' - 'udp_portaddr_for_each_entry_rcu' - 'usb_hub_for_each_child' @@ -686,6 +704,9 @@ ForEachMacros: - 'xbc_node_for_each_child' - 'xbc_node_for_each_key_value' - 'xbc_node_for_each_subkey' + - 'ynl_attr_for_each' + - 'ynl_attr_for_each_nested' + - 'ynl_attr_for_each_payload' - 'zorro_for_each_dev' IncludeBlocks: Preserve From 19c91bd8932a2e00f8de76022c780951166be9a0 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Fri, 12 Jul 2024 03:51:36 +0000 Subject: [PATCH 002/111] memblock tests: include memory_hotplug.h in mmzone.h as kernel dose In kernel code, memory_hotplug.h is included in mmzone.h instead of in init.h. Let's sync with kernel. This is a preparation for move init.h in common include directory. Signed-off-by: Wei Yang CC: Mike Rapoport Link: https://lore.kernel.org/r/20240712035138.24674-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/testing/memblock/linux/init.h | 1 - tools/testing/memblock/linux/mmzone.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/memblock/linux/init.h b/tools/testing/memblock/linux/init.h index 828e0ee0bc6c..4aeddce53310 100644 --- a/tools/testing/memblock/linux/init.h +++ b/tools/testing/memblock/linux/init.h @@ -4,7 +4,6 @@ #include #include -#include #define __section(section) __attribute__((__section__(section))) diff --git a/tools/testing/memblock/linux/mmzone.h b/tools/testing/memblock/linux/mmzone.h index 71546e15bdd3..bb682659a12d 100644 --- a/tools/testing/memblock/linux/mmzone.h +++ b/tools/testing/memblock/linux/mmzone.h @@ -3,6 +3,7 @@ #define _TOOLS_MMZONE_H #include +#include struct pglist_data *first_online_pgdat(void); struct pglist_data *next_online_pgdat(struct pglist_data *pgdat); From d68c08173b70952cd74bb45075b843f4a637a43b Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Fri, 12 Jul 2024 03:51:37 +0000 Subject: [PATCH 003/111] memblock tests: include export.h in linkage.h as kernel dose In kernel code, linkage.h includes export.h. Let's sync with kernel. This is a preparation for move init.h in common include directory. Signed-off-by: Wei Yang CC: Mike Rapoport Link: https://lore.kernel.org/r/20240712035138.24674-2-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/linkage.h | 2 ++ tools/testing/memblock/linux/init.h | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/include/linux/linkage.h b/tools/include/linux/linkage.h index bc763d500262..20dee24d7e1b 100644 --- a/tools/include/linux/linkage.h +++ b/tools/include/linux/linkage.h @@ -1,4 +1,6 @@ #ifndef _TOOLS_INCLUDE_LINUX_LINKAGE_H #define _TOOLS_INCLUDE_LINUX_LINKAGE_H +#include + #endif /* _TOOLS_INCLUDE_LINUX_LINKAGE_H */ diff --git a/tools/testing/memblock/linux/init.h b/tools/testing/memblock/linux/init.h index 4aeddce53310..bd74abc5cba6 100644 --- a/tools/testing/memblock/linux/init.h +++ b/tools/testing/memblock/linux/init.h @@ -3,7 +3,6 @@ #define _LINUX_INIT_H #include -#include #define __section(section) __attribute__((__section__(section))) From e2ae9cf39f8806d2d8f5eb0a22ba511804a804ec Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Fri, 12 Jul 2024 03:51:38 +0000 Subject: [PATCH 004/111] tools/testing: abstract two init.h into common include directory Currently we have two test suits define its own init.h. This is a little redundant. Let's create a init.h in common include directory and merge these two into it. Signed-off-by: Wei Yang CC: Mike Rapoport CC: Liam R. Howlett Link: https://lore.kernel.org/r/20240712035138.24674-3-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/compiler.h | 4 ---- tools/{testing/memblock => include}/linux/init.h | 14 +++++++++++--- tools/testing/radix-tree/linux/init.h | 2 -- tools/testing/radix-tree/maple.c | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) rename tools/{testing/memblock => include}/linux/init.h (81%) delete mode 100644 tools/testing/radix-tree/linux/init.h diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index 6f7f22ac9da5..4b5a45919ff8 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -126,10 +126,6 @@ # define unlikely(x) __builtin_expect(!!(x), 0) #endif -#ifndef __init -# define __init -#endif - #include /* diff --git a/tools/testing/memblock/linux/init.h b/tools/include/linux/init.h similarity index 81% rename from tools/testing/memblock/linux/init.h rename to tools/include/linux/init.h index bd74abc5cba6..7ed407976dda 100644 --- a/tools/testing/memblock/linux/init.h +++ b/tools/include/linux/init.h @@ -1,9 +1,17 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_INIT_H -#define _LINUX_INIT_H +#ifndef _TOOLS_LINUX_INIT_H_ +#define _TOOLS_LINUX_INIT_H_ #include +#ifndef __init +# define __init +#endif + +#ifndef __exit +# define __exit +#endif + #define __section(section) __attribute__((__section__(section))) #define __initconst @@ -29,4 +37,4 @@ struct obs_kernel_param { #define early_param(str, fn) \ __setup_param(str, fn, fn, 1) -#endif +#endif /* _TOOLS_LINUX_INIT_H_ */ diff --git a/tools/testing/radix-tree/linux/init.h b/tools/testing/radix-tree/linux/init.h deleted file mode 100644 index 81563c3dfce7..000000000000 --- a/tools/testing/radix-tree/linux/init.h +++ /dev/null @@ -1,2 +0,0 @@ -#define __init -#define __exit diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index cd1cf05503b4..3437292babff 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -14,7 +14,7 @@ #include "test.h" #include #include -#include "linux/init.h" +#include #define module_init(x) #define module_exit(x) From 39f64e402f659890a99d415eaf63a01f3b80a9a8 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 6 Aug 2024 01:03:15 +0000 Subject: [PATCH 005/111] memblock test: fix implicit declaration of function 'virt_to_phys' Commit 94ff46de4a73 ("memblock: Move late alloc warning down to phys alloc") introduce the usage of virt_to_phys(), which is not defined in memblock tests. Define it in mm.h to fix the build error. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240806010319.29194-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/mm.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/include/linux/mm.h b/tools/include/linux/mm.h index cad4f2927983..677c37e4a18c 100644 --- a/tools/include/linux/mm.h +++ b/tools/include/linux/mm.h @@ -25,6 +25,12 @@ static inline void *phys_to_virt(unsigned long address) return __va(address); } +#define virt_to_phys virt_to_phys +static inline phys_addr_t virt_to_phys(volatile void *address) +{ + return (phys_addr_t)address; +} + void reserve_bootmem_region(phys_addr_t start, phys_addr_t end, int nid); static inline void totalram_pages_inc(void) From 9f76c2ade323121f9006f6a529e0795317e16b5c Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 6 Aug 2024 01:03:16 +0000 Subject: [PATCH 006/111] memblock test: add the definition of __setup() Commit 1e4c64b71c9b ("mm/memblock: Add "reserve_mem" to reserved named memory at boot up") introduce usage of __setup(), which is not defined in memblock test. Define it in init.h to fix the build error. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240806010319.29194-2-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/init.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/include/linux/init.h b/tools/include/linux/init.h index 7ed407976dda..51b5cde28639 100644 --- a/tools/include/linux/init.h +++ b/tools/include/linux/init.h @@ -34,6 +34,9 @@ struct obs_kernel_param { __aligned(__alignof__(struct obs_kernel_param)) = \ { __setup_str_##unique_id, fn, early } +#define __setup(str, fn) \ + __setup_param(str, fn, fn, 0) + #define early_param(str, fn) \ __setup_param(str, fn, fn, 1) From a88cde5769d523e4ae6aad61237e4a5f6bd2309a Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 6 Aug 2024 01:03:17 +0000 Subject: [PATCH 007/111] memblock test: fix implicit declaration of function 'memparse' Commit 1e4c64b71c9b ("mm/memblock: Add "reserve_mem" to reserved named memory at boot up") introduce the usage of memparse(), which is not defined in memblock test. Add the definition and link it to fix the build. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240806010319.29194-3-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/string.h | 1 + tools/lib/cmdline.c | 53 +++++++++++++++++++++++++++ tools/testing/memblock/Makefile | 2 +- tools/testing/memblock/linux/kernel.h | 1 + 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 tools/lib/cmdline.c diff --git a/tools/include/linux/string.h b/tools/include/linux/string.h index db5c99318c79..fb8eda3019b5 100644 --- a/tools/include/linux/string.h +++ b/tools/include/linux/string.h @@ -47,4 +47,5 @@ extern char * __must_check skip_spaces(const char *); extern char *strim(char *); extern void *memchr_inv(const void *start, int c, size_t bytes); +extern unsigned long long memparse(const char *ptr, char **retptr); #endif /* _TOOLS_LINUX_STRING_H_ */ diff --git a/tools/lib/cmdline.c b/tools/lib/cmdline.c new file mode 100644 index 000000000000..c85f00f43c5e --- /dev/null +++ b/tools/lib/cmdline.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * From lib/cmdline.c + */ +#include + +#if __has_attribute(__fallthrough__) +# define fallthrough __attribute__((__fallthrough__)) +#else +# define fallthrough do {} while (0) /* fallthrough */ +#endif + +unsigned long long memparse(const char *ptr, char **retptr) +{ + char *endptr; /* local pointer to end of parsed string */ + + unsigned long long ret = strtoll(ptr, &endptr, 0); + + switch (*endptr) { + case 'E': + case 'e': + ret <<= 10; + fallthrough; + case 'P': + case 'p': + ret <<= 10; + fallthrough; + case 'T': + case 't': + ret <<= 10; + fallthrough; + case 'G': + case 'g': + ret <<= 10; + fallthrough; + case 'M': + case 'm': + ret <<= 10; + fallthrough; + case 'K': + case 'k': + ret <<= 10; + endptr++; + fallthrough; + default: + break; + } + + if (retptr) + *retptr = endptr; + + return ret; +} diff --git a/tools/testing/memblock/Makefile b/tools/testing/memblock/Makefile index 7a1ca694a982..d80982ccdc20 100644 --- a/tools/testing/memblock/Makefile +++ b/tools/testing/memblock/Makefile @@ -8,7 +8,7 @@ LDFLAGS += -fsanitize=address -fsanitize=undefined TARGETS = main TEST_OFILES = tests/alloc_nid_api.o tests/alloc_helpers_api.o tests/alloc_api.o \ tests/basic_api.o tests/common.o tests/alloc_exact_nid_api.o -DEP_OFILES = memblock.o lib/slab.o mmzone.o slab.o +DEP_OFILES = memblock.o lib/slab.o mmzone.o slab.o cmdline.o OFILES = main.o $(DEP_OFILES) $(TEST_OFILES) EXTR_SRC = ../../../mm/memblock.c diff --git a/tools/testing/memblock/linux/kernel.h b/tools/testing/memblock/linux/kernel.h index d2f148bd8902..c16d9cd2d1ae 100644 --- a/tools/testing/memblock/linux/kernel.h +++ b/tools/testing/memblock/linux/kernel.h @@ -8,5 +8,6 @@ #include #include #include +#include #endif From 8ac13bc7c266102f1068faafa5314522b68ebe65 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 6 Aug 2024 01:03:18 +0000 Subject: [PATCH 008/111] memblock test: fix implicit declaration of function 'isspace' Commit 1e4c64b71c9b ("mm/memblock: Add "reserve_mem" to reserved named memory at boot up") introduce usage of isspace(). Let's include in kernel.h to fix this. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240806010319.29194-4-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/testing/memblock/linux/kernel.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/memblock/linux/kernel.h b/tools/testing/memblock/linux/kernel.h index c16d9cd2d1ae..4d1012d5be6e 100644 --- a/tools/testing/memblock/linux/kernel.h +++ b/tools/testing/memblock/linux/kernel.h @@ -9,5 +9,6 @@ #include #include #include +#include #endif From 9e3d665384fca2a1c56283c7a79a968243ef4614 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 6 Aug 2024 01:03:19 +0000 Subject: [PATCH 009/111] memblock test: fix implicit declaration of function 'strscpy' Commit 1e4c64b71c9b ("mm/memblock: Add "reserve_mem" to reserved named memory at boot up") introduce the usage of strscpy, which breaks the memblock test. Let's define it as strcpy in userspace to fix it. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240806010319.29194-5-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- tools/include/linux/string.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/include/linux/string.h b/tools/include/linux/string.h index fb8eda3019b5..41e7fa734922 100644 --- a/tools/include/linux/string.h +++ b/tools/include/linux/string.h @@ -12,6 +12,8 @@ void argv_free(char **argv); int strtobool(const char *s, bool *res); +#define strscpy strcpy + /* * glibc based builds needs the extern while uClibc doesn't. * However uClibc headers also define __GLIBC__ hence the hack below From d0f8a8973f265f6a276f99d091af99edfb2b87de Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Thu, 8 Aug 2024 00:14:13 +0000 Subject: [PATCH 010/111] mm/memblock: introduce a new helper memblock_estimated_nr_free_pages() During bootup, system may need the number of free pages in the whole system to do some calculation before all pages are freed to buddy system. Usually this number is get from totalram_pages(). Since we plan to move the free pages accounting in __free_pages_core(), this value may not represent total free pages at the early stage, especially when CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled. Instead of using raw memblock api, let's introduce a new helper for user to get the estimated number of free pages from memblock point of view. Signed-off-by: Wei Yang CC: David Hildenbrand Reviewed-by: David Hildenbrand Link: https://lore.kernel.org/r/20240808001415.6298-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- include/linux/memblock.h | 1 + mm/memblock.c | 17 +++++++++++++++++ tools/include/linux/pfn.h | 1 + 3 files changed, 19 insertions(+) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index fc4d75c6cec3..673d5cae7c81 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -467,6 +467,7 @@ static inline __init_memblock bool memblock_bottom_up(void) phys_addr_t memblock_phys_mem_size(void); phys_addr_t memblock_reserved_size(void); +unsigned long memblock_estimated_nr_free_pages(void); phys_addr_t memblock_start_of_DRAM(void); phys_addr_t memblock_end_of_DRAM(void); void memblock_enforce_memory_limit(phys_addr_t memory_limit); diff --git a/mm/memblock.c b/mm/memblock.c index 3b9dc2d89b8a..213057603b65 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1731,6 +1731,23 @@ phys_addr_t __init_memblock memblock_reserved_size(void) return memblock.reserved.total_size; } +/** + * memblock_estimated_nr_free_pages - return estimated number of free pages + * from memblock point of view + * + * During bootup, subsystems might need a rough estimate of the number of free + * pages in the whole system, before precise numbers are available from the + * buddy. Especially with CONFIG_DEFERRED_STRUCT_PAGE_INIT, the numbers + * obtained from the buddy might be very imprecise during bootup. + * + * Return: + * An estimated number of free pages from memblock point of view. + */ +unsigned long __init memblock_estimated_nr_free_pages(void) +{ + return PHYS_PFN(memblock_phys_mem_size() - memblock_reserved_size()); +} + /* lowest address */ phys_addr_t __init_memblock memblock_start_of_DRAM(void) { diff --git a/tools/include/linux/pfn.h b/tools/include/linux/pfn.h index 7512a58189eb..f77a30d70152 100644 --- a/tools/include/linux/pfn.h +++ b/tools/include/linux/pfn.h @@ -7,4 +7,5 @@ #define PFN_UP(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT) #define PFN_DOWN(x) ((x) >> PAGE_SHIFT) #define PFN_PHYS(x) ((phys_addr_t)(x) << PAGE_SHIFT) +#define PHYS_PFN(x) ((unsigned long)((x) >> PAGE_SHIFT)) #endif From 0910bf0ef85c5404aac94394cb31e076e4eb03f1 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Thu, 8 Aug 2024 00:14:14 +0000 Subject: [PATCH 011/111] kernel/fork.c: get estimated free pages by memblock api Instead of getting estimated free pages from memblock directly, we have introduced an API, memblock_estimated_nr_free_pages(), which is more friendly for users. Just replace it with new API, no functional change. Signed-off-by: Wei Yang CC: Mike Rapoport CC: David Hildenbrand CC: Oleg Nesterov Acked-by: David Hildenbrand Link: https://lore.kernel.org/r/20240808001415.6298-2-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- kernel/fork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/fork.c b/kernel/fork.c index cc760491f201..d99f148d818b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -997,7 +997,7 @@ void __init __weak arch_task_cache_init(void) { } static void __init set_max_threads(unsigned int max_threads_suggested) { u64 threads; - unsigned long nr_pages = PHYS_PFN(memblock_phys_mem_size() - memblock_reserved_size()); + unsigned long nr_pages = memblock_estimated_nr_free_pages(); /* * The number of threads shall be limited such that the thread From cb088e38aab4c7e9ce711c18c66e851c8f4227bb Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Thu, 8 Aug 2024 00:14:15 +0000 Subject: [PATCH 012/111] s390/mm: get estimated free pages by memblock api Instead of getting estimated free pages from memblock directly, we have introduced an API, memblock_estimated_nr_free_pages(), which is more friendly for users. Just replace it with new API, no functional change. Signed-off-by: Wei Yang CC: Mike Rapoport CC: David Hildenbrand Acked-by: David Hildenbrand Link: https://lore.kernel.org/r/20240808001415.6298-3-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (Microsoft) --- arch/s390/mm/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index e3d258f9e726..651344206294 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -62,7 +62,7 @@ EXPORT_SYMBOL(zero_page_mask); static void __init setup_zero_pages(void) { - unsigned long total_pages = PHYS_PFN(memblock_phys_mem_size() - memblock_reserved_size()); + unsigned long total_pages = memblock_estimated_nr_free_pages(); unsigned int order; struct page *page; int i; From 08f983a55ccf0b015e4788d1a0de0da84e4a7626 Mon Sep 17 00:00:00 2001 From: Alex Mantel Date: Fri, 26 Jul 2024 21:24:42 -0700 Subject: [PATCH 013/111] rust: Implement the smart pointer `InPlaceInit` for `Arc` For pinned and unpinned initialization of structs, a trait named `InPlaceInit` exists for uniform access. `Arc` did not implement `InPlaceInit` yet, although the functions already existed. The main reason for that, was that the trait itself returned a `Pin`. The `Arc` implementation of the kernel is already implicitly pinned. To enable `Arc` to implement `InPlaceInit` and to have uniform access, for in-place and pinned in-place initialization, an associated type is introduced for `InPlaceInit`. The new implementation of `InPlaceInit` for `Arc` sets `Arc` as the associated type. Older implementations use an explicit `Pin` as the associated type. The implemented methods for `Arc` are mostly moved from a direct implementation on `Arc`. There should be no user impact. The implementation for `ListArc` is omitted, because it is not merged yet. Link: https://github.com/Rust-for-Linux/linux/issues/1079 Signed-off-by: Alex Mantel Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20240727042442.682109-1-alexmantel93@mailbox.org [ Removed "Rusts" (Benno). - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 41 ++++++++++++++++++++++++++++++++++++----- rust/kernel/sync/arc.rs | 25 ++----------------------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 495c09ebe3a3..771701805a97 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -213,6 +213,7 @@ use crate::{ alloc::{box_ext::BoxExt, AllocError, Flags}, error::{self, Error}, + sync::Arc, sync::UniqueArc, types::{Opaque, ScopeGuard}, }; @@ -1107,11 +1108,17 @@ unsafe impl PinInit for T { /// Smart pointer that can initialize memory in-place. pub trait InPlaceInit: Sized { + /// Pinned version of `Self`. + /// + /// If a type already implicitly pins its pointee, `Pin` is unnecessary. In this case use + /// `Self`, otherwise just use `Pin`. + type PinnedSelf; + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result where E: From; @@ -1119,7 +1126,7 @@ pub trait InPlaceInit: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn pin_init(init: impl PinInit, flags: Flags) -> error::Result> + fn pin_init(init: impl PinInit, flags: Flags) -> error::Result where Error: From, { @@ -1148,9 +1155,31 @@ pub trait InPlaceInit: Sized { } } -impl InPlaceInit for Box { +impl InPlaceInit for Arc { + type PinnedSelf = Self; + #[inline] - fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_pin_init(init, flags).map(|u| u.into()) + } + + #[inline] + fn try_init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + UniqueArc::try_init(init, flags).map(|u| u.into()) + } +} + +impl InPlaceInit for Box { + type PinnedSelf = Pin; + + #[inline] + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result where E: From, { @@ -1179,8 +1208,10 @@ impl InPlaceInit for Box { } impl InPlaceInit for UniqueArc { + type PinnedSelf = Pin; + #[inline] - fn try_pin_init(init: impl PinInit, flags: Flags) -> Result, E> + fn try_pin_init(init: impl PinInit, flags: Flags) -> Result where E: From, { diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 3673496c2363..3021f30fd822 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -12,12 +12,13 @@ //! 2. It does not support weak references, which allows it to be half the size. //! 3. It saturates the reference count instead of aborting when it goes over a threshold. //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. +//! 5. The object in [`Arc`] is pinned implicitly. //! //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ alloc::{box_ext::BoxExt, AllocError, Flags}, - error::{self, Error}, + bindings, init::{self, InPlaceInit, Init, PinInit}, try_init, types::{ForeignOwnable, Opaque}, @@ -209,28 +210,6 @@ impl Arc { // `Arc` object. Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) } - - /// Use the given initializer to in-place initialize a `T`. - /// - /// If `T: !Unpin` it will not be able to move afterwards. - #[inline] - pub fn pin_init(init: impl PinInit, flags: Flags) -> error::Result - where - Error: From, - { - UniqueArc::pin_init(init, flags).map(|u| u.into()) - } - - /// Use the given initializer to in-place initialize a `T`. - /// - /// This is equivalent to [`Arc::pin_init`], since an [`Arc`] is always pinned. - #[inline] - pub fn init(init: impl Init, flags: Flags) -> error::Result - where - Error: From, - { - UniqueArc::init(init, flags).map(|u| u.into()) - } } impl Arc { From 6c2d0ad53b8ff25cb1a12570191576d834e9108d Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Tue, 30 Jul 2024 13:06:32 +0000 Subject: [PATCH 014/111] rust: implement ForeignOwnable for Pin> We already implement ForeignOwnable for Box, but it may be useful to store pinned data in a ForeignOwnable container. This patch makes that possible. This will be used together with upcoming miscdev abstractions, which Binder will use when binderfs is disabled. Signed-off-by: Alice Ryhl Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20240730-foreign-ownable-pin-box-v1-1-b1d70cdae541@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/types.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index bd189d646adb..132ca1113083 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -9,6 +9,7 @@ use core::{ marker::{PhantomData, PhantomPinned}, mem::MaybeUninit, ops::{Deref, DerefMut}, + pin::Pin, ptr::NonNull, }; @@ -89,6 +90,32 @@ impl ForeignOwnable for Box { } } +impl ForeignOwnable for Pin> { + type Borrowed<'a> = Pin<&'a T>; + + fn into_foreign(self) -> *const core::ffi::c_void { + // SAFETY: We are still treating the box as pinned. + Box::into_raw(unsafe { Pin::into_inner_unchecked(self) }) as _ + } + + unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> Pin<&'a T> { + // SAFETY: The safety requirements for this function ensure that the object is still alive, + // so it is safe to dereference the raw pointer. + // The safety requirements of `from_foreign` also ensure that the object remains alive for + // the lifetime of the returned value. + let r = unsafe { &*ptr.cast() }; + + // SAFETY: This pointer originates from a `Pin>`. + unsafe { Pin::new_unchecked(r) } + } + + unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + unsafe { Pin::new_unchecked(Box::from_raw(ptr as _)) } + } +} + impl ForeignOwnable for () { type Borrowed<'a> = (); From 7adcdd572248591c3932e27d98b4a086662d5cbe Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Tue, 30 Jul 2024 18:23:04 +0000 Subject: [PATCH 015/111] rust: types: improve `ForeignOwnable` documentation There are no guarantees for the pointer returned by `into_foreign`. This is simply because there is no safety documentation stating any guarantees. Therefore dereferencing and all other operations for that pointer are not allowed in a general context (i.e. when the concrete type implementing the trait is not known). This might be confusing, therefore add normal documentation to state that there are no guarantees given for the pointer. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20240730182251.1466684-1-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/kernel/types.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 132ca1113083..ee7dd1f963ef 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -27,7 +27,10 @@ pub trait ForeignOwnable: Sized { /// Converts a Rust-owned object to a foreign-owned one. /// - /// The foreign representation is a pointer to void. + /// The foreign representation is a pointer to void. There are no guarantees for this pointer. + /// For example, it might be invalid, dangling or pointing to uninitialized memory. Using it in + /// any way except for [`ForeignOwnable::from_foreign`], [`ForeignOwnable::borrow`], + /// [`ForeignOwnable::try_from_foreign`] can result in undefined behavior. fn into_foreign(self) -> *const core::ffi::c_void; /// Borrows a foreign-owned object. From 7bc186731e87482662c4f86da455f435fe838fb6 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Tue, 30 Jul 2024 17:57:02 +0200 Subject: [PATCH 016/111] rust: error: allow `useless_conversion` for 32-bit builds For the new Rust support for 32-bit arm [1], Clippy warns: error: useless conversion to the same type: `i32` --> rust/kernel/error.rs:139:36 | 139 | unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } | ^^^^^^^^^^^^^ help: consider removing `.into()`: `self.0` | = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#useless_conversion = note: `-D clippy::useless-conversion` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::useless_conversion)]` The `self.0.into()` converts an `c_int` into `ERR_PTR`'s parameter which is a `c_long`. Thus, both types are `i32` in 32-bit. Therefore, allow it for those architectures. Link: https://lore.kernel.org/rust-for-linux/2dbd1491-149d-443c-9802-75786a6a3b73@gmail.com/ [1] Reviewed-by: Alice Ryhl Reviewed-by: Christian Schrefl Link: https://lore.kernel.org/r/20240730155702.1110144-1-ojeda@kernel.org [ Fixed typo in tag. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 145f5c397009..6f1587a2524e 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -135,8 +135,11 @@ impl Error { /// Returns the error encoded as a pointer. #[allow(dead_code)] pub(crate) fn to_ptr(self) -> *mut T { + #[cfg_attr(target_pointer_width = "32", allow(clippy::useless_conversion))] // SAFETY: `self.0` is a valid error due to its invariant. - unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } + unsafe { + bindings::ERR_PTR(self.0.into()) as *mut _ + } } /// Returns a string representing the error, if one exists. From f1385dc670fe66860bcec5dcba215364bf71b807 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Wed, 31 Jul 2024 08:54:28 -0400 Subject: [PATCH 017/111] init/Kconfig: Only block on RANDSTRUCT for RUST When enabling Rust in the kernel, we only need to block on the RANDSTRUCT feature and GCC plugin. The rest of the GCC plugins are reasonably safe to enable. [ Originally (years ago) we only had this restriction, but we ended up restricting also the rest of the GCC plugins 1) to be on the safe side, 2) since compiler plugin support could be going away in the kernel and 3) since mixed builds are best effort so far; so I asked Neal about his experience enabling the other plugins -- Neal says: When I originally wrote this patch two years ago to get things working, Fedora used all the GCC plugins, so I was trying to get GCC + Rust to work while minimizing the delta on build differences. This was the combination that worked. We've been carrying this patch in the Asahi tree for a year now. And while Fedora does not currently have GCC plugins enabled because it caused issues with some third-party modules (I think it was the NVIDIA driver, but I'm not sure), it was around long enough for me to know with some confidence that it was fine this way. - Miguel ] Signed-off-by: Neal Gompa Link: https://lore.kernel.org/r/20240731125615.3368813-1-neal@gompa.dev Signed-off-by: Miguel Ojeda --- init/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/Kconfig b/init/Kconfig index 5783a0b87517..839c83034006 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1900,7 +1900,7 @@ config RUST depends on RUST_IS_AVAILABLE depends on !CFI_CLANG depends on !MODVERSIONS - depends on !GCC_PLUGINS + depends on !GCC_PLUGIN_RANDSTRUCT depends on !RANDSTRUCT depends on !SHADOW_CALL_STACK depends on !DEBUG_INFO_BTF || PAHOLE_HAS_LANG_EXCLUDE From 876346536c1b59a5b1b5e44477b1b3ece77647fd Mon Sep 17 00:00:00 2001 From: Andreas Hindborg Date: Thu, 15 Aug 2024 10:30:26 +0000 Subject: [PATCH 018/111] rust: kbuild: split up helpers.c This patch splits up the rust helpers C file. When rebasing patch sets on upstream linux, merge conflicts in helpers.c is common and time consuming [1]. Thus, split the file so that each kernel component can live in a separate file. This patch lists helper files explicitly and thus conflicts in the file list is still likely. However, they should be more simple to resolve than the conflicts usually seen in helpers.c. [ Removed `README.md` and undeleted the original comment since now, in v3 of the series, we have a `helpers.c` again; which also allows us to keep the "Sorted alphabetically" line and makes the diff easier. In addition, updated the Documentation/ mentions of the file, reworded title and removed blank lines at the end of `page.c`. - Miguel ] Link: https://rust-for-linux.zulipchat.com/#narrow/stream/288089-General/topic/Splitting.20up.20helpers.2Ec/near/426694012 [1] Signed-off-by: Andreas Hindborg Reviewed-by: Gary Guo Acked-by: Dirk Behme Reviewed-by: Alice Ryhl Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20240815103016.2771842-1-nmi@metaspace.dk Signed-off-by: Miguel Ojeda --- Documentation/rust/general-information.rst | 4 +- rust/Makefile | 6 +- rust/helpers.c | 239 --------------------- rust/helpers/blk.c | 16 ++ rust/helpers/bug.c | 9 + rust/helpers/build_assert.c | 25 +++ rust/helpers/build_bug.c | 10 + rust/helpers/err.c | 22 ++ rust/helpers/helpers.c | 38 ++++ rust/helpers/kunit.c | 10 + rust/helpers/mutex.c | 10 + rust/helpers/page.c | 22 ++ rust/helpers/refcount.c | 22 ++ rust/helpers/signal.c | 10 + rust/helpers/slab.c | 10 + rust/helpers/spinlock.c | 27 +++ rust/helpers/task.c | 22 ++ rust/helpers/uaccess.c | 17 ++ rust/helpers/wait.c | 10 + rust/helpers/workqueue.c | 16 ++ 20 files changed, 301 insertions(+), 244 deletions(-) delete mode 100644 rust/helpers.c create mode 100644 rust/helpers/blk.c create mode 100644 rust/helpers/bug.c create mode 100644 rust/helpers/build_assert.c create mode 100644 rust/helpers/build_bug.c create mode 100644 rust/helpers/err.c create mode 100644 rust/helpers/helpers.c create mode 100644 rust/helpers/kunit.c create mode 100644 rust/helpers/mutex.c create mode 100644 rust/helpers/page.c create mode 100644 rust/helpers/refcount.c create mode 100644 rust/helpers/signal.c create mode 100644 rust/helpers/slab.c create mode 100644 rust/helpers/spinlock.c create mode 100644 rust/helpers/task.c create mode 100644 rust/helpers/uaccess.c create mode 100644 rust/helpers/wait.c create mode 100644 rust/helpers/workqueue.c diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst index e3f388ef4ee4..a82926d7b379 100644 --- a/Documentation/rust/general-information.rst +++ b/Documentation/rust/general-information.rst @@ -75,7 +75,7 @@ should provide as-safe-as-possible abstractions as needed. .. code-block:: rust/bindings/ - (rust/helpers.c) + (rust/helpers/) include/ -----+ <-+ | | @@ -112,7 +112,7 @@ output files in the ``rust/bindings/`` directory. For parts of the C header that ``bindgen`` does not auto generate, e.g. C ``inline`` functions or non-trivial macros, it is acceptable to add a small -wrapper function to ``rust/helpers.c`` to make it available for the Rust side as +wrapper function to ``rust/helpers/`` to make it available for the Rust side as well. Abstractions diff --git a/rust/Makefile b/rust/Makefile index 8de3ebba9551..07e670d1b6d5 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -8,8 +8,8 @@ always-$(CONFIG_RUST) += exports_core_generated.h # Missing prototypes are expected in the helpers since these are exported # for Rust only, thus there is no header nor prototypes. -obj-$(CONFIG_RUST) += helpers.o -CFLAGS_REMOVE_helpers.o = -Wmissing-prototypes -Wmissing-declarations +obj-$(CONFIG_RUST) += helpers/helpers.o +CFLAGS_REMOVE_helpers/helpers.o = -Wmissing-prototypes -Wmissing-declarations always-$(CONFIG_RUST) += libmacros.so no-clean-files += libmacros.so @@ -299,7 +299,7 @@ $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \ -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations $(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \ sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@ -$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE +$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE $(call if_changed_dep,bindgen) quiet_cmd_exports = EXPORTS $@ diff --git a/rust/helpers.c b/rust/helpers.c deleted file mode 100644 index 92d3c03ae1bd..000000000000 --- a/rust/helpers.c +++ /dev/null @@ -1,239 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions - * cannot be called either. This file explicitly creates functions ("helpers") - * that wrap those so that they can be called from Rust. - * - * Even though Rust kernel modules should never use the bindings directly, some - * of these helpers need to be exported because Rust generics and inlined - * functions may not get their code generated in the crate where they are - * defined. Other helpers, called from non-inline functions, may not be - * exported, in principle. However, in general, the Rust compiler does not - * guarantee codegen will be performed for a non-inline function either. - * Therefore, this file exports all the helpers. In the future, this may be - * revisited to reduce the number of exports after the compiler is informed - * about the places codegen is required. - * - * All symbols are exported as GPL-only to guarantee no GPL-only feature is - * accidentally exposed. - * - * Sorted alphabetically. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -__noreturn void rust_helper_BUG(void) -{ - BUG(); -} -EXPORT_SYMBOL_GPL(rust_helper_BUG); - -unsigned long rust_helper_copy_from_user(void *to, const void __user *from, - unsigned long n) -{ - return copy_from_user(to, from, n); -} -EXPORT_SYMBOL_GPL(rust_helper_copy_from_user); - -unsigned long rust_helper_copy_to_user(void __user *to, const void *from, - unsigned long n) -{ - return copy_to_user(to, from, n); -} -EXPORT_SYMBOL_GPL(rust_helper_copy_to_user); - -void rust_helper_mutex_lock(struct mutex *lock) -{ - mutex_lock(lock); -} -EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); - -void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, - struct lock_class_key *key) -{ -#ifdef CONFIG_DEBUG_SPINLOCK - __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); -#else - spin_lock_init(lock); -#endif -} -EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init); - -void rust_helper_spin_lock(spinlock_t *lock) -{ - spin_lock(lock); -} -EXPORT_SYMBOL_GPL(rust_helper_spin_lock); - -void rust_helper_spin_unlock(spinlock_t *lock) -{ - spin_unlock(lock); -} -EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); - -void rust_helper_init_wait(struct wait_queue_entry *wq_entry) -{ - init_wait(wq_entry); -} -EXPORT_SYMBOL_GPL(rust_helper_init_wait); - -int rust_helper_signal_pending(struct task_struct *t) -{ - return signal_pending(t); -} -EXPORT_SYMBOL_GPL(rust_helper_signal_pending); - -struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) -{ - return alloc_pages(gfp_mask, order); -} -EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); - -void *rust_helper_kmap_local_page(struct page *page) -{ - return kmap_local_page(page); -} -EXPORT_SYMBOL_GPL(rust_helper_kmap_local_page); - -void rust_helper_kunmap_local(const void *addr) -{ - kunmap_local(addr); -} -EXPORT_SYMBOL_GPL(rust_helper_kunmap_local); - -refcount_t rust_helper_REFCOUNT_INIT(int n) -{ - return (refcount_t)REFCOUNT_INIT(n); -} -EXPORT_SYMBOL_GPL(rust_helper_REFCOUNT_INIT); - -void rust_helper_refcount_inc(refcount_t *r) -{ - refcount_inc(r); -} -EXPORT_SYMBOL_GPL(rust_helper_refcount_inc); - -bool rust_helper_refcount_dec_and_test(refcount_t *r) -{ - return refcount_dec_and_test(r); -} -EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); - -__force void *rust_helper_ERR_PTR(long err) -{ - return ERR_PTR(err); -} -EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); - -bool rust_helper_IS_ERR(__force const void *ptr) -{ - return IS_ERR(ptr); -} -EXPORT_SYMBOL_GPL(rust_helper_IS_ERR); - -long rust_helper_PTR_ERR(__force const void *ptr) -{ - return PTR_ERR(ptr); -} -EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); - -const char *rust_helper_errname(int err) -{ - return errname(err); -} -EXPORT_SYMBOL_GPL(rust_helper_errname); - -struct task_struct *rust_helper_get_current(void) -{ - return current; -} -EXPORT_SYMBOL_GPL(rust_helper_get_current); - -void rust_helper_get_task_struct(struct task_struct *t) -{ - get_task_struct(t); -} -EXPORT_SYMBOL_GPL(rust_helper_get_task_struct); - -void rust_helper_put_task_struct(struct task_struct *t) -{ - put_task_struct(t); -} -EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); - -struct kunit *rust_helper_kunit_get_current_test(void) -{ - return kunit_get_current_test(); -} -EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test); - -void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, - bool onstack, const char *name, - struct lock_class_key *key) -{ - __init_work(work, onstack); - work->data = (atomic_long_t)WORK_DATA_INIT(); - lockdep_init_map(&work->lockdep_map, name, key, 0); - INIT_LIST_HEAD(&work->entry); - work->func = func; -} -EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key); - -void * __must_check __realloc_size(2) -rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) -{ - return krealloc(objp, new_size, flags); -} -EXPORT_SYMBOL_GPL(rust_helper_krealloc); - -/* - * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can - * use it in contexts where Rust expects a `usize` like slice (array) indices. - * `usize` is defined to be the same as C's `uintptr_t` type (can hold any - * pointer) but not necessarily the same as `size_t` (can hold the size of any - * single object). Most modern platforms use the same concrete integer type for - * both of them, but in case we find ourselves on a platform where - * that's not true, fail early instead of risking ABI or - * integer-overflow issues. - * - * If your platform fails this assertion, it means that you are in - * danger of integer-overflow bugs (even if you attempt to add - * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on - * your platform such that `size_t` matches `uintptr_t` (i.e., to increase - * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). - */ -static_assert( - sizeof(size_t) == sizeof(uintptr_t) && - __alignof__(size_t) == __alignof__(uintptr_t), - "Rust code expects C `size_t` to match Rust `usize`" -); - -// This will soon be moved to a separate file, so no need to merge with above. -#include -#include - -void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) -{ - return blk_mq_rq_to_pdu(rq); -} -EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_to_pdu); - -struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) -{ - return blk_mq_rq_from_pdu(pdu); -} -EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_from_pdu); diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c new file mode 100644 index 000000000000..d99c965eb59b --- /dev/null +++ b/rust/helpers/blk.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +{ + return blk_mq_rq_to_pdu(rq); +} +EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_to_pdu); + +struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) +{ + return blk_mq_rq_from_pdu(pdu); +} +EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_from_pdu); diff --git a/rust/helpers/bug.c b/rust/helpers/bug.c new file mode 100644 index 000000000000..e2afbad23dcd --- /dev/null +++ b/rust/helpers/bug.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__noreturn void rust_helper_BUG(void) +{ + BUG(); +} +EXPORT_SYMBOL_GPL(rust_helper_BUG); diff --git a/rust/helpers/build_assert.c b/rust/helpers/build_assert.c new file mode 100644 index 000000000000..6a54b2680b14 --- /dev/null +++ b/rust/helpers/build_assert.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* + * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can + * use it in contexts where Rust expects a `usize` like slice (array) indices. + * `usize` is defined to be the same as C's `uintptr_t` type (can hold any + * pointer) but not necessarily the same as `size_t` (can hold the size of any + * single object). Most modern platforms use the same concrete integer type for + * both of them, but in case we find ourselves on a platform where + * that's not true, fail early instead of risking ABI or + * integer-overflow issues. + * + * If your platform fails this assertion, it means that you are in + * danger of integer-overflow bugs (even if you attempt to add + * `--no-size_t-is-usize`). It may be easiest to change the kernel ABI on + * your platform such that `size_t` matches `uintptr_t` (i.e., to increase + * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). + */ +static_assert( + sizeof(size_t) == sizeof(uintptr_t) && + __alignof__(size_t) == __alignof__(uintptr_t), + "Rust code expects C `size_t` to match Rust `usize`" +); diff --git a/rust/helpers/build_bug.c b/rust/helpers/build_bug.c new file mode 100644 index 000000000000..f3106f248485 --- /dev/null +++ b/rust/helpers/build_bug.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +const char *rust_helper_errname(int err) +{ + return errname(err); +} +EXPORT_SYMBOL_GPL(rust_helper_errname); diff --git a/rust/helpers/err.c b/rust/helpers/err.c new file mode 100644 index 000000000000..fba4e0be64f5 --- /dev/null +++ b/rust/helpers/err.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +__force void *rust_helper_ERR_PTR(long err) +{ + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); + +bool rust_helper_IS_ERR(__force const void *ptr) +{ + return IS_ERR(ptr); +} +EXPORT_SYMBOL_GPL(rust_helper_IS_ERR); + +long rust_helper_PTR_ERR(__force const void *ptr) +{ + return PTR_ERR(ptr); +} +EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c new file mode 100644 index 000000000000..2b54f22e8774 --- /dev/null +++ b/rust/helpers/helpers.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions + * cannot be called either. This file explicitly creates functions ("helpers") + * that wrap those so that they can be called from Rust. + * + * Even though Rust kernel modules should never use the bindings directly, some + * of these helpers need to be exported because Rust generics and inlined + * functions may not get their code generated in the crate where they are + * defined. Other helpers, called from non-inline functions, may not be + * exported, in principle. However, in general, the Rust compiler does not + * guarantee codegen will be performed for a non-inline function either. + * Therefore, this file exports all the helpers. In the future, this may be + * revisited to reduce the number of exports after the compiler is informed + * about the places codegen is required. + * + * All symbols are exported as GPL-only to guarantee no GPL-only feature is + * accidentally exposed. + * + * Sorted alphabetically. + */ + +#include "blk.c" +#include "bug.c" +#include "build_assert.c" +#include "build_bug.c" +#include "err.c" +#include "kunit.c" +#include "mutex.c" +#include "page.c" +#include "refcount.c" +#include "signal.c" +#include "slab.c" +#include "spinlock.c" +#include "task.c" +#include "uaccess.c" +#include "wait.c" +#include "workqueue.c" diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c new file mode 100644 index 000000000000..905e4ff4424a --- /dev/null +++ b/rust/helpers/kunit.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +struct kunit *rust_helper_kunit_get_current_test(void) +{ + return kunit_get_current_test(); +} +EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test); diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c new file mode 100644 index 000000000000..29fd141c387d --- /dev/null +++ b/rust/helpers/mutex.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void rust_helper_mutex_lock(struct mutex *lock) +{ + mutex_lock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); diff --git a/rust/helpers/page.c b/rust/helpers/page.c new file mode 100644 index 000000000000..7fd333411a88 --- /dev/null +++ b/rust/helpers/page.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} +EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); + +void *rust_helper_kmap_local_page(struct page *page) +{ + return kmap_local_page(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap_local_page); + +void rust_helper_kunmap_local(const void *addr) +{ + kunmap_local(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap_local); diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c new file mode 100644 index 000000000000..13ab64805f77 --- /dev/null +++ b/rust/helpers/refcount.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +refcount_t rust_helper_REFCOUNT_INIT(int n) +{ + return (refcount_t)REFCOUNT_INIT(n); +} +EXPORT_SYMBOL_GPL(rust_helper_REFCOUNT_INIT); + +void rust_helper_refcount_inc(refcount_t *r) +{ + refcount_inc(r); +} +EXPORT_SYMBOL_GPL(rust_helper_refcount_inc); + +bool rust_helper_refcount_dec_and_test(refcount_t *r) +{ + return refcount_dec_and_test(r); +} +EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c new file mode 100644 index 000000000000..d44e8096b8a9 --- /dev/null +++ b/rust/helpers/signal.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +int rust_helper_signal_pending(struct task_struct *t) +{ + return signal_pending(t); +} +EXPORT_SYMBOL_GPL(rust_helper_signal_pending); diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c new file mode 100644 index 000000000000..3e0a1a173d8a --- /dev/null +++ b/rust/helpers/slab.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void * __must_check __realloc_size(2) +rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +{ + return krealloc(objp, new_size, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_krealloc); diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c new file mode 100644 index 000000000000..04fd8ddb4986 --- /dev/null +++ b/rust/helpers/spinlock.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_SPINLOCK + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +#else + spin_lock_init(lock); +#endif +} +EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init); + +void rust_helper_spin_lock(spinlock_t *lock) +{ + spin_lock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock); + +void rust_helper_spin_unlock(spinlock_t *lock) +{ + spin_unlock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); diff --git a/rust/helpers/task.c b/rust/helpers/task.c new file mode 100644 index 000000000000..b176c347f0d4 --- /dev/null +++ b/rust/helpers/task.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +struct task_struct *rust_helper_get_current(void) +{ + return current; +} +EXPORT_SYMBOL_GPL(rust_helper_get_current); + +void rust_helper_get_task_struct(struct task_struct *t) +{ + get_task_struct(t); +} +EXPORT_SYMBOL_GPL(rust_helper_get_task_struct); + +void rust_helper_put_task_struct(struct task_struct *t) +{ + put_task_struct(t); +} +EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c new file mode 100644 index 000000000000..3d004ac1c180 --- /dev/null +++ b/rust/helpers/uaccess.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +unsigned long rust_helper_copy_from_user(void *to, const void __user *from, + unsigned long n) +{ + return copy_from_user(to, from, n); +} +EXPORT_SYMBOL_GPL(rust_helper_copy_from_user); + +unsigned long rust_helper_copy_to_user(void __user *to, const void *from, + unsigned long n) +{ + return copy_to_user(to, from, n); +} +EXPORT_SYMBOL_GPL(rust_helper_copy_to_user); diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c new file mode 100644 index 000000000000..bf361f40c7cb --- /dev/null +++ b/rust/helpers/wait.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +{ + init_wait(wq_entry); +} +EXPORT_SYMBOL_GPL(rust_helper_init_wait); diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c new file mode 100644 index 000000000000..12e2ee66aa4f --- /dev/null +++ b/rust/helpers/workqueue.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, + bool onstack, const char *name, + struct lock_class_key *key) +{ + __init_work(work, onstack); + work->data = (atomic_long_t)WORK_DATA_INIT(); + lockdep_init_map(&work->lockdep_map, name, key, 0); + INIT_LIST_HEAD(&work->entry); + work->func = func; +} +EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key); From 289088d54623a1a50bb3ff79f7331bbe501ea591 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:18 +0200 Subject: [PATCH 019/111] rust: module: add static pointer to `{init,cleanup}_module()` Add the equivalent of the `___ADDRESSABLE()` annotation in the `module_{init,exit}` macros to the Rust `module!` macro. Without this, `objtool` would complain if enabled for Rust (under IBT builds), e.g.: samples/rust/rust_print.o: warning: objtool: cleanup_module(): not an indirect call target samples/rust/rust_print.o: warning: objtool: init_module(): not an indirect call target Tested-by: Alice Ryhl Tested-by: Benno Lossin Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20240725183325.122827-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/macros/module.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 411dc103d82e..571ffa2e189c 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -256,6 +256,12 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { unsafe {{ __init() }} }} + #[cfg(MODULE)] + #[doc(hidden)] + #[used] + #[link_section = \".init.data\"] + static __UNIQUE_ID___addressable_init_module: unsafe extern \"C\" fn() -> i32 = init_module; + #[cfg(MODULE)] #[doc(hidden)] #[no_mangle] @@ -269,6 +275,12 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { unsafe {{ __exit() }} }} + #[cfg(MODULE)] + #[doc(hidden)] + #[used] + #[link_section = \".exit.data\"] + static __UNIQUE_ID___addressable_cleanup_module: extern \"C\" fn() = cleanup_module; + // Built-in modules are initialized through an initcall pointer // and the identifiers need to be unique. #[cfg(not(MODULE))] From 284a3ac4a96c619af269dfbdef5431a9a2a34d3b Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:19 +0200 Subject: [PATCH 020/111] x86/rust: support MITIGATION_RETPOLINE Support `MITIGATION_RETPOLINE` by enabling the target features that Clang does. The existing target feature being enabled was a leftover from our old `rust` branch, and it is not enough: the target feature `retpoline-external-thunk` only implies `retpoline-indirect-calls`, but not `retpoline-indirect-branches` (see LLVM's `X86.td`), unlike Clang's flag of the same name `-mretpoline-external-thunk` which does imply both (see Clang's `lib/Driver/ToolChains/Arch/X86.cpp`). Without this, `objtool` would complain if enabled for Rust, e.g.: rust/core.o: warning: objtool: _R...escape_default+0x13: indirect jump found in RETPOLINE build In addition, change the comment to note that LLVM is the one disabling jump tables when retpoline is enabled, thus we do not need to use `-Zno-jump-tables` for Rust here -- see commit c58f2166ab39 ("Introduce the "retpoline" x86 mitigation technique ...") [1]: The goal is simple: avoid generating code which contains an indirect branch that could have its prediction poisoned by an attacker. In many cases, the compiler can simply use directed conditional branches and a small search tree. LLVM already has support for lowering switches in this way and the first step of this patch is to disable jump-table lowering of switches and introduce a pass to rewrite explicit indirectbr sequences into a switch over integers. As well as a live example at [2]. These should be eventually enabled via `-Ctarget-feature` when `rustc` starts recognizing them (or via a new dedicated flag) [3]. Cc: Daniel Borkmann Link: https://github.com/llvm/llvm-project/commit/c58f2166ab3987f37cb0d7815b561bff5a20a69a [1] Link: https://godbolt.org/z/G4YPr58qG [2] Link: https://github.com/rust-lang/rust/issues/116852 [3] Reviewed-by: Gary Guo Tested-by: Alice Ryhl Tested-by: Benno Lossin Link: https://github.com/Rust-for-Linux/linux/issues/945 Link: https://lore.kernel.org/r/20240725183325.122827-3-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- arch/x86/Makefile | 2 +- scripts/generate_rust_target.rs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 801fd85c3ef6..e8214bff1aeb 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -220,7 +220,7 @@ ifdef CONFIG_MITIGATION_RETPOLINE KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) # Additionally, avoid generating expensive indirect jumps which # are subject to retpolines for small number of switch cases. - # clang turns off jump table generation by default when under + # LLVM turns off jump table generation by default when under # retpoline builds, however, gcc does not for x86. This has # only been fixed starting from gcc stable version 8.4.0 and # onwards, but not for older ones. See gcc bug #86952. diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 404edf7587e0..836fdf622c2d 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -164,7 +164,14 @@ fn main() { ); let mut features = "-mmx,+soft-float".to_string(); if cfg.has("MITIGATION_RETPOLINE") { + // The kernel uses `-mretpoline-external-thunk` (for Clang), which Clang maps to the + // target feature of the same name plus the other two target features in + // `clang/lib/Driver/ToolChains/Arch/X86.cpp`. These should be eventually enabled via + // `-Ctarget-feature` when `rustc` starts recognizing them (or via a new dedicated + // flag); see https://github.com/rust-lang/rust/issues/116852. features += ",+retpoline-external-thunk"; + features += ",+retpoline-indirect-branches"; + features += ",+retpoline-indirect-calls"; } ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); From d7868550d5731e05148c881f035423f009a2b4d5 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:20 +0200 Subject: [PATCH 021/111] x86/rust: support MITIGATION_RETHUNK The Rust compiler added support for `-Zfunction-return=thunk-extern` [1] in 1.76.0 [2], i.e. the equivalent of `-mfunction-return=thunk-extern`. Thus add support for `MITIGATION_RETHUNK`. Without this, `objtool` would warn if enabled for Rust and already warns under IBT builds, e.g.: samples/rust/rust_print.o: warning: objtool: _R...init+0xa5c: 'naked' return found in RETHUNK build Link: https://github.com/rust-lang/rust/issues/116853 [1] Link: https://github.com/rust-lang/rust/pull/116892 [2] Reviewed-by: Gary Guo Tested-by: Alice Ryhl Tested-by: Benno Lossin Reviewed-by: Thomas Gleixner Link: https://github.com/Rust-for-Linux/linux/issues/945 Link: https://lore.kernel.org/r/20240725183325.122827-4-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- arch/x86/Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/Makefile b/arch/x86/Makefile index e8214bff1aeb..a1883a30a5d8 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -24,11 +24,15 @@ RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch-cs-prefix) ifdef CONFIG_MITIGATION_RETHUNK RETHUNK_CFLAGS := -mfunction-return=thunk-extern +RETHUNK_RUSTFLAGS := -Zfunction-return=thunk-extern RETPOLINE_CFLAGS += $(RETHUNK_CFLAGS) +RETPOLINE_RUSTFLAGS += $(RETHUNK_RUSTFLAGS) endif export RETHUNK_CFLAGS +export RETHUNK_RUSTFLAGS export RETPOLINE_CFLAGS +export RETPOLINE_RUSTFLAGS export RETPOLINE_VDSO_CFLAGS # For gcc stack alignment is specified with -mpreferred-stack-boundary, @@ -218,6 +222,7 @@ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables # Avoid indirect branches in kernel to deal with Spectre ifdef CONFIG_MITIGATION_RETPOLINE KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) + KBUILD_RUSTFLAGS += $(RETPOLINE_RUSTFLAGS) # Additionally, avoid generating expensive indirect jumps which # are subject to retpolines for small number of switch cases. # LLVM turns off jump table generation by default when under From fc582dfc1f20476cab9d43d0ee8ec0a6bfe13485 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:21 +0200 Subject: [PATCH 022/111] x86/rust: support MITIGATION_SLS Support `MITIGATION_SLS` by enabling the target features that Clang does. Without this, `objtool` would complain if enabled for Rust, e.g.: rust/core.o: warning: objtool: _R...next_up+0x44: missing int3 after ret These should be eventually enabled via `-Ctarget-feature` when `rustc` starts recognizing them (or via a new dedicated flag) [1]. Link: https://github.com/rust-lang/rust/issues/116851 [1] Reviewed-by: Gary Guo Tested-by: Alice Ryhl Tested-by: Benno Lossin Link: https://lore.kernel.org/r/20240725183325.122827-5-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- scripts/generate_rust_target.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 836fdf622c2d..863720777313 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -173,6 +173,14 @@ fn main() { features += ",+retpoline-indirect-branches"; features += ",+retpoline-indirect-calls"; } + if cfg.has("MITIGATION_SLS") { + // The kernel uses `-mharden-sls=all`, which Clang maps to both these target features in + // `clang/lib/Driver/ToolChains/Arch/X86.cpp`. These should be eventually enabled via + // `-Ctarget-feature` when `rustc` starts recognizing them (or via a new dedicated + // flag); see https://github.com/rust-lang/rust/issues/116851. + features += ",+harden-sls-ijmp"; + features += ",+harden-sls-ret"; + } ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); ts.push("target-pointer-width", "64"); From 56d680dd23c38067a32fb8aeb74d6ce838fcf26c Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:22 +0200 Subject: [PATCH 023/111] objtool/rust: list `noreturn` Rust functions Rust functions may be `noreturn` (i.e. diverging) by returning the "never" type, `!`, e.g. fn f() -> ! { loop {} } Thus list the known `noreturn` functions to avoid such warnings. Without this, `objtool` would complain if enabled for Rust, e.g.: rust/core.o: warning: objtool: _R...9panic_fmt() falls through to next function _R...18panic_nounwind_fmt() rust/alloc.o: warning: objtool: .text: unexpected end of section In order to do so, we cannot match symbols' names exactly, for two reasons: - Rust mangling scheme [1] contains disambiguators [2] which we cannot predict (e.g. they may vary depending on the compiler version). One possibility to solve this would be to parse v0 and ignore/zero those before comparison. - Some of the diverging functions come from `core`, i.e. the Rust standard library, which may change with each compiler version since they are implementation details (e.g. `panic_internals`). Thus, to workaround both issues, only part of the symbols are matched, instead of using the `NORETURN` macro in `noreturns.h`. Ideally, just like for the C side, we should have a better solution. For instance, the compiler could give us the list via something like: $ rustc --emit=noreturns ... [ Kees agrees this should be automated and Peter says: So it would be fairly simple to make objtool consume a magic section emitted by the compiler.. I think we've asked the compiler folks for that at some point even, but I don't have clear recollections. We will ask upstream Rust about it. And if they agree, then perhaps we can get Clang/GCC to implement something similar too -- for this sort of thing we can take advantage of the shorter cycles of `rustc` as well as their unstable features concept to experiment. Gary proposed using DWARF (though it would need to be available), and wrote a proof of concept script using the `object` and `gimli` crates: https://gist.github.com/nbdd0121/449692570622c2f46a29ad9f47c3379a - Miguel ] Link: https://rust-lang.github.io/rfcs/2603-rust-symbol-name-mangling-v0.html [1] Link: https://doc.rust-lang.org/rustc/symbol-mangling/v0.html#disambiguator [2] Acked-by: Peter Zijlstra (Intel) Tested-by: Alice Ryhl Reviewed-by: Kees Cook Tested-by: Benno Lossin Link: https://lore.kernel.org/r/20240725183325.122827-6-ojeda@kernel.org [ Added `len_mismatch_fail` symbol for new `kernel` crate code merged since then as well as 3 more `core::panicking` symbols that appear in `RUST_DEBUG_ASSERTIONS=y` builds. - Miguel ] Signed-off-by: Miguel Ojeda --- tools/objtool/check.c | 52 ++++++++++++++++++++++++++++++++++++++- tools/objtool/noreturns.h | 2 ++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 01237d167223..d086f207a3d3 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -177,6 +177,52 @@ static bool is_sibling_call(struct instruction *insn) return (is_static_jump(insn) && insn_call_dest(insn)); } +/* + * Checks if a string ends with another. + */ +static bool str_ends_with(const char *s, const char *sub) +{ + const int slen = strlen(s); + const int sublen = strlen(sub); + + if (sublen > slen) + return 0; + + return !memcmp(s + slen - sublen, sub, sublen); +} + +/* + * Checks if a function is a Rust "noreturn" one. + */ +static bool is_rust_noreturn(const struct symbol *func) +{ + /* + * If it does not start with "_R", then it is not a Rust symbol. + */ + if (strncmp(func->name, "_R", 2)) + return false; + + /* + * These are just heuristics -- we do not control the precise symbol + * name, due to the crate disambiguators (which depend on the compiler) + * as well as changes to the source code itself between versions (since + * these come from the Rust standard library). + */ + return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || + str_ends_with(func->name, "_4core6option13unwrap_failed") || + str_ends_with(func->name, "_4core6result13unwrap_failed") || + str_ends_with(func->name, "_4core9panicking5panic") || + str_ends_with(func->name, "_4core9panicking9panic_fmt") || + str_ends_with(func->name, "_4core9panicking14panic_explicit") || + str_ends_with(func->name, "_4core9panicking14panic_nounwind") || + str_ends_with(func->name, "_4core9panicking18panic_bounds_check") || + str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || + str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || + strstr(func->name, "_4core9panicking11panic_const24panic_const_") || + (strstr(func->name, "_4core5slice5index24slice_") && + str_ends_with(func->name, "_fail")); +} + /* * This checks to see if the given function is a "noreturn" function. * @@ -202,10 +248,14 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, if (!func) return false; - if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) + if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) { + if (is_rust_noreturn(func)) + return true; + for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) if (!strcmp(func->name, global_noreturns[i])) return true; + } if (func->bind == STB_WEAK) return false; diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h index 1e8141ef1b15..e7da92489167 100644 --- a/tools/objtool/noreturns.h +++ b/tools/objtool/noreturns.h @@ -39,6 +39,8 @@ NORETURN(panic) NORETURN(panic_smp_self_stop) NORETURN(rest_init) NORETURN(rewind_stack_and_make_dead) +NORETURN(rust_begin_unwind) +NORETURN(rust_helper_BUG) NORETURN(sev_es_terminate) NORETURN(snp_abort) NORETURN(start_kernel) From c4d7f546dd9aa9780716cdb07416ca97264dce43 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 25 Jul 2024 20:33:23 +0200 Subject: [PATCH 024/111] objtool/kbuild/rust: enable objtool for Rust Now that we should be `objtool`-warning free, enable `objtool` for Rust too. Before this patch series, we were already getting warnings under e.g. IBT builds, since those would see Rust code via `vmlinux.o`. Tested-by: Alice Ryhl Tested-by: Benno Lossin Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20240725183325.122827-7-ojeda@kernel.org [ Solved trivial conflict. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/Makefile | 22 ++++++++++++++-------- scripts/Makefile.build | 9 +++++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 07e670d1b6d5..99204e33f1dd 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -344,7 +344,8 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --crate-type rlib -L$(objtree)/$(obj) \ --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ --sysroot=/dev/null \ - $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) + $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ + $(cmd_objtool) rust-analyzer: $(Q)$(srctree)/scripts/generate_rust_analyzer.py \ @@ -366,44 +367,49 @@ ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) __ashlti3 __lshrti3 endif +define rule_rustc_library + $(call cmd_and_fixdep,rustc_library) + $(call cmd,gen_objtooldep) +endef + $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Wunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) $(obj)/core.o: private rustc_target_flags = $(core-cfgs) $(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),) $(obj)/core.o: scripts/target.json endif $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' $(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) $(obj)/alloc.o: private skip_clippy = 1 $(obj)/alloc.o: private skip_flags = -Wunreachable_pub $(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) $(obj)/alloc.o: $(RUST_LIB_SRC)/alloc/src/lib.rs $(obj)/compiler_builtins.o FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) $(obj)/build_error.o: $(src)/build_error.rs $(obj)/compiler_builtins.o FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) $(obj)/bindings.o: $(src)/bindings/lib.rs \ $(obj)/compiler_builtins.o \ $(obj)/bindings/bindings_generated.rs \ $(obj)/bindings/bindings_helpers_generated.rs FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) $(obj)/uapi.o: $(src)/uapi/lib.rs \ $(obj)/compiler_builtins.o \ $(obj)/uapi/uapi_generated.rs FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) $(obj)/kernel.o: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros --extern bindings --extern uapi $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE - +$(call if_changed_dep,rustc_library) + +$(call if_changed_rule,rustc_library) endif # CONFIG_RUST diff --git a/scripts/Makefile.build b/scripts/Makefile.build index efacca63c897..72b1232b1f7d 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -288,10 +288,15 @@ rust_common_cmd = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< + cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< $(cmd_objtool) + +define rule_rustc_o_rs + $(call cmd_and_fixdep,rustc_o_rs) + $(call cmd,gen_objtooldep) +endef $(obj)/%.o: $(obj)/%.rs FORCE - +$(call if_changed_dep,rustc_o_rs) + +$(call if_changed_rule,rustc_o_rs) quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ From e26fa546042add70944d018b930530d16b3cf626 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 17 Aug 2024 17:51:32 +0100 Subject: [PATCH 025/111] rust: kbuild: auto generate helper exports This removes the need to explicitly export all symbols. Generate helper exports similarly to what's currently done for Rust crates. These helpers are exclusively called from within Rust code and therefore can be treated similar as other Rust symbols. Signed-off-by: Gary Guo Reviewed-by: Boqun Feng Tested-by: Boqun Feng Link: https://lore.kernel.org/r/20240817165302.3852499-1-gary@garyguo.net [ Fixed dependency path, reworded slightly, edited comment a bit and rebased on top of the changes made when applying Andreas' patch (e.g. no `README.md` anymore, so moved the edits). - Miguel ] Signed-off-by: Miguel Ojeda --- rust/Makefile | 16 ++++++++++++++-- rust/exports.c | 1 + rust/helpers/blk.c | 2 -- rust/helpers/bug.c | 1 - rust/helpers/build_bug.c | 1 - rust/helpers/err.c | 3 --- rust/helpers/helpers.c | 13 ------------- rust/helpers/kunit.c | 1 - rust/helpers/mutex.c | 1 - rust/helpers/page.c | 3 --- rust/helpers/refcount.c | 3 --- rust/helpers/signal.c | 1 - rust/helpers/slab.c | 1 - rust/helpers/spinlock.c | 3 --- rust/helpers/task.c | 3 --- rust/helpers/uaccess.c | 2 -- rust/helpers/wait.c | 1 - rust/helpers/workqueue.c | 1 - 18 files changed, 15 insertions(+), 42 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index 99204e33f1dd..7ea5905f544c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -16,8 +16,8 @@ no-clean-files += libmacros.so always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o -always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \ - exports_kernel_generated.h +always-$(CONFIG_RUST) += exports_alloc_generated.h exports_helpers_generated.h \ + exports_bindings_generated.h exports_kernel_generated.h always-$(CONFIG_RUST) += uapi/uapi_generated.rs obj-$(CONFIG_RUST) += uapi.o @@ -313,6 +313,18 @@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE $(call if_changed,exports) +# Even though Rust kernel modules should never use the bindings directly, +# symbols from the `bindings` crate and the C helpers need to be exported +# because Rust generics and inlined functions may not get their code generated +# in the crate where they are defined. Other helpers, called from non-inline +# functions, may not be exported, in principle. However, in general, the Rust +# compiler does not guarantee codegen will be performed for a non-inline +# function either. Therefore, we export all symbols from helpers and bindings. +# In the future, this may be revisited to reduce the number of exports after +# the compiler is informed about the places codegen is required. +$(obj)/exports_helpers_generated.h: $(obj)/helpers/helpers.o FORCE + $(call if_changed,exports) + $(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE $(call if_changed,exports) diff --git a/rust/exports.c b/rust/exports.c index 3803c21d1403..e5695f3b45b7 100644 --- a/rust/exports.c +++ b/rust/exports.c @@ -17,6 +17,7 @@ #include "exports_core_generated.h" #include "exports_alloc_generated.h" +#include "exports_helpers_generated.h" #include "exports_bindings_generated.h" #include "exports_kernel_generated.h" diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c index d99c965eb59b..cc9f4e6a2d23 100644 --- a/rust/helpers/blk.c +++ b/rust/helpers/blk.c @@ -7,10 +7,8 @@ void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) { return blk_mq_rq_to_pdu(rq); } -EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_to_pdu); struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) { return blk_mq_rq_from_pdu(pdu); } -EXPORT_SYMBOL_GPL(rust_helper_blk_mq_rq_from_pdu); diff --git a/rust/helpers/bug.c b/rust/helpers/bug.c index e2afbad23dcd..e2d13babc737 100644 --- a/rust/helpers/bug.c +++ b/rust/helpers/bug.c @@ -6,4 +6,3 @@ __noreturn void rust_helper_BUG(void) { BUG(); } -EXPORT_SYMBOL_GPL(rust_helper_BUG); diff --git a/rust/helpers/build_bug.c b/rust/helpers/build_bug.c index f3106f248485..e994f7b5928c 100644 --- a/rust/helpers/build_bug.c +++ b/rust/helpers/build_bug.c @@ -7,4 +7,3 @@ const char *rust_helper_errname(int err) { return errname(err); } -EXPORT_SYMBOL_GPL(rust_helper_errname); diff --git a/rust/helpers/err.c b/rust/helpers/err.c index fba4e0be64f5..be3d45ef78a2 100644 --- a/rust/helpers/err.c +++ b/rust/helpers/err.c @@ -7,16 +7,13 @@ __force void *rust_helper_ERR_PTR(long err) { return ERR_PTR(err); } -EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); bool rust_helper_IS_ERR(__force const void *ptr) { return IS_ERR(ptr); } -EXPORT_SYMBOL_GPL(rust_helper_IS_ERR); long rust_helper_PTR_ERR(__force const void *ptr) { return PTR_ERR(ptr); } -EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 2b54f22e8774..173533616c91 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -4,19 +4,6 @@ * cannot be called either. This file explicitly creates functions ("helpers") * that wrap those so that they can be called from Rust. * - * Even though Rust kernel modules should never use the bindings directly, some - * of these helpers need to be exported because Rust generics and inlined - * functions may not get their code generated in the crate where they are - * defined. Other helpers, called from non-inline functions, may not be - * exported, in principle. However, in general, the Rust compiler does not - * guarantee codegen will be performed for a non-inline function either. - * Therefore, this file exports all the helpers. In the future, this may be - * revisited to reduce the number of exports after the compiler is informed - * about the places codegen is required. - * - * All symbols are exported as GPL-only to guarantee no GPL-only feature is - * accidentally exposed. - * * Sorted alphabetically. */ diff --git a/rust/helpers/kunit.c b/rust/helpers/kunit.c index 905e4ff4424a..9d725067eb3b 100644 --- a/rust/helpers/kunit.c +++ b/rust/helpers/kunit.c @@ -7,4 +7,3 @@ struct kunit *rust_helper_kunit_get_current_test(void) { return kunit_get_current_test(); } -EXPORT_SYMBOL_GPL(rust_helper_kunit_get_current_test); diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c index 29fd141c387d..200db7e6279f 100644 --- a/rust/helpers/mutex.c +++ b/rust/helpers/mutex.c @@ -7,4 +7,3 @@ void rust_helper_mutex_lock(struct mutex *lock) { mutex_lock(lock); } -EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); diff --git a/rust/helpers/page.c b/rust/helpers/page.c index 7fd333411a88..b3f2b8fbf87f 100644 --- a/rust/helpers/page.c +++ b/rust/helpers/page.c @@ -7,16 +7,13 @@ struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) { return alloc_pages(gfp_mask, order); } -EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); void *rust_helper_kmap_local_page(struct page *page) { return kmap_local_page(page); } -EXPORT_SYMBOL_GPL(rust_helper_kmap_local_page); void rust_helper_kunmap_local(const void *addr) { kunmap_local(addr); } -EXPORT_SYMBOL_GPL(rust_helper_kunmap_local); diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c index 13ab64805f77..f47afc148ec3 100644 --- a/rust/helpers/refcount.c +++ b/rust/helpers/refcount.c @@ -7,16 +7,13 @@ refcount_t rust_helper_REFCOUNT_INIT(int n) { return (refcount_t)REFCOUNT_INIT(n); } -EXPORT_SYMBOL_GPL(rust_helper_REFCOUNT_INIT); void rust_helper_refcount_inc(refcount_t *r) { refcount_inc(r); } -EXPORT_SYMBOL_GPL(rust_helper_refcount_inc); bool rust_helper_refcount_dec_and_test(refcount_t *r) { return refcount_dec_and_test(r); } -EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c index d44e8096b8a9..63c407f80c26 100644 --- a/rust/helpers/signal.c +++ b/rust/helpers/signal.c @@ -7,4 +7,3 @@ int rust_helper_signal_pending(struct task_struct *t) { return signal_pending(t); } -EXPORT_SYMBOL_GPL(rust_helper_signal_pending); diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index 3e0a1a173d8a..f043e087f9d6 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -7,4 +7,3 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) { return krealloc(objp, new_size, flags); } -EXPORT_SYMBOL_GPL(rust_helper_krealloc); diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 04fd8ddb4986..acc1376b833c 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -12,16 +12,13 @@ void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, spin_lock_init(lock); #endif } -EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init); void rust_helper_spin_lock(spinlock_t *lock) { spin_lock(lock); } -EXPORT_SYMBOL_GPL(rust_helper_spin_lock); void rust_helper_spin_unlock(spinlock_t *lock) { spin_unlock(lock); } -EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); diff --git a/rust/helpers/task.c b/rust/helpers/task.c index b176c347f0d4..7ac789232d11 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -7,16 +7,13 @@ struct task_struct *rust_helper_get_current(void) { return current; } -EXPORT_SYMBOL_GPL(rust_helper_get_current); void rust_helper_get_task_struct(struct task_struct *t) { get_task_struct(t); } -EXPORT_SYMBOL_GPL(rust_helper_get_task_struct); void rust_helper_put_task_struct(struct task_struct *t) { put_task_struct(t); } -EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index 3d004ac1c180..f49076f813cd 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -7,11 +7,9 @@ unsigned long rust_helper_copy_from_user(void *to, const void __user *from, { return copy_from_user(to, from, n); } -EXPORT_SYMBOL_GPL(rust_helper_copy_from_user); unsigned long rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) { return copy_to_user(to, from, n); } -EXPORT_SYMBOL_GPL(rust_helper_copy_to_user); diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c index bf361f40c7cb..c7336bbf2750 100644 --- a/rust/helpers/wait.c +++ b/rust/helpers/wait.c @@ -7,4 +7,3 @@ void rust_helper_init_wait(struct wait_queue_entry *wq_entry) { init_wait(wq_entry); } -EXPORT_SYMBOL_GPL(rust_helper_init_wait); diff --git a/rust/helpers/workqueue.c b/rust/helpers/workqueue.c index 12e2ee66aa4f..f59427acc323 100644 --- a/rust/helpers/workqueue.c +++ b/rust/helpers/workqueue.c @@ -13,4 +13,3 @@ void rust_helper_init_work_with_key(struct work_struct *work, work_func_t func, INIT_LIST_HEAD(&work->entry); work->func = func; } -EXPORT_SYMBOL_GPL(rust_helper_init_work_with_key); From 1d15880378662ade209eb9289f9f03c98b431254 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Fri, 9 Aug 2024 13:28:35 +0000 Subject: [PATCH 026/111] rust: sort blk includes in bindings_helper.h The headers in this file are sorted alphabetically, which makes it easy to quickly resolve conflicts by selecting all of the headers and invoking :'<,'>sort to sort them. To keep this technique to resolve conflicts working, also apply sorting to symbols that are not letters. This file is very prone to merge conflicts, so I think keeping conflict resolution really easy is more important than not messing with git blame history. These includes were originally introduced in commit 3253aba3408a ("rust: block: introduce `kernel::block::mq` module"). Signed-off-by: Alice Ryhl Acked-by: Danilo Krummrich Link: https://lore.kernel.org/r/20240809132835.274603-1-aliceryhl@google.com Signed-off-by: Miguel Ojeda --- rust/bindings/bindings_helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index b940a5777330..ae82e9c941af 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -7,8 +7,8 @@ */ #include -#include #include +#include #include #include #include From 76501d19c6af43054492420ae53a9bd2d4de50b3 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Wed, 14 Aug 2024 18:37:22 +0200 Subject: [PATCH 027/111] rust: enable bindgen's `--enable-function-attribute-detection` flag `bindgen` is able to detect certain function attributes and annotate functions correspondingly in its output for the Rust side, when the `--enable-function-attribute-detection` is passed. In particular, it is currently able to use `__must_check` in C (`#[must_use]` in Rust), which give us a bunch of annotations that are nice to have to prevent possible issues in Rust abstractions, e.g.: extern "C" { + #[must_use] pub fn kobject_add( kobj: *mut kobject, parent: *mut kobject, fmt: *const core::ffi::c_char, ... ) -> core::ffi::c_int; } Apparently, there are edge cases where this can make generation very slow, which is why it is behind a flag [1], but it does not seem to affect us in any major way at the moment. Thus enable it. Link: https://github.com/rust-lang/rust-bindgen/issues/1465 [1] Link: https://lore.kernel.org/rust-for-linux/CANiq72=u5Nrz_NW3U3_VqywJkD8pECA07q2pFDd1wjtXOWdkAQ@mail.gmail.com/ Reviewed-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Gary Guo Acked-by: Danilo Krummrich Link: https://lore.kernel.org/r/20240814163722.1550064-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/Makefile b/rust/Makefile index 7ea5905f544c..c24c3689e7b4 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -270,7 +270,7 @@ quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) \ --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \ - --no-debug '.*' \ + --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) From 01db99b272318da75a1aa5a81f75adb9d32f676e Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Mon, 19 Aug 2024 11:24:35 +0000 Subject: [PATCH 028/111] rust: kernel: add `drop_contents` to `BoxExt` Sometimes (see [1]) it is necessary to drop the value inside of a `Box`, but retain the allocation. For example to reuse the allocation in the future. Introduce a new function `drop_contents` that turns a `Box` into `Box>` by dropping the value. Link: https://lore.kernel.org/rust-for-linux/20240418-b4-rbtree-v3-5-323e134390ce@google.com/ [1] Signed-off-by: Benno Lossin Reviewed-by: Boqun Feng Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20240819112415.99810-1-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/box_ext.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs index 829cb1c1cf9e..b68ade26a42d 100644 --- a/rust/kernel/alloc/box_ext.rs +++ b/rust/kernel/alloc/box_ext.rs @@ -4,7 +4,7 @@ use super::{AllocError, Flags}; use alloc::boxed::Box; -use core::mem::MaybeUninit; +use core::{mem::MaybeUninit, ptr, result::Result}; /// Extensions to [`Box`]. pub trait BoxExt: Sized { @@ -17,6 +17,22 @@ pub trait BoxExt: Sized { /// /// The allocation may fail, in which case an error is returned. fn new_uninit(flags: Flags) -> Result>, AllocError>; + + /// Drops the contents, but keeps the allocation. + /// + /// # Examples + /// + /// ``` + /// use kernel::alloc::{flags, box_ext::BoxExt}; + /// let value = Box::new([0; 32], flags::GFP_KERNEL)?; + /// assert_eq!(*value, [0; 32]); + /// let value = Box::drop_contents(value); + /// // Now we can re-use `value`: + /// let value = Box::write(value, [1; 32]); + /// assert_eq!(*value, [1; 32]); + /// # Ok::<(), Error>(()) + /// ``` + fn drop_contents(this: Self) -> Box>; } impl BoxExt for Box { @@ -53,4 +69,17 @@ impl BoxExt for Box { // zero-sized types, we use `NonNull::dangling`. Ok(unsafe { Box::from_raw(ptr) }) } + + fn drop_contents(this: Self) -> Box> { + let ptr = Box::into_raw(this); + // SAFETY: `ptr` is valid, because it came from `Box::into_raw`. + unsafe { ptr::drop_in_place(ptr) }; + + // CAST: `MaybeUninit` is a transparent wrapper of `T`. + let ptr = ptr.cast::>(); + + // SAFETY: `ptr` is valid for writes, because it came from `Box::into_raw` and it is valid for + // reads, since the pointer came from `Box::into_raw` and the type is `MaybeUninit`. + unsafe { Box::from_raw(ptr) } + } } From 6d1c22d0ace31d096b0dab5318c6a0d3219d6456 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Mon, 19 Aug 2024 11:24:39 +0000 Subject: [PATCH 029/111] rust: init: add `write_[pin_]init` functions Sometimes it is necessary to split allocation and initialization into two steps. One such situation is when reusing existing allocations obtained via `Box::drop_contents`. See [1] for an example. In order to support this use case add `write_[pin_]init` functions to the pin-init API. These functions operate on already allocated smart pointers that wrap `MaybeUninit`. Link: https://lore.kernel.org/rust-for-linux/f026532f-8594-4f18-9aa5-57ad3f5bc592@proton.me/ [1] Signed-off-by: Benno Lossin Reviewed-by: Boqun Feng Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20240819112415.99810-2-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 84 ++++++++++++++++++++++++++++++------------ rust/kernel/prelude.rs | 2 +- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 771701805a97..a8068f99fcaa 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1183,13 +1183,7 @@ impl InPlaceInit for Box { where E: From, { - let mut this = as BoxExt<_>>::new_uninit(flags)?; - let slot = this.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { this.assume_init() }.into()) + as BoxExt<_>>::new_uninit(flags)?.write_pin_init(init) } #[inline] @@ -1197,13 +1191,7 @@ impl InPlaceInit for Box { where E: From, { - let mut this = as BoxExt<_>>::new_uninit(flags)?; - let slot = this.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid. - unsafe { init.__init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { this.assume_init() }) + as BoxExt<_>>::new_uninit(flags)?.write_init(init) } } @@ -1215,13 +1203,7 @@ impl InPlaceInit for UniqueArc { where E: From, { - let mut this = UniqueArc::new_uninit(flags)?; - let slot = this.as_mut_ptr(); - // SAFETY: When init errors/panics, slot will get deallocated but not dropped, - // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; - // SAFETY: All fields have been initialized. - Ok(unsafe { this.assume_init() }.into()) + UniqueArc::new_uninit(flags)?.write_pin_init(init) } #[inline] @@ -1229,13 +1211,67 @@ impl InPlaceInit for UniqueArc { where E: From, { - let mut this = UniqueArc::new_uninit(flags)?; - let slot = this.as_mut_ptr(); + UniqueArc::new_uninit(flags)?.write_init(init) + } +} + +/// Smart pointer containing uninitialized memory and that can write a value. +pub trait InPlaceWrite { + /// The type `Self` turns into when the contents are initialized. + type Initialized; + + /// Use the given initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_init(self, init: impl Init) -> Result; + + /// Use the given pin-initializer to write a value into `self`. + /// + /// Does not drop the current value and considers it as uninitialized memory. + fn write_pin_init(self, init: impl PinInit) -> Result, E>; +} + +impl InPlaceWrite for Box> { + type Initialized = Box; + + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized. - Ok(unsafe { this.assume_init() }) + Ok(unsafe { self.assume_init() }) + } + + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) + } +} + +impl InPlaceWrite for UniqueArc> { + type Initialized = UniqueArc; + + fn write_init(mut self, init: impl Init) -> Result { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }) + } + + fn write_pin_init(mut self, init: impl PinInit) -> Result, E> { + let slot = self.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { self.assume_init() }.into()) } } diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index b37a0b3180fb..4571daec0961 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -37,6 +37,6 @@ pub use super::error::{code::*, Error, Result}; pub use super::{str::CStr, ThisModule}; -pub use super::init::{InPlaceInit, Init, PinInit}; +pub use super::init::{InPlaceInit, InPlaceWrite, Init, PinInit}; pub use super::current; From c6945acad7a14b6afb390a4afc1d354ef5413ea6 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 30 Jul 2024 09:26:24 +0000 Subject: [PATCH 030/111] rust: support arrays in target JSON Some configuration options such as the supported sanitizer list are arrays. To support using Rust with sanitizers on x86, we must update the target.json generator to support this case. The Push trait is removed in favor of the From trait because the Push trait doesn't work well in the nested case where you are not really pushing values to a TargetSpec. Signed-off-by: Matthew Maurer Signed-off-by: Alice Ryhl Reviewed-by: Gary Guo Tested-by: Gatlin Newhouse Link: https://lore.kernel.org/r/20240730-target-json-arrays-v1-1-2b376fd0ecf4@google.com Signed-off-by: Miguel Ojeda --- scripts/generate_rust_target.rs | 102 ++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 863720777313..fbf723996d20 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -20,12 +20,28 @@ enum Value { Boolean(bool), Number(i32), String(String), + Array(Vec), Object(Object), } type Object = Vec<(String, Value)>; -/// Minimal "almost JSON" generator (e.g. no `null`s, no arrays, no escaping), +fn comma_sep( + seq: &[T], + formatter: &mut Formatter<'_>, + f: impl Fn(&mut Formatter<'_>, &T) -> Result, +) -> Result { + if let [ref rest @ .., ref last] = seq[..] { + for v in rest { + f(formatter, v)?; + formatter.write_str(",")?; + } + f(formatter, last)?; + } + Ok(()) +} + +/// Minimal "almost JSON" generator (e.g. no `null`s, no escaping), /// enough for this purpose. impl Display for Value { fn fmt(&self, formatter: &mut Formatter<'_>) -> Result { @@ -33,59 +49,67 @@ impl Display for Value { Value::Boolean(boolean) => write!(formatter, "{}", boolean), Value::Number(number) => write!(formatter, "{}", number), Value::String(string) => write!(formatter, "\"{}\"", string), + Value::Array(values) => { + formatter.write_str("[")?; + comma_sep(&values[..], formatter, |formatter, v| v.fmt(formatter))?; + formatter.write_str("]") + } Value::Object(object) => { formatter.write_str("{")?; - if let [ref rest @ .., ref last] = object[..] { - for (key, value) in rest { - write!(formatter, "\"{}\": {},", key, value)?; - } - write!(formatter, "\"{}\": {}", last.0, last.1)?; - } + comma_sep(&object[..], formatter, |formatter, v| { + write!(formatter, "\"{}\": {}", v.0, v.1) + })?; formatter.write_str("}") } } } } +impl From for Value { + fn from(value: bool) -> Self { + Self::Boolean(value) + } +} + +impl From for Value { + fn from(value: i32) -> Self { + Self::Number(value) + } +} + +impl From for Value { + fn from(value: String) -> Self { + Self::String(value) + } +} + +impl From<&str> for Value { + fn from(value: &str) -> Self { + Self::String(value.to_string()) + } +} + +impl From for Value { + fn from(object: Object) -> Self { + Self::Object(object) + } +} + +impl, const N: usize> From<[T; N]> for Value { + fn from(i: [T; N]) -> Self { + Self::Array(i.into_iter().map(|v| v.into()).collect()) + } +} + struct TargetSpec(Object); impl TargetSpec { fn new() -> TargetSpec { TargetSpec(Vec::new()) } -} -trait Push { - fn push(&mut self, key: &str, value: T); -} - -impl Push for TargetSpec { - fn push(&mut self, key: &str, value: bool) { - self.0.push((key.to_string(), Value::Boolean(value))); - } -} - -impl Push for TargetSpec { - fn push(&mut self, key: &str, value: i32) { - self.0.push((key.to_string(), Value::Number(value))); - } -} - -impl Push for TargetSpec { - fn push(&mut self, key: &str, value: String) { - self.0.push((key.to_string(), Value::String(value))); - } -} - -impl Push<&str> for TargetSpec { - fn push(&mut self, key: &str, value: &str) { - self.push(key, value.to_string()); - } -} - -impl Push for TargetSpec { - fn push(&mut self, key: &str, value: Object) { - self.0.push((key.to_string(), Value::Object(value))); + fn push(&mut self, key: &str, value: impl Into) { + self.0.push((key.to_string(), value.into())); } } From 0528ca0a4f858da3369d405af8c76b8248dfeb7b Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Wed, 14 Aug 2024 08:05:20 +0000 Subject: [PATCH 031/111] rust: init: add `assert_pinned` macro Add a macro to statically check if a field of a struct is marked with `#[pin]` ie that it is structurally pinned. This can be used when `unsafe` code needs to rely on fields being structurally pinned. The macro has a special "inline" mode for the case where the type depends on generic parameters from the surrounding scope. Signed-off-by: Benno Lossin Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-1-f5f5e8075da0@google.com [ Replaced `compile_fail` with `ignore` and a TODO note. Removed `pub` from example to clean `unreachable_pub` lint. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 68 ++++++++++++++++++++++++++++++++++ rust/kernel/init/__internal.rs | 29 +++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index a8068f99fcaa..a17ac8762d8f 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -743,6 +743,74 @@ macro_rules! try_init { }; } +/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is +/// structurally pinned. +/// +/// # Example +/// +/// This will succeed: +/// ``` +/// use kernel::assert_pinned; +/// #[pin_data] +/// struct MyStruct { +/// #[pin] +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// This will fail: +// TODO: replace with `compile_fail` when supported. +/// ```ignore +/// use kernel::assert_pinned; +/// #[pin_data] +/// struct MyStruct { +/// some_field: u64, +/// } +/// +/// assert_pinned!(MyStruct, some_field, u64); +/// ``` +/// +/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To +/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can +/// only be used when the macro is invoked from a function body. +/// ``` +/// use kernel::assert_pinned; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// elem: T, +/// } +/// +/// impl Foo { +/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> { +/// assert_pinned!(Foo, elem, T, inline); +/// +/// // SAFETY: The field is structurally pinned. +/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! assert_pinned { + ($ty:ty, $field:ident, $field_ty:ty, inline) => { + let _ = move |ptr: *mut $field_ty| { + // SAFETY: This code is unreachable. + let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() }; + let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new(); + // SAFETY: This code is unreachable. + unsafe { data.$field(ptr, init) }.ok(); + }; + }; + + ($ty:ty, $field:ident, $field_ty:ty) => { + const _: () = { + $crate::assert_pinned!($ty, $field, $field_ty, inline); + }; + }; +} + /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index db3372619ecd..13cefd37512f 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -228,3 +228,32 @@ impl OnlyCallFromDrop { Self(()) } } + +/// Initializer that always fails. +/// +/// Used by [`assert_pinned!`]. +/// +/// [`assert_pinned!`]: crate::assert_pinned +pub struct AlwaysFail { + _t: PhantomData, +} + +impl AlwaysFail { + /// Creates a new initializer that always fails. + pub fn new() -> Self { + Self { _t: PhantomData } + } +} + +impl Default for AlwaysFail { + fn default() -> Self { + Self::new() + } +} + +// SAFETY: `__pinned_init` always fails, which is always okay. +unsafe impl PinInit for AlwaysFail { + unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + Err(()) + } +} From 6cd341715558b8422f33509d9b99a1a0a5b4b29c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:21 +0000 Subject: [PATCH 032/111] rust: list: add ListArc The `ListArc` type can be thought of as a special reference to a refcounted object that owns the permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring that each object has only one `ListArc` reference, the owner of that reference is assured exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a `List`, the `List` takes ownership of the `ListArc` reference. There are various strategies for ensuring that a value has only one `ListArc` reference. The simplest is to convert a `UniqueArc` into a `ListArc`. However, the refcounted object could also keep track of whether a `ListArc` exists using a boolean, which could allow for the creation of new `ListArc` references from an `Arc` reference. Whatever strategy is used, the relevant tracking is referred to as "the tracking inside `T`", and the `ListArcSafe` trait (and its subtraits) are used to update the tracking when a `ListArc` is created or destroyed. Note that we allow the case where the tracking inside `T` thinks that a `ListArc` exists, but actually, there isn't a `ListArc`. However, we do not allow the opposite situation where a `ListArc` exists, but the tracking thinks it doesn't. This is because the former can at most result in us failing to create a `ListArc` when the operation could succeed, whereas the latter can result in the creation of two `ListArc` references. Only the latter situation can lead to memory safety issues. This patch introduces the `impl_list_arc_safe!` macro that allows you to implement `ListArcSafe` for types using the strategy where a `ListArc` can only be created from a `UniqueArc`. Other strategies are introduced in later patches. This is part of the linked list that Rust Binder will use for many different things. The strategy where a `ListArc` can only be created from a `UniqueArc` is actually sufficient for most of the objects that Rust Binder needs to insert into linked lists. Usually, these are todo items that are created and then immediately inserted into a queue. The const generic ID allows objects to have several prev/next pointer pairs so that the same object can be inserted into several different lists. You are able to have several `ListArc` references as long as they correspond to different pointer pairs. The ID itself is purely a compile-time concept and will not be present in the final binary. Both the `List` and the `ListArc` will need to agree on the ID for them to work together. Rust Binder uses this in a few places (e.g. death recipients) where the same object can be inserted into both generic todo lists and some other lists for tracking the status of the object. The ID is a const generic rather than a type parameter because the `pair_from_unique` method needs to be able to assert that the two ids are different. There's no easy way to assert that when using types instead of integers. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-2-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + rust/kernel/list.rs | 8 + rust/kernel/list/arc.rs | 352 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+) create mode 100644 rust/kernel/list.rs create mode 100644 rust/kernel/list/arc.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 274bdc1b0a82..9baea9e9ee1a 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -38,6 +38,7 @@ pub mod init; pub mod ioctl; #[cfg(CONFIG_KUNIT)] pub mod kunit; +pub mod list; #[cfg(CONFIG_NET)] pub mod net; pub mod page; diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs new file mode 100644 index 000000000000..fb16ea43b2ba --- /dev/null +++ b/rust/kernel/list.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A linked list implementation. + +mod arc; +pub use self::arc::{impl_list_arc_safe, ListArc, ListArcSafe}; diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs new file mode 100644 index 000000000000..966076da4a75 --- /dev/null +++ b/rust/kernel/list/arc.rs @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A wrapper around `Arc` for linked lists. + +use crate::alloc::{AllocError, Flags}; +use crate::prelude::*; +use crate::sync::{Arc, ArcBorrow, UniqueArc}; +use core::marker::Unsize; +use core::ops::Deref; +use core::pin::Pin; + +/// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for +/// this id. +/// +/// Types that implement this trait should include some kind of logic for keeping track of whether +/// a [`ListArc`] exists or not. We refer to this logic as "the tracking inside `T`". +/// +/// We allow the case where the tracking inside `T` thinks that a [`ListArc`] exists, but actually, +/// there isn't a [`ListArc`]. However, we do not allow the opposite situation where a [`ListArc`] +/// exists, but the tracking thinks it doesn't. This is because the former can at most result in us +/// failing to create a [`ListArc`] when the operation could succeed, whereas the latter can result +/// in the creation of two [`ListArc`] references. Only the latter situation can lead to memory +/// safety issues. +/// +/// A consequence of the above is that you may implement the tracking inside `T` by not actually +/// keeping track of anything. To do this, you always claim that a [`ListArc`] exists, even if +/// there isn't one. This implementation is allowed by the above rule, but it means that +/// [`ListArc`] references can only be created if you have ownership of *all* references to the +/// refcounted object, as you otherwise have no way of knowing whether a [`ListArc`] exists. +pub trait ListArcSafe { + /// Informs the tracking inside this type that it now has a [`ListArc`] reference. + /// + /// This method may be called even if the tracking inside this type thinks that a `ListArc` + /// reference exists. (But only if that's not actually the case.) + /// + /// # Safety + /// + /// Must not be called if a [`ListArc`] already exist for this value. + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>); + + /// Informs the tracking inside this type that there is no [`ListArc`] reference anymore. + /// + /// # Safety + /// + /// Must only be called if there is no [`ListArc`] reference, but the tracking thinks there is. + unsafe fn on_drop_list_arc(&self); +} + +/// Declares that this type supports [`ListArc`]. +/// +/// When using this macro, it will only be possible to create a [`ListArc`] from a [`UniqueArc`]. +#[macro_export] +macro_rules! impl_list_arc_safe { + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {} + unsafe fn on_drop_list_arc(&self) {} + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + + () => {}; +} +pub use impl_list_arc_safe; + +/// A wrapper around [`Arc`] that's guaranteed unique for the given id. +/// +/// The `ListArc` type can be thought of as a special reference to a refcounted object that owns the +/// permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring +/// that each object has only one `ListArc` reference, the owner of that reference is assured +/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a `List`, the +/// `List` takes ownership of the `ListArc` reference. +/// +/// There are various strategies to ensuring that a value has only one `ListArc` reference. The +/// simplest is to convert a [`UniqueArc`] into a `ListArc`. However, the refcounted object could +/// also keep track of whether a `ListArc` exists using a boolean, which could allow for the +/// creation of new `ListArc` references from an [`Arc`] reference. Whatever strategy is used, the +/// relevant tracking is referred to as "the tracking inside `T`", and the [`ListArcSafe`] trait +/// (and its subtraits) are used to update the tracking when a `ListArc` is created or destroyed. +/// +/// Note that we allow the case where the tracking inside `T` thinks that a `ListArc` exists, but +/// actually, there isn't a `ListArc`. However, we do not allow the opposite situation where a +/// `ListArc` exists, but the tracking thinks it doesn't. This is because the former can at most +/// result in us failing to create a `ListArc` when the operation could succeed, whereas the latter +/// can result in the creation of two `ListArc` references. +/// +/// While this `ListArc` is unique for the given id, there still might exist normal `Arc` +/// references to the object. +/// +/// # Invariants +/// +/// * Each reference counted object has at most one `ListArc` for each value of `ID`. +/// * The tracking inside `T` is aware that a `ListArc` reference exists. +#[repr(transparent)] +pub struct ListArc +where + T: ListArcSafe + ?Sized, +{ + arc: Arc, +} + +impl, const ID: u64> ListArc { + /// Constructs a new reference counted instance of `T`. + #[inline] + pub fn new(contents: T, flags: Flags) -> Result { + Ok(Self::from(UniqueArc::new(contents, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + // We don't implement `InPlaceInit` because `ListArc` is implicitly pinned. This is similar to + // what we do for `Arc`. + #[inline] + pub fn pin_init(init: impl PinInit, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_pin_init(init, flags)?)) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// This is equivalent to [`ListArc::pin_init`], since a [`ListArc`] is always pinned. + #[inline] + pub fn init(init: impl Init, flags: Flags) -> Result + where + E: From, + { + Ok(Self::from(UniqueArc::try_init(init, flags)?)) + } +} + +impl From> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(unique: UniqueArc) -> Self { + Self::from(Pin::from(unique)) + } +} + +impl From>> for ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Convert a pinned [`UniqueArc`] into a [`ListArc`]. + #[inline] + fn from(mut unique: Pin>) -> Self { + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { T::on_create_list_arc_from_unique(unique.as_mut()) }; + let arc = Arc::from(unique); + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc`, + // so we can create a `ListArc`. + unsafe { Self::transmute_from_arc(arc) } + } +} + +impl ListArc +where + T: ListArcSafe + ?Sized, +{ + /// Creates two `ListArc`s from a [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_unique(unique: UniqueArc) -> (Self, ListArc) + where + T: ListArcSafe, + { + Self::pair_from_pin_unique(Pin::from(unique)) + } + + /// Creates two `ListArc`s from a pinned [`UniqueArc`]. + /// + /// The two ids must be different. + #[inline] + pub fn pair_from_pin_unique( + mut unique: Pin>, + ) -> (Self, ListArc) + where + T: ListArcSafe, + { + build_assert!(ID != ID2); + + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + // SAFETY: We have a `UniqueArc`, so there is no `ListArc`. + unsafe { >::on_create_list_arc_from_unique(unique.as_mut()) }; + + let arc1 = Arc::from(unique); + let arc2 = Arc::clone(&arc1); + + // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc` + // for both IDs (which are different), so we can create two `ListArc`s. + unsafe { + ( + Self::transmute_from_arc(arc1), + ListArc::transmute_from_arc(arc2), + ) + } + } + + /// Transmutes an [`Arc`] into a `ListArc` without updating the tracking inside `T`. + /// + /// # Safety + /// + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + unsafe fn transmute_from_arc(arc: Arc) -> Self { + // INVARIANT: By the safety requirements, the invariants on `ListArc` are satisfied. + Self { arc } + } + + /// Transmutes a `ListArc` into an [`Arc`] without updating the tracking inside `T`. + /// + /// After this call, the tracking inside `T` will still think that there is a `ListArc` + /// reference. + #[inline] + fn transmute_to_arc(self) -> Arc { + // Use a transmute to skip destructor. + // + // SAFETY: ListArc is repr(transparent). + unsafe { core::mem::transmute(self) } + } + + /// Convert ownership of this `ListArc` into a raw pointer. + /// + /// The returned pointer is indistinguishable from pointers returned by [`Arc::into_raw`]. The + /// tracking inside `T` will still think that a `ListArc` exists after this call. + #[inline] + pub fn into_raw(self) -> *const T { + Arc::into_raw(Self::transmute_to_arc(self)) + } + + /// Take ownership of the `ListArc` from a raw pointer. + /// + /// # Safety + /// + /// * `ptr` must satisfy the safety requirements of [`Arc::from_raw`]. + /// * The value must not already have a `ListArc` reference. + /// * The tracking inside `T` must think that there is a `ListArc` reference. + #[inline] + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The pointer satisfies the safety requirements for `Arc::from_raw`. + let arc = unsafe { Arc::from_raw(ptr) }; + // SAFETY: The value doesn't already have a `ListArc` reference, but the tracking thinks it + // does. + unsafe { Self::transmute_from_arc(arc) } + } + + /// Converts the `ListArc` into an [`Arc`]. + #[inline] + pub fn into_arc(self) -> Arc { + let arc = Self::transmute_to_arc(self); + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is. + unsafe { T::on_drop_list_arc(&arc) }; + arc + } + + /// Clone a `ListArc` into an [`Arc`]. + #[inline] + pub fn clone_arc(&self) -> Arc { + self.arc.clone() + } + + /// Returns a reference to an [`Arc`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`&Arc`] (e.g., in a method + /// receiver), but we have a [`ListArc`] instead. + /// + /// [`&Arc`]: Arc + #[inline] + pub fn as_arc(&self) -> &Arc { + &self.arc + } + + /// Returns an [`ArcBorrow`] from the given [`ListArc`]. + /// + /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method + /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. + #[inline] + pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { + self.arc.as_arc_borrow() + } + + /// Compare whether two [`ListArc`] pointers reference the same underlying object. + #[inline] + pub fn ptr_eq(this: &Self, other: &Self) -> bool { + Arc::ptr_eq(&this.arc, &other.arc) + } +} + +impl Deref for ListArc +where + T: ListArcSafe + ?Sized, +{ + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.arc.deref() + } +} + +impl Drop for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is by the type + // invariants on `Self`. + unsafe { T::on_drop_list_arc(&self.arc) }; + } +} + +impl AsRef> for ListArc +where + T: ListArcSafe + ?Sized, +{ + #[inline] + fn as_ref(&self) -> &Arc { + self.as_arc() + } +} + +// This is to allow [`ListArc`] (and variants) to be used as the type of `self`. +impl core::ops::Receiver for ListArc where T: ListArcSafe + ?Sized {} + +// This is to allow coercion from `ListArc` to `ListArc` if `T` can be converted to the +// dynamically-sized type (DST) `U`. +impl core::ops::CoerceUnsized> for ListArc +where + T: ListArcSafe + Unsize + ?Sized, + U: ListArcSafe + ?Sized, +{ +} + +// This is to allow `ListArc` to be dispatched on when `ListArc` can be coerced into +// `ListArc`. +impl core::ops::DispatchFromDyn> for ListArc +where + T: ListArcSafe + Unsize + ?Sized, + U: ListArcSafe + ?Sized, +{ +} From a48026315cd7b6018bd74831a6dc0586adbba1b9 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:22 +0000 Subject: [PATCH 033/111] rust: list: add tracking for ListArc Add the ability to track whether a ListArc exists for a given value, allowing for the creation of ListArcs without going through UniqueArc. The `impl_list_arc_safe!` macro is extended with a `tracked_by` strategy that defers the tracking of ListArcs to a field of the struct. Additionally, the AtomicListArcTracker type is introduced, which can track whether a ListArc exists using an atomic. By deferring the tracking to a field of type AtomicListArcTracker, structs gain the ability to create ListArcs without going through a UniqueArc. Rust Binder uses this for some objects where we want to be able to insert them into a linked list at any time. Using the AtomicListArcTracker, we are able to check whether an item is already in the list, and if not, we can create a `ListArc` and push it. The macro has the ability to defer the tracking of ListArcs to a field, using whatever strategy that field has. Since we don't add any strategies other than AtomicListArcTracker, another similar option would be to hard-code that the field should be an AtomicListArcTracker. However, Rust Binder has a case where the AtomicListArcTracker is not stored directly in the struct, but in a sub-struct. Furthermore, the outer struct is generic: struct Wrapper { links: ListLinks, inner: T, } Here, the Wrapper struct implements ListArcSafe with `tracked_by inner`, and then the various types used with `inner` also uses the macro to implement ListArcSafe. Some of them use the untracked strategy, and some of them use tracked_by with an AtomicListArcTracker. This way, Wrapper just inherits whichever choice `inner` has made. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-3-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 2 +- rust/kernel/list/arc.rs | 171 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 170 insertions(+), 3 deletions(-) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index fb16ea43b2ba..8e1533ee987b 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -5,4 +5,4 @@ //! A linked list implementation. mod arc; -pub use self::arc::{impl_list_arc_safe, ListArc, ListArcSafe}; +pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index 966076da4a75..c5921a7d5966 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -7,9 +7,10 @@ use crate::alloc::{AllocError, Flags}; use crate::prelude::*; use crate::sync::{Arc, ArcBorrow, UniqueArc}; -use core::marker::Unsize; +use core::marker::{PhantomPinned, Unsize}; use core::ops::Deref; use core::pin::Pin; +use core::sync::atomic::{AtomicBool, Ordering}; /// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for /// this id. @@ -48,9 +49,38 @@ pub trait ListArcSafe { unsafe fn on_drop_list_arc(&self); } +/// Declares that this type is able to safely attempt to create `ListArc`s at any time. +/// +/// # Safety +/// +/// The guarantees of `try_new_list_arc` must be upheld. +pub unsafe trait TryNewListArc: ListArcSafe { + /// Attempts to convert an `Arc` into an `ListArc`. Returns `true` if the + /// conversion was successful. + /// + /// This method should not be called directly. Use [`ListArc::try_from_arc`] instead. + /// + /// # Guarantees + /// + /// If this call returns `true`, then there is no [`ListArc`] pointing to this value. + /// Additionally, this call will have transitioned the tracking inside `Self` from not thinking + /// that a [`ListArc`] exists, to thinking that a [`ListArc`] exists. + fn try_new_list_arc(&self) -> bool; +} + /// Declares that this type supports [`ListArc`]. /// -/// When using this macro, it will only be possible to create a [`ListArc`] from a [`UniqueArc`]. +/// This macro supports a few different strategies for implementing the tracking inside the type: +/// +/// * The `untracked` strategy does not actually keep track of whether a [`ListArc`] exists. When +/// using this strategy, the only way to create a [`ListArc`] is using a [`UniqueArc`]. +/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user much specify +/// which field to defer the tracking to. The field must implement [`ListArcSafe`]. If the field +/// implements [`TryNewListArc`], then the type will also implement [`TryNewListArc`]. +/// +/// The `tracked_by` strategy is usually used by deferring to a field of type +/// [`AtomicTracker`]. However, it is also possible to defer the tracking to another struct +/// using also using this macro. #[macro_export] macro_rules! impl_list_arc_safe { (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => { @@ -61,6 +91,39 @@ macro_rules! impl_list_arc_safe { $crate::list::impl_list_arc_safe! { $($rest)* } }; + (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { + tracked_by $field:ident : $fty:ty; + } $($rest:tt)*) => { + impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t { + unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) { + $crate::assert_pinned!($t, $field, $fty, inline); + + // SAFETY: This field is structurally pinned as per the above assertion. + let field = unsafe { + ::core::pin::Pin::map_unchecked_mut(self, |me| &mut me.$field) + }; + // SAFETY: The caller promises that there is no `ListArc`. + unsafe { + <$fty as $crate::list::ListArcSafe<$num>>::on_create_list_arc_from_unique(field) + }; + } + unsafe fn on_drop_list_arc(&self) { + // SAFETY: The caller promises that there is no `ListArc` reference, and also + // promises that the tracking thinks there is a `ListArc` reference. + unsafe { <$fty as $crate::list::ListArcSafe<$num>>::on_drop_list_arc(&self.$field) }; + } + } + unsafe impl$(<$($generics)*>)? $crate::list::TryNewListArc<$num> for $t + where + $fty: TryNewListArc<$num>, + { + fn try_new_list_arc(&self) -> bool { + <$fty as $crate::list::TryNewListArc<$num>>::try_new_list_arc(&self.$field) + } + } + $crate::list::impl_list_arc_safe! { $($rest)* } + }; + () => {}; } pub use impl_list_arc_safe; @@ -205,6 +268,52 @@ where } } + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc(arc: Arc) -> Result> + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Ok(unsafe { Self::transmute_from_arc(arc) }) + } else { + Err(arc) + } + } + + /// Try to create a new `ListArc`. + /// + /// This fails if this value already has a `ListArc`. + pub fn try_from_arc_borrow(arc: ArcBorrow<'_, T>) -> Option + where + T: TryNewListArc, + { + if arc.try_new_list_arc() { + // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think + // that a `ListArc` exists. This lets us create a `ListArc`. + Some(unsafe { Self::transmute_from_arc(Arc::from(arc)) }) + } else { + None + } + } + + /// Try to create a new `ListArc`. + /// + /// If it's not possible to create a new `ListArc`, then the `Arc` is dropped. This will never + /// run the destructor of the value. + pub fn try_from_arc_or_drop(arc: Arc) -> Option + where + T: TryNewListArc, + { + match Self::try_from_arc(arc) { + Ok(list_arc) => Some(list_arc), + Err(arc) => Arc::into_unique_or_drop(arc).map(Self::from), + } + } + /// Transmutes an [`Arc`] into a `ListArc` without updating the tracking inside `T`. /// /// # Safety @@ -350,3 +459,61 @@ where U: ListArcSafe + ?Sized, { } + +/// A utility for tracking whether a [`ListArc`] exists using an atomic. +/// +/// # Invariant +/// +/// If the boolean is `false`, then there is no [`ListArc`] for this value. +#[repr(transparent)] +pub struct AtomicTracker { + inner: AtomicBool, + // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. + _pin: PhantomPinned, +} + +impl AtomicTracker { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: AtomicBool::new(false), + _pin: PhantomPinned, + } + } + + fn project_inner(self: Pin<&mut Self>) -> &mut AtomicBool { + // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable + // reference to it even if we only have a pinned reference to `self`. + unsafe { &mut Pin::into_inner_unchecked(self).inner } + } +} + +impl ListArcSafe for AtomicTracker { + unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>) { + // INVARIANT: We just created a ListArc, so the boolean should be true. + *self.project_inner().get_mut() = true; + } + + unsafe fn on_drop_list_arc(&self) { + // INVARIANT: We just dropped a ListArc, so the boolean should be false. + self.inner.store(false, Ordering::Release); + } +} + +// SAFETY: If this method returns `true`, then by the type invariant there is no `ListArc` before +// this call, so it is okay to create a new `ListArc`. +// +// The acquire ordering will synchronize with the release store from the destruction of any +// previous `ListArc`, so if there was a previous `ListArc`, then the destruction of the previous +// `ListArc` happens-before the creation of the new `ListArc`. +unsafe impl TryNewListArc for AtomicTracker { + fn try_new_list_arc(&self) -> bool { + // INVARIANT: If this method returns true, then the boolean used to be false, and is no + // longer false, so it is okay for the caller to create a new [`ListArc`]. + self.inner + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } +} From 14176295fe56ce9506e650dce436d2bdadec93b5 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:23 +0000 Subject: [PATCH 034/111] rust: list: add struct with prev/next pointers Define the ListLinks struct, which wraps the prev/next pointers that will be used to insert values into a List in a future patch. Also define the ListItem trait, which is implemented by structs that have a ListLinks field. The ListItem trait provides four different methods that are all essentially container_of or the reverse of container_of. Two of them are used before inserting/after removing an item from the list, and the two others are used when looking at a value without changing whether it is in a list. This distinction is introduced because it is needed for the patch that adds support for heterogeneous lists, which are implemented by adding a third pointer field with a fat pointer to the full struct. When inserting into the heterogeneous list, the pointer-to-self is updated to have the right vtable, and the container_of operation is implemented by just returning that pointer instead of using the real container_of operation. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-4-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 119 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 8e1533ee987b..074ae863ff5a 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -4,5 +4,124 @@ //! A linked list implementation. +use crate::init::PinInit; +use crate::types::Opaque; +use core::ptr; + mod arc; pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; + +/// Implemented by types where a [`ListArc`] can be inserted into a `List`. +/// +/// # Safety +/// +/// Implementers must ensure that they provide the guarantees documented on methods provided by +/// this trait. +/// +/// [`ListArc`]: ListArc +pub unsafe trait ListItem: ListArcSafe { + /// Views the [`ListLinks`] for this value. + /// + /// # Guarantees + /// + /// If there is a previous call to `prepare_to_insert` and there is no call to `post_remove` + /// since the most recent such call, then this returns the same pointer as the one returned by + /// the most recent call to `prepare_to_insert`. + /// + /// Otherwise, the returned pointer points at a read-only [`ListLinks`] with two null pointers. + /// + /// # Safety + /// + /// The provided pointer must point at a valid value. (It need not be in an `Arc`.) + unsafe fn view_links(me: *const Self) -> *mut ListLinks; + + /// View the full value given its [`ListLinks`] field. + /// + /// Can only be used when the value is in a list. + /// + /// # Guarantees + /// + /// * Returns the same pointer as the one passed to the most recent call to `prepare_to_insert`. + /// * The returned pointer is valid until the next call to `post_remove`. + /// + /// # Safety + /// + /// * The provided pointer must originate from the most recent call to `prepare_to_insert`, or + /// from a call to `view_links` that happened after the most recent call to + /// `prepare_to_insert`. + /// * Since the most recent call to `prepare_to_insert`, the `post_remove` method must not have + /// been called. + unsafe fn view_value(me: *mut ListLinks) -> *const Self; + + /// This is called when an item is inserted into a `List`. + /// + /// # Guarantees + /// + /// The caller is granted exclusive access to the returned [`ListLinks`] until `post_remove` is + /// called. + /// + /// # Safety + /// + /// * The provided pointer must point at a valid value in an [`Arc`]. + /// * Calls to `prepare_to_insert` and `post_remove` on the same value must alternate. + /// * The caller must own the [`ListArc`] for this value. + /// * The caller must not give up ownership of the [`ListArc`] unless `post_remove` has been + /// called after this call to `prepare_to_insert`. + /// + /// [`Arc`]: crate::sync::Arc + unsafe fn prepare_to_insert(me: *const Self) -> *mut ListLinks; + + /// This undoes a previous call to `prepare_to_insert`. + /// + /// # Guarantees + /// + /// The returned pointer is the pointer that was originally passed to `prepare_to_insert`. + /// + /// # Safety + /// + /// The provided pointer must be the pointer returned by the most recent call to + /// `prepare_to_insert`. + unsafe fn post_remove(me: *mut ListLinks) -> *const Self; +} + +#[repr(C)] +#[derive(Copy, Clone)] +struct ListLinksFields { + next: *mut ListLinksFields, + prev: *mut ListLinksFields, +} + +/// The prev/next pointers for an item in a linked list. +/// +/// # Invariants +/// +/// The fields are null if and only if this item is not in a list. +#[repr(transparent)] +pub struct ListLinks { + // This type is `!Unpin` for aliasing reasons as the pointers are part of an intrusive linked + // list. + #[allow(dead_code)] + inner: Opaque, +} + +// SAFETY: The only way to access/modify the pointers inside of `ListLinks` is via holding the +// associated `ListArc`. Since that type correctly implements `Send`, it is impossible to +// move this an instance of this type to a different thread if the pointees are `!Send`. +unsafe impl Send for ListLinks {} +// SAFETY: The type is opaque so immutable references to a ListLinks are useless. Therefore, it's +// okay to have immutable access to a ListLinks from several threads at once. +unsafe impl Sync for ListLinks {} + +impl ListLinks { + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + } + } +} From 40c53294596b4a7fe2ae126d7aab986752496c31 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:24 +0000 Subject: [PATCH 035/111] rust: list: add macro for implementing ListItem Adds a macro for safely implementing the ListItem trait. As part of the implementation of the macro, we also provide a HasListLinks trait similar to the workqueue's HasWorkItem trait. The HasListLinks trait is only necessary if you are implementing ListItem using the impl_list_item macro. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-5-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 3 + rust/kernel/list/impl_list_item_mod.rs | 143 +++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 rust/kernel/list/impl_list_item_mod.rs diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 074ae863ff5a..670d53989b8f 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -8,6 +8,9 @@ use crate::init::PinInit; use crate::types::Opaque; use core::ptr; +mod impl_list_item_mod; +pub use self::impl_list_item_mod::{impl_has_list_links, impl_list_item, HasListLinks}; + mod arc; pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs new file mode 100644 index 000000000000..1bcb14774aeb --- /dev/null +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! Helpers for implementing list traits safely. + +use crate::list::ListLinks; + +/// Declares that this type has a `ListLinks` field at a fixed offset. +/// +/// This trait is only used to help implement `ListItem` safely. If `ListItem` is implemented +/// manually, then this trait is not needed. Use the [`impl_has_list_links!`] macro to implement +/// this trait. +/// +/// # Safety +/// +/// All values of this type must have a `ListLinks` field at the given offset. +/// +/// The behavior of `raw_get_list_links` must not be changed. +pub unsafe trait HasListLinks { + /// The offset of the `ListLinks` field. + const OFFSET: usize; + + /// Returns a pointer to the [`ListLinks`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + /// + /// [`ListLinks`]: ListLinks + // We don't really need this method, but it's necessary for the implementation of + // `impl_has_list_links!` to be correct. + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut ListLinks { + // SAFETY: The caller promises that the pointer is valid. The implementer promises that the + // `OFFSET` constant is correct. + unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut ListLinks } + } +} + +/// Implements the [`HasListLinks`] trait for the given type. +#[macro_export] +macro_rules! impl_has_list_links { + ($(impl$(<$($implarg:ident),*>)? + HasListLinks$(<$id:tt>)? + for $self:ident $(<$($selfarg:ty),*>)? + { self$(.$field:ident)* } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + // + // The behavior of `raw_get_list_links` is not changed since the `addr_of_mut!` macro is + // equivalent to the pointer offset operation in the trait definition. + unsafe impl$(<$($implarg),*>)? $crate::list::HasListLinks$(<$id>)? for + $self $(<$($selfarg),*>)? + { + const OFFSET: usize = ::core::mem::offset_of!(Self, $($field).*) as usize; + + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + // SAFETY: The caller promises that the pointer is not dangling. We know that this + // expression doesn't follow any pointers, as the `offset_of!` invocation above + // would otherwise not compile. + unsafe { ::core::ptr::addr_of_mut!((*ptr)$(.$field)*) } + } + } + )*}; +} +pub use impl_has_list_links; + +/// Implements the [`ListItem`] trait for the given type. +/// +/// Requires that the type implements [`HasListLinks`]. Use the [`impl_has_list_links!`] macro to +/// implement that trait. +/// +/// [`ListItem`]: crate::list::ListItem +#[macro_export] +macro_rules! impl_list_item { + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $t:ty { + using ListLinks; + })* + ) => {$( + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $t { + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // is implemented in terms of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller guarantees that `me` points at a valid value of type `Self`. + unsafe { + >::raw_get_list_links(me.cast_mut()) + } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which just added + // `offset` to the pointer passed to `prepare_to_insert`. This method subtracts + // `offset` from `me` so it returns the pointer originally passed to + // `prepare_to_insert`. + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + let offset = >::OFFSET; + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field at offset `offset` in a value of type `Self`. Thus, + // subtracting `offset` from `me` is still in-bounds of the allocation. + unsafe { (me as *const u8).sub(offset) as *const Self } + } + + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value. + unsafe { >::view_links(me) } + } + + // GUARANTEES: + // * `me` originates from the most recent call to `prepare_to_insert`, which just added + // `offset` to the pointer passed to `prepare_to_insert`. This method subtracts + // `offset` from `me` so it returns the pointer originally passed to + // `prepare_to_insert`. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + let offset = >::OFFSET; + // SAFETY: `me` originates from the most recent call to `prepare_to_insert`, so it + // points at the field at offset `offset` in a value of type `Self`. Thus, + // subtracting `offset` from `me` is still in-bounds of the allocation. + unsafe { (me as *const u8).sub(offset) as *const Self } + } + } + )*}; +} +pub use impl_list_item; From db841866ecc01ca01ab93282d7809b87568a18ff Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:25 +0000 Subject: [PATCH 036/111] rust: list: add List Add the actual linked list itself. The linked list uses the following design: The List type itself just has a single pointer to the first element of the list. And the actual list items then form a cycle. So the last item is `first->prev`. This is slightly different from the usual kernel linked list. Matching that exactly would amount to giving List two pointers, and having it be part of the cycle of items. This alternate design has the advantage that the cycle is never completely empty, which can reduce the number of branches in some cases. However, it also has the disadvantage that List must be pinned, which this design is trying to avoid. Having the list items form a cycle rather than having null pointers at the beginning/end is convenient for several reasons. For one, it lets us store only one pointer in List, and it simplifies the implementation of several functions. Unfortunately, the `remove` function that removes an arbitrary element from the list has to be unsafe. This is needed because there is no way to handle the case where you pass an element from the wrong list. For example, if it is the first element of some other list, then that other list's `first` pointer would not be updated. Similarly, it could be a data race if you try to remove it from two different lists in parallel. (There's no problem with passing `remove` an item that's not in any list. Additionally, other removal methods such as `pop_front` need not be unsafe, as they can't be used to remove items from another list.) A future patch in this series will introduce support for cursors that can be used to remove arbitrary items without unsafe code. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-6-f5f5e8075da0@google.com [ Fixed a few typos. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 330 +++++++++++++++++++++++++++++++++++++++- rust/kernel/list/arc.rs | 6 +- 2 files changed, 331 insertions(+), 5 deletions(-) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 670d53989b8f..a87deadcfcc2 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -6,6 +6,7 @@ use crate::init::PinInit; use crate::types::Opaque; +use core::marker::PhantomData; use core::ptr; mod impl_list_item_mod; @@ -14,7 +15,42 @@ pub use self::impl_list_item_mod::{impl_has_list_links, impl_list_item, HasListL mod arc; pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; -/// Implemented by types where a [`ListArc`] can be inserted into a `List`. +/// A linked list. +/// +/// All elements in this linked list will be [`ListArc`] references to the value. Since a value can +/// only have one `ListArc` (for each pair of prev/next pointers), this ensures that the same +/// prev/next pointers are not used for several linked lists. +/// +/// # Invariants +/// +/// * If the list is empty, then `first` is null. Otherwise, `first` points at the `ListLinks` +/// field of the first element in the list. +/// * All prev/next pointers in `ListLinks` fields of items in the list are valid and form a cycle. +/// * For every item in the list, the list owns the associated [`ListArc`] reference and has +/// exclusive access to the `ListLinks` field. +pub struct List, const ID: u64 = 0> { + first: *mut ListLinksFields, + _ty: PhantomData>, +} + +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Send for List +where + ListArc: Send, + T: ?Sized + ListItem, +{ +} +// SAFETY: This is a container of `ListArc`, and access to the container allows the same +// type of access to the `ListArc` elements. +unsafe impl Sync for List +where + ListArc: Sync, + T: ?Sized + ListItem, +{ +} + +/// Implemented by types where a [`ListArc`] can be inserted into a [`List`]. /// /// # Safety /// @@ -56,7 +92,7 @@ pub unsafe trait ListItem: ListArcSafe { /// been called. unsafe fn view_value(me: *mut ListLinks) -> *const Self; - /// This is called when an item is inserted into a `List`. + /// This is called when an item is inserted into a [`List`]. /// /// # Guarantees /// @@ -103,7 +139,6 @@ struct ListLinksFields { pub struct ListLinks { // This type is `!Unpin` for aliasing reasons as the pointers are part of an intrusive linked // list. - #[allow(dead_code)] inner: Opaque, } @@ -127,4 +162,293 @@ impl ListLinks { }), } } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn fields(me: *mut Self) -> *mut ListLinksFields { + // SAFETY: The caller promises that the pointer is valid. + unsafe { Opaque::raw_get(ptr::addr_of!((*me).inner)) } + } + + /// # Safety + /// + /// `me` must be dereferenceable. + #[inline] + unsafe fn from_fields(me: *mut ListLinksFields) -> *mut Self { + me.cast() + } +} + +impl, const ID: u64> List { + /// Creates a new empty list. + pub const fn new() -> Self { + Self { + first: ptr::null_mut(), + _ty: PhantomData, + } + } + + /// Returns whether this list is empty. + pub fn is_empty(&self) -> bool { + self.first.is_null() + } + + /// Add the provided item to the back of the list. + pub fn push_back(&mut self, item: ListArc) { + let raw_item = ListArc::into_raw(item); + // SAFETY: + // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. + // * Since we have ownership of the `ListArc`, `post_remove` must have been called after + // the most recent call to `prepare_to_insert`, if any. + // * We own the `ListArc`. + // * Removing items from this list is always done using `remove_internal_inner`, which + // calls `post_remove` before giving up ownership. + let list_links = unsafe { T::prepare_to_insert(raw_item) }; + // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. + let item = unsafe { ListLinks::fields(list_links) }; + + if self.first.is_null() { + self.first = item; + // SAFETY: The caller just gave us ownership of these fields. + // INVARIANT: A linked list with one item should be cyclic. + unsafe { + (*item).next = item; + (*item).prev = item; + } + } else { + let next = self.first; + // SAFETY: By the type invariant, this pointer is valid or null. We just checked that + // it's not null, so it must be valid. + let prev = unsafe { (*next).prev }; + // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us + // ownership of the fields on `item`. + // INVARIANT: This correctly inserts `item` between `prev` and `next`. + unsafe { + (*item).next = next; + (*item).prev = prev; + (*prev).next = item; + (*next).prev = item; + } + } + } + + /// Add the provided item to the front of the list. + pub fn push_front(&mut self, item: ListArc) { + let raw_item = ListArc::into_raw(item); + // SAFETY: + // * We just got `raw_item` from a `ListArc`, so it's in an `Arc`. + // * If this requirement is violated, then the previous caller of `prepare_to_insert` + // violated the safety requirement that they can't give up ownership of the `ListArc` + // until they call `post_remove`. + // * We own the `ListArc`. + // * Removing items] from this list is always done using `remove_internal_inner`, which + // calls `post_remove` before giving up ownership. + let list_links = unsafe { T::prepare_to_insert(raw_item) }; + // SAFETY: We have not yet called `post_remove`, so `list_links` is still valid. + let item = unsafe { ListLinks::fields(list_links) }; + + if self.first.is_null() { + // SAFETY: The caller just gave us ownership of these fields. + // INVARIANT: A linked list with one item should be cyclic. + unsafe { + (*item).next = item; + (*item).prev = item; + } + } else { + let next = self.first; + // SAFETY: We just checked that `next` is non-null. + let prev = unsafe { (*next).prev }; + // SAFETY: Pointers in a linked list are never dangling, and the caller just gave us + // ownership of the fields on `item`. + // INVARIANT: This correctly inserts `item` between `prev` and `next`. + unsafe { + (*item).next = next; + (*item).prev = prev; + (*prev).next = item; + (*next).prev = item; + } + } + self.first = item; + } + + /// Removes the last item from this list. + pub fn pop_back(&mut self) -> Option> { + if self.first.is_null() { + return None; + } + + // SAFETY: We just checked that the list is not empty. + let last = unsafe { (*self.first).prev }; + // SAFETY: The last item of this list is in this list. + Some(unsafe { self.remove_internal(last) }) + } + + /// Removes the first item from this list. + pub fn pop_front(&mut self) -> Option> { + if self.first.is_null() { + return None; + } + + // SAFETY: The first item of this list is in this list. + Some(unsafe { self.remove_internal(self.first) }) + } + + /// Removes the provided item from this list and returns it. + /// + /// This returns `None` if the item is not in the list. (Note that by the safety requirements, + /// this means that the item is not in any list.) + /// + /// # Safety + /// + /// `item` must not be in a different linked list (with the same id). + pub unsafe fn remove(&mut self, item: &T) -> Option> { + let mut item = unsafe { ListLinks::fields(T::view_links(item)) }; + // SAFETY: The user provided a reference, and reference are never dangling. + // + // As for why this is not a data race, there are two cases: + // + // * If `item` is not in any list, then these fields are read-only and null. + // * If `item` is in this list, then we have exclusive access to these fields since we + // have a mutable reference to the list. + // + // In either case, there's no race. + let ListLinksFields { next, prev } = unsafe { *item }; + + debug_assert_eq!(next.is_null(), prev.is_null()); + if !next.is_null() { + // This is really a no-op, but this ensures that `item` is a raw pointer that was + // obtained without going through a pointer->reference->pointer conversion roundtrip. + // This ensures that the list is valid under the more restrictive strict provenance + // ruleset. + // + // SAFETY: We just checked that `next` is not null, and it's not dangling by the + // list invariants. + unsafe { + debug_assert_eq!(item, (*next).prev); + item = (*next).prev; + } + + // SAFETY: We just checked that `item` is in a list, so the caller guarantees that it + // is in this list. The pointers are in the right order. + Some(unsafe { self.remove_internal_inner(item, next, prev) }) + } else { + None + } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// `item` must point at an item in this list. + unsafe fn remove_internal(&mut self, item: *mut ListLinksFields) -> ListArc { + // SAFETY: The caller promises that this pointer is not dangling, and there's no data race + // since we have a mutable reference to the list containing `item`. + let ListLinksFields { next, prev } = unsafe { *item }; + // SAFETY: The pointers are ok and in the right order. + unsafe { self.remove_internal_inner(item, next, prev) } + } + + /// Removes the provided item from the list. + /// + /// # Safety + /// + /// The `item` pointer must point at an item in this list, and we must have `(*item).next == + /// next` and `(*item).prev == prev`. + unsafe fn remove_internal_inner( + &mut self, + item: *mut ListLinksFields, + next: *mut ListLinksFields, + prev: *mut ListLinksFields, + ) -> ListArc { + // SAFETY: We have exclusive access to the pointers of items in the list, and the prev/next + // pointers are always valid for items in a list. + // + // INVARIANT: There are three cases: + // * If the list has at least three items, then after removing the item, `prev` and `next` + // will be next to each other. + // * If the list has two items, then the remaining item will point at itself. + // * If the list has one item, then `next == prev == item`, so these writes have no + // effect. The list remains unchanged and `item` is still in the list for now. + unsafe { + (*next).prev = prev; + (*prev).next = next; + } + // SAFETY: We have exclusive access to items in the list. + // INVARIANT: `item` is being removed, so the pointers should be null. + unsafe { + (*item).prev = ptr::null_mut(); + (*item).next = ptr::null_mut(); + } + // INVARIANT: There are three cases: + // * If `item` was not the first item, then `self.first` should remain unchanged. + // * If `item` was the first item and there is another item, then we just updated + // `prev->next` to `next`, which is the new first item, and setting `item->next` to null + // did not modify `prev->next`. + // * If `item` was the only item in the list, then `prev == item`, and we just set + // `item->next` to null, so this correctly sets `first` to null now that the list is + // empty. + if self.first == item { + // SAFETY: The `prev` pointer is the value that `item->prev` had when it was in this + // list, so it must be valid. There is no race since `prev` is still in the list and we + // still have exclusive access to the list. + self.first = unsafe { (*prev).next }; + } + + // SAFETY: `item` used to be in the list, so it is dereferenceable by the type invariants + // of `List`. + let list_links = unsafe { ListLinks::from_fields(item) }; + // SAFETY: Any pointer in the list originates from a `prepare_to_insert` call. + let raw_item = unsafe { T::post_remove(list_links) }; + // SAFETY: The above call to `post_remove` guarantees that we can recreate the `ListArc`. + unsafe { ListArc::from_raw(raw_item) } + } + + /// Moves all items from `other` into `self`. + /// + /// The items of `other` are added to the back of `self`, so the last item of `other` becomes + /// the last item of `self`. + pub fn push_all_back(&mut self, other: &mut List) { + // First, we insert the elements into `self`. At the end, we make `other` empty. + if self.is_empty() { + // INVARIANT: All of the elements in `other` become elements of `self`. + self.first = other.first; + } else if !other.is_empty() { + let other_first = other.first; + // SAFETY: The other list is not empty, so this pointer is valid. + let other_last = unsafe { (*other_first).prev }; + let self_first = self.first; + // SAFETY: The self list is not empty, so this pointer is valid. + let self_last = unsafe { (*self_first).prev }; + + // SAFETY: We have exclusive access to both lists, so we can update the pointers. + // INVARIANT: This correctly sets the pointers to merge both lists. We do not need to + // update `self.first` because the first element of `self` does not change. + unsafe { + (*self_first).prev = other_last; + (*other_last).next = self_first; + (*self_last).next = other_first; + (*other_first).prev = self_last; + } + } + + // INVARIANT: The other list is now empty, so update its pointer. + other.first = ptr::null_mut(); + } +} + +impl, const ID: u64> Default for List { + fn default() -> Self { + List::new() + } +} + +impl, const ID: u64> Drop for List { + fn drop(&mut self) { + while let Some(item) = self.pop_front() { + drop(item); + } + } } diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index c5921a7d5966..d801b9dc6291 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -133,8 +133,8 @@ pub use impl_list_arc_safe; /// The `ListArc` type can be thought of as a special reference to a refcounted object that owns the /// permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring /// that each object has only one `ListArc` reference, the owner of that reference is assured -/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a `List`, the -/// `List` takes ownership of the `ListArc` reference. +/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a [`List`], +/// the [`List`] takes ownership of the `ListArc` reference. /// /// There are various strategies to ensuring that a value has only one `ListArc` reference. The /// simplest is to convert a [`UniqueArc`] into a `ListArc`. However, the refcounted object could @@ -156,6 +156,8 @@ pub use impl_list_arc_safe; /// /// * Each reference counted object has at most one `ListArc` for each value of `ID`. /// * The tracking inside `T` is aware that a `ListArc` reference exists. +/// +/// [`List`]: crate::list::List #[repr(transparent)] pub struct ListArc where From deeecc9c1b979f45ca7b97255763505e5430cce5 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:26 +0000 Subject: [PATCH 037/111] rust: list: add iterators Rust Binder has lists containing stuff such as all contexts or all processes, and sometimes needs to iterate over them. This patch enables Rust Binder to do that using a normal for loop. The iterator returns the ArcBorrow type, so it is possible to grab a refcount to values while iterating. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-7-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 102 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index a87deadcfcc2..a215f77a9de4 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -5,7 +5,9 @@ //! A linked list implementation. use crate::init::PinInit; +use crate::sync::ArcBorrow; use crate::types::Opaque; +use core::iter::{DoubleEndedIterator, FusedIterator}; use core::marker::PhantomData; use core::ptr; @@ -437,6 +439,17 @@ impl, const ID: u64> List { // INVARIANT: The other list is now empty, so update its pointer. other.first = ptr::null_mut(); } + + /// Creates an iterator over the list. + pub fn iter(&self) -> Iter<'_, T, ID> { + // INVARIANT: If the list is empty, both pointers are null. Otherwise, both pointers point + // at the first element of the same list. + Iter { + current: self.first, + stop: self.first, + _ty: PhantomData, + } + } } impl, const ID: u64> Default for List { @@ -452,3 +465,92 @@ impl, const ID: u64> Drop for List { } } } + +/// An iterator over a [`List`]. +/// +/// # Invariants +/// +/// * There must be a [`List`] that is immutably borrowed for the duration of `'a`. +/// * The `current` pointer is null or points at a value in that [`List`]. +/// * The `stop` pointer is equal to the `first` field of that [`List`]. +#[derive(Clone)] +pub struct Iter<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + current: *mut ListLinksFields, + stop: *mut ListLinksFields, + _ty: PhantomData<&'a ListArc>, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Iterator for Iter<'a, T, ID> { + type Item = ArcBorrow<'a, T>; + + fn next(&mut self) -> Option> { + if self.current.is_null() { + return None; + } + + let current = self.current; + + // SAFETY: We just checked that `current` is not null, so it is in a list, and hence not + // dangling. There's no race because the iterator holds an immutable borrow to the list. + let next = unsafe { (*current).next }; + // INVARIANT: If `current` was the last element of the list, then this updates it to null. + // Otherwise, we update it to the next element. + self.current = if next != self.stop { + next + } else { + ptr::null_mut() + }; + + // SAFETY: The `current` pointer points at a value in the list. + let item = unsafe { T::view_value(ListLinks::from_fields(current)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the list. However, the `ArcBorrow` is annotated with the iterator's + // lifetime, and the list is immutably borrowed for that lifetime. + // * Values in a list never have a `UniqueArc` reference. + Some(unsafe { ArcBorrow::from_raw(item) }) + } +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> FusedIterator for Iter<'a, T, ID> {} + +impl<'a, T: ?Sized + ListItem, const ID: u64> IntoIterator for &'a List { + type IntoIter = Iter<'a, T, ID>; + type Item = ArcBorrow<'a, T>; + + fn into_iter(self) -> Iter<'a, T, ID> { + self.iter() + } +} + +/// An owning iterator into a [`List`]. +pub struct IntoIter, const ID: u64 = 0> { + list: List, +} + +impl, const ID: u64> Iterator for IntoIter { + type Item = ListArc; + + fn next(&mut self) -> Option> { + self.list.pop_front() + } +} + +impl, const ID: u64> FusedIterator for IntoIter {} + +impl, const ID: u64> DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option> { + self.list.pop_back() + } +} + +impl, const ID: u64> IntoIterator for List { + type IntoIter = IntoIter; + type Item = ListArc; + + fn into_iter(self) -> IntoIter { + IntoIter { list: self } + } +} From 9078a4f956dbef9366e1657915c883b380e6db39 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:27 +0000 Subject: [PATCH 038/111] rust: list: add cursor The cursor is very similar to the list iterator, but it has one important feature that the iterator doesn't: it can be used to remove items from the linked list. This feature cannot be added to the iterator because the references you get from the iterator are considered borrows of the original list, rather than borrows of the iterator. This means that there's no way to prevent code like this: let item = iter.next(); iter.remove(); use(item); If `iter` was a cursor instead of an iterator, then `item` will be considered a borrow of `iter`. Since `remove` destroys `iter`, this means that the borrow-checker will prevent uses of `item` after the call to `remove`. So there is a trade-off between supporting use in traditional for loops, and supporting removal of elements as you iterate. Iterators and cursors represents two different choices on that spectrum. Rust Binder needs cursors for the list of death notifications that a process is currently handling. When userspace tells Binder that it has finished processing the death notification, Binder will iterate the list to search for the relevant item and remove it. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-8-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 82 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index a215f77a9de4..6017f2e4ba3e 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -440,6 +440,20 @@ impl, const ID: u64> List { other.first = ptr::null_mut(); } + /// Returns a cursor to the first element of the list. + /// + /// If the list is empty, this returns `None`. + pub fn cursor_front(&mut self) -> Option> { + if self.first.is_null() { + None + } else { + Some(Cursor { + current: self.first, + list: self, + }) + } + } + /// Creates an iterator over the list. pub fn iter(&self) -> Iter<'_, T, ID> { // INVARIANT: If the list is empty, both pointers are null. Otherwise, both pointers point @@ -514,6 +528,74 @@ impl<'a, T: ?Sized + ListItem, const ID: u64> Iterator for Iter<'a, T, ID> { } } +/// A cursor into a [`List`]. +/// +/// # Invariants +/// +/// The `current` pointer points a value in `list`. +pub struct Cursor<'a, T: ?Sized + ListItem, const ID: u64 = 0> { + current: *mut ListLinksFields, + list: &'a mut List, +} + +impl<'a, T: ?Sized + ListItem, const ID: u64> Cursor<'a, T, ID> { + /// Access the current element of this cursor. + pub fn current(&self) -> ArcBorrow<'_, T> { + // SAFETY: The `current` pointer points a value in the list. + let me = unsafe { T::view_value(ListLinks::from_fields(self.current)) }; + // SAFETY: + // * All values in a list are stored in an `Arc`. + // * The value cannot be removed from the list for the duration of the lifetime annotated + // on the returned `ArcBorrow`, because removing it from the list would require mutable + // access to the cursor or the list. However, the `ArcBorrow` holds an immutable borrow + // on the cursor, which in turn holds a mutable borrow on the list, so any such + // mutable access requires first releasing the immutable borrow on the cursor. + // * Values in a list never have a `UniqueArc` reference, because the list has a `ListArc` + // reference, and `UniqueArc` references must be unique. + unsafe { ArcBorrow::from_raw(me) } + } + + /// Move the cursor to the next element. + pub fn next(self) -> Option> { + // SAFETY: The `current` field is always in a list. + let next = unsafe { (*self.current).next }; + + if next == self.list.first { + None + } else { + // INVARIANT: Since `self.current` is in the `list`, its `next` pointer is also in the + // `list`. + Some(Cursor { + current: next, + list: self.list, + }) + } + } + + /// Move the cursor to the previous element. + pub fn prev(self) -> Option> { + // SAFETY: The `current` field is always in a list. + let prev = unsafe { (*self.current).prev }; + + if self.current == self.list.first { + None + } else { + // INVARIANT: Since `self.current` is in the `list`, its `prev` pointer is also in the + // `list`. + Some(Cursor { + current: prev, + list: self.list, + }) + } + } + + /// Remove the current element from the list. + pub fn remove(self) -> ListArc { + // SAFETY: The `current` pointer always points at a member of the list. + unsafe { self.list.remove_internal(self.current) } + } +} + impl<'a, T: ?Sized + ListItem, const ID: u64> FusedIterator for Iter<'a, T, ID> {} impl<'a, T: ?Sized + ListItem, const ID: u64> IntoIterator for &'a List { From 2003c04b059759b0ec3bff108f24ded9de86a726 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:28 +0000 Subject: [PATCH 039/111] rust: list: support heterogeneous lists Support linked lists that can hold many different structs at once. This is generally done using trait objects. The main challenge is figuring what the struct is given only a pointer to the ListLinks. We do this by storing a pointer to the struct next to the ListLinks field. The container_of operation will then just read that pointer. When the type is a trait object, that pointer will be a fat pointer whose metadata is a vtable that tells you what kind of struct it is. Heterogeneous lists are heavily used by Rust Binder. There are a lot of so-called todo lists containing various events that need to be delivered to userspace next time userspace calls into the driver. And there are quite a few different todo item types: incoming transaction, changes to refcounts, death notifications, and more. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-9-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 47 ++++++++- rust/kernel/list/impl_list_item_mod.rs | 131 +++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 6017f2e4ba3e..432a75a58d02 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -12,7 +12,9 @@ use core::marker::PhantomData; use core::ptr; mod impl_list_item_mod; -pub use self::impl_list_item_mod::{impl_has_list_links, impl_list_item, HasListLinks}; +pub use self::impl_list_item_mod::{ + impl_has_list_links, impl_has_list_links_self_ptr, impl_list_item, HasListLinks, HasSelfPtr, +}; mod arc; pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; @@ -183,6 +185,49 @@ impl ListLinks { } } +/// Similar to [`ListLinks`], but also contains a pointer to the full value. +/// +/// This type can be used instead of [`ListLinks`] to support lists with trait objects. +#[repr(C)] +pub struct ListLinksSelfPtr { + /// The `ListLinks` field inside this value. + /// + /// This is public so that it can be used with `impl_has_list_links!`. + pub inner: ListLinks, + // UnsafeCell is not enough here because we use `Opaque::uninit` as a dummy value, and + // `ptr::null()` doesn't work for `T: ?Sized`. + self_ptr: Opaque<*const T>, +} + +// SAFETY: The fields of a ListLinksSelfPtr can be moved across thread boundaries. +unsafe impl Send for ListLinksSelfPtr {} +// SAFETY: The type is opaque so immutable references to a ListLinksSelfPtr are useless. Therefore, +// it's okay to have immutable access to a ListLinks from several threads at once. +// +// Note that `inner` being a public field does not prevent this type from being opaque, since +// `inner` is a opaque type. +unsafe impl Sync for ListLinksSelfPtr {} + +impl ListLinksSelfPtr { + /// The offset from the [`ListLinks`] to the self pointer field. + pub const LIST_LINKS_SELF_PTR_OFFSET: usize = core::mem::offset_of!(Self, self_ptr); + + /// Creates a new initializer for this type. + pub fn new() -> impl PinInit { + // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will + // not be constructed in an `Arc` that already has a `ListArc`. + Self { + inner: ListLinks { + inner: Opaque::new(ListLinksFields { + prev: ptr::null_mut(), + next: ptr::null_mut(), + }), + }, + self_ptr: Opaque::uninit(), + } + } +} + impl, const ID: u64> List { /// Creates a new empty list. pub const fn new() -> Self { diff --git a/rust/kernel/list/impl_list_item_mod.rs b/rust/kernel/list/impl_list_item_mod.rs index 1bcb14774aeb..a0438537cee1 100644 --- a/rust/kernel/list/impl_list_item_mod.rs +++ b/rust/kernel/list/impl_list_item_mod.rs @@ -68,6 +68,49 @@ macro_rules! impl_has_list_links { } pub use impl_has_list_links; +/// Declares that the `ListLinks` field in this struct is inside a `ListLinksSelfPtr`. +/// +/// # Safety +/// +/// The `ListLinks` field of this struct at the offset `HasListLinks::OFFSET` must be +/// inside a `ListLinksSelfPtr`. +pub unsafe trait HasSelfPtr +where + Self: HasListLinks, +{ +} + +/// Implements the [`HasListLinks`] and [`HasSelfPtr`] traits for the given type. +#[macro_export] +macro_rules! impl_has_list_links_self_ptr { + ($(impl$({$($implarg:tt)*})? + HasSelfPtr<$item_type:ty $(, $id:tt)?> + for $self:ident $(<$($selfarg:ty),*>)? + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_list_links` only compiles if the field has the + // right type. + unsafe impl$(<$($implarg)*>)? $crate::list::HasSelfPtr<$item_type $(, $id)?> for + $self $(<$($selfarg),*>)? + {} + + unsafe impl$(<$($implarg)*>)? $crate::list::HasListLinks$(<$id>)? for + $self $(<$($selfarg),*>)? + { + const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; + + #[inline] + unsafe fn raw_get_list_links(ptr: *mut Self) -> *mut $crate::list::ListLinks$(<$id>)? { + // SAFETY: The caller promises that the pointer is not dangling. + let ptr: *mut $crate::list::ListLinksSelfPtr<$item_type $(, $id)?> = + unsafe { ::core::ptr::addr_of_mut!((*ptr).$field) }; + ptr.cast() + } + } + )*}; +} +pub use impl_has_list_links_self_ptr; + /// Implements the [`ListItem`] trait for the given type. /// /// Requires that the type implements [`HasListLinks`]. Use the [`impl_has_list_links!`] macro to @@ -139,5 +182,93 @@ macro_rules! impl_list_item { } } )*}; + + ( + $(impl$({$($generics:tt)*})? ListItem<$num:tt> for $t:ty { + using ListLinksSelfPtr; + })* + ) => {$( + // SAFETY: See GUARANTEES comment on each method. + unsafe impl$(<$($generics)*>)? $crate::list::ListItem<$num> for $t { + // GUARANTEES: + // This implementation of `ListItem` will not give out exclusive access to the same + // `ListLinks` several times because calls to `prepare_to_insert` and `post_remove` + // must alternate and exclusive access is given up when `post_remove` is called. + // + // Other invocations of `impl_list_item!` also cannot give out exclusive access to the + // same `ListLinks` because you can only implement `ListItem` once for each value of + // `ID`, and the `ListLinks` fields only work with the specified `ID`. + unsafe fn prepare_to_insert(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + let links_field = unsafe { >::view_links(me) }; + + let spoff = $crate::list::ListLinksSelfPtr::::LIST_LINKS_SELF_PTR_OFFSET; + // Goes via the offset as the field is private. + // + // SAFETY: The constant is equal to `offset_of!(ListLinksSelfPtr, self_ptr)`, so + // the pointer stays in bounds of the allocation. + let self_ptr = unsafe { (links_field as *const u8).add(spoff) } + as *const $crate::types::Opaque<*const Self>; + let cell_inner = $crate::types::Opaque::raw_get(self_ptr); + + // SAFETY: This value is not accessed in any other places than `prepare_to_insert`, + // `post_remove`, or `view_value`. By the safety requirements of those methods, + // none of these three methods may be called in parallel with this call to + // `prepare_to_insert`, so this write will not race with any other access to the + // value. + unsafe { ::core::ptr::write(cell_inner, me) }; + + links_field + } + + // GUARANTEES: + // * This returns the same pointer as `prepare_to_insert` because `prepare_to_insert` + // returns the return value of `view_links`. + // * By the type invariants of `ListLinks`, the `ListLinks` has two null pointers when + // this value is not in a list. + unsafe fn view_links(me: *const Self) -> *mut $crate::list::ListLinks<$num> { + // SAFETY: The caller promises that `me` points at a valid value of type `Self`. + unsafe { >::raw_get_list_links(me.cast_mut()) } + } + + // This function is also used as the implementation of `post_remove`, so the caller + // may choose to satisfy the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + // + // GUARANTEES: (always) + // * This returns the same pointer as the one passed to the most recent call to + // `prepare_to_insert` since that call wrote that pointer to this location. The value + // is only modified in `prepare_to_insert`, so it has not been modified since the + // most recent call. + // + // GUARANTEES: (only when using the `view_value` safety requirements) + // * The pointer remains valid until the next call to `post_remove` because the caller + // of the most recent call to `prepare_to_insert` promised to retain ownership of the + // `ListArc` containing `Self` until the next call to `post_remove`. The value cannot + // be destroyed while a `ListArc` reference exists. + unsafe fn view_value(links_field: *mut $crate::list::ListLinks<$num>) -> *const Self { + let spoff = $crate::list::ListLinksSelfPtr::::LIST_LINKS_SELF_PTR_OFFSET; + // SAFETY: The constant is equal to `offset_of!(ListLinksSelfPtr, self_ptr)`, so + // the pointer stays in bounds of the allocation. + let self_ptr = unsafe { (links_field as *const u8).add(spoff) } + as *const ::core::cell::UnsafeCell<*const Self>; + let cell_inner = ::core::cell::UnsafeCell::raw_get(self_ptr); + // SAFETY: This is not a data race, because the only function that writes to this + // value is `prepare_to_insert`, but by the safety requirements the + // `prepare_to_insert` method may not be called in parallel with `view_value` or + // `post_remove`. + unsafe { ::core::ptr::read(cell_inner) } + } + + // GUARANTEES: + // The first guarantee of `view_value` is exactly what `post_remove` guarantees. + unsafe fn post_remove(me: *mut $crate::list::ListLinks<$num>) -> *const Self { + // SAFETY: This specific implementation of `view_value` allows the caller to + // promise the safety requirements of `post_remove` instead of the safety + // requirements for `view_value`. + unsafe { >::view_value(me) } + } + } + )*}; } pub use impl_list_item; From b204bbc53f958fc3119d63bf2cda5a526e7267a4 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 14 Aug 2024 08:05:29 +0000 Subject: [PATCH 040/111] rust: list: add ListArcField One way to explain what `ListArc` does is that it controls exclusive access to the prev/next pointer field in a refcounted object. The feature of having a special reference to a refcounted object with exclusive access to specific fields is useful for other things, so provide a general utility for that. This is used by Rust Binder to keep track of which processes have a reference to a given node. This involves an object for each process/node pair, that is referenced by both the process and the node. For some fields in this object, only the process's reference needs to access them (and it needs mutable access), so Binder uses a ListArc to give the process's reference exclusive access. Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240814-linked-list-v5-10-f5f5e8075da0@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/list.rs | 3 ++ rust/kernel/list/arc_field.rs | 96 +++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 rust/kernel/list/arc_field.rs diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 432a75a58d02..5b4aec29eb67 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -19,6 +19,9 @@ pub use self::impl_list_item_mod::{ mod arc; pub use self::arc::{impl_list_arc_safe, AtomicTracker, ListArc, ListArcSafe, TryNewListArc}; +mod arc_field; +pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; + /// A linked list. /// /// All elements in this linked list will be [`ListArc`] references to the value. Since a value can diff --git a/rust/kernel/list/arc_field.rs b/rust/kernel/list/arc_field.rs new file mode 100644 index 000000000000..2330f673427a --- /dev/null +++ b/rust/kernel/list/arc_field.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2024 Google LLC. + +//! A field that is exclusively owned by a [`ListArc`]. +//! +//! This can be used to have reference counted struct where one of the reference counted pointers +//! has exclusive access to a field of the struct. +//! +//! [`ListArc`]: crate::list::ListArc + +use core::cell::UnsafeCell; + +/// A field owned by a specific [`ListArc`]. +/// +/// [`ListArc`]: crate::list::ListArc +pub struct ListArcField { + value: UnsafeCell, +} + +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Send for ListArcField {} +// SAFETY: If the inner type is thread-safe, then it's also okay for `ListArc` to be thread-safe. +unsafe impl Sync for ListArcField {} + +impl ListArcField { + /// Creates a new `ListArcField`. + pub fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + } + } + + /// Access the value when we have exclusive access to the `ListArcField`. + /// + /// This allows access to the field using an `UniqueArc` instead of a `ListArc`. + pub fn get_mut(&mut self) -> &mut T { + self.value.get_mut() + } + + /// Unsafely assert that you have shared access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have shared access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + pub unsafe fn assert_ref(&self) -> &T { + // SAFETY: The caller has shared access to the `ListArc`, so they also have shared access + // to this field. + unsafe { &*self.value.get() } + } + + /// Unsafely assert that you have mutable access to the `ListArc` for this field. + /// + /// # Safety + /// + /// The caller must have mutable access to the `ListArc` containing the struct with this + /// field for the duration of the returned reference. + #[allow(clippy::mut_from_ref)] + pub unsafe fn assert_mut(&self) -> &mut T { + // SAFETY: The caller has exclusive access to the `ListArc`, so they also have exclusive + // access to this field. + unsafe { &mut *self.value.get() } + } +} + +/// Defines getters for a [`ListArcField`]. +#[macro_export] +macro_rules! define_list_arc_field_getter { + ($pub:vis fn $name:ident(&self $(<$id:tt>)?) -> &$typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a $crate::list::ListArc) -> &'a $typ { + let field = &(&**self).$field; + // SAFETY: We have a shared reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_ref(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + ($pub:vis fn $name:ident(&mut self $(<$id:tt>)?) -> &mut $typ:ty { $field:ident } + $($rest:tt)* + ) => { + $pub fn $name<'a>(self: &'a mut $crate::list::ListArc) -> &'a mut $typ { + let field = &(&**self).$field; + // SAFETY: We have a mutable reference to the `ListArc`. + unsafe { $crate::list::ListArcField::<$typ $(, $id)?>::assert_mut(field) } + } + + $crate::list::define_list_arc_field_getter!($($rest)*); + }; + + () => {}; +} +pub use define_list_arc_field_getter; From f363930484d733e784fce12355d32d04e5baa313 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 16 Aug 2024 20:15:14 +0200 Subject: [PATCH 041/111] docs: rust: quick-start: add Debian Testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Debian Testing is now also providing recent Rust releases (outside of the freeze period), like Debian Unstable (Sid). Thus add it to the list. Cc: Fabian Grünbichler Signed-off-by: Miguel Ojeda --- Documentation/rust/quick-start.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst index d06a36106cd4..ed5af2b5339a 100644 --- a/Documentation/rust/quick-start.rst +++ b/Documentation/rust/quick-start.rst @@ -39,8 +39,8 @@ of the box, e.g.:: Debian ****** -Debian Unstable (Sid), outside of the freeze period, provides recent Rust -releases and thus it should generally work out of the box, e.g.:: +Debian Testing and Debian Unstable (Sid), outside of the freeze period, provide +recent Rust releases and thus they should generally work out of the box, e.g.:: apt install rustc rust-src bindgen rustfmt rust-clippy From c73051168e7fc0c78e58791097cbc3a3ec95839e Mon Sep 17 00:00:00 2001 From: Michael Vetter Date: Tue, 20 Aug 2024 09:26:43 +0200 Subject: [PATCH 042/111] rust: kernel: use docs.kernel.org links in code documentation Use links to docs.kernel.org instead of www.kernel.org/doc/html/latest in the code documentation. The links are shorter and cleaner. Link: https://github.com/Rust-for-Linux/linux/issues/1101 Signed-off-by: Michael Vetter [ Reworded slightly. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/print.rs | 20 ++++++++++---------- rust/kernel/std_vendor.rs | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index a78aa3514a0a..508b0221256c 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -4,7 +4,7 @@ //! //! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h) //! -//! Reference: +//! Reference: use core::{ ffi::{c_char, c_void}, @@ -197,7 +197,7 @@ macro_rules! print_macro ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg +/// [`pr_emerg`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_emerg /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -221,7 +221,7 @@ macro_rules! pr_emerg ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_alert`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_alert +/// [`pr_alert`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_alert /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -245,7 +245,7 @@ macro_rules! pr_alert ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_crit`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_crit +/// [`pr_crit`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_crit /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -269,7 +269,7 @@ macro_rules! pr_crit ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_err`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_err +/// [`pr_err`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_err /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -293,7 +293,7 @@ macro_rules! pr_err ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_warn`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_warn +/// [`pr_warn`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_warn /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -317,7 +317,7 @@ macro_rules! pr_warn ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_notice`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_notice +/// [`pr_notice`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_notice /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -341,7 +341,7 @@ macro_rules! pr_notice ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info +/// [`pr_info`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_info /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -367,7 +367,7 @@ macro_rules! pr_info ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// -/// [`pr_debug`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_debug +/// [`pr_debug`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_debug /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples @@ -395,7 +395,7 @@ macro_rules! pr_debug ( /// `alloc::format!` for information about the formatting syntax. /// /// [`pr_info!`]: crate::pr_info! -/// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont +/// [`pr_cont`]: https://docs.kernel.org/core-api/printk-basics.html#c.pr_cont /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// /// # Examples diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 39679a960c1a..67bf9d37ddb5 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -136,7 +136,7 @@ /// /// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html /// [`eprintln`]: https://doc.rust-lang.org/std/macro.eprintln.html -/// [`printk`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html +/// [`printk`]: https://docs.kernel.org/core-api/printk-basics.html /// [`pr_info`]: crate::pr_info! /// [`pr_debug`]: crate::pr_debug! #[macro_export] From 96fff2dc2954dcaf7affe7da212aba3f5107d06d Mon Sep 17 00:00:00 2001 From: Kartik Prajapati Date: Wed, 21 Aug 2024 19:58:58 +0000 Subject: [PATCH 043/111] rust: types: add `ARef::into_raw` Add a method for `ARef` that is analogous to `Arc::into_raw`. It is the inverse operation of `ARef::from_raw`, and allows you to convert the `ARef` back into a raw pointer while retaining ownership of the refcount. This new function will be used by [1] for converting the type in an `ARef` using `ARef::from_raw(ARef::into_raw(me).cast())`. Alice has also needed the same function for other use-cases in the past, but [1] is the first to go upstream. This was implemented independently by Kartik and Alice. The two versions were merged by Alice, so all mistakes are Alice's. Link: https://lore.kernel.org/r/20240801-vma-v3-1-db6c1c0afda9@google.com [1] Link: https://github.com/Rust-for-Linux/linux/issues/1044 Signed-off-by: Kartik Prajapati Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Reviewed-by: Benno Lossin [ Reworded to correct the author reference and changed tag to Link since it is not a bug. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/kernel/types.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ee7dd1f963ef..9e7ca066355c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -7,7 +7,7 @@ use alloc::boxed::Box; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, - mem::MaybeUninit, + mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, ptr::NonNull, @@ -396,6 +396,35 @@ impl ARef { _p: PhantomData, } } + + /// Consumes the `ARef`, returning a raw pointer. + /// + /// This function does not change the refcount. After calling this function, the caller is + /// responsible for the refcount previously managed by the `ARef`. + /// + /// # Examples + /// + /// ``` + /// use core::ptr::NonNull; + /// use kernel::types::{ARef, AlwaysRefCounted}; + /// + /// struct Empty {} + /// + /// unsafe impl AlwaysRefCounted for Empty { + /// fn inc_ref(&self) {} + /// unsafe fn dec_ref(_obj: NonNull) {} + /// } + /// + /// let mut data = Empty {}; + /// let ptr = NonNull::::new(&mut data as *mut _).unwrap(); + /// let data_ref: ARef = unsafe { ARef::from_raw(ptr) }; + /// let raw_ptr: NonNull = ARef::into_raw(data_ref); + /// + /// assert_eq!(ptr, raw_ptr); + /// ``` + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } } impl Clone for ARef { From 0eef6ec5a8e5bc2d633d6716f2640de8d6b45da7 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 18 Aug 2024 16:11:59 +0200 Subject: [PATCH 044/111] docs: rust: link to https://rust.docs.kernel.org The Rust code documentation (i.e. `rustdoc`-generated docs) is now available at: https://rust.docs.kernel.org Thus document it and remove the `TODO` line. The generation uses a particular kernel configuration, based on x86_64, which may get tweaked over time. Older tags, and how they are generated, may also change in the future. We may consider freezing them at some point, but for the moment, the content should not be considered immutable. Thanks Konstantin for the support setting it up! Cc: Konstantin Ryabitsev Acked-by: Konstantin Ryabitsev Link: https://lore.kernel.org/r/20240818141200.386899-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Documentation/rust/general-information.rst | 13 ++++++++++--- Documentation/rust/index.rst | 3 +++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst index a82926d7b379..2dec01888d53 100644 --- a/Documentation/rust/general-information.rst +++ b/Documentation/rust/general-information.rst @@ -22,10 +22,17 @@ Rust kernel code is documented using ``rustdoc``, its built-in documentation generator. The generated HTML docs include integrated search, linked items (e.g. types, -functions, constants), source code, etc. They may be read at (TODO: link when -in mainline and generated alongside the rest of the documentation): +functions, constants), source code, etc. They may be read at: - http://kernel.org/ + https://rust.docs.kernel.org + +For linux-next, please see: + + https://rust.docs.kernel.org/next/ + +There are also tags for each main release, e.g.: + + https://rust.docs.kernel.org/6.10/ The docs can also be easily generated and read locally. This is quite fast (same order as compiling the code itself) and no special tools or environment diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst index 46d35bd395cf..5e51bfb248a4 100644 --- a/Documentation/rust/index.rst +++ b/Documentation/rust/index.rst @@ -32,6 +32,9 @@ configurations. .. only:: not rustdoc and html This documentation does not include rustdoc generated information. + A pregenerated version is provided at: + + https://rust.docs.kernel.org .. toctree:: :maxdepth: 1 From b2bf463ed9a8131ad4e91a11af8c9a4ec84b876a Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 18 Aug 2024 16:12:00 +0200 Subject: [PATCH 045/111] docs: rust: improve main page introducing a "Code documentation" section Clean the "Rust" main page by introducing a 'Code documentation" section to separate it from the rest of the text above. In addition, introduce the "Rust code documentation" term, which may be clearer than referring to a potentially unknown tool. Furthermore, for the HTML case, homogenize both `rustdoc` and non-`rustdoc` cases and use the term introduced above instead. Then, always generate the pregenerated version part, since now there is a section that is always generated and thus makes sense to do so. Finally, finish the new section with a link to more details about the Rust code documentation. The intention is that: - The non-HTML case mentions the code documentation too, making it more prominent for readers of non-HTML docs. - Both HTML cases read more naturally. - The pregenerated version is always mentioned, since it is likely useful for readers of non-HTML docs. Link: https://lore.kernel.org/r/20240818141200.386899-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Documentation/rust/general-information.rst | 2 ++ Documentation/rust/index.rst | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst index 2dec01888d53..2d76e09da121 100644 --- a/Documentation/rust/general-information.rst +++ b/Documentation/rust/general-information.rst @@ -15,6 +15,8 @@ but not `std `_. Crates for use in the kernel must opt into this behavior using the ``#![no_std]`` attribute. +.. _rust_code_documentation: + Code documentation ------------------ diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst index 5e51bfb248a4..55dcde9e9e7e 100644 --- a/Documentation/rust/index.rst +++ b/Documentation/rust/index.rst @@ -25,16 +25,27 @@ support is still in development/experimental, especially for certain kernel configurations. +Code documentation +------------------ + +Given a kernel configuration, the kernel may generate Rust code documentation, +i.e. HTML rendered by the ``rustdoc`` tool. + .. only:: rustdoc and html - You can also browse `rustdoc documentation `_. + This kernel documentation was built with `Rust code documentation + `_. .. only:: not rustdoc and html - This documentation does not include rustdoc generated information. - A pregenerated version is provided at: + This kernel documentation was not built with Rust code documentation. - https://rust.docs.kernel.org +A pregenerated version is provided at: + + https://rust.docs.kernel.org + +Please see the :ref:`Code documentation ` section for +more details. .. toctree:: :maxdepth: 1 From 6e6efc5fef4a1cdcccca3cffd5b73fd25d093352 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 18 Aug 2024 16:12:49 +0200 Subject: [PATCH 046/111] rust: enable rustdoc's `--generate-link-to-definition` In Rust 1.56.0 [1], rustdoc introduced the "jump to definition" feature [2], i.e. the unstable flag `--generate-link-to-definition`. It adds links to the source view of the documentation. For instance, in the source view of `rust/kernel/sync.rs`, for this code: impl Default for LockClassKey { fn default() -> Self { Self::new() } } It will add three hyperlinks: - `Default` points to the rendered "Trait `core::default::Default`" page (not the source view, since it goes to another crate, though this may change). - `LockClassKey` points to the `pub struct LockClassKey(...);` line in the same page, highlighting the line number. - `Self::new()` points to the `pub const fn new() -> Self { ... }` associated function, highlighting its line numbers (i.e. for the full function). This makes the source view more useful and a bit closer to the experience in e.g. the Elixir Cross Referencer [3]. I have provisionally enabled it for rust.docs.kernel.org [4] -- one can take a look at the source view there for an example of how it looks like. Thus enable it. Cc: Guillaume Gomez Link: https://github.com/rust-lang/rust/pull/84176 [1] Link: https://github.com/rust-lang/rust/issues/89095 [2] Link: https://elixir.bootlin.com [3] Link: https://rust.docs.kernel.org [4] Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20240818141249.387166-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/Makefile b/rust/Makefile index c24c3689e7b4..e13d14ec5fe7 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -63,6 +63,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \ $(rustc_target_flags) -L$(objtree)/$(obj) \ + -Zunstable-options --generate-link-to-definition \ --output $(rustdoc_output) \ --crate-name $(subst rustdoc-,,$@) \ $(if $(rustdoc_host),,--sysroot=/dev/null) \ From a0d13aac7022f95ec161c18d18e3d81172666ed8 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 22 Aug 2024 16:37:53 +0000 Subject: [PATCH 047/111] rust: rbtree: add red-black tree implementation backed by the C version The rust rbtree exposes a map-like interface over keys and values, backed by the kernel red-black tree implementation. Values can be inserted, deleted, and retrieved from a `RBTree` by key. This base abstraction is used by binder to store key/value pairs and perform lookups, for example the patch "[PATCH RFC 03/20] rust_binder: add threading support" in the binder RFC [1]. Link: https://lore.kernel.org/rust-for-linux/20231101-rust-binder-v1-3-08ba9197f637@google.com/ [1] Signed-off-by: Wedson Almeida Filho Reviewed-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Boqun Feng Reviewed-by: Benno Lossin Signed-off-by: Matt Gilbride Link: https://lore.kernel.org/r/20240822-b4-rbtree-v12-1-014561758a57@google.com [ Updated link to docs.kernel.org. - Miguel ] Signed-off-by: Miguel Ojeda --- rust/helpers/helpers.c | 1 + rust/helpers/rbtree.c | 9 + rust/kernel/lib.rs | 1 + rust/kernel/rbtree.rs | 432 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 443 insertions(+) create mode 100644 rust/helpers/rbtree.c create mode 100644 rust/kernel/rbtree.rs diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 173533616c91..30f40149f3a9 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -15,6 +15,7 @@ #include "kunit.c" #include "mutex.c" #include "page.c" +#include "rbtree.c" #include "refcount.c" #include "signal.c" #include "slab.c" diff --git a/rust/helpers/rbtree.c b/rust/helpers/rbtree.c new file mode 100644 index 000000000000..6d404b84a9b5 --- /dev/null +++ b/rust/helpers/rbtree.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_rb_link_node(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) +{ + rb_link_node(node, parent, rb_link); +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9baea9e9ee1a..f10b06a78b9d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -44,6 +44,7 @@ pub mod net; pub mod page; pub mod prelude; pub mod print; +pub mod rbtree; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs new file mode 100644 index 000000000000..cf25437c795f --- /dev/null +++ b/rust/kernel/rbtree.rs @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Red-black trees. +//! +//! C header: [`include/linux/rbtree.h`](srctree/include/linux/rbtree.h) +//! +//! Reference: + +use crate::{alloc::Flags, bindings, container_of, error::Result, prelude::*}; +use alloc::boxed::Box; +use core::{ + cmp::{Ord, Ordering}, + marker::PhantomData, + mem::MaybeUninit, + ptr::{addr_of_mut, NonNull}, +}; + +/// A red-black tree with owned nodes. +/// +/// It is backed by the kernel C red-black trees. +/// +/// # Examples +/// +/// In the example below we do several operations on a tree. We note that insertions may fail if +/// the system is out of memory. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &100); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30).unwrap(), &300); +/// } +/// +/// // Replace one of the elements. +/// tree.try_create_and_insert(10, 1000, flags::GFP_KERNEL)?; +/// +/// // Check that the tree reflects the replacement. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &1000); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30).unwrap(), &300); +/// } +/// +/// // Change the value of one of the elements. +/// *tree.get_mut(&30).unwrap() = 3000; +/// +/// // Check that the tree reflects the update. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &1000); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30).unwrap(), &3000); +/// } +/// +/// // Remove an element. +/// tree.remove(&10); +/// +/// // Check that the tree reflects the removal. +/// { +/// assert_eq!(tree.get(&10), None); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30).unwrap(), &3000); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// In the example below, we first allocate a node, acquire a spinlock, then insert the node into +/// the tree. This is useful when the insertion context does not allow sleeping, for example, when +/// holding a spinlock. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNode}, sync::SpinLock}; +/// +/// fn insert_test(tree: &SpinLock>) -> Result { +/// // Pre-allocate node. This may fail (as it allocates memory). +/// let node = RBTreeNode::new(10, 100, flags::GFP_KERNEL)?; +/// +/// // Insert node while holding the lock. It is guaranteed to succeed with no allocation +/// // attempts. +/// let mut guard = tree.lock(); +/// guard.insert(node); +/// Ok(()) +/// } +/// ``` +/// +/// In the example below, we reuse an existing node allocation from an element we removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::{RBTree, RBTreeNodeReservation}}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Check the nodes we just inserted. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &100); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30).unwrap(), &300); +/// } +/// +/// // Remove a node, getting back ownership of it. +/// let existing = tree.remove(&30).unwrap(); +/// +/// // Check that the tree reflects the removal. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &100); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// assert_eq!(tree.get(&30), None); +/// } +/// +/// // Create a preallocated reservation that we can re-use later. +/// let reservation = RBTreeNodeReservation::new(flags::GFP_KERNEL)?; +/// +/// // Insert a new node into the tree, reusing the previous allocation. This is guaranteed to +/// // succeed (no memory allocations). +/// tree.insert(reservation.into_node(15, 150)); +/// +/// // Check that the tree reflect the new insertion. +/// { +/// assert_eq!(tree.get(&10).unwrap(), &100); +/// assert_eq!(tree.get(&15).unwrap(), &150); +/// assert_eq!(tree.get(&20).unwrap(), &200); +/// } +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// # Invariants +/// +/// Non-null parent/children pointers stored in instances of the `rb_node` C struct are always +/// valid, and pointing to a field of our internal representation of a node. +pub struct RBTree { + root: bindings::rb_root, + _p: PhantomData>, +} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Send condition as would be used for a struct with K and V fields. +unsafe impl Send for RBTree {} + +// SAFETY: An [`RBTree`] allows the same kinds of access to its values that a struct allows to its +// fields, so we use the same Sync condition as would be used for a struct with K and V fields. +unsafe impl Sync for RBTree {} + +impl RBTree { + /// Creates a new and empty tree. + pub fn new() -> Self { + Self { + // INVARIANT: There are no nodes in the tree, so the invariant holds vacuously. + root: bindings::rb_root::default(), + _p: PhantomData, + } + } +} + +impl RBTree +where + K: Ord, +{ + /// Tries to insert a new value into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// Returns an error if it cannot allocate memory for the new node. + pub fn try_create_and_insert( + &mut self, + key: K, + value: V, + flags: Flags, + ) -> Result>> { + Ok(self.insert(RBTreeNode::new(key, value, flags)?)) + } + + /// Inserts a new node into the tree. + /// + /// It overwrites a node if one already exists with the same key and returns it (containing the + /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. + /// + /// This function always succeeds. + pub fn insert(&mut self, RBTreeNode { node }: RBTreeNode) -> Option> { + let node = Box::into_raw(node); + // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when + // the node is removed or replaced. + let node_links = unsafe { addr_of_mut!((*node).links) }; + + // The parameters of `bindings::rb_link_node` are as follows: + // - `node`: A pointer to an uninitialized node being inserted. + // - `parent`: A pointer to an existing node in the tree. One of its child pointers must be + // null, and `node` will become a child of `parent` by replacing that child pointer + // with a pointer to `node`. + // - `rb_link`: A pointer to either the left-child or right-child field of `parent`. This + // specifies which child of `parent` should hold `node` after this call. The + // value of `*rb_link` must be null before the call to `rb_link_node`. If the + // red/black tree is empty, then it’s also possible for `parent` to be null. In + // this case, `rb_link` is a pointer to the `root` field of the red/black tree. + // + // We will traverse the tree looking for a node that has a null pointer as its child, + // representing an empty subtree where we can insert our new node. We need to make sure + // that we preserve the ordering of the nodes in the tree. In each iteration of the loop + // we store `parent` and `child_field_of_parent`, and the new `node` will go somewhere + // in the subtree of `parent` that `child_field_of_parent` points at. Once + // we find an empty subtree, we can insert the new node using `rb_link_node`. + let mut parent = core::ptr::null_mut(); + let mut child_field_of_parent: &mut *mut bindings::rb_node = &mut self.root.rb_node; + while !child_field_of_parent.is_null() { + parent = *child_field_of_parent; + + // We need to determine whether `node` should be the left or right child of `parent`, + // so we will compare with the `key` field of `parent` a.k.a. `this` below. + // + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(parent, Node, links) }; + + // SAFETY: `this` is a non-null node so it is valid by the type invariants. `node` is + // valid until the node is removed. + match unsafe { (*node).key.cmp(&(*this).key) } { + // We would like `node` to be the left child of `parent`. Move to this child to check + // whether we can use it, or continue searching, at the next iteration. + // + // SAFETY: `parent` is a non-null node so it is valid by the type invariants. + Ordering::Less => child_field_of_parent = unsafe { &mut (*parent).rb_left }, + // We would like `node` to be the right child of `parent`. Move to this child to check + // whether we can use it, or continue searching, at the next iteration. + // + // SAFETY: `parent` is a non-null node so it is valid by the type invariants. + Ordering::Greater => child_field_of_parent = unsafe { &mut (*parent).rb_right }, + Ordering::Equal => { + // There is an existing node in the tree with this key, and that node is + // `parent`. Thus, we are replacing parent with a new node. + // + // INVARIANT: We are replacing an existing node with a new one, which is valid. + // It remains valid because we "forgot" it with `Box::into_raw`. + // SAFETY: All pointers are non-null and valid. + unsafe { bindings::rb_replace_node(parent, node_links, &mut self.root) }; + + // INVARIANT: The node is being returned and the caller may free it, however, + // it was removed from the tree. So the invariants still hold. + return Some(RBTreeNode { + // SAFETY: `this` was a node in the tree, so it is valid. + node: unsafe { Box::from_raw(this.cast_mut()) }, + }); + } + } + } + + // INVARIANT: We are linking in a new node, which is valid. It remains valid because we + // "forgot" it with `Box::into_raw`. + // SAFETY: All pointers are non-null and valid (`*child_field_of_parent` is null, but `child_field_of_parent` is a + // mutable reference). + unsafe { bindings::rb_link_node(node_links, parent, child_field_of_parent) }; + + // SAFETY: All pointers are valid. `node` has just been inserted into the tree. + unsafe { bindings::rb_insert_color(node_links, &mut self.root) }; + None + } + + /// Returns a node with the given key, if one exists. + fn find(&self, key: &K) -> Option>> { + let mut node = self.root.rb_node; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }; + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + node = match key.cmp(unsafe { &(*this).key }) { + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + Ordering::Less => unsafe { (*node).rb_left }, + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + Ordering::Greater => unsafe { (*node).rb_right }, + Ordering::Equal => return NonNull::new(this.cast_mut()), + } + } + None + } + + /// Returns a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option<&V> { + // SAFETY: The `find` return value is a node in the tree, so it is valid. + self.find(key).map(|node| unsafe { &node.as_ref().value }) + } + + /// Returns a mutable reference to the value corresponding to the key. + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + // SAFETY: The `find` return value is a node in the tree, so it is valid. + self.find(key) + .map(|mut node| unsafe { &mut node.as_mut().value }) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the node that was removed if one exists, or [`None`] otherwise. + fn remove_node(&mut self, key: &K) -> Option> { + let mut node = self.find(key)?; + + // SAFETY: The `find` return value is a node in the tree, so it is valid. + unsafe { bindings::rb_erase(&mut node.as_mut().links, &mut self.root) }; + + // INVARIANT: The node is being returned and the caller may free it, however, it was + // removed from the tree. So the invariants still hold. + Some(RBTreeNode { + // SAFETY: The `find` return value was a node in the tree, so it is valid. + node: unsafe { Box::from_raw(node.as_ptr()) }, + }) + } + + /// Removes the node with the given key from the tree. + /// + /// It returns the value that was removed if one exists, or [`None`] otherwise. + pub fn remove(&mut self, key: &K) -> Option { + self.remove_node(key).map(|node| node.node.value) + } +} + +impl Default for RBTree { + fn default() -> Self { + Self::new() + } +} + +impl Drop for RBTree { + fn drop(&mut self) { + // SAFETY: `root` is valid as it's embedded in `self` and we have a valid `self`. + let mut next = unsafe { bindings::rb_first_postorder(&self.root) }; + + // INVARIANT: The loop invariant is that all tree nodes from `next` in postorder are valid. + while !next.is_null() { + // SAFETY: All links fields we create are in a `Node`. + let this = unsafe { container_of!(next, Node, links) }; + + // Find out what the next node is before disposing of the current one. + // SAFETY: `next` and all nodes in postorder are still valid. + next = unsafe { bindings::rb_next_postorder(next) }; + + // INVARIANT: This is the destructor, so we break the type invariant during clean-up, + // but it is not observable. The loop invariant is still maintained. + + // SAFETY: `this` is valid per the loop invariant. + unsafe { drop(Box::from_raw(this.cast_mut())) }; + } + } +} + +/// A memory reservation for a red-black tree node. +/// +/// +/// It contains the memory needed to hold a node that can be inserted into a red-black tree. One +/// can be obtained by directly allocating it ([`RBTreeNodeReservation::new`]). +pub struct RBTreeNodeReservation { + node: Box>>, +} + +impl RBTreeNodeReservation { + /// Allocates memory for a node to be eventually initialised and inserted into the tree via a + /// call to [`RBTree::insert`]. + pub fn new(flags: Flags) -> Result> { + Ok(RBTreeNodeReservation { + node: as BoxExt<_>>::new_uninit(flags)?, + }) + } +} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. Those can always +// be moved across threads. +unsafe impl Send for RBTreeNodeReservation {} + +// SAFETY: This doesn't actually contain K or V, and is just a memory allocation. +unsafe impl Sync for RBTreeNodeReservation {} + +impl RBTreeNodeReservation { + /// Initialises a node reservation. + /// + /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. + pub fn into_node(self, key: K, value: V) -> RBTreeNode { + let node = Box::write( + self.node, + Node { + key, + value, + links: bindings::rb_node::default(), + }, + ); + RBTreeNode { node } + } +} + +/// A red-black tree node. +/// +/// The node is fully initialised (with key and value) and can be inserted into a tree without any +/// extra allocations or failure paths. +pub struct RBTreeNode { + node: Box>, +} + +impl RBTreeNode { + /// Allocates and initialises a node that can be inserted into the tree via + /// [`RBTree::insert`]. + pub fn new(key: K, value: V, flags: Flags) -> Result> { + Ok(RBTreeNodeReservation::new(flags)?.into_node(key, value)) + } +} + +// SAFETY: If K and V can be sent across threads, then it's also okay to send [`RBTreeNode`] across +// threads. +unsafe impl Send for RBTreeNode {} + +// SAFETY: If K and V can be accessed without synchronization, then it's also okay to access +// [`RBTreeNode`] without synchronization. +unsafe impl Sync for RBTreeNode {} + +struct Node { + links: bindings::rb_node, + key: K, + value: V, +} From e601f1bb8e859758289b1c52503bc626168fba59 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 22 Aug 2024 16:37:54 +0000 Subject: [PATCH 048/111] rust: rbtree: add iterator - Add Iterator implementation for `RBTree`, allowing iteration over (key, value) pairs in key order. - Add individual `keys()` and `values()` functions to iterate over keys or values alone. - Update doctests to use iteration instead of explicitly getting items. Iteration is needed by the binder driver to enumerate all values in a tree for oneway spam detection [1]. Link: https://lore.kernel.org/rust-for-linux/20231101-rust-binder-v1-17-08ba9197f637@google.com/ [1] Signed-off-by: Wedson Almeida Filho Reviewed-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Benno Lossin Reviewed-by: Boqun Feng Signed-off-by: Matt Gilbride Link: https://lore.kernel.org/r/20240822-b4-rbtree-v12-2-014561758a57@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/rbtree.rs | 130 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 112 insertions(+), 18 deletions(-) diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index cf25437c795f..ca19d79053de 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -42,14 +42,30 @@ use core::{ /// assert_eq!(tree.get(&30).unwrap(), &300); /// } /// +/// // Iterate over the nodes we just inserted. +/// { +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &100)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert!(iter.next().is_none()); +/// } +/// +/// // Print all elements. +/// for (key, value) in &tree { +/// pr_info!("{} = {}\n", key, value); +/// } +/// /// // Replace one of the elements. /// tree.try_create_and_insert(10, 1000, flags::GFP_KERNEL)?; /// /// // Check that the tree reflects the replacement. /// { -/// assert_eq!(tree.get(&10).unwrap(), &1000); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30).unwrap(), &300); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &1000)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert!(iter.next().is_none()); /// } /// /// // Change the value of one of the elements. @@ -57,9 +73,11 @@ use core::{ /// /// // Check that the tree reflects the update. /// { -/// assert_eq!(tree.get(&10).unwrap(), &1000); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30).unwrap(), &3000); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &1000)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next().unwrap(), (&30, &3000)); +/// assert!(iter.next().is_none()); /// } /// /// // Remove an element. @@ -67,9 +85,10 @@ use core::{ /// /// // Check that the tree reflects the removal. /// { -/// assert_eq!(tree.get(&10), None); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30).unwrap(), &3000); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next().unwrap(), (&30, &3000)); +/// assert!(iter.next().is_none()); /// } /// /// # Ok::<(), Error>(()) @@ -109,9 +128,11 @@ use core::{ /// /// // Check the nodes we just inserted. /// { -/// assert_eq!(tree.get(&10).unwrap(), &100); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30).unwrap(), &300); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &100)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert_eq!(iter.next().unwrap(), (&30, &300)); +/// assert!(iter.next().is_none()); /// } /// /// // Remove a node, getting back ownership of it. @@ -119,9 +140,10 @@ use core::{ /// /// // Check that the tree reflects the removal. /// { -/// assert_eq!(tree.get(&10).unwrap(), &100); -/// assert_eq!(tree.get(&20).unwrap(), &200); -/// assert_eq!(tree.get(&30), None); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &100)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert!(iter.next().is_none()); /// } /// /// // Create a preallocated reservation that we can re-use later. @@ -133,9 +155,11 @@ use core::{ /// /// // Check that the tree reflect the new insertion. /// { -/// assert_eq!(tree.get(&10).unwrap(), &100); -/// assert_eq!(tree.get(&15).unwrap(), &150); -/// assert_eq!(tree.get(&20).unwrap(), &200); +/// let mut iter = tree.iter(); +/// assert_eq!(iter.next().unwrap(), (&10, &100)); +/// assert_eq!(iter.next().unwrap(), (&15, &150)); +/// assert_eq!(iter.next().unwrap(), (&20, &200)); +/// assert!(iter.next().is_none()); /// } /// /// # Ok::<(), Error>(()) @@ -167,6 +191,26 @@ impl RBTree { _p: PhantomData, } } + + /// Returns an iterator over the tree nodes, sorted by key. + pub fn iter(&self) -> Iter<'_, K, V> { + // INVARIANT: `bindings::rb_first` returns a valid pointer to a tree node given a valid pointer to a tree root. + Iter { + _tree: PhantomData, + // SAFETY: `self.root` is a valid pointer to the tree root. + next: unsafe { bindings::rb_first(&self.root) }, + } + } + + /// Returns an iterator over the keys of the nodes in the tree, in sorted order. + pub fn keys(&self) -> impl Iterator { + self.iter().map(|(k, _)| k) + } + + /// Returns an iterator over the values of the nodes in the tree, sorted by key. + pub fn values(&self) -> impl Iterator { + self.iter().map(|(_, v)| v) + } } impl RBTree @@ -358,6 +402,56 @@ impl Drop for RBTree { } } +impl<'a, K, V> IntoIterator for &'a RBTree { + type Item = (&'a K, &'a V); + type IntoIter = Iter<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter`]. +/// +/// # Invariants +/// - `self.next` is a valid pointer. +/// - `self.next` points to a node stored inside of a valid `RBTree`. +pub struct Iter<'a, K, V> { + _tree: PhantomData<&'a RBTree>, + next: *mut bindings::rb_node, +} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Send for Iter<'a, K, V> {} + +// SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same +// thread safety requirements as immutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for Iter<'a, K, V> {} + +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + if self.next.is_null() { + return None; + } + + // SAFETY: By the type invariant of `Iter`, `self.next` is a valid node in an `RBTree`, + // and by the type invariant of `RBTree`, all nodes point to the links field of `Node` objects. + let cur = unsafe { container_of!(self.next, Node, links) }; + + // SAFETY: `self.next` is a valid tree node by the type invariants. + self.next = unsafe { bindings::rb_next(self.next) }; + + // SAFETY: By the same reasoning above, it is safe to dereference the node. Additionally, + // it is ok to return a reference to members because the iterator must outlive it. + Some(unsafe { (&(*cur).key, &(*cur).value) }) + } +} + /// A memory reservation for a red-black tree node. /// /// From cf5397d1776489e1c66b7db01f6a58c431ab08f1 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 22 Aug 2024 16:37:55 +0000 Subject: [PATCH 049/111] rust: rbtree: add mutable iterator Add mutable Iterator implementation for `RBTree`, allowing iteration over (key, value) pairs in key order. Only values are mutable, as mutating keys implies modifying a node's position in the tree. Mutable iteration is used by the binder driver during shutdown to clean up the tree maintained by the "range allocator" [1]. Link: https://lore.kernel.org/rust-for-linux/20231101-rust-binder-v1-6-08ba9197f637@google.com/ [1] Signed-off-by: Wedson Almeida Filho Reviewed-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Boqun Feng Reviewed-by: Benno Lossin Signed-off-by: Matt Gilbride Link: https://lore.kernel.org/r/20240822-b4-rbtree-v12-3-014561758a57@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/rbtree.rs | 103 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index ca19d79053de..b07a21b5d8b0 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -12,7 +12,7 @@ use core::{ cmp::{Ord, Ordering}, marker::PhantomData, mem::MaybeUninit, - ptr::{addr_of_mut, NonNull}, + ptr::{addr_of_mut, from_mut, NonNull}, }; /// A red-black tree with owned nodes. @@ -194,11 +194,31 @@ impl RBTree { /// Returns an iterator over the tree nodes, sorted by key. pub fn iter(&self) -> Iter<'_, K, V> { - // INVARIANT: `bindings::rb_first` returns a valid pointer to a tree node given a valid pointer to a tree root. Iter { _tree: PhantomData, - // SAFETY: `self.root` is a valid pointer to the tree root. - next: unsafe { bindings::rb_first(&self.root) }, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(&self.root) }, + _phantom: PhantomData, + }, + } + } + + /// Returns a mutable iterator over the tree nodes, sorted by key. + pub fn iter_mut(&mut self) -> IterMut<'_, K, V> { + IterMut { + _tree: PhantomData, + // INVARIANT: + // - `self.root` is a valid pointer to a tree root. + // - `bindings::rb_first` produces a valid pointer to a node given `root` is valid. + iter_raw: IterRaw { + // SAFETY: by the invariants, all pointers are valid. + next: unsafe { bindings::rb_first(from_mut(&mut self.root)) }, + _phantom: PhantomData, + }, } } @@ -211,6 +231,11 @@ impl RBTree { pub fn values(&self) -> impl Iterator { self.iter().map(|(_, v)| v) } + + /// Returns a mutable iterator over the values of the nodes in the tree, sorted by key. + pub fn values_mut(&mut self) -> impl Iterator { + self.iter_mut().map(|(_, v)| v) + } } impl RBTree @@ -414,13 +439,9 @@ impl<'a, K, V> IntoIterator for &'a RBTree { /// An iterator over the nodes of a [`RBTree`]. /// /// Instances are created by calling [`RBTree::iter`]. -/// -/// # Invariants -/// - `self.next` is a valid pointer. -/// - `self.next` points to a node stored inside of a valid `RBTree`. pub struct Iter<'a, K, V> { _tree: PhantomData<&'a RBTree>, - next: *mut bindings::rb_node, + iter_raw: IterRaw, } // SAFETY: The [`Iter`] gives out immutable references to K and V, so it has the same @@ -434,21 +455,75 @@ unsafe impl<'a, K: Sync, V: Sync> Sync for Iter<'a, K, V> {} impl<'a, K, V> Iterator for Iter<'a, K, V> { type Item = (&'a K, &'a V); + fn next(&mut self) -> Option { + // SAFETY: Due to `self._tree`, `k` and `v` are valid for the lifetime of `'a`. + self.iter_raw.next().map(|(k, v)| unsafe { (&*k, &*v) }) + } +} + +impl<'a, K, V> IntoIterator for &'a mut RBTree { + type Item = (&'a K, &'a mut V); + type IntoIter = IterMut<'a, K, V>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +/// A mutable iterator over the nodes of a [`RBTree`]. +/// +/// Instances are created by calling [`RBTree::iter_mut`]. +pub struct IterMut<'a, K, V> { + _tree: PhantomData<&'a mut RBTree>, + iter_raw: IterRaw, +} + +// SAFETY: The [`IterMut`] has exclusive access to both `K` and `V`, so it is sufficient to require them to be `Send`. +// The iterator only gives out immutable references to the keys, but since the iterator has excusive access to those same +// keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the user. +unsafe impl<'a, K: Send, V: Send> Send for IterMut<'a, K, V> {} + +// SAFETY: The [`IterMut`] gives out immutable references to K and mutable references to V, so it has the same +// thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for IterMut<'a, K, V> {} + +impl<'a, K, V> Iterator for IterMut<'a, K, V> { + type Item = (&'a K, &'a mut V); + + fn next(&mut self) -> Option { + self.iter_raw.next().map(|(k, v)| + // SAFETY: Due to `&mut self`, we have exclusive access to `k` and `v`, for the lifetime of `'a`. + unsafe { (&*k, &mut *v) }) + } +} + +/// A raw iterator over the nodes of a [`RBTree`]. +/// +/// # Invariants +/// - `self.next` is a valid pointer. +/// - `self.next` points to a node stored inside of a valid `RBTree`. +struct IterRaw { + next: *mut bindings::rb_node, + _phantom: PhantomData (K, V)>, +} + +impl Iterator for IterRaw { + type Item = (*mut K, *mut V); + fn next(&mut self) -> Option { if self.next.is_null() { return None; } - // SAFETY: By the type invariant of `Iter`, `self.next` is a valid node in an `RBTree`, + // SAFETY: By the type invariant of `IterRaw`, `self.next` is a valid node in an `RBTree`, // and by the type invariant of `RBTree`, all nodes point to the links field of `Node` objects. - let cur = unsafe { container_of!(self.next, Node, links) }; + let cur = unsafe { container_of!(self.next, Node, links) }.cast_mut(); // SAFETY: `self.next` is a valid tree node by the type invariants. self.next = unsafe { bindings::rb_next(self.next) }; - // SAFETY: By the same reasoning above, it is safe to dereference the node. Additionally, - // it is ok to return a reference to members because the iterator must outlive it. - Some(unsafe { (&(*cur).key, &(*cur).value) }) + // SAFETY: By the same reasoning above, it is safe to dereference the node. + Some(unsafe { (addr_of_mut!((*cur).key), addr_of_mut!((*cur).value)) }) } } From 98c14e40e07a077827f6842e8f31d191cb82576c Mon Sep 17 00:00:00 2001 From: Matt Gilbride Date: Thu, 22 Aug 2024 16:37:56 +0000 Subject: [PATCH 050/111] rust: rbtree: add cursor Add a cursor interface to `RBTree`, supporting the following use cases: - Inspect the current node pointed to by the cursor, inspect/move to it's neighbors in sort order (bidirectionally). - Mutate the tree itself by removing the current node pointed to by the cursor, or one of its neighbors. Add functions to obtain a cursor to the tree by key: - The node with the smallest key - The node with the largest key - The node matching the given key, or the one with the next larger key The cursor abstraction is needed by the binder driver to efficiently search for nodes and (conditionally) modify them, as well as their neighbors [1]. Link: https://lore.kernel.org/rust-for-linux/20231101-rust-binder-v1-6-08ba9197f637@google.com/ [1] Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Boqun Feng Reviewed-by: Benno Lossin Signed-off-by: Matt Gilbride Link: https://lore.kernel.org/r/20240822-b4-rbtree-v12-4-014561758a57@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/rbtree.rs | 523 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 523 insertions(+) diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index b07a21b5d8b0..64f1611758bb 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -236,6 +236,36 @@ impl RBTree { pub fn values_mut(&mut self) -> impl Iterator { self.iter_mut().map(|(_, v)| v) } + + /// Returns a cursor over the tree nodes, starting with the smallest key. + pub fn cursor_front(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node + let current = unsafe { bindings::rb_first(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + tree: self, + } + }) + } + + /// Returns a cursor over the tree nodes, starting with the largest key. + pub fn cursor_back(&mut self) -> Option> { + let root = addr_of_mut!(self.root); + // SAFETY: `self.root` is always a valid root node + let current = unsafe { bindings::rb_last(root) }; + NonNull::new(current).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + tree: self, + } + }) + } } impl RBTree @@ -396,6 +426,67 @@ where pub fn remove(&mut self, key: &K) -> Option { self.remove_node(key).map(|node| node.node.value) } + + /// Returns a cursor over the tree nodes based on the given key. + /// + /// If the given key exists, the cursor starts there. + /// Otherwise it starts with the first larger key in sort order. + /// If there is no larger key, it returns [`None`]. + pub fn cursor_lower_bound(&mut self, key: &K) -> Option> + where + K: Ord, + { + let mut node = self.root.rb_node; + let mut best_match: Option>> = None; + while !node.is_null() { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node, Node, links) }.cast_mut(); + // SAFETY: `this` is a non-null node so it is valid by the type invariants. + let this_key = unsafe { &(*this).key }; + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let left_child = unsafe { (*node).rb_left }; + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + let right_child = unsafe { (*node).rb_right }; + match key.cmp(this_key) { + Ordering::Equal => { + best_match = NonNull::new(this); + break; + } + Ordering::Greater => { + node = right_child; + } + Ordering::Less => { + let is_better_match = match best_match { + None => true, + Some(best) => { + // SAFETY: `best` is a non-null node so it is valid by the type invariants. + let best_key = unsafe { &(*best.as_ptr()).key }; + best_key > this_key + } + }; + if is_better_match { + best_match = NonNull::new(this); + } + node = left_child; + } + }; + } + + let best = best_match?; + + // SAFETY: `best` is a non-null node so it is valid by the type invariants. + let links = unsafe { addr_of_mut!((*best.as_ptr()).links) }; + + NonNull::new(links).map(|current| { + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self`. + Cursor { + current, + tree: self, + } + }) + } } impl Default for RBTree { @@ -427,6 +518,433 @@ impl Drop for RBTree { } } +/// A bidirectional cursor over the tree nodes, sorted by key. +/// +/// # Examples +/// +/// In the following example, we obtain a cursor to the first element in the tree. +/// The cursor allows us to iterate bidirectionally over key/value pairs in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Move the cursor, updating it to the 2nd element. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Peek at the next element without impacting the cursor. +/// let next = cursor.peek_next().unwrap(); +/// assert_eq!(next, (&30, &300)); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Moving past the last element causes the cursor to return [`None`]. +/// cursor = cursor.move_next().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// let cursor = cursor.move_next(); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// A cursor can also be obtained at the last element in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// let mut cursor = tree.cursor_back().unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Obtaining a cursor returns [`None`] if the tree is empty. +/// +/// ``` +/// use kernel::rbtree::RBTree; +/// +/// let mut tree: RBTree = RBTree::new(); +/// assert!(tree.cursor_front().is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`RBTree::cursor_lower_bound`] can be used to start at an arbitrary node in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert five elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(40, 400, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(50, 500, flags::GFP_KERNEL)?; +/// +/// // If the provided key exists, a cursor to that key is returned. +/// let cursor = tree.cursor_lower_bound(&20).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // If the provided key doesn't exist, a cursor to the first larger element in sort order is returned. +/// let cursor = tree.cursor_lower_bound(&25).unwrap(); +/// let current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // If there is no larger key, [`None`] is returned. +/// let cursor = tree.cursor_lower_bound(&55); +/// assert!(cursor.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The cursor allows mutation of values in the tree. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Retrieve a cursor. +/// let mut cursor = tree.cursor_front().unwrap(); +/// +/// // Get a mutable reference to the current value. +/// let (k, v) = cursor.current_mut(); +/// *v = 1000; +/// +/// // The updated value is reflected in the tree. +/// let updated = tree.get(&10).unwrap(); +/// assert_eq!(updated, &1000); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// It also allows node removal. The following examples demonstrate the behavior of removing the current node. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Remove the first element. +/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// cursor = cursor.remove_current().0.unwrap(); +/// +/// // If a node exists after the current element, it is returned. +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Get a cursor to the last element, and remove it. +/// cursor = tree.cursor_back().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Since there is no next node, the previous node is returned. +/// cursor = cursor.remove_current().0.unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&20, &200)); +/// +/// // Removing the last element in the tree returns [`None`]. +/// assert!(cursor.remove_current().0.is_none()); +/// +/// # Ok::<(), Error>(()) +/// ``` +/// +/// Nodes adjacent to the current node can also be removed. +/// +/// ``` +/// use kernel::{alloc::flags, rbtree::RBTree}; +/// +/// // Create a new tree. +/// let mut tree = RBTree::new(); +/// +/// // Insert three elements. +/// tree.try_create_and_insert(10, 100, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(20, 200, flags::GFP_KERNEL)?; +/// tree.try_create_and_insert(30, 300, flags::GFP_KERNEL)?; +/// +/// // Get a cursor to the first element. +/// let mut cursor = tree.cursor_front().unwrap(); +/// let mut current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_prev` from the first element returns [`None`]. +/// assert!(cursor.remove_prev().is_none()); +/// +/// // Get a cursor to the last element. +/// cursor = tree.cursor_back().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&30, &300)); +/// +/// // Calling `remove_prev` removes and returns the middle element. +/// assert_eq!(cursor.remove_prev().unwrap().to_key_value(), (20, 200)); +/// +/// // Calling `remove_next` from the last element returns [`None`]. +/// assert!(cursor.remove_next().is_none()); +/// +/// // Move to the first element +/// cursor = cursor.move_prev().unwrap(); +/// current = cursor.current(); +/// assert_eq!(current, (&10, &100)); +/// +/// // Calling `remove_next` removes and returns the last element. +/// assert_eq!(cursor.remove_next().unwrap().to_key_value(), (30, 300)); +/// +/// # Ok::<(), Error>(()) +/// +/// ``` +/// +/// # Invariants +/// - `current` points to a node that is in the same [`RBTree`] as `tree`. +pub struct Cursor<'a, K, V> { + tree: &'a mut RBTree, + current: NonNull, +} + +// SAFETY: The [`Cursor`] has exclusive access to both `K` and `V`, so it is sufficient to require them to be `Send`. +// The cursor only gives out immutable references to the keys, but since it has excusive access to those same +// keys, `Send` is sufficient. `Sync` would be okay, but it is more restrictive to the user. +unsafe impl<'a, K: Send, V: Send> Send for Cursor<'a, K, V> {} + +// SAFETY: The [`Cursor`] gives out immutable references to K and mutable references to V, +// so it has the same thread safety requirements as mutable references. +unsafe impl<'a, K: Sync, V: Sync> Sync for Cursor<'a, K, V> {} + +impl<'a, K, V> Cursor<'a, K, V> { + /// The current node + pub fn current(&self) -> (&K, &V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an immutable reference by the function signature. + unsafe { Self::to_key_value(self.current) } + } + + /// The current node, with a mutable value + pub fn current_mut(&mut self) -> (&K, &mut V) { + // SAFETY: + // - `self.current` is a valid node by the type invariants. + // - We have an mutable reference by the function signature. + unsafe { Self::to_key_value_mut(self.current) } + } + + /// Remove the current node from the tree. + /// + /// Returns a tuple where the first element is a cursor to the next node, if it exists, + /// else the previous node, else [`None`] (if the tree becomes empty). The second element + /// is the removed node. + pub fn remove_current(self) -> (Option, RBTreeNode) { + let prev = self.get_neighbor_raw(Direction::Prev); + let next = self.get_neighbor_raw(Direction::Next); + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(self.current.as_ptr(), Node, links) }.cast_mut(); + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { Box::from_raw(this) }; + let node = RBTreeNode { node }; + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(&mut (*this).links, addr_of_mut!(self.tree.root)) }; + + let current = match (prev, next) { + (_, Some(next)) => next, + (Some(prev), None) => prev, + (None, None) => { + return (None, node); + } + }; + + ( + // INVARIANT: + // - `current` is a valid node in the [`RBTree`] pointed to by `self.tree`. + Some(Self { + current, + tree: self.tree, + }), + node, + ) + } + + /// Remove the previous node, returning it if it exists. + pub fn remove_prev(&mut self) -> Option> { + self.remove_neighbor(Direction::Prev) + } + + /// Remove the next node, returning it if it exists. + pub fn remove_next(&mut self) -> Option> { + self.remove_neighbor(Direction::Next) + } + + fn remove_neighbor(&mut self, direction: Direction) -> Option> { + if let Some(neighbor) = self.get_neighbor_raw(direction) { + let neighbor = neighbor.as_ptr(); + // SAFETY: The reference to the tree used to create the cursor outlives the cursor, so + // the tree cannot change. By the tree invariant, all nodes are valid. + unsafe { bindings::rb_erase(neighbor, addr_of_mut!(self.tree.root)) }; + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(neighbor, Node, links) }.cast_mut(); + // SAFETY: `this` is valid by the type invariants as described above. + let node = unsafe { Box::from_raw(this) }; + return Some(RBTreeNode { node }); + } + None + } + + /// Move the cursor to the previous node, returning [`None`] if it doesn't exist. + pub fn move_prev(self) -> Option { + self.mv(Direction::Prev) + } + + /// Move the cursor to the next node, returning [`None`] if it doesn't exist. + pub fn move_next(self) -> Option { + self.mv(Direction::Next) + } + + fn mv(self, direction: Direction) -> Option { + // INVARIANT: + // - `neighbor` is a valid node in the [`RBTree`] pointed to by `self.tree`. + self.get_neighbor_raw(direction).map(|neighbor| Self { + tree: self.tree, + current: neighbor, + }) + } + + /// Access the previous node without moving the cursor. + pub fn peek_prev(&self) -> Option<(&K, &V)> { + self.peek(Direction::Prev) + } + + /// Access the previous node without moving the cursor. + pub fn peek_next(&self) -> Option<(&K, &V)> { + self.peek(Direction::Next) + } + + fn peek(&self, direction: Direction) -> Option<(&K, &V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have an immutable reference to `self`. + unsafe { Self::to_key_value(neighbor) } + }) + } + + /// Access the previous node mutably without moving the cursor. + pub fn peek_prev_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Prev) + } + + /// Access the next node mutably without moving the cursor. + pub fn peek_next_mut(&mut self) -> Option<(&K, &mut V)> { + self.peek_mut(Direction::Next) + } + + fn peek_mut(&mut self, direction: Direction) -> Option<(&K, &mut V)> { + self.get_neighbor_raw(direction).map(|neighbor| { + // SAFETY: + // - `neighbor` is a valid tree node. + // - By the function signature, we have a mutable reference to `self`. + unsafe { Self::to_key_value_mut(neighbor) } + }) + } + + fn get_neighbor_raw(&self, direction: Direction) -> Option> { + // SAFETY: `self.current` is valid by the type invariants. + let neighbor = unsafe { + match direction { + Direction::Prev => bindings::rb_prev(self.current.as_ptr()), + Direction::Next => bindings::rb_next(self.current.as_ptr()), + } + }; + + NonNull::new(neighbor) + } + + /// SAFETY: + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to `node` for the duration of 'b. + unsafe fn to_key_value<'b>(node: NonNull) -> (&'b K, &'b V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees immutable access to `node`. + (k, unsafe { &*v }) + } + + /// SAFETY: + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has mutable access to `node` for the duration of 'b. + unsafe fn to_key_value_mut<'b>(node: NonNull) -> (&'b K, &'b mut V) { + // SAFETY: the caller guarantees that `node` is a valid pointer in an `RBTree`. + let (k, v) = unsafe { Self::to_key_value_raw(node) }; + // SAFETY: the caller guarantees mutable access to `node`. + (k, unsafe { &mut *v }) + } + + /// SAFETY: + /// - `node` must be a valid pointer to a node in an [`RBTree`]. + /// - The caller has immutable access to the key for the duration of 'b. + unsafe fn to_key_value_raw<'b>(node: NonNull) -> (&'b K, *mut V) { + // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` + // point to the links field of `Node` objects. + let this = unsafe { container_of!(node.as_ptr(), Node, links) }.cast_mut(); + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let k = unsafe { &(*this).key }; + // SAFETY: The passed `node` is the current node or a non-null neighbor, + // thus `this` is valid by the type invariants. + let v = unsafe { addr_of_mut!((*this).value) }; + (k, v) + } +} + +/// Direction for [`Cursor`] operations. +enum Direction { + /// the node immediately before, in sort order + Prev, + /// the node immediately after, in sort order + Next, +} + impl<'a, K, V> IntoIterator for &'a RBTree { type Item = (&'a K, &'a V); type IntoIter = Iter<'a, K, V>; @@ -584,6 +1102,11 @@ impl RBTreeNode { pub fn new(key: K, value: V, flags: Flags) -> Result> { Ok(RBTreeNodeReservation::new(flags)?.into_node(key, value)) } + + /// Get the key and value from inside the node. + pub fn to_key_value(self) -> (K, V) { + (self.node.key, self.node.value) + } } // SAFETY: If K and V can be sent across threads, then it's also okay to send [`RBTreeNode`] across From a335e95914046c6bed45c0d17cabcd483682cf5e Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 22 Aug 2024 16:37:57 +0000 Subject: [PATCH 051/111] rust: rbtree: add `RBTree::entry` This mirrors the entry API [1] from the Rust standard library on `RBTree`. This API can be used to access the entry at a specific key and make modifications depending on whether the key is vacant or occupied. This API is useful because it can often be used to avoid traversing the tree multiple times. This is used by binder to look up and conditionally access or insert a value, depending on whether it is there or not [2]. Link: https://doc.rust-lang.org/stable/std/collections/btree_map/enum.Entry.html [1] Link: https://android-review.googlesource.com/c/kernel/common/+/2849906 [2] Signed-off-by: Alice Ryhl Tested-by: Alice Ryhl Reviewed-by: Boqun Feng Reviewed-by: Benno Lossin Signed-off-by: Matt Gilbride Link: https://lore.kernel.org/r/20240822-b4-rbtree-v12-5-014561758a57@google.com Signed-off-by: Miguel Ojeda --- rust/kernel/rbtree.rs | 307 +++++++++++++++++++++++++++++++----------- 1 file changed, 231 insertions(+), 76 deletions(-) diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index 64f1611758bb..48ceb9560bf5 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -293,12 +293,19 @@ where /// key/value pair). Returns [`None`] if a node with the same key didn't already exist. /// /// This function always succeeds. - pub fn insert(&mut self, RBTreeNode { node }: RBTreeNode) -> Option> { - let node = Box::into_raw(node); - // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when - // the node is removed or replaced. - let node_links = unsafe { addr_of_mut!((*node).links) }; + pub fn insert(&mut self, node: RBTreeNode) -> Option> { + match self.raw_entry(&node.node.key) { + RawEntry::Occupied(entry) => Some(entry.replace(node)), + RawEntry::Vacant(entry) => { + entry.insert(node); + None + } + } + } + fn raw_entry(&mut self, key: &K) -> RawEntry<'_, K, V> { + let raw_self: *mut RBTree = self; + // The returned `RawEntry` is used to call either `rb_link_node` or `rb_replace_node`. // The parameters of `bindings::rb_link_node` are as follows: // - `node`: A pointer to an uninitialized node being inserted. // - `parent`: A pointer to an existing node in the tree. One of its child pointers must be @@ -317,62 +324,56 @@ where // in the subtree of `parent` that `child_field_of_parent` points at. Once // we find an empty subtree, we can insert the new node using `rb_link_node`. let mut parent = core::ptr::null_mut(); - let mut child_field_of_parent: &mut *mut bindings::rb_node = &mut self.root.rb_node; - while !child_field_of_parent.is_null() { - parent = *child_field_of_parent; + let mut child_field_of_parent: &mut *mut bindings::rb_node = + // SAFETY: `raw_self` is a valid pointer to the `RBTree` (created from `self` above). + unsafe { &mut (*raw_self).root.rb_node }; + while !(*child_field_of_parent).is_null() { + let curr = *child_field_of_parent; + // SAFETY: All links fields we create are in a `Node`. + let node = unsafe { container_of!(curr, Node, links) }; - // We need to determine whether `node` should be the left or right child of `parent`, - // so we will compare with the `key` field of `parent` a.k.a. `this` below. - // - // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` - // point to the links field of `Node` objects. - let this = unsafe { container_of!(parent, Node, links) }; - - // SAFETY: `this` is a non-null node so it is valid by the type invariants. `node` is - // valid until the node is removed. - match unsafe { (*node).key.cmp(&(*this).key) } { - // We would like `node` to be the left child of `parent`. Move to this child to check - // whether we can use it, or continue searching, at the next iteration. - // - // SAFETY: `parent` is a non-null node so it is valid by the type invariants. - Ordering::Less => child_field_of_parent = unsafe { &mut (*parent).rb_left }, - // We would like `node` to be the right child of `parent`. Move to this child to check - // whether we can use it, or continue searching, at the next iteration. - // - // SAFETY: `parent` is a non-null node so it is valid by the type invariants. - Ordering::Greater => child_field_of_parent = unsafe { &mut (*parent).rb_right }, + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + match key.cmp(unsafe { &(*node).key }) { + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Less => child_field_of_parent = unsafe { &mut (*curr).rb_left }, + // SAFETY: `curr` is a non-null node so it is valid by the type invariants. + Ordering::Greater => child_field_of_parent = unsafe { &mut (*curr).rb_right }, Ordering::Equal => { - // There is an existing node in the tree with this key, and that node is - // `parent`. Thus, we are replacing parent with a new node. - // - // INVARIANT: We are replacing an existing node with a new one, which is valid. - // It remains valid because we "forgot" it with `Box::into_raw`. - // SAFETY: All pointers are non-null and valid. - unsafe { bindings::rb_replace_node(parent, node_links, &mut self.root) }; - - // INVARIANT: The node is being returned and the caller may free it, however, - // it was removed from the tree. So the invariants still hold. - return Some(RBTreeNode { - // SAFETY: `this` was a node in the tree, so it is valid. - node: unsafe { Box::from_raw(this.cast_mut()) }, - }); + return RawEntry::Occupied(OccupiedEntry { + rbtree: self, + node_links: curr, + }) } } + parent = curr; } - // INVARIANT: We are linking in a new node, which is valid. It remains valid because we - // "forgot" it with `Box::into_raw`. - // SAFETY: All pointers are non-null and valid (`*child_field_of_parent` is null, but `child_field_of_parent` is a - // mutable reference). - unsafe { bindings::rb_link_node(node_links, parent, child_field_of_parent) }; - - // SAFETY: All pointers are valid. `node` has just been inserted into the tree. - unsafe { bindings::rb_insert_color(node_links, &mut self.root) }; - None + RawEntry::Vacant(RawVacantEntry { + rbtree: raw_self, + parent, + child_field_of_parent, + _phantom: PhantomData, + }) } - /// Returns a node with the given key, if one exists. - fn find(&self, key: &K) -> Option>> { + /// Gets the given key's corresponding entry in the map for in-place manipulation. + pub fn entry(&mut self, key: K) -> Entry<'_, K, V> { + match self.raw_entry(&key) { + RawEntry::Occupied(entry) => Entry::Occupied(entry), + RawEntry::Vacant(entry) => Entry::Vacant(VacantEntry { raw: entry, key }), + } + } + + /// Used for accessing the given node, if it exists. + pub fn find_mut(&mut self, key: &K) -> Option> { + match self.raw_entry(key) { + RawEntry::Occupied(entry) => Some(entry), + RawEntry::Vacant(_entry) => None, + } + } + + /// Returns a reference to the value corresponding to the key. + pub fn get(&self, key: &K) -> Option<&V> { let mut node = self.root.rb_node; while !node.is_null() { // SAFETY: By the type invariant of `Self`, all non-null `rb_node` pointers stored in `self` @@ -384,47 +385,30 @@ where Ordering::Less => unsafe { (*node).rb_left }, // SAFETY: `node` is a non-null node so it is valid by the type invariants. Ordering::Greater => unsafe { (*node).rb_right }, - Ordering::Equal => return NonNull::new(this.cast_mut()), + // SAFETY: `node` is a non-null node so it is valid by the type invariants. + Ordering::Equal => return Some(unsafe { &(*this).value }), } } None } - /// Returns a reference to the value corresponding to the key. - pub fn get(&self, key: &K) -> Option<&V> { - // SAFETY: The `find` return value is a node in the tree, so it is valid. - self.find(key).map(|node| unsafe { &node.as_ref().value }) - } - /// Returns a mutable reference to the value corresponding to the key. pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { - // SAFETY: The `find` return value is a node in the tree, so it is valid. - self.find(key) - .map(|mut node| unsafe { &mut node.as_mut().value }) + self.find_mut(key).map(|node| node.into_mut()) } /// Removes the node with the given key from the tree. /// /// It returns the node that was removed if one exists, or [`None`] otherwise. - fn remove_node(&mut self, key: &K) -> Option> { - let mut node = self.find(key)?; - - // SAFETY: The `find` return value is a node in the tree, so it is valid. - unsafe { bindings::rb_erase(&mut node.as_mut().links, &mut self.root) }; - - // INVARIANT: The node is being returned and the caller may free it, however, it was - // removed from the tree. So the invariants still hold. - Some(RBTreeNode { - // SAFETY: The `find` return value was a node in the tree, so it is valid. - node: unsafe { Box::from_raw(node.as_ptr()) }, - }) + pub fn remove_node(&mut self, key: &K) -> Option> { + self.find_mut(key).map(OccupiedEntry::remove_node) } /// Removes the node with the given key from the tree. /// /// It returns the value that was removed if one exists, or [`None`] otherwise. pub fn remove(&mut self, key: &K) -> Option { - self.remove_node(key).map(|node| node.node.value) + self.find_mut(key).map(OccupiedEntry::remove) } /// Returns a cursor over the tree nodes based on the given key. @@ -1117,6 +1101,177 @@ unsafe impl Send for RBTreeNode {} // [`RBTreeNode`] without synchronization. unsafe impl Sync for RBTreeNode {} +impl RBTreeNode { + /// Drop the key and value, but keep the allocation. + /// + /// It then becomes a reservation that can be re-initialised into a different node (i.e., with + /// a different key and/or value). + /// + /// The existing key and value are dropped in-place as part of this operation, that is, memory + /// may be freed (but only for the key/value; memory for the node itself is kept for reuse). + pub fn into_reservation(self) -> RBTreeNodeReservation { + RBTreeNodeReservation { + node: Box::drop_contents(self.node), + } + } +} + +/// A view into a single entry in a map, which may either be vacant or occupied. +/// +/// This enum is constructed from the [`RBTree::entry`]. +/// +/// [`entry`]: fn@RBTree::entry +pub enum Entry<'a, K, V> { + /// This [`RBTree`] does not have a node with this key. + Vacant(VacantEntry<'a, K, V>), + /// This [`RBTree`] already has a node with this key. + Occupied(OccupiedEntry<'a, K, V>), +} + +/// Like [`Entry`], except that it doesn't have ownership of the key. +enum RawEntry<'a, K, V> { + Vacant(RawVacantEntry<'a, K, V>), + Occupied(OccupiedEntry<'a, K, V>), +} + +/// A view into a vacant entry in a [`RBTree`]. It is part of the [`Entry`] enum. +pub struct VacantEntry<'a, K, V> { + key: K, + raw: RawVacantEntry<'a, K, V>, +} + +/// Like [`VacantEntry`], but doesn't hold on to the key. +/// +/// # Invariants +/// - `parent` may be null if the new node becomes the root. +/// - `child_field_of_parent` is a valid pointer to the left-child or right-child of `parent`. If `parent` is +/// null, it is a pointer to the root of the [`RBTree`]. +struct RawVacantEntry<'a, K, V> { + rbtree: *mut RBTree, + /// The node that will become the parent of the new node if we insert one. + parent: *mut bindings::rb_node, + /// This points to the left-child or right-child field of `parent`, or `root` if `parent` is + /// null. + child_field_of_parent: *mut *mut bindings::rb_node, + _phantom: PhantomData<&'a mut RBTree>, +} + +impl<'a, K, V> RawVacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + /// + /// The `node` must have a key such that inserting it here does not break the ordering of this + /// [`RBTree`]. + fn insert(self, node: RBTreeNode) -> &'a mut V { + let node = Box::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when + // the node is removed or replaced. + let node_links = unsafe { addr_of_mut!((*node).links) }; + + // INVARIANT: We are linking in a new node, which is valid. It remains valid because we + // "forgot" it with `Box::into_raw`. + // SAFETY: The type invariants of `RawVacantEntry` are exactly the safety requirements of `rb_link_node`. + unsafe { bindings::rb_link_node(node_links, self.parent, self.child_field_of_parent) }; + + // SAFETY: All pointers are valid. `node` has just been inserted into the tree. + unsafe { bindings::rb_insert_color(node_links, addr_of_mut!((*self.rbtree).root)) }; + + // SAFETY: The node is valid until we remove it from the tree. + unsafe { &mut (*node).value } + } +} + +impl<'a, K, V> VacantEntry<'a, K, V> { + /// Inserts the given node into the [`RBTree`] at this entry. + pub fn insert(self, value: V, reservation: RBTreeNodeReservation) -> &'a mut V { + self.raw.insert(reservation.into_node(self.key, value)) + } +} + +/// A view into an occupied entry in a [`RBTree`]. It is part of the [`Entry`] enum. +/// +/// # Invariants +/// - `node_links` is a valid, non-null pointer to a tree node in `self.rbtree` +pub struct OccupiedEntry<'a, K, V> { + rbtree: &'a mut RBTree, + /// The node that this entry corresponds to. + node_links: *mut bindings::rb_node, +} + +impl<'a, K, V> OccupiedEntry<'a, K, V> { + /// Gets a reference to the value in the entry. + pub fn get(&self) -> &V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have shared access to the underlying tree, and can thus give out a shared reference. + unsafe { &(*container_of!(self.node_links, Node, links)).value } + } + + /// Gets a mutable reference to the value in the entry. + pub fn get_mut(&mut self) -> &mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - We have exclusive access to the underlying tree, and can thus give out a mutable reference. + unsafe { &mut (*(container_of!(self.node_links, Node, links).cast_mut())).value } + } + + /// Converts the entry into a mutable reference to its value. + /// + /// If you need multiple references to the `OccupiedEntry`, see [`self#get_mut`]. + pub fn into_mut(self) -> &'a mut V { + // SAFETY: + // - `self.node_links` is a valid pointer to a node in the tree. + // - This consumes the `&'a mut RBTree`, therefore it can give out a mutable reference that lives for `'a`. + unsafe { &mut (*(container_of!(self.node_links, Node, links).cast_mut())).value } + } + + /// Remove this entry from the [`RBTree`]. + pub fn remove_node(self) -> RBTreeNode { + // SAFETY: The node is a node in the tree, so it is valid. + unsafe { bindings::rb_erase(self.node_links, &mut self.rbtree.root) }; + + // INVARIANT: The node is being returned and the caller may free it, however, it was + // removed from the tree. So the invariants still hold. + RBTreeNode { + // SAFETY: The node was a node in the tree, but we removed it, so we can convert it + // back into a box. + node: unsafe { + Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) + }, + } + } + + /// Takes the value of the entry out of the map, and returns it. + pub fn remove(self) -> V { + self.remove_node().node.value + } + + /// Swap the current node for the provided node. + /// + /// The key of both nodes must be equal. + fn replace(self, node: RBTreeNode) -> RBTreeNode { + let node = Box::into_raw(node.node); + + // SAFETY: `node` is valid at least until we call `Box::from_raw`, which only happens when + // the node is removed or replaced. + let new_node_links = unsafe { addr_of_mut!((*node).links) }; + + // SAFETY: This updates the pointers so that `new_node_links` is in the tree where + // `self.node_links` used to be. + unsafe { + bindings::rb_replace_node(self.node_links, new_node_links, &mut self.rbtree.root) + }; + + // SAFETY: + // - `self.node_ptr` produces a valid pointer to a node in the tree. + // - Now that we removed this entry from the tree, we can convert the node to a box. + let old_node = + unsafe { Box::from_raw(container_of!(self.node_links, Node, links).cast_mut()) }; + + RBTreeNode { node: old_node } + } +} + struct Node { links: bindings::rb_node, key: K, From 68d3b6aa08708bb3907c2c13eaf4b3ccf4805160 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 19:32:54 +0200 Subject: [PATCH 052/111] MAINTAINERS: add Trevor Gross as Rust reviewer Trevor has been involved with the Rust for Linux project for more than a year now. He has been active reviewing Rust code in the mailing list, and he already is a formal reviewer of the Rust PHY library and the two PHY drivers. In addition, he is also part of several upstream Rust teams: compiler-contributors team (contributors to the Rust compiler on a regular basis), libs-contributors (contributors to the Rust standard library on a regular basis), crate-maintainers (maintainers of official Rust crates), the binary size working group and the Rust for Linux ping group. His expertise with the language will be very useful to have around in the future if Rust keeps growing within the kernel, thus add him to the `RUST` entry as a reviewer. Acked-by: Boqun Feng Acked-by: Trevor Gross Link: https://lore.kernel.org/r/20240902173255.1105340-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index f328373463b0..77b395476a80 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19914,6 +19914,7 @@ R: Björn Roy Baron R: Benno Lossin R: Andreas Hindborg R: Alice Ryhl +R: Trevor Gross L: rust-for-linux@vger.kernel.org S: Supported W: https://rust-for-linux.com From ab309b6e084c70a29f9fa3cee797572bd2340901 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Wed, 4 Sep 2024 16:42:29 +0200 Subject: [PATCH 053/111] rust: avoid `box_uninit_write` feature Like commit 0903b9e2a46c ("rust: alloc: eschew `Box>::write`"), but for the new `rbtree` and `alloc` code. That is, `feature(new_uninit)` [1] got partially stabilized [2] for Rust 1.82.0 (expected to be released on 2024-10-17), but it did not include `Box>::write`, which got split into `feature(box_uninit_write)` [3]. To avoid relying on a new unstable feature, rewrite the `write` + `assume_init` pair manually. Link: https://github.com/rust-lang/rust/issues/63291 [1] Link: https://github.com/rust-lang/rust/pull/129401 [2] Link: https://github.com/rust-lang/rust/issues/129397 [3] Reviewed-by: Alice Ryhl Reviewed-by: Matt Gilbride Link: https://lore.kernel.org/r/20240904144229.18592-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/kernel/alloc/box_ext.rs | 6 ++++-- rust/kernel/rbtree.rs | 17 ++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs index b68ade26a42d..5b1550d620fd 100644 --- a/rust/kernel/alloc/box_ext.rs +++ b/rust/kernel/alloc/box_ext.rs @@ -26,9 +26,11 @@ pub trait BoxExt: Sized { /// use kernel::alloc::{flags, box_ext::BoxExt}; /// let value = Box::new([0; 32], flags::GFP_KERNEL)?; /// assert_eq!(*value, [0; 32]); - /// let value = Box::drop_contents(value); + /// let mut value = Box::drop_contents(value); /// // Now we can re-use `value`: - /// let value = Box::write(value, [1; 32]); + /// value.write([1; 32]); + /// // SAFETY: We just wrote to it. + /// let value = unsafe { value.assume_init() }; /// assert_eq!(*value, [1; 32]); /// # Ok::<(), Error>(()) /// ``` diff --git a/rust/kernel/rbtree.rs b/rust/kernel/rbtree.rs index 48ceb9560bf5..25eb36fd1cdc 100644 --- a/rust/kernel/rbtree.rs +++ b/rust/kernel/rbtree.rs @@ -1059,15 +1059,14 @@ impl RBTreeNodeReservation { /// Initialises a node reservation. /// /// It then becomes an [`RBTreeNode`] that can be inserted into a tree. - pub fn into_node(self, key: K, value: V) -> RBTreeNode { - let node = Box::write( - self.node, - Node { - key, - value, - links: bindings::rb_node::default(), - }, - ); + pub fn into_node(mut self, key: K, value: V) -> RBTreeNode { + self.node.write(Node { + key, + value, + links: bindings::rb_node::default(), + }); + // SAFETY: We just wrote to it. + let node = unsafe { self.node.assume_init() }; RBTreeNode { node } } } From 6e74c6b5a42e6a7313fcd29e814f211b392a00f5 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 18:55:28 +0200 Subject: [PATCH 054/111] kbuild: rust: add `CONFIG_RUSTC_VERSION` Now that we support several Rust versions, introduce `CONFIG_RUSTC_VERSION` so that it can be used in Kconfig to enable and disable configuration options based on the `rustc` version. The approach taken resembles `pahole`'s -- see commit 613fe1692377 ("kbuild: Add CONFIG_PAHOLE_VERSION"), i.e. a simple version parsing without trying to identify several kinds of compilers, since so far there is only one (`rustc`). However, unlike `pahole`'s, we also print a zero if executing failed for any reason, rather than checking if the command is found and executable (which still leaves things like a file that exists and is executable, but e.g. is built for another platform [1]). An equivalent approach to the one here was also submitted for `pahole` [2]. Link: https://lore.kernel.org/rust-for-linux/CANiq72=4vX_tJMJLE6e+bg7ZECHkS-AQpm8GBzuK75G1EB7+Nw@mail.gmail.com/ [1] Link: https://lore.kernel.org/linux-kbuild/20240728125527.690726-1-ojeda@kernel.org/ [2] Reviewed-by: Nicolas Schier Tested-by: Alice Ryhl Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240902165535.1101978-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- init/Kconfig | 7 +++++++ scripts/rustc-version.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100755 scripts/rustc-version.sh diff --git a/init/Kconfig b/init/Kconfig index 839c83034006..38c1cfcce821 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -60,6 +60,13 @@ config LLD_VERSION default $(ld-version) if LD_IS_LLD default 0 +config RUSTC_VERSION + int + default $(shell,$(srctree)/scripts/rustc-version.sh $(RUSTC)) + help + It does not depend on `RUST` since that one may need to use the version + in a `depends on`. + config RUST_IS_AVAILABLE def_bool $(success,$(srctree)/scripts/rust_is_available.sh) help diff --git a/scripts/rustc-version.sh b/scripts/rustc-version.sh new file mode 100755 index 000000000000..4e22593e2eab --- /dev/null +++ b/scripts/rustc-version.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Usage: $ ./rustc-version.sh rustc +# +# Print the Rust compiler version in a 6 or 7-digit form. + +# Convert the version string x.y.z to a canonical up-to-7-digits form. +# +# Note that this function uses one more digit (compared to other +# instances in other version scripts) to give a bit more space to +# `rustc` since it will reach 1.100.0 in late 2026. +get_canonical_version() +{ + IFS=. + set -- $1 + echo $((100000 * $1 + 100 * $2 + $3)) +} + +if output=$("$@" --version 2>/dev/null); then + set -- $output + get_canonical_version $2 +else + echo 0 + exit 1 +fi From 5134a335cfe6ebdd2420e15b5f6c06915040aa51 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 18:55:29 +0200 Subject: [PATCH 055/111] kbuild: rust: re-run Kconfig if the version text changes Re-run Kconfig if we detect the Rust compiler has changed via the version text, like it is done for C. Unlike C, and unlike `RUSTC_VERSION`, the `RUSTC_VERSION_TEXT` is kept under `depends on RUST`, since it should not be needed unless `RUST` is enabled. Reviewed-by: Nicolas Schier Tested-by: Alice Ryhl Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240902165535.1101978-3-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Makefile | 8 +++++--- init/Kconfig | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 68ebd6d6b444..1883aeb919a4 100644 --- a/Makefile +++ b/Makefile @@ -648,9 +648,11 @@ endif # The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. # Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. -# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), -# and from include/config/auto.conf.cmd to detect the compiler upgrade. +# CC_VERSION_TEXT and RUSTC_VERSION_TEXT are referenced from Kconfig (so they +# need export), and from include/config/auto.conf.cmd to detect the compiler +# upgrade. CC_VERSION_TEXT = $(subst $(pound),,$(shell LC_ALL=C $(CC) --version 2>/dev/null | head -n 1)) +RUSTC_VERSION_TEXT = $(subst $(pound),,$(shell $(RUSTC) --version 2>/dev/null)) ifneq ($(findstring clang,$(CC_VERSION_TEXT)),) include $(srctree)/scripts/Makefile.clang @@ -671,7 +673,7 @@ ifdef config-build # KBUILD_DEFCONFIG may point out an alternative default configuration # used for 'make defconfig' include $(srctree)/arch/$(SRCARCH)/Makefile -export KBUILD_DEFCONFIG KBUILD_KCONFIG CC_VERSION_TEXT +export KBUILD_DEFCONFIG KBUILD_KCONFIG CC_VERSION_TEXT RUSTC_VERSION_TEXT config: outputmakefile scripts_basic FORCE $(Q)$(MAKE) $(build)=scripts/kconfig $@ diff --git a/init/Kconfig b/init/Kconfig index 38c1cfcce821..c6b744a48dcc 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1927,7 +1927,9 @@ config RUST config RUSTC_VERSION_TEXT string depends on RUST - default "$(shell,$(RUSTC) --version 2>/dev/null)" + default "$(RUSTC_VERSION_TEXT)" + help + See `CC_VERSION_TEXT`. config BINDGEN_VERSION_TEXT string From ac3e972629a69e118e3867531df936a6ce5e5f5a Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 18:55:30 +0200 Subject: [PATCH 056/111] kbuild: rust: rebuild if the version text changes Now that `RUSTC_VERSION_TEXT` exists, use it to rebuild `core` when the version text changes (which in turn will trigger a rebuild of all the kernel Rust code). This also applies to proc macros (which only work with the `rustc` that compiled them), via the already existing dependency on `core.o`. That is cleaned up in the next commit. However, this does not cover host programs written in Rust, which is the same case in the C side. This is accomplished by referencing directly the generated file, instead of using the `fixdep` header trick, since we cannot change the Rust standard library sources. This is not too much of a burden, since it only needs to be done for `core`. Tested-by: Alice Ryhl Reviewed-by: Nicolas Schier Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240902165535.1101978-4-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/Makefile b/rust/Makefile index e13d14ec5fe7..bb57a7c30f1a 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -389,7 +389,8 @@ $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Wunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \ + $(wildcard $(objtree)/include/config/RUSTC_VERSION_TEXT) FORCE +$(call if_changed_rule,rustc_library) ifneq ($(or $(CONFIG_X86_64),$(CONFIG_X86_32)),) $(obj)/core.o: scripts/target.json From aeb0e24abbebebff3b5ac65486c933d0ecd5cf81 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 18:55:31 +0200 Subject: [PATCH 057/111] kbuild: rust: replace proc macros dependency on `core.o` with the version text With the `RUSTC_VERSION_TEXT` rebuild support in place, now proc macros can depend on that instead of `core.o`. This means that both the `core` and `macros` crates can be built in parallel, and that touching `core.o` does not trigger a rebuild of the proc macros. This could be accomplished using the same approach as for `core` (i.e. depending directly on `include/config/RUSTC_VERSION_TEXT`). However, that is considered an implementation detail [1], and thus it is best to avoid it. Instead, let fixdep find a string that we explicitly write down in the source code for this purpose (like it is done for `include/linux/compiler-version.h`), which we can easily do (unlike for `core`) since this is our own source code. Suggested-by: Masahiro Yamada Link: https://lore.kernel.org/rust-for-linux/CAK7LNAQBG0nDupXSgAAk-6nOqeqGVkr3H1RjYaqRJ1OxmLm6xA@mail.gmail.com/ [1] Reviewed-by: Nicolas Schier Tested-by: Alice Ryhl Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240902165535.1101978-5-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- rust/Makefile | 4 +--- rust/macros/lib.rs | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index bb57a7c30f1a..4eae318f36ff 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -342,9 +342,7 @@ quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ --crate-name $(patsubst lib%.so,%,$(notdir $@)) $< # Procedural macros can only be used with the `rustc` that compiled it. -# Therefore, to get `libmacros.so` automatically recompiled when the compiler -# version changes, we add `core.o` as a dependency (even if it is not needed). -$(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE +$(obj)/libmacros.so: $(src)/macros/lib.rs FORCE +$(call if_changed_dep,rustc_procmacro) quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 5be0cb9db3ee..a626b1145e5c 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -2,6 +2,10 @@ //! Crate for all kernel procedural macros. +// When fixdep scans this, it will find this string `CONFIG_RUSTC_VERSION_TEXT` +// and thus add a dependency on `include/config/RUSTC_VERSION_TEXT`, which is +// touched by Kconfig when the version string from the compiler changes. + #[macro_use] mod quote; mod concat_idents; From 93dc3be19450447a3a7090bd1dfb9f3daac3e8d2 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 2 Sep 2024 18:55:33 +0200 Subject: [PATCH 058/111] docs: rust: include other expressions in conditional compilation section Expand the conditional compilation section to explain how to support other expressions, such as testing whether `RUSTC_VERSION` is at least a given version, which requires a numerical comparison that Rust's `cfg` predicates do not support (yet?). Reviewed-by: Nicolas Schier Tested-by: Alice Ryhl Acked-by: Masahiro Yamada Link: https://lore.kernel.org/r/20240902165535.1101978-7-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- Documentation/rust/general-information.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst index 2d76e09da121..6146b49b6a98 100644 --- a/Documentation/rust/general-information.rst +++ b/Documentation/rust/general-information.rst @@ -151,3 +151,11 @@ configuration: #[cfg(CONFIG_X="y")] // Enabled as a built-in (`y`) #[cfg(CONFIG_X="m")] // Enabled as a module (`m`) #[cfg(not(CONFIG_X))] // Disabled + +For other predicates that Rust's ``cfg`` does not support, e.g. expressions with +numerical comparisons, one may define a new Kconfig symbol: + +.. code-block:: kconfig + + config RUSTC_VERSION_MIN_107900 + def_bool y if RUSTC_VERSION >= 107900 From d077242d68a31075ef5f5da041bf8f6fc19aa231 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 29 Aug 2024 08:22:45 +0000 Subject: [PATCH 059/111] rust: support for shadow call stack sanitizer Add all of the flags that are needed to support the shadow call stack (SCS) sanitizer with Rust, and updates Kconfig to allow only configurations that work. The -Zfixed-x18 flag is required to use SCS on arm64, and requires rustc version 1.80.0 or greater. This restriction is reflected in Kconfig. When CONFIG_DYNAMIC_SCS is enabled, the build will be configured to include unwind tables in the build artifacts. Dynamic SCS uses the unwind tables at boot to find all places that need to be patched. The -Cforce-unwind-tables=y flag ensures that unwind tables are available for Rust code. In non-dynamic mode, the -Zsanitizer=shadow-call-stack flag is what enables the SCS sanitizer. Using this flag requires rustc version 1.82.0 or greater on the targets used by Rust in the kernel. This restriction is reflected in Kconfig. It is possible to avoid the requirement of rustc 1.80.0 by using -Ctarget-feature=+reserve-x18 instead of -Zfixed-x18. However, this flag emits a warning during the build, so this patch does not add support for using it and instead requires 1.80.0 or greater. The dependency is placed on `select HAVE_RUST` to avoid a situation where enabling Rust silently turns off the sanitizer. Instead, turning on the sanitizer results in Rust being disabled. We generally do not want changes to CONFIG_RUST to result in any mitigations being changed or turned off. At the time of writing, rustc 1.82.0 only exists via the nightly release channel. There is a chance that the -Zsanitizer=shadow-call-stack flag will end up needing 1.83.0 instead, but I think it is small. Reviewed-by: Sami Tolvanen Reviewed-by: Ard Biesheuvel Reviewed-by: Kees Cook Acked-by: Will Deacon Signed-off-by: Alice Ryhl Link: https://lore.kernel.org/r/20240829-shadow-call-stack-v7-1-2f62a4432abf@google.com [ Fixed indentation using spaces. - Miguel ] Signed-off-by: Miguel Ojeda --- Makefile | 1 + arch/arm64/Kconfig | 14 +++++++++++++- arch/arm64/Makefile | 3 +++ arch/riscv/Kconfig | 9 ++++++++- init/Kconfig | 1 - 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1883aeb919a4..ec9cec3b650d 100644 --- a/Makefile +++ b/Makefile @@ -929,6 +929,7 @@ ifdef CONFIG_SHADOW_CALL_STACK ifndef CONFIG_DYNAMIC_SCS CC_FLAGS_SCS := -fsanitize=shadow-call-stack KBUILD_CFLAGS += $(CC_FLAGS_SCS) +KBUILD_RUSTFLAGS += -Zsanitizer=shadow-call-stack endif export CC_FLAGS_SCS endif diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a2f8ff354ca6..d56400bfd20a 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -231,7 +231,7 @@ config ARM64 select HAVE_FUNCTION_ARG_ACCESS_API select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ - select HAVE_RUST if CPU_LITTLE_ENDIAN + select HAVE_RUST if RUSTC_SUPPORTS_ARM64 select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select HAVE_KPROBES @@ -265,6 +265,18 @@ config ARM64 help ARM 64-bit (AArch64) Linux support. +config RUSTC_SUPPORTS_ARM64 + def_bool y + depends on CPU_LITTLE_ENDIAN + # Shadow call stack is only supported on certain rustc versions. + # + # When using the UNWIND_PATCH_PAC_INTO_SCS option, rustc version 1.80+ is + # required due to use of the -Zfixed-x18 flag. + # + # Otherwise, rustc version 1.82+ is required due to use of the + # -Zsanitizer=shadow-call-stack flag. + depends on !SHADOW_CALL_STACK || RUSTC_VERSION >= 108200 || RUSTC_VERSION >= 108000 && UNWIND_PATCH_PAC_INTO_SCS + config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS def_bool CC_IS_CLANG # https://github.com/ClangBuiltLinux/linux/issues/1507 diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index f6bc3da1ef11..b058c4803efb 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -57,9 +57,11 @@ KBUILD_AFLAGS += $(call cc-option,-mabi=lp64) ifneq ($(CONFIG_UNWIND_TABLES),y) KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables KBUILD_AFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables +KBUILD_RUSTFLAGS += -Cforce-unwind-tables=n else KBUILD_CFLAGS += -fasynchronous-unwind-tables KBUILD_AFLAGS += -fasynchronous-unwind-tables +KBUILD_RUSTFLAGS += -Cforce-unwind-tables=y -Zuse-sync-unwind=n endif ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) @@ -114,6 +116,7 @@ endif ifeq ($(CONFIG_SHADOW_CALL_STACK), y) KBUILD_CFLAGS += -ffixed-x18 +KBUILD_RUSTFLAGS += -Zfixed-x18 endif ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 0f3cd7c3a436..6d9234216c69 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -172,7 +172,7 @@ config RISCV select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RETHOOK if !XIP_KERNEL select HAVE_RSEQ - select HAVE_RUST if 64BIT + select HAVE_RUST if RUSTC_SUPPORTS_RISCV select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_STACKPROTECTOR @@ -202,6 +202,13 @@ config RISCV select UACCESS_MEMCPY if !MMU select ZONE_DMA32 if 64BIT +config RUSTC_SUPPORTS_RISCV + def_bool y + depends on 64BIT + # Shadow call stack requires rustc version 1.82+ due to use of the + # -Zsanitizer=shadow-call-stack flag. + depends on !SHADOW_CALL_STACK || RUSTC_VERSION >= 108200 + config CLANG_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_CLANG # https://github.com/ClangBuiltLinux/linux/issues/1817 diff --git a/init/Kconfig b/init/Kconfig index c6b744a48dcc..9bcda3b0a20f 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1909,7 +1909,6 @@ config RUST depends on !MODVERSIONS depends on !GCC_PLUGIN_RANDSTRUCT depends on !RANDSTRUCT - depends on !SHADOW_CALL_STACK depends on !DEBUG_INFO_BTF || PAHOLE_HAS_LANG_EXCLUDE help Enables Rust support in the kernel. From ce4a2620985cdf06c200ec0b6dce80374237697c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 1 Aug 2024 13:35:17 +0000 Subject: [PATCH 060/111] cfi: add CONFIG_CFI_ICALL_NORMALIZE_INTEGERS Introduce a Kconfig option for enabling the experimental option to normalize integer types. This ensures that integer types of the same size and signedness are considered compatible by the Control Flow Integrity sanitizer. The security impact of this flag is minimal. When Sami Tolvanen looked into it, he found that integer normalization reduced the number of unique type hashes in the kernel by ~1%, which is acceptable. This option exists for compatibility with Rust, as C and Rust do not have the same set of integer types. There are cases where C has two different integer types of the same size and signedness, but Rust only has one integer type of that size and signedness. When Rust calls into C functions using such types in their signature, this results in CFI failures. One example is 'unsigned long long' and 'unsigned long' which are both 64-bit on LP64 targets, so on those targets this flag will give both types the same CFI tag. This flag changes the ABI heavily. It is not applied automatically when CONFIG_RUST is turned on to make sure that the CONFIG_RUST option does not change the ABI of C code. For example, some build may need to make other changes atomically with toggling this flag. Having it be a separate option makes it possible to first turn on normalized integer tags, and then later turn on CONFIG_RUST. Similarly, when turning on CONFIG_RUST in a build, you may need a few attempts where the RUST=y commit gets reverted a few times. It is inconvenient if reverting RUST=y also requires reverting the changes you made to support normalized integer tags. To avoid having this flag impact builds that don't care about this, the next patch in this series will make CONFIG_RUST turn on this option using `select` rather than `depends on`. Signed-off-by: Alice Ryhl Reviewed-by: Sami Tolvanen Tested-by: Gatlin Newhouse Acked-by: Kees Cook Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20240801-kcfi-v2-1-c93caed3d121@google.com Signed-off-by: Miguel Ojeda --- Makefile | 3 +++ arch/Kconfig | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/Makefile b/Makefile index ec9cec3b650d..35253bff5ca2 100644 --- a/Makefile +++ b/Makefile @@ -954,6 +954,9 @@ endif ifdef CONFIG_CFI_CLANG CC_FLAGS_CFI := -fsanitize=kcfi +ifdef CONFIG_CFI_ICALL_NORMALIZE_INTEGERS + CC_FLAGS_CFI += -fsanitize-cfi-icall-experimental-normalize-integers +endif KBUILD_CFLAGS += $(CC_FLAGS_CFI) export CC_FLAGS_CFI endif diff --git a/arch/Kconfig b/arch/Kconfig index 975dd22a2dbd..ee58df8b1080 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -826,6 +826,22 @@ config CFI_CLANG https://clang.llvm.org/docs/ControlFlowIntegrity.html +config CFI_ICALL_NORMALIZE_INTEGERS + bool "Normalize CFI tags for integers" + depends on CFI_CLANG + depends on $(cc-option,-fsanitize=kcfi -fsanitize-cfi-icall-experimental-normalize-integers) + help + This option normalizes the CFI tags for integer types so that all + integer types of the same size and signedness receive the same CFI + tag. + + The option is separate from CONFIG_RUST because it affects the ABI. + When working with build systems that care about the ABI, it is + convenient to be able to turn on this flag first, before Rust is + turned on. + + This option is necessary for using CFI with Rust. If unsure, say N. + config CFI_PERMISSIVE bool "Use CFI in permissive mode" depends on CFI_CLANG From d21dffe51baabf6729a95585181507f24bd695a0 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Mon, 29 Jul 2024 14:49:26 +0800 Subject: [PATCH 061/111] arch/sparc: remove unused varible paddrbase in function leon_swprobe() commit f22ed71cd602 ("sparc32,leon: SRMMU MMU Table probe fix") change return value from paddrbase to 'pte', but left the varible here. That causes a build warning for this varible, so we may remove it. make --keep-going CROSS_COMPILE=/home/alexs/0day/gcc-14.1.0-nolibc/sparc-linux/bin/sparc-linux- --jobs=16 KCFLAGS= -Wtautological-compare -Wno-error=return-type -Wreturn-type -Wcast-function-type -funsigned-char -Wundef -fstrict-flex-arrays=3 -Wformat-overflow -Wformat-truncation -Wrestrict -Wenum-conversion W=1 O=sparc ARCH=sparc defconfig SHELL=/bin/bash arch/sparc/mm/ mm/ -s :1519:2: warning: #warning syscall clone3 not implemented [-Wcpp] ../arch/sparc/mm/leon_mm.c: In function 'leon_swprobe': ../arch/sparc/mm/leon_mm.c:42:32: warning: variable 'paddrbase' set but not used [-Wunused-but-set-variable] 42 | unsigned int lvl, pte, paddrbase; | ^~~~~~~~~ Signed-off-by: Alex Shi To: linux-kernel@vger.kernel.org To: sparclinux@vger.kernel.org To: Christian Brauner To: Andreas Larsson To: David S. Miller Reviewed-by: Andreas Larsson Tested-by: Andreas Larsson Link: https://lore.kernel.org/r/20240729064926.3126528-1-alexs@kernel.org Signed-off-by: Andreas Larsson --- arch/sparc/mm/leon_mm.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c index ec61ff1f96b7..1dc9b3d70eda 100644 --- a/arch/sparc/mm/leon_mm.c +++ b/arch/sparc/mm/leon_mm.c @@ -39,12 +39,10 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) unsigned int ctxtbl; unsigned int pgd, pmd, ped; unsigned int ptr; - unsigned int lvl, pte, paddrbase; + unsigned int lvl, pte; unsigned int ctx; unsigned int paddr_calc; - paddrbase = 0; - if (srmmu_swprobe_trace) printk(KERN_INFO "swprobe: trace on\n"); @@ -73,7 +71,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: pgd is entry level 3\n"); lvl = 3; pte = pgd; - paddrbase = pgd & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((pgd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -96,7 +93,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: pmd is entry level 2\n"); lvl = 2; pte = pmd; - paddrbase = pmd & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((pmd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -124,7 +120,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: ped is entry level 1\n"); lvl = 1; pte = ped; - paddrbase = ped & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((ped & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -147,7 +142,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: ptr is entry level 0\n"); lvl = 0; pte = ptr; - paddrbase = ptr & _SRMMU_PTE_PMASK_LEON; goto ready; } if (srmmu_swprobe_trace) From ca627e636551e74b528f150d744f67d9a63f0ae7 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Thu, 12 Sep 2024 21:00:44 +0200 Subject: [PATCH 062/111] rust: cfi: add support for CFI_CLANG with Rust Make it possible to use the Control Flow Integrity (CFI) sanitizer when Rust is enabled. Enabling CFI with Rust requires that CFI is configured to normalize integer types so that all integer types of the same size and signedness are compatible under CFI. Rust and C use the same LLVM backend for code generation, so Rust KCFI is compatible with the KCFI used in the kernel for C. In the case of FineIBT, CFI also depends on -Zpatchable-function-entry for rewriting the function prologue, so we set that flag for Rust as well. The flag for FineIBT requires rustc 1.80.0 or later, so include a Kconfig requirement for that. Enabling Rust will select CFI_ICALL_NORMALIZE_INTEGERS because the flag is required to use Rust with CFI. Using select rather than `depends on` avoids the case where Rust is not visible in menuconfig due to CFI_ICALL_NORMALIZE_INTEGERS not being enabled. One disadvantage of select is that RUST must `depends on` all of the things that CFI_ICALL_NORMALIZE_INTEGERS depends on to avoid invalid configurations. Alice has been using KCFI on her phone for several months, so it is reasonably well tested on arm64. Signed-off-by: Matthew Maurer Co-developed-by: Alice Ryhl Signed-off-by: Alice Ryhl Reviewed-by: Sami Tolvanen Tested-by: Gatlin Newhouse Acked-by: Kees Cook Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20240801-kcfi-v2-2-c93caed3d121@google.com [ Replaced `!FINEIBT` requirement with `!CALL_PADDING` to prevent a build error on older Rust compilers. Fixed typo. - Miguel ] Signed-off-by: Miguel Ojeda --- Makefile | 7 +++++++ arch/x86/Makefile | 4 ++++ init/Kconfig | 4 +++- rust/Makefile | 2 +- scripts/generate_rust_target.rs | 1 + 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 35253bff5ca2..08ba14ef128e 100644 --- a/Makefile +++ b/Makefile @@ -957,6 +957,13 @@ CC_FLAGS_CFI := -fsanitize=kcfi ifdef CONFIG_CFI_ICALL_NORMALIZE_INTEGERS CC_FLAGS_CFI += -fsanitize-cfi-icall-experimental-normalize-integers endif +ifdef CONFIG_RUST + # Always pass -Zsanitizer-cfi-normalize-integers as CONFIG_RUST selects + # CONFIG_CFI_ICALL_NORMALIZE_INTEGERS. + RUSTC_FLAGS_CFI := -Zsanitizer=kcfi -Zsanitizer-cfi-normalize-integers + KBUILD_RUSTFLAGS += $(RUSTC_FLAGS_CFI) + export RUSTC_FLAGS_CFI +endif KBUILD_CFLAGS += $(CC_FLAGS_CFI) export CC_FLAGS_CFI endif diff --git a/arch/x86/Makefile b/arch/x86/Makefile index a1883a30a5d8..cd75e78a06c1 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -242,6 +242,10 @@ ifdef CONFIG_CALL_PADDING PADDING_CFLAGS := -fpatchable-function-entry=$(CONFIG_FUNCTION_PADDING_BYTES),$(CONFIG_FUNCTION_PADDING_BYTES) KBUILD_CFLAGS += $(PADDING_CFLAGS) export PADDING_CFLAGS + +PADDING_RUSTFLAGS := -Zpatchable-function-entry=$(CONFIG_FUNCTION_PADDING_BYTES),$(CONFIG_FUNCTION_PADDING_BYTES) +KBUILD_RUSTFLAGS += $(PADDING_RUSTFLAGS) +export PADDING_RUSTFLAGS endif KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE) diff --git a/init/Kconfig b/init/Kconfig index 9bcda3b0a20f..53f4589b7847 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1905,11 +1905,13 @@ config RUST bool "Rust support" depends on HAVE_RUST depends on RUST_IS_AVAILABLE - depends on !CFI_CLANG depends on !MODVERSIONS depends on !GCC_PLUGIN_RANDSTRUCT depends on !RANDSTRUCT depends on !DEBUG_INFO_BTF || PAHOLE_HAS_LANG_EXCLUDE + depends on !CFI_CLANG || RUSTC_VERSION >= 107900 && $(cc-option,-fsanitize=kcfi -fsanitize-cfi-icall-experimental-normalize-integers) + select CFI_ICALL_NORMALIZE_INTEGERS if CFI_CLANG + depends on !CALL_PADDING || RUSTC_VERSION >= 108000 help Enables Rust support in the kernel. diff --git a/rust/Makefile b/rust/Makefile index 4eae318f36ff..dd76dc27d666 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -306,7 +306,7 @@ $(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers/helpers.c FORCE quiet_cmd_exports = EXPORTS $@ cmd_exports = \ $(NM) -p --defined-only $< \ - | awk '/ (T|R|D) / {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@ + | awk '$$2~/(T|R|D)/ && $$3!~/__cfi/ {printf "EXPORT_SYMBOL_RUST_GPL(%s);\n",$$3}' > $@ $(obj)/exports_core_generated.h: $(obj)/core.o FORCE $(call if_changed,exports) diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index fbf723996d20..087c1d13d33b 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -207,6 +207,7 @@ fn main() { } ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); + ts.push("supported-sanitizers", ["kcfi"]); ts.push("target-pointer-width", "64"); } else if cfg.has("X86_32") { // This only works on UML, as i386 otherwise needs regparm support in rustc From cc1d98f9fe30467a2224184336b3166ef4adbc25 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 14 Aug 2024 18:10:52 +0200 Subject: [PATCH 063/111] kasan: simplify and clarify Makefile When KASAN support was being added to the Linux kernel, GCC did not yet support all of the KASAN-related compiler options. Thus, the KASAN Makefile had to probe the compiler for supported options. Nowadays, the Linux kernel GCC version requirement is 5.1+, and thus we don't need the probing of the -fasan-shadow-offset parameter: it exists in all 5.1+ GCCs. Simplify the KASAN Makefile to drop CFLAGS_KASAN_MINIMAL. Also add a few more comments and unify the indentation. Signed-off-by: Andrey Konovalov Acked-by: Marco Elver Link: https://lore.kernel.org/r/20240814161052.10374-1-andrey.konovalov@linux.dev Signed-off-by: Miguel Ojeda --- scripts/Makefile.kasan | 43 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 390658a2d5b7..aab4154af00a 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -22,30 +22,31 @@ endif ifdef CONFIG_KASAN_GENERIC ifdef CONFIG_KASAN_INLINE + # When the number of memory accesses in a function is less than this + # call threshold number, the compiler will use inline instrumentation. + # 10000 is chosen offhand as a sufficiently large number to make all + # kernel functions to be instrumented inline. call_threshold := 10000 else call_threshold := 0 endif -CFLAGS_KASAN_MINIMAL := -fsanitize=kernel-address +# First, enable -fsanitize=kernel-address together with providing the shadow +# mapping offset, as for GCC, -fasan-shadow-offset fails without -fsanitize +# (GCC accepts the shadow mapping offset via -fasan-shadow-offset instead of +# a --param like the other KASAN parameters). +# Instead of ifdef-checking the compiler, rely on cc-option. +CFLAGS_KASAN := $(call cc-option, -fsanitize=kernel-address \ + -fasan-shadow-offset=$(KASAN_SHADOW_OFFSET), \ + $(call cc-option, -fsanitize=kernel-address \ + -mllvm -asan-mapping-offset=$(KASAN_SHADOW_OFFSET))) -# -fasan-shadow-offset fails without -fsanitize -CFLAGS_KASAN_SHADOW := $(call cc-option, -fsanitize=kernel-address \ - -fasan-shadow-offset=$(KASAN_SHADOW_OFFSET), \ - $(call cc-option, -fsanitize=kernel-address \ - -mllvm -asan-mapping-offset=$(KASAN_SHADOW_OFFSET))) - -ifeq ($(strip $(CFLAGS_KASAN_SHADOW)),) - CFLAGS_KASAN := $(CFLAGS_KASAN_MINIMAL) -else - # Now add all the compiler specific options that are valid standalone - CFLAGS_KASAN := $(CFLAGS_KASAN_SHADOW) \ - $(call cc-param,asan-globals=1) \ - $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ - $(call cc-param,asan-instrument-allocas=1) -endif - -CFLAGS_KASAN += $(call cc-param,asan-stack=$(stack_enable)) +# Now, add other parameters enabled similarly in both GCC and Clang. +# As some of them are not supported by older compilers, use cc-param. +CFLAGS_KASAN += $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ + $(call cc-param,asan-stack=$(stack_enable)) \ + $(call cc-param,asan-instrument-allocas=1) \ + $(call cc-param,asan-globals=1) # Instrument memcpy/memset/memmove calls by using instrumented __asan_mem*() # instead. With compilers that don't support this option, compiler-inserted @@ -57,9 +58,9 @@ endif # CONFIG_KASAN_GENERIC ifdef CONFIG_KASAN_SW_TAGS ifdef CONFIG_KASAN_INLINE - instrumentation_flags := $(call cc-param,hwasan-mapping-offset=$(KASAN_SHADOW_OFFSET)) + instrumentation_flags := $(call cc-param,hwasan-mapping-offset=$(KASAN_SHADOW_OFFSET)) else - instrumentation_flags := $(call cc-param,hwasan-instrument-with-calls=1) + instrumentation_flags := $(call cc-param,hwasan-instrument-with-calls=1) endif CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ @@ -70,7 +71,7 @@ CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ # Instrument memcpy/memset/memmove calls by using instrumented __hwasan_mem*(). ifeq ($(call clang-min-version, 150000)$(call gcc-min-version, 130000),y) -CFLAGS_KASAN += $(call cc-param,hwasan-kernel-mem-intrinsic-prefix=1) + CFLAGS_KASAN += $(call cc-param,hwasan-kernel-mem-intrinsic-prefix=1) endif endif # CONFIG_KASAN_SW_TAGS From c42297438aee70e2d391225de3d35ffeb2bdbaf9 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 20 Aug 2024 19:48:56 +0000 Subject: [PATCH 064/111] kbuild: rust: Define probing macros for rustc Creates flag probe macro variants for `rustc`. These are helpful because: 1. The kernel now supports a minimum `rustc` version rather than a single version. 2. `rustc` links against a range of LLVM revisions, occasionally even ones without an official release number. Since the availability of some Rust flags depends on which LLVM it has been linked against, probing is necessary. Signed-off-by: Matthew Maurer Link: https://github.com/Rust-for-Linux/linux/pull/1087 Link: https://lore.kernel.org/r/20240820194910.187826-2-mmaurer@google.com Signed-off-by: Miguel Ojeda --- scripts/Kconfig.include | 8 ++++++++ scripts/Makefile.compiler | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include index 3500a3d62f0d..785a491e5996 100644 --- a/scripts/Kconfig.include +++ b/scripts/Kconfig.include @@ -64,3 +64,11 @@ ld-version := $(shell,set -- $(ld-info) && echo $2) cc-option-bit = $(if-success,$(CC) -Werror $(1) -E -x c /dev/null -o /dev/null,$(1)) m32-flag := $(cc-option-bit,-m32) m64-flag := $(cc-option-bit,-m64) + +# $(rustc-option,) +# Return y if the Rust compiler supports , n otherwise +# Calls to this should be guarded so that they are not evaluated if +# CONFIG_RUST_IS_AVAILABLE is not set. +# If you are testing for unstable features, consider testing RUSTC_VERSION +# instead, as features may have different completeness while available. +rustc-option = $(success,trap "rm -rf .tmp_$$" EXIT; mkdir .tmp_$$; $(RUSTC) $(1) --crate-type=rlib /dev/null --out-dir=.tmp_$$ -o .tmp_$$/tmp.rlib) diff --git a/scripts/Makefile.compiler b/scripts/Makefile.compiler index 92be0c9a13ee..057305eae85c 100644 --- a/scripts/Makefile.compiler +++ b/scripts/Makefile.compiler @@ -72,3 +72,18 @@ clang-min-version = $(call test-ge, $(CONFIG_CLANG_VERSION), $1) # ld-option # Usage: KBUILD_LDFLAGS += $(call ld-option, -X, -Y) ld-option = $(call try-run, $(LD) $(KBUILD_LDFLAGS) $(1) -v,$(1),$(2),$(3)) + +# __rustc-option +# Usage: MY_RUSTFLAGS += $(call __rustc-option,$(RUSTC),$(MY_RUSTFLAGS),-Cinstrument-coverage,-Zinstrument-coverage) +__rustc-option = $(call try-run,\ + $(1) $(2) $(3) --crate-type=rlib /dev/null --out-dir=$$TMPOUT -o "$$TMP",$(3),$(4)) + +# rustc-option +# Usage: rustflags-y += $(call rustc-option,-Cinstrument-coverage,-Zinstrument-coverage) +rustc-option = $(call __rustc-option, $(RUSTC),\ + $(KBUILD_RUSTFLAGS),$(1),$(2)) + +# rustc-option-yn +# Usage: flag := $(call rustc-option-yn,-Cinstrument-coverage) +rustc-option-yn = $(call try-run,\ + $(RUSTC) $(KBUILD_RUSTFLAGS) $(1) --crate-type=rlib /dev/null --out-dir=$$TMPOUT -o "$$TMP",y,n) From f64e2f3a66e30319023b7924d438d159ac742d63 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 20 Aug 2024 19:48:57 +0000 Subject: [PATCH 065/111] rust: kasan: Rust does not support KHWASAN Rust does not yet have support for software tags. Prevent RUST from being selected if KASAN_SW_TAGS is enabled. Signed-off-by: Matthew Maurer Reviewed-by: Andrey Konovalov Link: https://lore.kernel.org/r/20240820194910.187826-3-mmaurer@google.com Signed-off-by: Miguel Ojeda --- init/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/init/Kconfig b/init/Kconfig index 53f4589b7847..4ea2a161d362 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1912,6 +1912,7 @@ config RUST depends on !CFI_CLANG || RUSTC_VERSION >= 107900 && $(cc-option,-fsanitize=kcfi -fsanitize-cfi-icall-experimental-normalize-integers) select CFI_ICALL_NORMALIZE_INTEGERS if CFI_CLANG depends on !CALL_PADDING || RUSTC_VERSION >= 108000 + depends on !KASAN_SW_TAGS help Enables Rust support in the kernel. From e3117404b41124c88a4d834fc3222669a880addc Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 20 Aug 2024 19:48:58 +0000 Subject: [PATCH 066/111] kbuild: rust: Enable KASAN support Rust supports KASAN via LLVM, but prior to this patch, the flags aren't set properly. Suggested-by: Miguel Ojeda Signed-off-by: Matthew Maurer Reviewed-by: Andrey Konovalov Link: https://lore.kernel.org/r/20240820194910.187826-4-mmaurer@google.com [ Applied "SW_TAGS KASAN" nit. - Miguel ] Signed-off-by: Miguel Ojeda --- scripts/Makefile.kasan | 57 ++++++++++++++++++++++++--------- scripts/Makefile.lib | 3 ++ scripts/generate_rust_target.rs | 2 +- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index aab4154af00a..693dbbebebba 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -12,6 +12,11 @@ endif KASAN_SHADOW_OFFSET ?= $(CONFIG_KASAN_SHADOW_OFFSET) cc-param = $(call cc-option, -mllvm -$(1), $(call cc-option, --param $(1))) +rustc-param = $(call rustc-option, -Cllvm-args=-$(1),) + +check-args = $(foreach arg,$(2),$(call $(1),$(arg))) + +kasan_params := ifdef CONFIG_KASAN_STACK stack_enable := 1 @@ -41,39 +46,59 @@ CFLAGS_KASAN := $(call cc-option, -fsanitize=kernel-address \ $(call cc-option, -fsanitize=kernel-address \ -mllvm -asan-mapping-offset=$(KASAN_SHADOW_OFFSET))) -# Now, add other parameters enabled similarly in both GCC and Clang. -# As some of them are not supported by older compilers, use cc-param. -CFLAGS_KASAN += $(call cc-param,asan-instrumentation-with-call-threshold=$(call_threshold)) \ - $(call cc-param,asan-stack=$(stack_enable)) \ - $(call cc-param,asan-instrument-allocas=1) \ - $(call cc-param,asan-globals=1) +# The minimum supported `rustc` version has a minimum supported LLVM +# version late enough that we can assume support for -asan-mapping-offset. +RUSTFLAGS_KASAN := -Zsanitizer=kernel-address \ + -Zsanitizer-recover=kernel-address \ + -Cllvm-args=-asan-mapping-offset=$(KASAN_SHADOW_OFFSET) + +# Now, add other parameters enabled similarly in GCC, Clang, and rustc. +# As some of them are not supported by older compilers, these will be filtered +# through `cc-param` or `rust-param` as applicable. +kasan_params += asan-instrumentation-with-call-threshold=$(call_threshold) \ + asan-stack=$(stack_enable) \ + asan-instrument-allocas=1 \ + asan-globals=1 # Instrument memcpy/memset/memmove calls by using instrumented __asan_mem*() # instead. With compilers that don't support this option, compiler-inserted # memintrinsics won't be checked by KASAN on GENERIC_ENTRY architectures. -CFLAGS_KASAN += $(call cc-param,asan-kernel-mem-intrinsic-prefix=1) +kasan_params += asan-kernel-mem-intrinsic-prefix=1 endif # CONFIG_KASAN_GENERIC ifdef CONFIG_KASAN_SW_TAGS +CFLAGS_KASAN := -fsanitize=kernel-hwaddress + +# This sets flags that will enable SW_TAGS KASAN once enabled in Rust. These +# will not work today, and is guarded against in dependencies for CONFIG_RUST. +RUSTFLAGS_KASAN := -Zsanitizer=kernel-hwaddress \ + -Zsanitizer-recover=kernel-hwaddress + ifdef CONFIG_KASAN_INLINE - instrumentation_flags := $(call cc-param,hwasan-mapping-offset=$(KASAN_SHADOW_OFFSET)) + kasan_params += hwasan-mapping-offset=$(KASAN_SHADOW_OFFSET) else - instrumentation_flags := $(call cc-param,hwasan-instrument-with-calls=1) + kasan_params += hwasan-instrument-with-calls=1 endif -CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ - $(call cc-param,hwasan-instrument-stack=$(stack_enable)) \ - $(call cc-param,hwasan-use-short-granules=0) \ - $(call cc-param,hwasan-inline-all-checks=0) \ - $(instrumentation_flags) +kasan_params += hwasan-instrument-stack=$(stack_enable) \ + hwasan-use-short-granules=0 \ + hwasan-inline-all-checks=0 # Instrument memcpy/memset/memmove calls by using instrumented __hwasan_mem*(). ifeq ($(call clang-min-version, 150000)$(call gcc-min-version, 130000),y) - CFLAGS_KASAN += $(call cc-param,hwasan-kernel-mem-intrinsic-prefix=1) + kasan_params += hwasan-kernel-mem-intrinsic-prefix=1 endif endif # CONFIG_KASAN_SW_TAGS -export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE +# Add all as-supported KASAN LLVM parameters requested by the configuration. +CFLAGS_KASAN += $(call check-args, cc-param, $(kasan_params)) + +ifdef CONFIG_RUST + # Avoid calling `rustc-param` unless Rust is enabled. + RUSTFLAGS_KASAN += $(call check-args, rustc-param, $(kasan_params)) +endif # CONFIG_RUST + +export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE RUSTFLAGS_KASAN diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index fe3668dc4954..27999da3d382 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -167,6 +167,9 @@ ifneq ($(CONFIG_KASAN_HW_TAGS),y) _c_flags += $(if $(patsubst n%,, \ $(KASAN_SANITIZE_$(target-stem).o)$(KASAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_KASAN), $(CFLAGS_KASAN_NOSANITIZE)) +_rust_flags += $(if $(patsubst n%,, \ + $(KASAN_SANITIZE_$(target-stem).o)$(KASAN_SANITIZE)$(is-kernel-object)), \ + $(RUSTFLAGS_KASAN)) endif endif diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 087c1d13d33b..0d00ac3723b5 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -207,7 +207,7 @@ fn main() { } ts.push("features", features); ts.push("llvm-target", "x86_64-linux-gnu"); - ts.push("supported-sanitizers", ["kcfi"]); + ts.push("supported-sanitizers", ["kcfi", "kernel-address"]); ts.push("target-pointer-width", "64"); } else if cfg.has("X86_32") { // This only works on UML, as i386 otherwise needs regparm support in rustc From a2f11547052001bd448ccec81dd1e68409078fbb Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 20 Aug 2024 19:48:59 +0000 Subject: [PATCH 067/111] kasan: rust: Add KASAN smoke test via UAF Adds a smoke test to ensure that KASAN in Rust is actually detecting a Rust-native UAF. There is significant room to expand this test suite, but this will at least ensure that flags are having the intended effect. The rename from kasan_test.c to kasan_test_c.c is in order to allow the single kasan_test.ko test suite to contain both a .o file produced by the C compiler and one produced by rustc. Signed-off-by: Matthew Maurer Reviewed-by: Andrey Konovalov Link: https://lore.kernel.org/r/20240820194910.187826-5-mmaurer@google.com [ Applied empty line nit, removed double empty line, applied `rustfmt` and formatted crate comment. - Miguel ] Signed-off-by: Miguel Ojeda --- mm/kasan/Makefile | 8 +++++++- mm/kasan/kasan.h | 6 ++++++ mm/kasan/{kasan_test.c => kasan_test_c.c} | 11 +++++++++++ mm/kasan/kasan_test_rust.rs | 21 +++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) rename mm/kasan/{kasan_test.c => kasan_test_c.c} (99%) create mode 100644 mm/kasan/kasan_test_rust.rs diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index 7634dd2a6128..b88543e5c0cc 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -44,7 +44,8 @@ ifndef CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX CFLAGS_KASAN_TEST += -fno-builtin endif -CFLAGS_kasan_test.o := $(CFLAGS_KASAN_TEST) +CFLAGS_kasan_test_c.o := $(CFLAGS_KASAN_TEST) +RUSTFLAGS_kasan_test_rust.o := $(RUSTFLAGS_KASAN) CFLAGS_kasan_test_module.o := $(CFLAGS_KASAN_TEST) obj-y := common.o report.o @@ -52,5 +53,10 @@ obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quaran obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o tags.o report_tags.o obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o tags.o report_tags.o +kasan_test-objs := kasan_test_c.o +ifdef CONFIG_RUST + kasan_test-objs += kasan_test_rust.o +endif + obj-$(CONFIG_KASAN_KUNIT_TEST) += kasan_test.o obj-$(CONFIG_KASAN_MODULE_TEST) += kasan_test_module.o diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index fb2b9ac0659a..f438a6cdc964 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -555,6 +555,12 @@ static inline bool kasan_arch_is_ready(void) { return true; } void kasan_kunit_test_suite_start(void); void kasan_kunit_test_suite_end(void); +#ifdef CONFIG_RUST +char kasan_test_rust_uaf(void); +#else +static inline char kasan_test_rust_uaf(void) { return '\0'; } +#endif + #else /* CONFIG_KASAN_KUNIT_TEST */ static inline void kasan_kunit_test_suite_start(void) { } diff --git a/mm/kasan/kasan_test.c b/mm/kasan/kasan_test_c.c similarity index 99% rename from mm/kasan/kasan_test.c rename to mm/kasan/kasan_test_c.c index 7b32be2a3cf0..0fd445a3be1c 100644 --- a/mm/kasan/kasan_test.c +++ b/mm/kasan/kasan_test_c.c @@ -1899,6 +1899,16 @@ static void match_all_mem_tag(struct kunit *test) kfree(ptr); } +/* + * Check that Rust performing a use-after-free using `unsafe` is detected. + * This is a smoke test to make sure that Rust is being sanitized properly. + */ +static void rust_uaf(struct kunit *test) +{ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_RUST); + KUNIT_EXPECT_KASAN_FAIL(test, kasan_test_rust_uaf()); +} + static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kmalloc_oob_right), KUNIT_CASE(kmalloc_oob_left), @@ -1971,6 +1981,7 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(match_all_not_assigned), KUNIT_CASE(match_all_ptr_tag), KUNIT_CASE(match_all_mem_tag), + KUNIT_CASE(rust_uaf), {} }; diff --git a/mm/kasan/kasan_test_rust.rs b/mm/kasan/kasan_test_rust.rs new file mode 100644 index 000000000000..caa7175964ef --- /dev/null +++ b/mm/kasan/kasan_test_rust.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helper crate for KASAN testing. +//! +//! Provides behavior to check the sanitization of Rust code. + +use core::ptr::addr_of_mut; +use kernel::prelude::*; + +/// Trivial UAF - allocate a big vector, grab a pointer partway through, +/// drop the vector, and touch it. +#[no_mangle] +pub extern "C" fn kasan_test_rust_uaf() -> u8 { + let mut v: Vec = Vec::new(); + for _ in 0..4096 { + v.push(0x42, GFP_KERNEL).unwrap(); + } + let ptr: *mut u8 = addr_of_mut!(v[2048]); + drop(v); + unsafe { *ptr } +} From 699d53f04829d6b8855ff458f86e4b75ef3e5f0c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 19 Sep 2024 20:55:57 +0200 Subject: [PATCH 068/111] powerpc/vdso32: Fix use of crtsavres for PPC64 crtsavres.S content is encloded by a #ifndef CONFIG_PPC64 To be used on VDSO32 on PPC64 it's content must available on PPC64 as well. Replace #ifndef CONFIG_PPC64 by #ifndef __powerpc64__ as __powerpc64__ is not set when building VDSO32 on PPC64. Reported-by: Christian Zigotzky Closes: https://lore.kernel.org/linuxppc-dev/047b7503-af0c-4bb0-b12a-2f6b1e461752@csgroup.eu/T/ Fixes: b163596a5b6f ("powerpc/vdso32: Add crtsavres") Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/aded2b257018fe654db759fdfa4ab1a0b5426b1b.1726772140.git.christophe.leroy@csgroup.eu --- arch/powerpc/lib/crtsavres.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/lib/crtsavres.S b/arch/powerpc/lib/crtsavres.S index 7e5e1c28e56a..8967903c15e9 100644 --- a/arch/powerpc/lib/crtsavres.S +++ b/arch/powerpc/lib/crtsavres.S @@ -46,7 +46,7 @@ .section ".text" -#ifndef CONFIG_PPC64 +#ifndef __powerpc64__ /* Routines for saving integer registers, called by the compiler. */ /* Called with r11 pointing to the stack header word of the caller of the */ From 3af2e2f68cc6baf0a11f662d30b0bf981f77bfea Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Tue, 17 Sep 2024 09:24:45 -0400 Subject: [PATCH 069/111] powerpc/pseries/eeh: move pseries_eeh_err_inject() outside CONFIG_DEBUG_FS block Makes pseries_eeh_err_inject() available even when debugfs is disabled (CONFIG_DEBUG_FS=n). It moves eeh_debugfs_break_device() and eeh_pe_inject_mmio_error() out of the CONFIG_DEBUG_FS block and renames it as eeh_break_device(). Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202409170509.VWC6jadC-lkp@intel.com/ Fixes: b0e2b828dfca ("powerpc/pseries/eeh: Fix pseries_eeh_err_inject") Signed-off-by: Narayana Murty N Reviewed-by: Ritesh Harjani (IBM) Signed-off-by: Michael Ellerman Link: https://msgid.link/20240917132445.3868016-1-nnmlinux@linux.ibm.com --- arch/powerpc/kernel/eeh.c | 198 +++++++++++++++++++------------------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 0e59b8fd9bc6..83fe99861eb1 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1574,6 +1574,104 @@ static int proc_eeh_show(struct seq_file *m, void *v) } #endif /* CONFIG_PROC_FS */ +static int eeh_break_device(struct pci_dev *pdev) +{ + struct resource *bar = NULL; + void __iomem *mapped; + u16 old, bit; + int i, pos; + + /* Do we have an MMIO BAR to disable? */ + for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { + struct resource *r = &pdev->resource[i]; + + if (!r->flags || !r->start) + continue; + if (r->flags & IORESOURCE_IO) + continue; + if (r->flags & IORESOURCE_UNSET) + continue; + + bar = r; + break; + } + + if (!bar) { + pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); + return -ENXIO; + } + + pci_err(pdev, "Going to break: %pR\n", bar); + + if (pdev->is_virtfn) { +#ifndef CONFIG_PCI_IOV + return -ENXIO; +#else + /* + * VFs don't have a per-function COMMAND register, so the best + * we can do is clear the Memory Space Enable bit in the PF's + * SRIOV control reg. + * + * Unfortunately, this requires that we have a PF (i.e doesn't + * work for a passed-through VF) and it has the potential side + * effect of also causing an EEH on every other VF under the + * PF. Oh well. + */ + pdev = pdev->physfn; + if (!pdev) + return -ENXIO; /* passed through VFs have no PF */ + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + pos += PCI_SRIOV_CTRL; + bit = PCI_SRIOV_CTRL_MSE; +#endif /* !CONFIG_PCI_IOV */ + } else { + bit = PCI_COMMAND_MEMORY; + pos = PCI_COMMAND; + } + + /* + * Process here is: + * + * 1. Disable Memory space. + * + * 2. Perform an MMIO to the device. This should result in an error + * (CA / UR) being raised by the device which results in an EEH + * PE freeze. Using the in_8() accessor skips the eeh detection hook + * so the freeze hook so the EEH Detection machinery won't be + * triggered here. This is to match the usual behaviour of EEH + * where the HW will asynchronously freeze a PE and it's up to + * the kernel to notice and deal with it. + * + * 3. Turn Memory space back on. This is more important for VFs + * since recovery will probably fail if we don't. For normal + * the COMMAND register is reset as a part of re-initialising + * the device. + * + * Breaking stuff is the point so who cares if it's racy ;) + */ + pci_read_config_word(pdev, pos, &old); + + mapped = ioremap(bar->start, PAGE_SIZE); + if (!mapped) { + pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); + return -ENXIO; + } + + pci_write_config_word(pdev, pos, old & ~bit); + in_8(mapped); + pci_write_config_word(pdev, pos, old); + + iounmap(mapped); + + return 0; +} + +int eeh_pe_inject_mmio_error(struct pci_dev *pdev) +{ + return eeh_break_device(pdev); +} + #ifdef CONFIG_DEBUG_FS @@ -1725,99 +1823,6 @@ static const struct file_operations eeh_dev_check_fops = { .read = eeh_debugfs_dev_usage, }; -static int eeh_debugfs_break_device(struct pci_dev *pdev) -{ - struct resource *bar = NULL; - void __iomem *mapped; - u16 old, bit; - int i, pos; - - /* Do we have an MMIO BAR to disable? */ - for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { - struct resource *r = &pdev->resource[i]; - - if (!r->flags || !r->start) - continue; - if (r->flags & IORESOURCE_IO) - continue; - if (r->flags & IORESOURCE_UNSET) - continue; - - bar = r; - break; - } - - if (!bar) { - pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); - return -ENXIO; - } - - pci_err(pdev, "Going to break: %pR\n", bar); - - if (pdev->is_virtfn) { -#ifndef CONFIG_PCI_IOV - return -ENXIO; -#else - /* - * VFs don't have a per-function COMMAND register, so the best - * we can do is clear the Memory Space Enable bit in the PF's - * SRIOV control reg. - * - * Unfortunately, this requires that we have a PF (i.e doesn't - * work for a passed-through VF) and it has the potential side - * effect of also causing an EEH on every other VF under the - * PF. Oh well. - */ - pdev = pdev->physfn; - if (!pdev) - return -ENXIO; /* passed through VFs have no PF */ - - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); - pos += PCI_SRIOV_CTRL; - bit = PCI_SRIOV_CTRL_MSE; -#endif /* !CONFIG_PCI_IOV */ - } else { - bit = PCI_COMMAND_MEMORY; - pos = PCI_COMMAND; - } - - /* - * Process here is: - * - * 1. Disable Memory space. - * - * 2. Perform an MMIO to the device. This should result in an error - * (CA / UR) being raised by the device which results in an EEH - * PE freeze. Using the in_8() accessor skips the eeh detection hook - * so the freeze hook so the EEH Detection machinery won't be - * triggered here. This is to match the usual behaviour of EEH - * where the HW will asynchronously freeze a PE and it's up to - * the kernel to notice and deal with it. - * - * 3. Turn Memory space back on. This is more important for VFs - * since recovery will probably fail if we don't. For normal - * the COMMAND register is reset as a part of re-initialising - * the device. - * - * Breaking stuff is the point so who cares if it's racy ;) - */ - pci_read_config_word(pdev, pos, &old); - - mapped = ioremap(bar->start, PAGE_SIZE); - if (!mapped) { - pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); - return -ENXIO; - } - - pci_write_config_word(pdev, pos, old & ~bit); - in_8(mapped); - pci_write_config_word(pdev, pos, old); - - iounmap(mapped); - - return 0; -} - static ssize_t eeh_dev_break_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) @@ -1829,7 +1834,7 @@ static ssize_t eeh_dev_break_write(struct file *filp, if (IS_ERR(pdev)) return PTR_ERR(pdev); - ret = eeh_debugfs_break_device(pdev); + ret = eeh_break_device(pdev); pci_dev_put(pdev); if (ret < 0) @@ -1844,11 +1849,6 @@ static const struct file_operations eeh_dev_break_fops = { .read = eeh_debugfs_dev_usage, }; -int eeh_pe_inject_mmio_error(struct pci_dev *pdev) -{ - return eeh_debugfs_break_device(pdev); -} - static ssize_t eeh_dev_can_recover(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) From 8f6a7c9467eaf39da4c14e5474e46190ab3fb529 Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Wed, 4 Sep 2024 20:34:57 +0800 Subject: [PATCH 070/111] nfs: fix memory leak in error path of nfs4_do_reclaim Commit c77e22834ae9 ("NFSv4: Fix a potential sleep while atomic in nfs4_do_reclaim()") separate out the freeing of the state owners from nfs4_purge_state_owners() and finish it outside the rcu lock. However, the error path is omitted. As a result, the state owners in "freeme" will not be released. Fix it by adding freeing in the error path. Fixes: c77e22834ae9 ("NFSv4: Fix a potential sleep while atomic in nfs4_do_reclaim()") Signed-off-by: Li Lingfeng Cc: stable@vger.kernel.org # v5.3+ Signed-off-by: Anna Schumaker --- fs/nfs/nfs4state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 877f682b45f2..30aba1dedaba 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -1957,6 +1957,7 @@ restart: set_bit(ops->owner_flag_bit, &sp->so_flags); nfs4_put_state_owner(sp); status = nfs4_recovery_handle_error(clp, status); + nfs4_free_state_owners(&freeme); return (status != 0) ? status : -EAGAIN; } From d98f72272500f505cd7e152ffa456e64ee3855f0 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 6 Sep 2024 12:32:03 +1000 Subject: [PATCH 071/111] nfs: simplify and guarantee owner uniqueness. I have evidence of an Linux NFS client getting NFS4ERR_BAD_SEQID to a v4.0 LOCK request to a Linux server (which had fixed the problem with RELEASE_LOCKOWNER bug fixed). The LOCK request presented a "new" lock owner so there are two seq ids in the request: that for the open file, and that for the new lock. Given the context I am confident that the new lock owner was reported to have the wrong seqid. As lock owner identifiers are reused, the server must still have a lock owner active which the client thinks is no longer active. I wasn't able to determine a root-cause but the simplest fix seems to be to ensure lock owners are always unique much as open owners are (thanks to a time stamp). The easiest way to ensure uniqueness is with a 64bit counter for each server. That will never cycle (if updated once a nanosecond the last 584 years. A single NFS server would not handle open/lock requests nearly that fast, and a Linux node is unlikely to have an uptime approaching that). This patch removes the 2 ida and instead uses a per-server atomic64_t to provide uniqueness. Note that the lock owner already encodes the id as 64 bits even though it is a 32bit value. So changing to a 64bit value does not change the encoding of the lock owner. The open owner encoding is now 4 bytes larger. Signed-off-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/client.c | 6 ++---- fs/nfs/nfs4_fs.h | 2 +- fs/nfs/nfs4state.c | 15 ++------------- fs/nfs/nfs4xdr.c | 6 +++--- include/linux/nfs_fs_sb.h | 3 +-- include/linux/nfs_xdr.h | 2 +- 6 files changed, 10 insertions(+), 24 deletions(-) diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 8286edd6062d..3fea7aa1366f 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -997,8 +997,8 @@ struct nfs_server *nfs_alloc_server(void) init_waitqueue_head(&server->write_congestion_wait); atomic_long_set(&server->writeback, 0); - ida_init(&server->openowner_id); - ida_init(&server->lockowner_id); + atomic64_set(&server->owner_ctr, 0); + pnfs_init_server(server); rpc_init_wait_queue(&server->uoc_rpcwaitq, "NFS UOC"); @@ -1037,8 +1037,6 @@ void nfs_free_server(struct nfs_server *server) } ida_free(&s_sysfs_ids, server->s_sysfs_id); - ida_destroy(&server->lockowner_id); - ida_destroy(&server->openowner_id); put_cred(server->cred); nfs_release_automount_timer(); call_rcu(&server->rcu, delayed_free); diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index c2045a2a9d0f..7d383d29a995 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -83,7 +83,7 @@ struct nfs4_minor_version_ops { #define NFS_SEQID_CONFIRMED 1 struct nfs_seqid_counter { ktime_t create_time; - int owner_id; + u64 owner_id; int flags; u32 counter; spinlock_t lock; /* Protects the list */ diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 30aba1dedaba..2ef656ca6371 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -501,11 +501,7 @@ nfs4_alloc_state_owner(struct nfs_server *server, sp = kzalloc(sizeof(*sp), gfp_flags); if (!sp) return NULL; - sp->so_seqid.owner_id = ida_alloc(&server->openowner_id, gfp_flags); - if (sp->so_seqid.owner_id < 0) { - kfree(sp); - return NULL; - } + sp->so_seqid.owner_id = atomic64_inc_return(&server->owner_ctr); sp->so_server = server; sp->so_cred = get_cred(cred); spin_lock_init(&sp->so_lock); @@ -536,7 +532,6 @@ static void nfs4_free_state_owner(struct nfs4_state_owner *sp) { nfs4_destroy_seqid_counter(&sp->so_seqid); put_cred(sp->so_cred); - ida_free(&sp->so_server->openowner_id, sp->so_seqid.owner_id); kfree(sp); } @@ -879,19 +874,13 @@ static struct nfs4_lock_state *nfs4_alloc_lock_state(struct nfs4_state *state, f refcount_set(&lsp->ls_count, 1); lsp->ls_state = state; lsp->ls_owner = owner; - lsp->ls_seqid.owner_id = ida_alloc(&server->lockowner_id, GFP_KERNEL_ACCOUNT); - if (lsp->ls_seqid.owner_id < 0) - goto out_free; + lsp->ls_seqid.owner_id = atomic64_inc_return(&server->owner_ctr); INIT_LIST_HEAD(&lsp->ls_locks); return lsp; -out_free: - kfree(lsp); - return NULL; } void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) { - ida_free(&server->lockowner_id, lsp->ls_seqid.owner_id); nfs4_destroy_seqid_counter(&lsp->ls_seqid); kfree(lsp); } diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index 7704a4509676..88bcbcba1381 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -1424,12 +1424,12 @@ static inline void encode_openhdr(struct xdr_stream *xdr, const struct nfs_opena */ encode_nfs4_seqid(xdr, arg->seqid); encode_share_access(xdr, arg->share_access); - p = reserve_space(xdr, 36); + p = reserve_space(xdr, 40); p = xdr_encode_hyper(p, arg->clientid); - *p++ = cpu_to_be32(24); + *p++ = cpu_to_be32(28); p = xdr_encode_opaque_fixed(p, "open id:", 8); *p++ = cpu_to_be32(arg->server->s_dev); - *p++ = cpu_to_be32(arg->id.uniquifier); + p = xdr_encode_hyper(p, arg->id.uniquifier); xdr_encode_hyper(p, arg->id.create_time); } diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 1df86ab98c77..e1e47ebd83ef 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -234,8 +234,7 @@ struct nfs_server { /* the following fields are protected by nfs_client->cl_lock */ struct rb_root state_owners; #endif - struct ida openowner_id; - struct ida lockowner_id; + atomic64_t owner_ctr; struct list_head state_owners_lru; struct list_head layouts; struct list_head delegations; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 45623af3e7b8..96ba04ab24f3 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -446,7 +446,7 @@ struct nfs42_clone_res { struct stateowner_id { __u64 create_time; - __u32 uniquifier; + __u64 uniquifier; }; struct nfs4_open_delegation { From bb8e4ce500cb09950c2b51a3be0e94e337a215aa Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Wed, 14 Aug 2024 12:01:28 +0200 Subject: [PATCH 072/111] nfs: Annotate struct nfs_cache_array with __counted_by() Add the __counted_by compiler attribute to the flexible array member array to improve access bounds-checking via CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE. Increment size before adding a new struct to the array. Signed-off-by: Thorsten Blum Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 4cb97ef41350..492cffd9d3d8 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -151,7 +151,7 @@ struct nfs_cache_array { unsigned char folio_full : 1, folio_is_eof : 1, cookies_are_ordered : 1; - struct nfs_cache_array_entry array[]; + struct nfs_cache_array_entry array[] __counted_by(size); }; struct nfs_readdir_descriptor { @@ -328,7 +328,8 @@ static int nfs_readdir_folio_array_append(struct folio *folio, goto out; } - cache_entry = &array->array[array->size]; + array->size++; + cache_entry = &array->array[array->size - 1]; cache_entry->cookie = array->last_cookie; cache_entry->ino = entry->ino; cache_entry->d_type = entry->d_type; @@ -337,7 +338,6 @@ static int nfs_readdir_folio_array_append(struct folio *folio, array->last_cookie = entry->cookie; if (array->last_cookie <= cache_entry->cookie) array->cookies_are_ordered = 0; - array->size++; if (entry->eof != 0) nfs_readdir_array_set_eof(array); out: From e343678ee990912a132e94236fe028efe78696b0 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sun, 1 Sep 2024 18:19:13 +0200 Subject: [PATCH 073/111] nfs: Remove unnecessary NULL check before kfree() Since kfree() already checks if its argument is NULL, an additional check before calling kfree() is unnecessary and can be removed. Remove it and thus also the following Coccinelle/coccicheck warning reported by ifnullfree.cocci: WARNING: NULL check before some freeing functions is not needed Reviewed-by: Benjamin Coddington Signed-off-by: Thorsten Blum Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/read.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/nfs/read.c b/fs/nfs/read.c index a6103333b666..81bd1b9aba17 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -48,8 +48,7 @@ static struct nfs_pgio_header *nfs_readhdr_alloc(void) static void nfs_readhdr_free(struct nfs_pgio_header *rhdr) { - if (rhdr->res.scratch != NULL) - kfree(rhdr->res.scratch); + kfree(rhdr->res.scratch); kmem_cache_free(nfs_rdata_cachep, rhdr); } From 9090a7f78623b5312562861d54a4476d905c7c4c Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 16 Aug 2024 10:07:40 +0800 Subject: [PATCH 074/111] SUNRPC: Fix -Wformat-truncation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Increase size of the servername array to avoid truncated output warning. net/sunrpc/clnt.c:582:75: error：‘%s’ directive output may be truncated writing up to 107 bytes into a region of size 48 [-Werror=format-truncation=] 582 | snprintf(servername, sizeof(servername), "%s", | ^~ net/sunrpc/clnt.c:582:33: note:‘snprintf’ output between 1 and 108 bytes into a destination of size 48 582 | snprintf(servername, sizeof(servername), "%s", | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 583 | sun->sun_path); Signed-off-by: Kunwu Chan Suggested-by: NeilBrown Signed-off-by: Anna Schumaker --- net/sunrpc/clnt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 09f29a95f2bc..c7cea069a5df 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -546,7 +546,7 @@ struct rpc_clnt *rpc_create(struct rpc_create_args *args) .connect_timeout = args->connect_timeout, .reconnect_timeout = args->reconnect_timeout, }; - char servername[48]; + char servername[RPC_MAXNETNAMELEN]; struct rpc_clnt *clnt; int i; From 0b108e83795c9c23101f584ef7e3ab4f1f120ef0 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 19 Aug 2024 08:58:59 -0700 Subject: [PATCH 075/111] SUNRPC: convert RPC_TASK_* constants to enum The RPC_TASK_* constants are defined as macros, which means that most kernel builds will not contain their definitions in the debuginfo. However, it's quite useful for debuggers to be able to view the task state constant and interpret it correctly. Conversion to an enum will ensure the constants are present in debuginfo and can be interpreted by debuggers without needing to hard-code them and track their changes. Signed-off-by: Stephen Brennan Signed-off-by: Anna Schumaker --- include/linux/sunrpc/sched.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index 0c77ba488bba..fec1e8a1570c 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -151,13 +151,15 @@ struct rpc_task_setup { #define RPC_WAS_SENT(t) ((t)->tk_flags & RPC_TASK_SENT) #define RPC_IS_MOVEABLE(t) ((t)->tk_flags & RPC_TASK_MOVEABLE) -#define RPC_TASK_RUNNING 0 -#define RPC_TASK_QUEUED 1 -#define RPC_TASK_ACTIVE 2 -#define RPC_TASK_NEED_XMIT 3 -#define RPC_TASK_NEED_RECV 4 -#define RPC_TASK_MSG_PIN_WAIT 5 -#define RPC_TASK_SIGNALLED 6 +enum { + RPC_TASK_RUNNING, + RPC_TASK_QUEUED, + RPC_TASK_ACTIVE, + RPC_TASK_NEED_XMIT, + RPC_TASK_NEED_RECV, + RPC_TASK_MSG_PIN_WAIT, + RPC_TASK_SIGNALLED, +}; #define rpc_test_and_set_running(t) \ test_and_set_bit(RPC_TASK_RUNNING, &(t)->tk_runstate) From 2e001972e8ebc318de3b5542887ac06ea309aff8 Mon Sep 17 00:00:00 2001 From: Siddh Raman Pant Date: Mon, 2 Sep 2024 15:02:48 +0530 Subject: [PATCH 076/111] SUNRPC: clnt.c: Remove misleading comment destroy_wait doesn't store all RPC clients. There was a list named "all_clients" above it, which got moved to struct sunrpc_net in 2012, but the comment was never removed. Fixes: 70abc49b4f4a ("SUNRPC: make SUNPRC clients list per network namespace context") Signed-off-by: Siddh Raman Pant Signed-off-by: Anna Schumaker --- net/sunrpc/clnt.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index c7cea069a5df..5e9f36505ab7 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -48,13 +48,8 @@ # define RPCDBG_FACILITY RPCDBG_CALL #endif -/* - * All RPC clients are linked into this list - */ - static DECLARE_WAIT_QUEUE_HEAD(destroy_wait); - static void call_start(struct rpc_task *task); static void call_reserve(struct rpc_task *task); static void call_reserveresult(struct rpc_task *task); From 64a3ab99675ef7b833f64cd3b84db66b6a351529 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Tue, 27 Aug 2024 17:52:18 +0800 Subject: [PATCH 077/111] net/sunrpc: make use of the helper macro LIST_HEAD() list_head can be initialized automatically with LIST_HEAD() instead of calling INIT_LIST_HEAD(). Here we can simplify the code. Signed-off-by: Hongbo Li Signed-off-by: Anna Schumaker --- net/sunrpc/cache.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 95ff74706104..4f31e73dc34d 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -731,11 +731,10 @@ static bool cache_defer_req(struct cache_req *req, struct cache_head *item) static void cache_revisit_request(struct cache_head *item) { struct cache_deferred_req *dreq; - struct list_head pending; struct hlist_node *tmp; int hash = DFR_HASH(item); + LIST_HEAD(pending); - INIT_LIST_HEAD(&pending); spin_lock(&cache_defer_lock); hlist_for_each_entry_safe(dreq, tmp, &cache_defer_hash[hash], hash) @@ -756,10 +755,8 @@ static void cache_revisit_request(struct cache_head *item) void cache_clean_deferred(void *owner) { struct cache_deferred_req *dreq, *tmp; - struct list_head pending; + LIST_HEAD(pending); - - INIT_LIST_HEAD(&pending); spin_lock(&cache_defer_lock); list_for_each_entry_safe(dreq, tmp, &cache_defer_list, recent) { @@ -1085,9 +1082,8 @@ static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch) { struct cache_queue *cq, *tmp; struct cache_request *cr; - struct list_head dequeued; + LIST_HEAD(dequeued); - INIT_LIST_HEAD(&dequeued); spin_lock(&queue_lock); list_for_each_entry_safe(cq, tmp, &detail->queue, list) if (!cq->reader) { From 40c80881ebef97a2ad4bba1e34edb80fd8585e8c Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Mon, 26 Aug 2024 11:21:57 +0800 Subject: [PATCH 078/111] nfs: Remove obsoleted declaration for nfs_read_prepare The nfs_read_prepare() have been removed since commit a4cdda59111f ("NFS: Create a common pgio_rpc_prepare function"), and now it is useless, so remove it. Signed-off-by: Gaosheng Cui Signed-off-by: Anna Schumaker --- fs/nfs/internal.h | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 5902a9beca1f..b3dc7c84eef9 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -505,7 +505,6 @@ extern int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, struct nfs_open_context *ctx, struct folio *folio); extern void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio); -extern void nfs_read_prepare(struct rpc_task *task, void *calldata); extern void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio); /* super.c */ From 03e02b94171b1985dd0aa184296fe94425b855a3 Mon Sep 17 00:00:00 2001 From: Zhaoyang Huang Date: Fri, 30 Aug 2024 11:27:47 +0800 Subject: [PATCH 079/111] fs: nfs: fix missing refcnt by replacing folio_set_private by folio_attach_private This patch is inspired by a code review of fs codes which aims at folio's extra refcnt that could introduce unwanted behavious when judging refcnt, such as[1].That is, the folio passed to mapping_evict_folio carries the refcnts from find_lock_entries, page_cache, corresponding to PTEs and folio's private if has. However, current code doesn't take the refcnt for folio's private which could have mapping_evict_folio miss the one to only PTE and lead to call filemap_release_folio wrongly. [1] long mapping_evict_folio(struct address_space *mapping, struct folio *folio) { ... //current code will misjudge here if there is one pte on the folio which is be deemed as the one as folio's private if (folio_ref_count(folio) > folio_nr_pages(folio) + folio_has_private(folio) + 1) return 0; if (!filemap_release_folio(folio, 0)) return 0; return remove_mapping(mapping, folio); } Signed-off-by: Zhaoyang Huang Signed-off-by: Anna Schumaker --- fs/nfs/write.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/nfs/write.c b/fs/nfs/write.c index d074d0ceb4f0..80c6ded5f74c 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -772,8 +772,7 @@ static void nfs_inode_add_request(struct nfs_page *req) nfs_lock_request(req); spin_lock(&mapping->i_private_lock); set_bit(PG_MAPPED, &req->wb_flags); - folio_set_private(folio); - folio->private = req; + folio_attach_private(folio, req); spin_unlock(&mapping->i_private_lock); atomic_long_inc(&nfsi->nrequests); /* this a head request for a page group - mark it as having an @@ -797,8 +796,7 @@ static void nfs_inode_remove_request(struct nfs_page *req) spin_lock(&mapping->i_private_lock); if (likely(folio)) { - folio->private = NULL; - folio_clear_private(folio); + folio_detach_private(folio); clear_bit(PG_MAPPED, &req->wb_head->wb_flags); } spin_unlock(&mapping->i_private_lock); From af94dca79b1296a6db7b8b47cd43be8e94fce8bb Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 9 Sep 2024 13:47:07 -0400 Subject: [PATCH 080/111] NFSv4: Fail mounts if the lease setup times out If the server is down when the client is trying to mount, so that the calls to exchange_id or create_session fail, then we should allow the mount system call to fail rather than hang and block other mount/umount calls. Reported-by: Oleksandr Tymoshenko Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/nfs4state.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 2ef656ca6371..581864a15888 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2013,6 +2013,12 @@ static int nfs4_handle_reclaim_lease_error(struct nfs_client *clp, int status) nfs_mark_client_ready(clp, -EPERM); clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); return -EPERM; + case -ETIMEDOUT: + if (clp->cl_cons_state == NFS_CS_SESSION_INITING) { + nfs_mark_client_ready(clp, -EIO); + return -EIO; + } + fallthrough; case -EACCES: case -NFS4ERR_DELAY: case -EAGAIN: From 615e693b14ba22e1332c3bd5a4e038284bbc3e07 Mon Sep 17 00:00:00 2001 From: Roi Azarzar Date: Sun, 15 Sep 2024 10:27:35 +0000 Subject: [PATCH 081/111] NFSv4.2: Fix detection of "Proxying of Times" server support According to draft-ietf-nfsv4-delstid-07: If a server informs the client via the fattr4_open_arguments attribute that it supports OPEN_ARGS_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS and it returns a valid delegation stateid for an OPEN operation which sets the OPEN4_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS flag, then it MUST query the client via a CB_GETATTR for the fattr4_time_deleg_access (see Section 5.2) attribute and fattr4_time_deleg_modify attribute (see Section 5.2). Thus, we should look that the server supports proxying of times via OPEN4_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS. We want to be extra pedantic and continue to check that FATTR4_TIME_DELEG_ACCESS and FATTR4_TIME_DELEG_MODIFY are set. The server needs to expose both for the client to correctly detect "Proxying of Times" support. Signed-off-by: Roi Azarzar Reviewed-by: Jeff Layton Fixes: dcb3c20f7419 ("NFSv4: Add a capability for delegated attributes") Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index b8ffbe52ba15..cd2fbde2e6d7 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -3904,6 +3904,18 @@ static void nfs4_close_context(struct nfs_open_context *ctx, int is_sync) #define FATTR4_WORD2_NFS41_MASK (2*FATTR4_WORD2_SUPPATTR_EXCLCREAT - 1UL) #define FATTR4_WORD2_NFS42_MASK (2*FATTR4_WORD2_OPEN_ARGUMENTS - 1UL) +#define FATTR4_WORD2_NFS42_TIME_DELEG_MASK \ + (FATTR4_WORD2_TIME_DELEG_MODIFY|FATTR4_WORD2_TIME_DELEG_ACCESS) +static bool nfs4_server_delegtime_capable(struct nfs4_server_caps_res *res) +{ + u32 share_access_want = res->open_caps.oa_share_access_want[0]; + u32 attr_bitmask = res->attr_bitmask[2]; + + return (share_access_want & NFS4_SHARE_WANT_DELEG_TIMESTAMPS) && + ((attr_bitmask & FATTR4_WORD2_NFS42_TIME_DELEG_MASK) == + FATTR4_WORD2_NFS42_TIME_DELEG_MASK); +} + static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) { u32 minorversion = server->nfs_client->cl_minorversion; @@ -3982,8 +3994,6 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f #endif if (res.attr_bitmask[0] & FATTR4_WORD0_FS_LOCATIONS) server->caps |= NFS_CAP_FS_LOCATIONS; - if (res.attr_bitmask[2] & FATTR4_WORD2_TIME_DELEG_MODIFY) - server->caps |= NFS_CAP_DELEGTIME; if (!(res.attr_bitmask[0] & FATTR4_WORD0_FILEID)) server->fattr_valid &= ~NFS_ATTR_FATTR_FILEID; if (!(res.attr_bitmask[1] & FATTR4_WORD1_MODE)) @@ -4011,6 +4021,8 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f if (res.open_caps.oa_share_access_want[0] & NFS4_SHARE_WANT_OPEN_XOR_DELEGATION) server->caps |= NFS_CAP_OPEN_XOR; + if (nfs4_server_delegtime_capable(&res)) + server->caps |= NFS_CAP_DELEGTIME; memcpy(server->cache_consistency_bitmask, res.attr_bitmask, sizeof(server->cache_consistency_bitmask)); server->cache_consistency_bitmask[0] &= FATTR4_WORD0_CHANGE|FATTR4_WORD0_SIZE; From 6d26c5e4d83cd39006bfd2ecbc97fbfac34aaefa Mon Sep 17 00:00:00 2001 From: Li Lingfeng Date: Sat, 24 Aug 2024 09:43:35 +0800 Subject: [PATCH 082/111] nfs: fix the comment of nfs_get_root The comment for nfs_get_root() needs to be updated as it would also be used by NFS4 as follows: @x[ nfs_get_root+1 nfs_get_tree_common+1819 nfs_get_tree+2594 vfs_get_tree+73 fc_mount+23 do_nfs4_mount+498 nfs4_try_get_tree+134 nfs_get_tree+2562 vfs_get_tree+73 path_mount+2776 do_mount+226 __se_sys_mount+343 __x64_sys_mount+106 do_syscall_64+69 entry_SYSCALL_64_after_hwframe+97 , mount.nfs4]: 1 Signed-off-by: Li Lingfeng Acked-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/getroot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/nfs/getroot.c b/fs/nfs/getroot.c index 11ff2b2e060f..f13d25d95b85 100644 --- a/fs/nfs/getroot.c +++ b/fs/nfs/getroot.c @@ -62,7 +62,7 @@ static int nfs_superblock_set_dummy_root(struct super_block *sb, struct inode *i } /* - * get an NFS2/NFS3 root dentry from the root filehandle + * get a root dentry from the root filehandle */ int nfs_get_root(struct super_block *s, struct fs_context *fc) { From dfb07e990a0d019d7ae9b78dd4260620ce32e79a Mon Sep 17 00:00:00 2001 From: Dan Aloni Date: Wed, 24 Jul 2024 14:07:12 +0300 Subject: [PATCH 083/111] nfs: add 'noalignwrite' option for lock-less 'lost writes' prevention There are some applications that write to predefined non-overlapping file offsets from multiple clients and therefore don't need to rely on file locking. However, if these applications want non-aligned offsets and sizes they need to either use locks or risk data corruption, as the NFS client defaults to extending writes to whole pages. This commit adds a new mount option `noalignwrite`, which allows to turn that off and avoid the need of locking, as long as these applications don't overlap on offsets. Signed-off-by: Dan Aloni Reviewed-by: Jeff Layton Reviewed-by: Sagi Grimberg Reviewed-by: Christoph Hellwig Signed-off-by: Anna Schumaker --- fs/nfs/fs_context.c | 8 ++++++++ fs/nfs/super.c | 3 +++ fs/nfs/write.c | 3 +++ include/linux/nfs_fs_sb.h | 1 + 4 files changed, 15 insertions(+) diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 6c9f3f6645dd..7e000d782e28 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -49,6 +49,7 @@ enum nfs_param { Opt_bsize, Opt_clientaddr, Opt_cto, + Opt_alignwrite, Opt_fg, Opt_fscache, Opt_fscache_flag, @@ -149,6 +150,7 @@ static const struct fs_parameter_spec nfs_fs_parameters[] = { fsparam_u32 ("bsize", Opt_bsize), fsparam_string("clientaddr", Opt_clientaddr), fsparam_flag_no("cto", Opt_cto), + fsparam_flag_no("alignwrite", Opt_alignwrite), fsparam_flag ("fg", Opt_fg), fsparam_flag_no("fsc", Opt_fscache_flag), fsparam_string("fsc", Opt_fscache), @@ -592,6 +594,12 @@ static int nfs_fs_context_parse_param(struct fs_context *fc, else ctx->flags |= NFS_MOUNT_TRUNK_DISCOVERY; break; + case Opt_alignwrite: + if (result.negated) + ctx->flags |= NFS_MOUNT_NO_ALIGNWRITE; + else + ctx->flags &= ~NFS_MOUNT_NO_ALIGNWRITE; + break; case Opt_ac: if (result.negated) ctx->flags |= NFS_MOUNT_NOAC; diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 97b386032b71..9723b6c53397 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -551,6 +551,9 @@ static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss, else seq_puts(m, ",local_lock=posix"); + if (nfss->flags & NFS_MOUNT_NO_ALIGNWRITE) + seq_puts(m, ",noalignwrite"); + if (nfss->flags & NFS_MOUNT_WRITE_EAGER) { if (nfss->flags & NFS_MOUNT_WRITE_WAIT) seq_puts(m, ",write=wait"); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 80c6ded5f74c..61b580bcb356 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1295,7 +1295,10 @@ static int nfs_can_extend_write(struct file *file, struct folio *folio, struct file_lock_context *flctx = locks_inode_context(inode); struct file_lock *fl; int ret; + unsigned int mntflags = NFS_SERVER(inode)->flags; + if (mntflags & NFS_MOUNT_NO_ALIGNWRITE) + return 0; if (file->f_flags & O_DSYNC) return 0; if (!nfs_folio_write_uptodate(folio, pagelen)) diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index e1e47ebd83ef..c49bfdded5c1 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -158,6 +158,7 @@ struct nfs_server { #define NFS_MOUNT_WRITE_WAIT 0x02000000 #define NFS_MOUNT_TRUNK_DISCOVERY 0x04000000 #define NFS_MOUNT_SHUTDOWN 0x08000000 +#define NFS_MOUNT_NO_ALIGNWRITE 0x10000000 unsigned int fattr_valid; /* Valid attributes */ unsigned int caps; /* server capabilities */ From 4806ded4c14c5e8fdc6ce885d83221a78c06a428 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:35 -0400 Subject: [PATCH 084/111] nfs_common: factor out nfs_errtbl and nfs_stat_to_errno Common nfs_stat_to_errno() is used by both fs/nfs/nfs2xdr.c and fs/nfs/nfs3xdr.c Will also be used by fs/nfsd/localio.c Signed-off-by: Mike Snitzer Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/Kconfig | 1 + fs/nfs/nfs2xdr.c | 70 +----------------------- fs/nfs/nfs3xdr.c | 108 +++++++------------------------------ fs/nfs/nfs4xdr.c | 4 +- fs/nfs_common/Makefile | 2 + fs/nfs_common/common.c | 67 +++++++++++++++++++++++ fs/nfsd/Kconfig | 1 + include/linux/nfs_common.h | 16 ++++++ 8 files changed, 109 insertions(+), 160 deletions(-) create mode 100644 fs/nfs_common/common.c create mode 100644 include/linux/nfs_common.h diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index 57249f040dfc..0eb20012792f 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -4,6 +4,7 @@ config NFS_FS depends on INET && FILE_LOCKING && MULTIUSER select LOCKD select SUNRPC + select NFS_COMMON select NFS_ACL_SUPPORT if NFS_V3_ACL help Choose Y here if you want to access files residing on other diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c index c19093814296..6e75c6c2d234 100644 --- a/fs/nfs/nfs2xdr.c +++ b/fs/nfs/nfs2xdr.c @@ -22,14 +22,12 @@ #include #include #include +#include #include "nfstrace.h" #include "internal.h" #define NFSDBG_FACILITY NFSDBG_XDR -/* Mapping from NFS error code to "errno" error code. */ -#define errno_NFSERR_IO EIO - /* * Declare the space requirements for NFS arguments and replies as * number of 32bit-words @@ -64,8 +62,6 @@ #define NFS_readdirres_sz (1+NFS_pagepad_sz) #define NFS_statfsres_sz (1+NFS_info_sz) -static int nfs_stat_to_errno(enum nfs_stat); - /* * Encode/decode NFSv2 basic data types * @@ -1054,70 +1050,6 @@ out_default: return nfs_stat_to_errno(status); } - -/* - * We need to translate between nfs status return values and - * the local errno values which may not be the same. - */ -static const struct { - int stat; - int errno; -} nfs_errtbl[] = { - { NFS_OK, 0 }, - { NFSERR_PERM, -EPERM }, - { NFSERR_NOENT, -ENOENT }, - { NFSERR_IO, -errno_NFSERR_IO}, - { NFSERR_NXIO, -ENXIO }, -/* { NFSERR_EAGAIN, -EAGAIN }, */ - { NFSERR_ACCES, -EACCES }, - { NFSERR_EXIST, -EEXIST }, - { NFSERR_XDEV, -EXDEV }, - { NFSERR_NODEV, -ENODEV }, - { NFSERR_NOTDIR, -ENOTDIR }, - { NFSERR_ISDIR, -EISDIR }, - { NFSERR_INVAL, -EINVAL }, - { NFSERR_FBIG, -EFBIG }, - { NFSERR_NOSPC, -ENOSPC }, - { NFSERR_ROFS, -EROFS }, - { NFSERR_MLINK, -EMLINK }, - { NFSERR_NAMETOOLONG, -ENAMETOOLONG }, - { NFSERR_NOTEMPTY, -ENOTEMPTY }, - { NFSERR_DQUOT, -EDQUOT }, - { NFSERR_STALE, -ESTALE }, - { NFSERR_REMOTE, -EREMOTE }, -#ifdef EWFLUSH - { NFSERR_WFLUSH, -EWFLUSH }, -#endif - { NFSERR_BADHANDLE, -EBADHANDLE }, - { NFSERR_NOT_SYNC, -ENOTSYNC }, - { NFSERR_BAD_COOKIE, -EBADCOOKIE }, - { NFSERR_NOTSUPP, -ENOTSUPP }, - { NFSERR_TOOSMALL, -ETOOSMALL }, - { NFSERR_SERVERFAULT, -EREMOTEIO }, - { NFSERR_BADTYPE, -EBADTYPE }, - { NFSERR_JUKEBOX, -EJUKEBOX }, - { -1, -EIO } -}; - -/** - * nfs_stat_to_errno - convert an NFS status code to a local errno - * @status: NFS status code to convert - * - * Returns a local errno value, or -EIO if the NFS status code is - * not recognized. This function is used jointly by NFSv2 and NFSv3. - */ -static int nfs_stat_to_errno(enum nfs_stat status) -{ - int i; - - for (i = 0; nfs_errtbl[i].stat != -1; i++) { - if (nfs_errtbl[i].stat == (int)status) - return nfs_errtbl[i].errno; - } - dprintk("NFS: Unrecognized nfs status value: %u\n", status); - return nfs_errtbl[i].errno; -} - #define PROC(proc, argtype, restype, timer) \ [NFSPROC_##proc] = { \ .p_proc = NFSPROC_##proc, \ diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c index 60f032be805a..4ae01c10b7e2 100644 --- a/fs/nfs/nfs3xdr.c +++ b/fs/nfs/nfs3xdr.c @@ -21,14 +21,13 @@ #include #include #include +#include + #include "nfstrace.h" #include "internal.h" #define NFSDBG_FACILITY NFSDBG_XDR -/* Mapping from NFS error code to "errno" error code. */ -#define errno_NFSERR_IO EIO - /* * Declare the space requirements for NFS arguments and replies as * number of 32bit-words @@ -91,8 +90,6 @@ NFS3_pagepad_sz) #define ACL3_setaclres_sz (1+NFS3_post_op_attr_sz) -static int nfs3_stat_to_errno(enum nfs_stat); - /* * Map file type to S_IFMT bits */ @@ -1406,7 +1403,7 @@ static int nfs3_xdr_dec_getattr3res(struct rpc_rqst *req, out: return error; out_default: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1445,7 +1442,7 @@ static int nfs3_xdr_dec_setattr3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1495,7 +1492,7 @@ out_default: error = decode_post_op_attr(xdr, result->dir_attr, userns); if (unlikely(error)) goto out; - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1537,7 +1534,7 @@ static int nfs3_xdr_dec_access3res(struct rpc_rqst *req, out: return error; out_default: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1578,7 +1575,7 @@ static int nfs3_xdr_dec_readlink3res(struct rpc_rqst *req, out: return error; out_default: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1658,7 +1655,7 @@ static int nfs3_xdr_dec_read3res(struct rpc_rqst *req, struct xdr_stream *xdr, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1728,7 +1725,7 @@ static int nfs3_xdr_dec_write3res(struct rpc_rqst *req, struct xdr_stream *xdr, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1795,7 +1792,7 @@ out_default: error = decode_wcc_data(xdr, result->dir_attr, userns); if (unlikely(error)) goto out; - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1835,7 +1832,7 @@ static int nfs3_xdr_dec_remove3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1881,7 +1878,7 @@ static int nfs3_xdr_dec_rename3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -1926,7 +1923,7 @@ static int nfs3_xdr_dec_link3res(struct rpc_rqst *req, struct xdr_stream *xdr, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /** @@ -2101,7 +2098,7 @@ out_default: error = decode_post_op_attr(xdr, result->dir_attr, rpc_rqst_userns(req)); if (unlikely(error)) goto out; - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -2167,7 +2164,7 @@ static int nfs3_xdr_dec_fsstat3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -2243,7 +2240,7 @@ static int nfs3_xdr_dec_fsinfo3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -2304,7 +2301,7 @@ static int nfs3_xdr_dec_pathconf3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } /* @@ -2350,7 +2347,7 @@ static int nfs3_xdr_dec_commit3res(struct rpc_rqst *req, out: return error; out_status: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } #ifdef CONFIG_NFS_V3_ACL @@ -2416,7 +2413,7 @@ static int nfs3_xdr_dec_getacl3res(struct rpc_rqst *req, out: return error; out_default: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } static int nfs3_xdr_dec_setacl3res(struct rpc_rqst *req, @@ -2435,76 +2432,11 @@ static int nfs3_xdr_dec_setacl3res(struct rpc_rqst *req, out: return error; out_default: - return nfs3_stat_to_errno(status); + return nfs_stat_to_errno(status); } #endif /* CONFIG_NFS_V3_ACL */ - -/* - * We need to translate between nfs status return values and - * the local errno values which may not be the same. - */ -static const struct { - int stat; - int errno; -} nfs_errtbl[] = { - { NFS_OK, 0 }, - { NFSERR_PERM, -EPERM }, - { NFSERR_NOENT, -ENOENT }, - { NFSERR_IO, -errno_NFSERR_IO}, - { NFSERR_NXIO, -ENXIO }, -/* { NFSERR_EAGAIN, -EAGAIN }, */ - { NFSERR_ACCES, -EACCES }, - { NFSERR_EXIST, -EEXIST }, - { NFSERR_XDEV, -EXDEV }, - { NFSERR_NODEV, -ENODEV }, - { NFSERR_NOTDIR, -ENOTDIR }, - { NFSERR_ISDIR, -EISDIR }, - { NFSERR_INVAL, -EINVAL }, - { NFSERR_FBIG, -EFBIG }, - { NFSERR_NOSPC, -ENOSPC }, - { NFSERR_ROFS, -EROFS }, - { NFSERR_MLINK, -EMLINK }, - { NFSERR_NAMETOOLONG, -ENAMETOOLONG }, - { NFSERR_NOTEMPTY, -ENOTEMPTY }, - { NFSERR_DQUOT, -EDQUOT }, - { NFSERR_STALE, -ESTALE }, - { NFSERR_REMOTE, -EREMOTE }, -#ifdef EWFLUSH - { NFSERR_WFLUSH, -EWFLUSH }, -#endif - { NFSERR_BADHANDLE, -EBADHANDLE }, - { NFSERR_NOT_SYNC, -ENOTSYNC }, - { NFSERR_BAD_COOKIE, -EBADCOOKIE }, - { NFSERR_NOTSUPP, -ENOTSUPP }, - { NFSERR_TOOSMALL, -ETOOSMALL }, - { NFSERR_SERVERFAULT, -EREMOTEIO }, - { NFSERR_BADTYPE, -EBADTYPE }, - { NFSERR_JUKEBOX, -EJUKEBOX }, - { -1, -EIO } -}; - -/** - * nfs3_stat_to_errno - convert an NFS status code to a local errno - * @status: NFS status code to convert - * - * Returns a local errno value, or -EIO if the NFS status code is - * not recognized. This function is used jointly by NFSv2 and NFSv3. - */ -static int nfs3_stat_to_errno(enum nfs_stat status) -{ - int i; - - for (i = 0; nfs_errtbl[i].stat != -1; i++) { - if (nfs_errtbl[i].stat == (int)status) - return nfs_errtbl[i].errno; - } - dprintk("NFS: Unrecognized nfs status value: %u\n", status); - return nfs_errtbl[i].errno; -} - - #define PROC(proc, argtype, restype, timer) \ [NFS3PROC_##proc] = { \ .p_proc = NFS3PROC_##proc, \ diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index 88bcbcba1381..b61ffe6b9851 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "nfs4_fs.h" #include "nfs4trace.h" @@ -63,9 +64,6 @@ #define NFSDBG_FACILITY NFSDBG_XDR -/* Mapping from NFS error code to "errno" error code. */ -#define errno_NFSERR_IO EIO - struct compound_hdr; static int nfs4_stat_to_errno(int); static void encode_layoutget(struct xdr_stream *xdr, diff --git a/fs/nfs_common/Makefile b/fs/nfs_common/Makefile index 119c75ab9fd0..e58b01bb8dda 100644 --- a/fs/nfs_common/Makefile +++ b/fs/nfs_common/Makefile @@ -8,3 +8,5 @@ nfs_acl-objs := nfsacl.o obj-$(CONFIG_GRACE_PERIOD) += grace.o obj-$(CONFIG_NFS_V4_2_SSC_HELPER) += nfs_ssc.o + +obj-$(CONFIG_NFS_COMMON) += common.o diff --git a/fs/nfs_common/common.c b/fs/nfs_common/common.c new file mode 100644 index 000000000000..a4ee95da2174 --- /dev/null +++ b/fs/nfs_common/common.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +/* + * We need to translate between nfs status return values and + * the local errno values which may not be the same. + */ +static const struct { + int stat; + int errno; +} nfs_errtbl[] = { + { NFS_OK, 0 }, + { NFSERR_PERM, -EPERM }, + { NFSERR_NOENT, -ENOENT }, + { NFSERR_IO, -errno_NFSERR_IO}, + { NFSERR_NXIO, -ENXIO }, +/* { NFSERR_EAGAIN, -EAGAIN }, */ + { NFSERR_ACCES, -EACCES }, + { NFSERR_EXIST, -EEXIST }, + { NFSERR_XDEV, -EXDEV }, + { NFSERR_NODEV, -ENODEV }, + { NFSERR_NOTDIR, -ENOTDIR }, + { NFSERR_ISDIR, -EISDIR }, + { NFSERR_INVAL, -EINVAL }, + { NFSERR_FBIG, -EFBIG }, + { NFSERR_NOSPC, -ENOSPC }, + { NFSERR_ROFS, -EROFS }, + { NFSERR_MLINK, -EMLINK }, + { NFSERR_NAMETOOLONG, -ENAMETOOLONG }, + { NFSERR_NOTEMPTY, -ENOTEMPTY }, + { NFSERR_DQUOT, -EDQUOT }, + { NFSERR_STALE, -ESTALE }, + { NFSERR_REMOTE, -EREMOTE }, +#ifdef EWFLUSH + { NFSERR_WFLUSH, -EWFLUSH }, +#endif + { NFSERR_BADHANDLE, -EBADHANDLE }, + { NFSERR_NOT_SYNC, -ENOTSYNC }, + { NFSERR_BAD_COOKIE, -EBADCOOKIE }, + { NFSERR_NOTSUPP, -ENOTSUPP }, + { NFSERR_TOOSMALL, -ETOOSMALL }, + { NFSERR_SERVERFAULT, -EREMOTEIO }, + { NFSERR_BADTYPE, -EBADTYPE }, + { NFSERR_JUKEBOX, -EJUKEBOX }, + { -1, -EIO } +}; + +/** + * nfs_stat_to_errno - convert an NFS status code to a local errno + * @status: NFS status code to convert + * + * Returns a local errno value, or -EIO if the NFS status code is + * not recognized. This function is used jointly by NFSv2 and NFSv3. + */ +int nfs_stat_to_errno(enum nfs_stat status) +{ + int i; + + for (i = 0; nfs_errtbl[i].stat != -1; i++) { + if (nfs_errtbl[i].stat == (int)status) + return nfs_errtbl[i].errno; + } + return nfs_errtbl[i].errno; +} +EXPORT_SYMBOL_GPL(nfs_stat_to_errno); diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig index ec2ab6429e00..c0bd1509ccd4 100644 --- a/fs/nfsd/Kconfig +++ b/fs/nfsd/Kconfig @@ -7,6 +7,7 @@ config NFSD select LOCKD select SUNRPC select EXPORTFS + select NFS_COMMON select NFS_ACL_SUPPORT if NFSD_V2_ACL select NFS_ACL_SUPPORT if NFSD_V3_ACL depends on MULTIUSER diff --git a/include/linux/nfs_common.h b/include/linux/nfs_common.h new file mode 100644 index 000000000000..3395c4a4d372 --- /dev/null +++ b/include/linux/nfs_common.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This file contains constants and methods used by both NFS client and server. + */ +#ifndef _LINUX_NFS_COMMON_H +#define _LINUX_NFS_COMMON_H + +#include +#include + +/* Mapping from NFS error code to "errno" error code. */ +#define errno_NFSERR_IO EIO + +int nfs_stat_to_errno(enum nfs_stat status); + +#endif /* _LINUX_NFS_COMMON_H */ From 1fcb16674e37e434efe68ec3e142229f35b6b9e1 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:36 -0400 Subject: [PATCH 085/111] nfs_common: factor out nfs4_errtbl and nfs4_stat_to_errno Common nfs4_stat_to_errno() is used by fs/nfs/nfs4xdr.c and will be used by fs/nfs/localio.c Signed-off-by: Mike Snitzer Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfs/nfs4xdr.c | 67 -------------------------------------- fs/nfs_common/common.c | 67 ++++++++++++++++++++++++++++++++++++++ include/linux/nfs_common.h | 1 + 3 files changed, 68 insertions(+), 67 deletions(-) diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index b61ffe6b9851..18d268029b1c 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -65,7 +65,6 @@ #define NFSDBG_FACILITY NFSDBG_XDR struct compound_hdr; -static int nfs4_stat_to_errno(int); static void encode_layoutget(struct xdr_stream *xdr, const struct nfs4_layoutget_args *args, struct compound_hdr *hdr); @@ -7619,72 +7618,6 @@ int nfs4_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry, return 0; } -/* - * We need to translate between nfs status return values and - * the local errno values which may not be the same. - */ -static struct { - int stat; - int errno; -} nfs_errtbl[] = { - { NFS4_OK, 0 }, - { NFS4ERR_PERM, -EPERM }, - { NFS4ERR_NOENT, -ENOENT }, - { NFS4ERR_IO, -errno_NFSERR_IO}, - { NFS4ERR_NXIO, -ENXIO }, - { NFS4ERR_ACCESS, -EACCES }, - { NFS4ERR_EXIST, -EEXIST }, - { NFS4ERR_XDEV, -EXDEV }, - { NFS4ERR_NOTDIR, -ENOTDIR }, - { NFS4ERR_ISDIR, -EISDIR }, - { NFS4ERR_INVAL, -EINVAL }, - { NFS4ERR_FBIG, -EFBIG }, - { NFS4ERR_NOSPC, -ENOSPC }, - { NFS4ERR_ROFS, -EROFS }, - { NFS4ERR_MLINK, -EMLINK }, - { NFS4ERR_NAMETOOLONG, -ENAMETOOLONG }, - { NFS4ERR_NOTEMPTY, -ENOTEMPTY }, - { NFS4ERR_DQUOT, -EDQUOT }, - { NFS4ERR_STALE, -ESTALE }, - { NFS4ERR_BADHANDLE, -EBADHANDLE }, - { NFS4ERR_BAD_COOKIE, -EBADCOOKIE }, - { NFS4ERR_NOTSUPP, -ENOTSUPP }, - { NFS4ERR_TOOSMALL, -ETOOSMALL }, - { NFS4ERR_SERVERFAULT, -EREMOTEIO }, - { NFS4ERR_BADTYPE, -EBADTYPE }, - { NFS4ERR_LOCKED, -EAGAIN }, - { NFS4ERR_SYMLINK, -ELOOP }, - { NFS4ERR_OP_ILLEGAL, -EOPNOTSUPP }, - { NFS4ERR_DEADLOCK, -EDEADLK }, - { NFS4ERR_NOXATTR, -ENODATA }, - { NFS4ERR_XATTR2BIG, -E2BIG }, - { -1, -EIO } -}; - -/* - * Convert an NFS error code to a local one. - * This one is used jointly by NFSv2 and NFSv3. - */ -static int -nfs4_stat_to_errno(int stat) -{ - int i; - for (i = 0; nfs_errtbl[i].stat != -1; i++) { - if (nfs_errtbl[i].stat == stat) - return nfs_errtbl[i].errno; - } - if (stat <= 10000 || stat > 10100) { - /* The server is looney tunes. */ - return -EREMOTEIO; - } - /* If we cannot translate the error, the recovery routines should - * handle it. - * Note: remaining NFSv4 error codes have values > 10000, so should - * not conflict with native Linux error codes. - */ - return -stat; -} - #ifdef CONFIG_NFS_V4_2 #include "nfs42xdr.c" #endif /* CONFIG_NFS_V4_2 */ diff --git a/fs/nfs_common/common.c b/fs/nfs_common/common.c index a4ee95da2174..34a115176f97 100644 --- a/fs/nfs_common/common.c +++ b/fs/nfs_common/common.c @@ -2,6 +2,7 @@ #include #include +#include /* * We need to translate between nfs status return values and @@ -65,3 +66,69 @@ int nfs_stat_to_errno(enum nfs_stat status) return nfs_errtbl[i].errno; } EXPORT_SYMBOL_GPL(nfs_stat_to_errno); + +/* + * We need to translate between nfs v4 status return values and + * the local errno values which may not be the same. + */ +static const struct { + int stat; + int errno; +} nfs4_errtbl[] = { + { NFS4_OK, 0 }, + { NFS4ERR_PERM, -EPERM }, + { NFS4ERR_NOENT, -ENOENT }, + { NFS4ERR_IO, -errno_NFSERR_IO}, + { NFS4ERR_NXIO, -ENXIO }, + { NFS4ERR_ACCESS, -EACCES }, + { NFS4ERR_EXIST, -EEXIST }, + { NFS4ERR_XDEV, -EXDEV }, + { NFS4ERR_NOTDIR, -ENOTDIR }, + { NFS4ERR_ISDIR, -EISDIR }, + { NFS4ERR_INVAL, -EINVAL }, + { NFS4ERR_FBIG, -EFBIG }, + { NFS4ERR_NOSPC, -ENOSPC }, + { NFS4ERR_ROFS, -EROFS }, + { NFS4ERR_MLINK, -EMLINK }, + { NFS4ERR_NAMETOOLONG, -ENAMETOOLONG }, + { NFS4ERR_NOTEMPTY, -ENOTEMPTY }, + { NFS4ERR_DQUOT, -EDQUOT }, + { NFS4ERR_STALE, -ESTALE }, + { NFS4ERR_BADHANDLE, -EBADHANDLE }, + { NFS4ERR_BAD_COOKIE, -EBADCOOKIE }, + { NFS4ERR_NOTSUPP, -ENOTSUPP }, + { NFS4ERR_TOOSMALL, -ETOOSMALL }, + { NFS4ERR_SERVERFAULT, -EREMOTEIO }, + { NFS4ERR_BADTYPE, -EBADTYPE }, + { NFS4ERR_LOCKED, -EAGAIN }, + { NFS4ERR_SYMLINK, -ELOOP }, + { NFS4ERR_OP_ILLEGAL, -EOPNOTSUPP }, + { NFS4ERR_DEADLOCK, -EDEADLK }, + { NFS4ERR_NOXATTR, -ENODATA }, + { NFS4ERR_XATTR2BIG, -E2BIG }, + { -1, -EIO } +}; + +/* + * Convert an NFS error code to a local one. + * This one is used by NFSv4. + */ +int nfs4_stat_to_errno(int stat) +{ + int i; + for (i = 0; nfs4_errtbl[i].stat != -1; i++) { + if (nfs4_errtbl[i].stat == stat) + return nfs4_errtbl[i].errno; + } + if (stat <= 10000 || stat > 10100) { + /* The server is looney tunes. */ + return -EREMOTEIO; + } + /* If we cannot translate the error, the recovery routines should + * handle it. + * Note: remaining NFSv4 error codes have values > 10000, so should + * not conflict with native Linux error codes. + */ + return -stat; +} +EXPORT_SYMBOL_GPL(nfs4_stat_to_errno); diff --git a/include/linux/nfs_common.h b/include/linux/nfs_common.h index 3395c4a4d372..5fc02df88252 100644 --- a/include/linux/nfs_common.h +++ b/include/linux/nfs_common.h @@ -12,5 +12,6 @@ #define errno_NFSERR_IO EIO int nfs_stat_to_errno(enum nfs_stat status); +int nfs4_stat_to_errno(int stat); #endif /* _LINUX_NFS_COMMON_H */ From 1545e488b1f908b10f6dff0c278c6b7a37122de8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:37 -0400 Subject: [PATCH 086/111] nfs: factor out {encode,decode}_opaque_fixed to nfs_xdr.h Eliminates duplicate functions in various files to allow for additional callers. Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/flexfilelayout/flexfilelayout.c | 6 ------ fs/nfs/nfs4xdr.c | 13 ------------- include/linux/nfs_xdr.h | 20 +++++++++++++++++++- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 39ba9f4208aa..d4d551ffea7b 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -2086,12 +2086,6 @@ static int ff_layout_encode_ioerr(struct xdr_stream *xdr, return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors); } -static void -encode_opaque_fixed(struct xdr_stream *xdr, const void *buf, size_t len) -{ - WARN_ON_ONCE(xdr_stream_encode_opaque_fixed(xdr, buf, len) < 0); -} - static void ff_layout_encode_ff_iostat_head(struct xdr_stream *xdr, const nfs4_stateid *stateid, diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index 18d268029b1c..1a909408cf5a 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -972,11 +972,6 @@ static __be32 *reserve_space(struct xdr_stream *xdr, size_t nbytes) return p; } -static void encode_opaque_fixed(struct xdr_stream *xdr, const void *buf, size_t len) -{ - WARN_ON_ONCE(xdr_stream_encode_opaque_fixed(xdr, buf, len) < 0); -} - static void encode_string(struct xdr_stream *xdr, unsigned int len, const char *str) { WARN_ON_ONCE(xdr_stream_encode_opaque(xdr, str, len) < 0); @@ -4406,14 +4401,6 @@ static int decode_access(struct xdr_stream *xdr, u32 *supported, u32 *access) return 0; } -static int decode_opaque_fixed(struct xdr_stream *xdr, void *buf, size_t len) -{ - ssize_t ret = xdr_stream_decode_opaque_fixed(xdr, buf, len); - if (unlikely(ret < 0)) - return -EIO; - return 0; -} - static int decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) { return decode_opaque_fixed(xdr, stateid, NFS4_STATEID_SIZE); diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 96ba04ab24f3..12d8e47bc5a3 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1853,6 +1853,24 @@ struct nfs_rpc_ops { void (*disable_swap)(struct inode *inode); }; +/* + * Helper functions used by NFS client and/or server + */ +static inline void encode_opaque_fixed(struct xdr_stream *xdr, + const void *buf, size_t len) +{ + WARN_ON_ONCE(xdr_stream_encode_opaque_fixed(xdr, buf, len) < 0); +} + +static inline int decode_opaque_fixed(struct xdr_stream *xdr, + void *buf, size_t len) +{ + ssize_t ret = xdr_stream_decode_opaque_fixed(xdr, buf, len); + if (unlikely(ret < 0)) + return -EIO; + return 0; +} + /* * Function vectors etc. for the NFS client */ @@ -1866,4 +1884,4 @@ extern const struct rpc_version nfs_version4; extern const struct rpc_version nfsacl_version3; extern const struct rpc_program nfsacl_program; -#endif +#endif /* _LINUX_NFS_XDR_H */ From 0a183f24a7ae48b7c0f2327989754b5db5de02b8 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 5 Sep 2024 15:09:38 -0400 Subject: [PATCH 087/111] NFSD: Handle @rqstp == NULL in check_nfsd_access() LOCALIO-initiated open operations are not running in an nfsd thread and thus do not have an associated svc_rqst context. Signed-off-by: NeilBrown Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Signed-off-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/export.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c index 7bb4f2075ac5..c82d8e3e0d4f 100644 --- a/fs/nfsd/export.c +++ b/fs/nfsd/export.c @@ -1074,10 +1074,30 @@ static struct svc_export *exp_find(struct cache_detail *cd, return exp; } +/** + * check_nfsd_access - check if access to export is allowed. + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp (will be NULL for LOCALIO). + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_wrongsec if access is denied + */ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) { struct exp_flavor_info *f, *end = exp->ex_flavors + exp->ex_nflavors; - struct svc_xprt *xprt = rqstp->rq_xprt; + struct svc_xprt *xprt; + + /* + * If rqstp is NULL, this is a LOCALIO request which will only + * ever use a filehandle/credential pair for which access has + * been affirmed (by ACCESS or OPEN NFS requests) over the + * wire. So there is no need for further checks here. + */ + if (!rqstp) + return nfs_ok; + + xprt = rqstp->rq_xprt; if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_NONE) { if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) @@ -1098,17 +1118,17 @@ __be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp) ok: /* legacy gss-only clients are always OK: */ if (exp->ex_client == rqstp->rq_gssclient) - return 0; + return nfs_ok; /* ip-address based client; check sec= export option: */ for (f = exp->ex_flavors; f < end; f++) { if (f->pseudoflavor == rqstp->rq_cred.cr_flavor) - return 0; + return nfs_ok; } /* defaults in absence of sec= options: */ if (exp->ex_nflavors == 0) { if (rqstp->rq_cred.cr_flavor == RPC_AUTH_NULL || rqstp->rq_cred.cr_flavor == RPC_AUTH_UNIX) - return 0; + return nfs_ok; } /* If the compound op contains a spo_must_allowed op, @@ -1118,7 +1138,7 @@ ok: */ if (nfsd4_spo_must_allow(rqstp)) - return 0; + return nfs_ok; denied: return nfserr_wrongsec; From b0d87dbd8bd311d4126f5b34a8494043c487695f Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 5 Sep 2024 15:09:39 -0400 Subject: [PATCH 088/111] NFSD: Refactor nfsd_setuser_and_check_port() There are several places where __fh_verify unconditionally dereferences rqstp to check that the connection is suitably secure. They look at rqstp->rq_xprt which is not meaningful in the target use case of "localio" NFS in which the client talks directly to the local server. Prepare these to always succeed when rqstp is NULL. Signed-off-by: NeilBrown Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Signed-off-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/nfsfh.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 50d23d56f403..4b964a71a504 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -87,23 +87,24 @@ nfsd_mode_check(struct dentry *dentry, umode_t requested) return nfserr_wrong_type; } -static bool nfsd_originating_port_ok(struct svc_rqst *rqstp, int flags) +static bool nfsd_originating_port_ok(struct svc_rqst *rqstp, + struct svc_cred *cred, + struct svc_export *exp) { - if (flags & NFSEXP_INSECURE_PORT) + if (nfsexp_flags(cred, exp) & NFSEXP_INSECURE_PORT) return true; /* We don't require gss requests to use low ports: */ - if (rqstp->rq_cred.cr_flavor >= RPC_AUTH_GSS) + if (cred->cr_flavor >= RPC_AUTH_GSS) return true; return test_bit(RQ_SECURE, &rqstp->rq_flags); } static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, + struct svc_cred *cred, struct svc_export *exp) { - int flags = nfsexp_flags(&rqstp->rq_cred, exp); - /* Check if the request originated from a secure port. */ - if (!nfsd_originating_port_ok(rqstp, flags)) { + if (rqstp && !nfsd_originating_port_ok(rqstp, cred, exp)) { RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]); dprintk("nfsd: request from insecure port %s!\n", svc_print_addr(rqstp, buf, sizeof(buf))); @@ -111,7 +112,7 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, } /* Set user creds for this exportpoint */ - return nfserrno(nfsd_setuser(&rqstp->rq_cred, exp)); + return nfserrno(nfsd_setuser(cred, exp)); } static inline __be32 check_pseudo_root(struct dentry *dentry, @@ -219,7 +220,7 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) put_cred(override_creds(new)); put_cred(new); } else { - error = nfsd_setuser_and_check_port(rqstp, exp); + error = nfsd_setuser_and_check_port(rqstp, &rqstp->rq_cred, exp); if (error) goto out; } @@ -358,7 +359,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) if (error) goto out; - error = nfsd_setuser_and_check_port(rqstp, exp); + error = nfsd_setuser_and_check_port(rqstp, &rqstp->rq_cred, exp); if (error) goto out; From 7c0b07b49b2da108438e2504452effe0e6cbb764 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 5 Sep 2024 15:09:40 -0400 Subject: [PATCH 089/111] NFSD: Avoid using rqstp->rq_vers in nfsd_set_fh_dentry() Currently, fh_verify() makes some daring assumptions about which version of file handle the caller wants, based on the things it can find in the passed-in rqstp. The about-to-be-introduced LOCALIO use case sometimes has no svc_rqst context, so this logic won't work in that case. Instead, examine the passed-in file handle. It's .max_size field should carry information to allow nfsd_set_fh_dentry() to initialize the file handle appropriately. The file handle used by lockd and the one created by write_filehandle never need any of the version-specific fields (which affect things like write and getattr requests and pre/post attributes). Signed-off-by: Chuck Lever Signed-off-by: Mike Snitzer Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfsd/nfsfh.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 4b964a71a504..60c2395d7af7 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -267,20 +267,20 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) fhp->fh_dentry = dentry; fhp->fh_export = exp; - switch (rqstp->rq_vers) { - case 4: + switch (fhp->fh_maxsize) { + case NFS4_FHSIZE: if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOATOMIC_ATTR) fhp->fh_no_atomic_attr = true; fhp->fh_64bit_cookies = true; break; - case 3: + case NFS3_FHSIZE: if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; fhp->fh_64bit_cookies = true; if (exp->ex_flags & NFSEXP_V4ROOT) goto out; break; - case 2: + case NFS_FHSIZE: fhp->fh_no_wcc = true; if (EX_WGATHER(exp)) fhp->fh_use_wgather = true; From 71c61a0077eb11e8bcc4d03e92ebc0c16df46abe Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 5 Sep 2024 15:09:41 -0400 Subject: [PATCH 090/111] NFSD: Short-circuit fh_verify tracepoints for LOCALIO LOCALIO will be able to call fh_verify() with a NULL rqstp. In this case, the existing trace points need to be skipped because they want to dereference the address fields in the passed-in rqstp. Temporarily make these trace points conditional to avoid a seg fault in this case. Putting the "rqstp != NULL" check in the trace points themselves makes the check more efficient. Signed-off-by: Chuck Lever Signed-off-by: Mike Snitzer Acked-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfsd/trace.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 7ab66497e261..e339d04ff32f 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -193,7 +193,7 @@ TRACE_EVENT(nfsd_compound_encode_err, { S_IFIFO, "FIFO" }, \ { S_IFSOCK, "SOCK" }) -TRACE_EVENT(nfsd_fh_verify, +TRACE_EVENT_CONDITION(nfsd_fh_verify, TP_PROTO( const struct svc_rqst *rqstp, const struct svc_fh *fhp, @@ -201,6 +201,7 @@ TRACE_EVENT(nfsd_fh_verify, int access ), TP_ARGS(rqstp, fhp, type, access), + TP_CONDITION(rqstp != NULL), TP_STRUCT__entry( __field(unsigned int, netns_ino) __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) @@ -239,7 +240,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify_err, __be32 error ), TP_ARGS(rqstp, fhp, type, access, error), - TP_CONDITION(error), + TP_CONDITION(rqstp != NULL && error), TP_STRUCT__entry( __field(unsigned int, netns_ino) __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) @@ -295,12 +296,13 @@ DECLARE_EVENT_CLASS(nfsd_fh_err_class, __entry->status) ) -#define DEFINE_NFSD_FH_ERR_EVENT(name) \ -DEFINE_EVENT(nfsd_fh_err_class, nfsd_##name, \ - TP_PROTO(struct svc_rqst *rqstp, \ - struct svc_fh *fhp, \ - int status), \ - TP_ARGS(rqstp, fhp, status)) +#define DEFINE_NFSD_FH_ERR_EVENT(name) \ +DEFINE_EVENT_CONDITION(nfsd_fh_err_class, nfsd_##name, \ + TP_PROTO(struct svc_rqst *rqstp, \ + struct svc_fh *fhp, \ + int status), \ + TP_ARGS(rqstp, fhp, status), \ + TP_CONDITION(rqstp != NULL)) DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badexport); DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badhandle); From 5e66d2d92a1c331ed86d943ce3703c654876b20a Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 5 Sep 2024 15:09:42 -0400 Subject: [PATCH 091/111] nfsd: factor out __fh_verify to allow NULL rqstp to be passed __fh_verify() offers an interface like fh_verify() but doesn't require a struct svc_rqst *, instead it also takes the specific parts as explicit required arguments. So it is safe to call __fh_verify() with a NULL rqstp, but the net, cred, and client args must not be NULL. __fh_verify() does not use SVC_NET(), nor does the functions it calls. Rather than using rqstp->rq_client pass the client and gssclient explicitly to __fh_verify and then to nfsd_set_fh_dentry(). Lastly, it should be noted that the previous commit prepared for 4 associated tracepoints to only be used if rqstp is not NULL (this is a stop-gap that should be properly fixed so localio also benefits from the utility these tracepoints provide when debugging fh_verify issues). Signed-off-by: NeilBrown Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Signed-off-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/nfsfh.c | 91 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index 60c2395d7af7..a77af71892a3 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -142,7 +142,11 @@ static inline __be32 check_pseudo_root(struct dentry *dentry, * dentry. On success, the results are used to set fh_export and * fh_dentry. */ -static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) +static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, + struct svc_cred *cred, + struct auth_domain *client, + struct auth_domain *gssclient, + struct svc_fh *fhp) { struct knfsd_fh *fh = &fhp->fh_handle; struct fid *fid = NULL; @@ -184,8 +188,8 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) data_left -= len; if (data_left < 0) return error; - exp = rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), - rqstp->rq_client, rqstp->rq_gssclient, + exp = rqst_exp_find(rqstp ? &rqstp->rq_chandle : NULL, + net, client, gssclient, fh->fh_fsid_type, fh->fh_fsid); fid = (struct fid *)(fh->fh_fsid + len); @@ -220,7 +224,7 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) put_cred(override_creds(new)); put_cred(new); } else { - error = nfsd_setuser_and_check_port(rqstp, &rqstp->rq_cred, exp); + error = nfsd_setuser_and_check_port(rqstp, cred, exp); if (error) goto out; } @@ -295,42 +299,33 @@ out: } /** - * fh_verify - filehandle lookup and access checking - * @rqstp: pointer to current rpc request + * __fh_verify - filehandle lookup and access checking + * @rqstp: RPC transaction context, or NULL + * @net: net namespace in which to perform the export lookup + * @cred: RPC user credential + * @client: RPC auth domain + * @gssclient: RPC GSS auth domain, or NULL * @fhp: filehandle to be verified * @type: expected type of object pointed to by filehandle * @access: type of access needed to object * - * Look up a dentry from the on-the-wire filehandle, check the client's - * access to the export, and set the current task's credentials. - * - * Regardless of success or failure of fh_verify(), fh_put() should be - * called on @fhp when the caller is finished with the filehandle. - * - * fh_verify() may be called multiple times on a given filehandle, for - * example, when processing an NFSv4 compound. The first call will look - * up a dentry using the on-the-wire filehandle. Subsequent calls will - * skip the lookup and just perform the other checks and possibly change - * the current task's credentials. - * - * @type specifies the type of object expected using one of the S_IF* - * constants defined in include/linux/stat.h. The caller may use zero - * to indicate that it doesn't care, or a negative integer to indicate - * that it expects something not of the given type. - * - * @access is formed from the NFSD_MAY_* constants defined in - * fs/nfsd/vfs.h. + * See fh_verify() for further descriptions of @fhp, @type, and @access. */ -__be32 -fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) +static __be32 +__fh_verify(struct svc_rqst *rqstp, + struct net *net, struct svc_cred *cred, + struct auth_domain *client, + struct auth_domain *gssclient, + struct svc_fh *fhp, umode_t type, int access) { - struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); struct svc_export *exp = NULL; struct dentry *dentry; __be32 error; if (!fhp->fh_dentry) { - error = nfsd_set_fh_dentry(rqstp, fhp); + error = nfsd_set_fh_dentry(rqstp, net, cred, client, + gssclient, fhp); if (error) goto out; } @@ -359,7 +354,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) if (error) goto out; - error = nfsd_setuser_and_check_port(rqstp, &rqstp->rq_cred, exp); + error = nfsd_setuser_and_check_port(rqstp, cred, exp); if (error) goto out; @@ -389,7 +384,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) skip_pseudoflavor_check: /* Finally, check access permissions. */ - error = nfsd_permission(&rqstp->rq_cred, exp, dentry, access); + error = nfsd_permission(cred, exp, dentry, access); out: trace_nfsd_fh_verify_err(rqstp, fhp, type, access, error); if (error == nfserr_stale) @@ -397,6 +392,40 @@ out: return error; } +/** + * fh_verify - filehandle lookup and access checking + * @rqstp: pointer to current rpc request + * @fhp: filehandle to be verified + * @type: expected type of object pointed to by filehandle + * @access: type of access needed to object + * + * Look up a dentry from the on-the-wire filehandle, check the client's + * access to the export, and set the current task's credentials. + * + * Regardless of success or failure of fh_verify(), fh_put() should be + * called on @fhp when the caller is finished with the filehandle. + * + * fh_verify() may be called multiple times on a given filehandle, for + * example, when processing an NFSv4 compound. The first call will look + * up a dentry using the on-the-wire filehandle. Subsequent calls will + * skip the lookup and just perform the other checks and possibly change + * the current task's credentials. + * + * @type specifies the type of object expected using one of the S_IF* + * constants defined in include/linux/stat.h. The caller may use zero + * to indicate that it doesn't care, or a negative integer to indicate + * that it expects something not of the given type. + * + * @access is formed from the NFSD_MAY_* constants defined in + * fs/nfsd/vfs.h. + */ +__be32 +fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) +{ + return __fh_verify(rqstp, SVC_NET(rqstp), &rqstp->rq_cred, + rqstp->rq_client, rqstp->rq_gssclient, + fhp, type, access); +} /* * Compose a file handle for an NFS reply. From c63f0e48febfaea8a3cd4146abda9cc7a329b0e3 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 5 Sep 2024 15:09:43 -0400 Subject: [PATCH 092/111] nfsd: add nfsd_file_acquire_local() nfsd_file_acquire_local() can be used to look up a file by filehandle without having a struct svc_rqst. This can be used by NFS LOCALIO to allow the NFS client to bypass the NFS protocol to directly access a file provided by the NFS server which is running in the same kernel. In nfsd_file_do_acquire() care is taken to always use fh_verify() if rqstp is not NULL (as is the case for non-LOCALIO callers). Otherwise the non-LOCALIO callers will not supply the correct and required arguments to __fh_verify (e.g. gssclient isn't passed). Introduce fh_verify_local() wrapper around __fh_verify to make it clear that LOCALIO is intended caller. Also, use GC for nfsd_file returned by nfsd_file_acquire_local. GC offers performance improvements if/when a file is reopened before launderette cleans it from the filecache's LRU. Suggested-by: Jeff Layton # use filecache's GC Signed-off-by: NeilBrown Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Signed-off-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/filecache.c | 71 ++++++++++++++++++++++++++++++++++++++++----- fs/nfsd/filecache.h | 3 ++ fs/nfsd/nfsfh.c | 23 +++++++++++++++ fs/nfsd/nfsfh.h | 2 ++ 4 files changed, 92 insertions(+), 7 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 24e8f1fbcebb..caeffc43ca4b 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -982,12 +982,14 @@ nfsd_file_is_cached(struct inode *inode) } static __be32 -nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, +nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net, + struct svc_cred *cred, + struct auth_domain *client, + struct svc_fh *fhp, unsigned int may_flags, struct file *file, struct nfsd_file **pnf, bool want_gc) { unsigned char need = may_flags & NFSD_FILE_MAY_MASK; - struct net *net = SVC_NET(rqstp); struct nfsd_file *new, *nf; bool stale_retry = true; bool open_retry = true; @@ -996,8 +998,13 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, int ret; retry: - status = fh_verify(rqstp, fhp, S_IFREG, - may_flags|NFSD_MAY_OWNER_OVERRIDE); + if (rqstp) { + status = fh_verify(rqstp, fhp, S_IFREG, + may_flags|NFSD_MAY_OWNER_OVERRIDE); + } else { + status = fh_verify_local(net, cred, client, fhp, S_IFREG, + may_flags|NFSD_MAY_OWNER_OVERRIDE); + } if (status != nfs_ok) return status; inode = d_inode(fhp->fh_dentry); @@ -1143,7 +1150,8 @@ __be32 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct nfsd_file **pnf) { - return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true); + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, NULL, pnf, true); } /** @@ -1167,7 +1175,55 @@ __be32 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct nfsd_file **pnf) { - return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false); + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, NULL, pnf, false); +} + +/** + * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio + * @net: The network namespace in which to perform a lookup + * @cred: the user credential with which to validate access + * @client: the auth_domain for LOCALIO lookup + * @fhp: the NFS filehandle of the file to be opened + * @may_flags: NFSD_MAY_ settings for the file + * @pnf: OUT: new or found "struct nfsd_file" object + * + * This file lookup interface provide access to a file given the + * filehandle and credential. No connection-based authorisation + * is performed and in that way it is quite different to other + * file access mediated by nfsd. It allows a kernel module such as the NFS + * client to reach across network and filesystem namespaces to access + * a file. The security implications of this should be carefully + * considered before use. + * + * The nfsd_file object returned by this API is reference-counted + * and garbage-collected. The object is retained for a few + * seconds after the final nfsd_file_put() in case the caller + * wants to re-use it. + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf) +{ + /* + * Save creds before calling nfsd_file_do_acquire() (which calls + * nfsd_setuser). Important because caller (LOCALIO) is from + * client context. + */ + const struct cred *save_cred = get_current_cred(); + __be32 beres; + + beres = nfsd_file_do_acquire(NULL, net, cred, client, + fhp, may_flags, NULL, pnf, true); + revert_creds(save_cred); + return beres; } /** @@ -1193,7 +1249,8 @@ nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct file *file, struct nfsd_file **pnf) { - return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false); + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, file, pnf, false); } /* diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index 3fbec24eea6c..26ada78b8c1e 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -66,5 +66,8 @@ __be32 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, __be32 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct file *file, struct nfsd_file **nfp); +__be32 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf); int nfsd_file_cache_stats_show(struct seq_file *m, void *v); #endif /* _FS_NFSD_FILECACHE_H */ diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index a77af71892a3..40ad58a6a036 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -392,6 +392,29 @@ out: return error; } +/** + * fh_verify_local - filehandle lookup and access checking + * @net: net namespace in which to perform the export lookup + * @cred: RPC user credential + * @client: RPC auth domain + * @fhp: filehandle to be verified + * @type: expected type of object pointed to by filehandle + * @access: type of access needed to object + * + * This API can be used by callers who do not have an RPC + * transaction context (ie are not running in an nfsd thread). + * + * See fh_verify() for further descriptions of @fhp, @type, and @access. + */ +__be32 +fh_verify_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + umode_t type, int access) +{ + return __fh_verify(NULL, net, cred, client, NULL, + fhp, type, access); +} + /** * fh_verify - filehandle lookup and access checking * @rqstp: pointer to current rpc request diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index 8d46e203d139..5b7394801dc4 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -217,6 +217,8 @@ extern char * SVCFH_fmt(struct svc_fh *fhp); * Function prototypes */ __be32 fh_verify(struct svc_rqst *, struct svc_fh *, umode_t, int); +__be32 fh_verify_local(struct net *, struct svc_cred *, struct auth_domain *, + struct svc_fh *, umode_t, int); __be32 fh_compose(struct svc_fh *, struct svc_export *, struct dentry *, struct svc_fh *); __be32 fh_update(struct svc_fh *); void fh_put(struct svc_fh *); From 47e988147f409f70e0f01a5e6dc5940375e02343 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:44 -0400 Subject: [PATCH 093/111] nfsd: add nfsd_serv_try_get and nfsd_serv_put Introduce nfsd_serv_try_get and nfsd_serv_put and update the nfsd code to prevent nfsd_destroy_serv from destroying nn->nfsd_serv until any caller of nfsd_serv_try_get releases their reference using nfsd_serv_put. A percpu_ref is used to implement the interlock between nfsd_destroy_serv and any caller of nfsd_serv_try_get. This interlock is needed to properly wait for the completion of client initiated localio calls to nfsd (that are _not_ in the context of nfsd). Signed-off-by: Mike Snitzer Reviewed-by: Chuck Lever Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- fs/nfsd/netns.h | 8 +++++++- fs/nfsd/nfssvc.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 37b8bfdcfeea..a228b9d6a6e1 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -139,7 +140,9 @@ struct nfsd_net { struct svc_info nfsd_info; #define nfsd_serv nfsd_info.serv - + struct percpu_ref nfsd_serv_ref; + struct completion nfsd_serv_confirm_done; + struct completion nfsd_serv_free_done; /* * clientid and stateid data for construction of net unique COPY @@ -222,6 +225,9 @@ struct nfsd_net { extern bool nfsd_support_version(int vers); extern unsigned int nfsd_net_id; +bool nfsd_serv_try_get(struct net *net); +void nfsd_serv_put(struct net *net); + void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn); void nfsd_reset_write_verifier(struct nfsd_net *nn); #endif /* __NFSD_NETNS_H__ */ diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index defc430f912f..307e365798f5 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -193,6 +193,34 @@ int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change return 0; } +bool nfsd_serv_try_get(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + return (nn && percpu_ref_tryget_live(&nn->nfsd_serv_ref)); +} + +void nfsd_serv_put(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + percpu_ref_put(&nn->nfsd_serv_ref); +} + +static void nfsd_serv_done(struct percpu_ref *ref) +{ + struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_serv_ref); + + complete(&nn->nfsd_serv_confirm_done); +} + +static void nfsd_serv_free(struct percpu_ref *ref) +{ + struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_serv_ref); + + complete(&nn->nfsd_serv_free_done); +} + /* * Maximum number of nfsd processes */ @@ -392,6 +420,7 @@ static void nfsd_shutdown_net(struct net *net) lockd_down(net); nn->lockd_up = false; } + percpu_ref_exit(&nn->nfsd_serv_ref); nn->nfsd_net_up = false; nfsd_shutdown_generic(); } @@ -471,6 +500,13 @@ void nfsd_destroy_serv(struct net *net) struct nfsd_net *nn = net_generic(net, nfsd_net_id); struct svc_serv *serv = nn->nfsd_serv; + lockdep_assert_held(&nfsd_mutex); + + percpu_ref_kill_and_confirm(&nn->nfsd_serv_ref, nfsd_serv_done); + wait_for_completion(&nn->nfsd_serv_confirm_done); + wait_for_completion(&nn->nfsd_serv_free_done); + /* percpu_ref_exit is called in nfsd_shutdown_net */ + spin_lock(&nfsd_notifier_lock); nn->nfsd_serv = NULL; spin_unlock(&nfsd_notifier_lock); @@ -595,6 +631,13 @@ int nfsd_create_serv(struct net *net) if (nn->nfsd_serv) return 0; + error = percpu_ref_init(&nn->nfsd_serv_ref, nfsd_serv_free, + 0, GFP_KERNEL); + if (error) + return error; + init_completion(&nn->nfsd_serv_free_done); + init_completion(&nn->nfsd_serv_confirm_done); + if (nfsd_max_blksize == 0) nfsd_max_blksize = nfsd_get_default_max_blksize(); nfsd_reset_versions(nn); From 2c8919848de1e5c881fe3473645c26a1c2a927f2 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:45 -0400 Subject: [PATCH 094/111] SUNRPC: remove call_allocate() BUG_ONs Remove BUG_ON if p_arglen=0 to allow RPC with void arg. Remove BUG_ON if p_replen=0 to allow RPC with void return. The former was needed for the first revision of the LOCALIO protocol which had an RPC that took a void arg: /* raw RFC 9562 UUID */ typedef u8 uuid_t; program NFS_LOCALIO_PROGRAM { version LOCALIO_V1 { void NULL(void) = 0; uuid_t GETUUID(void) = 1; } = 1; } = 400122; The latter is needed for the final revision of the LOCALIO protocol which has a UUID_IS_LOCAL RPC which returns a void: /* raw RFC 9562 UUID */ typedef u8 uuid_t; program NFS_LOCALIO_PROGRAM { version LOCALIO_V1 { void NULL(void) = 0; void UUID_IS_LOCAL(uuid_t) = 1; } = 1; } = 400122; There is really no value in triggering a BUG_ON in response to either of these previously unsupported conditions. NeilBrown would like the entire 'if (proc->p_proc != 0)' branch removed (not just the one BUG_ON that must be removed for LOCALIO's immediate needs of returning void). Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- net/sunrpc/clnt.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 5e9f36505ab7..0090162ee8c3 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1888,12 +1888,6 @@ call_allocate(struct rpc_task *task) if (req->rq_buffer) return; - if (proc->p_proc != 0) { - BUG_ON(proc->p_arglen == 0); - if (proc->p_decode != NULL) - BUG_ON(proc->p_replen == 0); - } - /* * Calculate the size (in quads) of the RPC call * and reply headers, and convert both values From 199f2128741077087a2ab33889a6868830465033 Mon Sep 17 00:00:00 2001 From: Weston Andros Adamson Date: Thu, 5 Sep 2024 15:09:46 -0400 Subject: [PATCH 095/111] SUNRPC: add svcauth_map_clnt_to_svc_cred_local Add new funtion svcauth_map_clnt_to_svc_cred_local which maps a generic cred to a svc_cred suitable for use in nfsd. This is needed by the localio code to map nfs client creds to nfs server credentials. Following from net/sunrpc/auth_unix.c:unx_marshal() it is clear that ->fsuid and ->fsgid must be used (rather than ->uid and ->gid). In addition, these uid and gid must be translated with from_kuid_munged() so local client uses correct uid and gid when acting as local server. Jeff Layton noted: This is where the magic happens. Since we're working in kuid_t/kgid_t, we don't need to worry about further idmapping. Suggested-by: NeilBrown # to approximate unx_marshal() Signed-off-by: Weston Andros Adamson Signed-off-by: Trond Myklebust Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Reviewed-by: Chuck Lever Reviewed-by: Jeff Layton Reviewed-by: NeilBrown Signed-off-by: Anna Schumaker --- include/linux/sunrpc/svcauth.h | 5 +++++ net/sunrpc/svcauth.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/include/linux/sunrpc/svcauth.h b/include/linux/sunrpc/svcauth.h index 63cf6fb26dcc..2e111153f7cd 100644 --- a/include/linux/sunrpc/svcauth.h +++ b/include/linux/sunrpc/svcauth.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -157,6 +158,10 @@ extern enum svc_auth_status svc_set_client(struct svc_rqst *rqstp); extern int svc_auth_register(rpc_authflavor_t flavor, struct auth_ops *aops); extern void svc_auth_unregister(rpc_authflavor_t flavor); +extern void svcauth_map_clnt_to_svc_cred_local(struct rpc_clnt *clnt, + const struct cred *, + struct svc_cred *); + extern struct auth_domain *unix_domain_find(char *name); extern void auth_domain_put(struct auth_domain *item); extern struct auth_domain *auth_domain_lookup(char *name, struct auth_domain *new); diff --git a/net/sunrpc/svcauth.c b/net/sunrpc/svcauth.c index 93d9e949e265..55b4d2874188 100644 --- a/net/sunrpc/svcauth.c +++ b/net/sunrpc/svcauth.c @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -175,6 +176,33 @@ rpc_authflavor_t svc_auth_flavor(struct svc_rqst *rqstp) } EXPORT_SYMBOL_GPL(svc_auth_flavor); +/** + * svcauth_map_clnt_to_svc_cred_local - maps a generic cred + * to a svc_cred suitable for use in nfsd. + * @clnt: rpc_clnt associated with nfs client + * @cred: generic cred associated with nfs client + * @svc: returned svc_cred that is suitable for use in nfsd + */ +void svcauth_map_clnt_to_svc_cred_local(struct rpc_clnt *clnt, + const struct cred *cred, + struct svc_cred *svc) +{ + struct user_namespace *userns = clnt->cl_cred ? + clnt->cl_cred->user_ns : &init_user_ns; + + memset(svc, 0, sizeof(struct svc_cred)); + + svc->cr_uid = KUIDT_INIT(from_kuid_munged(userns, cred->fsuid)); + svc->cr_gid = KGIDT_INIT(from_kgid_munged(userns, cred->fsgid)); + svc->cr_flavor = clnt->cl_auth->au_flavor; + if (cred->group_info) + svc->cr_group_info = get_group_info(cred->group_info); + /* These aren't relevant for local (network is bypassed) */ + svc->cr_principal = NULL; + svc->cr_gss_mech = NULL; +} +EXPORT_SYMBOL_GPL(svcauth_map_clnt_to_svc_cred_local); + /************************************************** * 'auth_domains' are stored in a hash table indexed by name. * When the last reference to an 'auth_domain' is dropped, From 86ab08beb3f07f6e51922a8b8f662a5ec7012d35 Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Thu, 5 Sep 2024 15:09:47 -0400 Subject: [PATCH 096/111] SUNRPC: replace program list with program array A service created with svc_create_pooled() can be given a linked list of programs and all of these will be served. Using a linked list makes it cumbersome when there are several programs that can be optionally selected with CONFIG settings. After this patch is applied, API consumers must use only svc_create_pooled() when creating an RPC service that listens for more than one RPC program. Signed-off-by: NeilBrown Signed-off-by: Mike Snitzer Acked-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/nfsctl.c | 2 +- fs/nfsd/nfsd.h | 2 +- fs/nfsd/nfssvc.c | 38 ++++++++++----------- include/linux/sunrpc/svc.h | 7 ++-- net/sunrpc/svc.c | 68 ++++++++++++++++++++++---------------- net/sunrpc/svc_xprt.c | 2 +- net/sunrpc/svcauth_unix.c | 3 +- 7 files changed, 67 insertions(+), 55 deletions(-) diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 1c9e5b4bcb0a..64c1b4d649bc 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -2246,7 +2246,7 @@ static __net_init int nfsd_net_init(struct net *net) if (retval) goto out_repcache_error; memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats)); - nn->nfsd_svcstats.program = &nfsd_program; + nn->nfsd_svcstats.program = &nfsd_programs[0]; for (i = 0; i < sizeof(nn->nfsd_versions); i++) nn->nfsd_versions[i] = nfsd_support_version(i); for (i = 0; i < sizeof(nn->nfsd4_minorversions); i++) diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 4ccbf014a2c7..b0d3e82d6dcd 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -85,7 +85,7 @@ struct nfsd_genl_rqstp { u32 rq_opnum[NFSD_MAX_OPS_PER_COMPOUND]; }; -extern struct svc_program nfsd_program; +extern struct svc_program nfsd_programs[]; extern const struct svc_version nfsd_version2, nfsd_version3, nfsd_version4; extern struct mutex nfsd_mutex; extern spinlock_t nfsd_drc_lock; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 307e365798f5..fa34de5e59bd 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -35,7 +35,6 @@ #define NFSDDBG_FACILITY NFSDDBG_SVC atomic_t nfsd_th_cnt = ATOMIC_INIT(0); -extern struct svc_program nfsd_program; static int nfsd(void *vrqstp); #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) static int nfsd_acl_rpcbind_set(struct net *, @@ -90,20 +89,9 @@ static const struct svc_version *nfsd_acl_version[] = { # endif }; -#define NFSD_ACL_MINVERS 2 +#define NFSD_ACL_MINVERS 2 #define NFSD_ACL_NRVERS ARRAY_SIZE(nfsd_acl_version) -static struct svc_program nfsd_acl_program = { - .pg_prog = NFS_ACL_PROGRAM, - .pg_nvers = NFSD_ACL_NRVERS, - .pg_vers = nfsd_acl_version, - .pg_name = "nfsacl", - .pg_class = "nfsd", - .pg_authenticate = &svc_set_client, - .pg_init_request = nfsd_acl_init_request, - .pg_rpcbind_set = nfsd_acl_rpcbind_set, -}; - #endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = { @@ -116,18 +104,29 @@ static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = { #endif }; -struct svc_program nfsd_program = { -#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) - .pg_next = &nfsd_acl_program, -#endif +struct svc_program nfsd_programs[] = { + { .pg_prog = NFS_PROGRAM, /* program number */ .pg_nvers = NFSD_MAXVERS+1, /* nr of entries in nfsd_version */ .pg_vers = nfsd_version, /* version table */ .pg_name = "nfsd", /* program name */ .pg_class = "nfsd", /* authentication class */ - .pg_authenticate = &svc_set_client, /* export authentication */ + .pg_authenticate = svc_set_client, /* export authentication */ .pg_init_request = nfsd_init_request, .pg_rpcbind_set = nfsd_rpcbind_set, + }, +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) + { + .pg_prog = NFS_ACL_PROGRAM, + .pg_nvers = NFSD_ACL_NRVERS, + .pg_vers = nfsd_acl_version, + .pg_name = "nfsacl", + .pg_class = "nfsd", + .pg_authenticate = svc_set_client, + .pg_init_request = nfsd_acl_init_request, + .pg_rpcbind_set = nfsd_acl_rpcbind_set, + }, +#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ }; bool nfsd_support_version(int vers) @@ -641,7 +640,8 @@ int nfsd_create_serv(struct net *net) if (nfsd_max_blksize == 0) nfsd_max_blksize = nfsd_get_default_max_blksize(); nfsd_reset_versions(nn); - serv = svc_create_pooled(&nfsd_program, &nn->nfsd_svcstats, + serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs), + &nn->nfsd_svcstats, nfsd_max_blksize, nfsd); if (serv == NULL) return -ENOMEM; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index c419a61f60e5..e68fecf6eab5 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -67,9 +67,10 @@ enum { * We currently do not support more than one RPC program per daemon. */ struct svc_serv { - struct svc_program * sv_program; /* RPC program */ + struct svc_program * sv_programs; /* RPC programs */ struct svc_stat * sv_stats; /* RPC statistics */ spinlock_t sv_lock; + unsigned int sv_nprogs; /* Number of sv_programs */ unsigned int sv_nrthreads; /* # of server threads */ unsigned int sv_maxconn; /* max connections allowed or * '0' causing max to be based @@ -360,10 +361,9 @@ struct svc_process_info { }; /* - * List of RPC programs on the same transport endpoint + * RPC program - an array of these can use the same transport endpoint */ struct svc_program { - struct svc_program * pg_next; /* other programs (same xprt) */ u32 pg_prog; /* program number */ unsigned int pg_lovers; /* lowest version */ unsigned int pg_hivers; /* highest version */ @@ -441,6 +441,7 @@ bool svc_rqst_replace_page(struct svc_rqst *rqstp, void svc_rqst_release_pages(struct svc_rqst *rqstp); void svc_exit_thread(struct svc_rqst *); struct svc_serv * svc_create_pooled(struct svc_program *prog, + unsigned int nprog, struct svc_stat *stats, unsigned int bufsize, int (*threadfn)(void *data)); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 9aff845196ce..7e7f4e0390c7 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -440,10 +440,11 @@ EXPORT_SYMBOL_GPL(svc_rpcb_cleanup); static int svc_uses_rpcbind(struct svc_serv *serv) { - struct svc_program *progp; - unsigned int i; + unsigned int p, i; + + for (p = 0; p < serv->sv_nprogs; p++) { + struct svc_program *progp = &serv->sv_programs[p]; - for (progp = serv->sv_program; progp; progp = progp->pg_next) { for (i = 0; i < progp->pg_nvers; i++) { if (progp->pg_vers[i] == NULL) continue; @@ -480,7 +481,7 @@ __svc_init_bc(struct svc_serv *serv) * Create an RPC service */ static struct svc_serv * -__svc_create(struct svc_program *prog, struct svc_stat *stats, +__svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, unsigned int bufsize, int npools, int (*threadfn)(void *data)) { struct svc_serv *serv; @@ -491,7 +492,8 @@ __svc_create(struct svc_program *prog, struct svc_stat *stats, if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL))) return NULL; serv->sv_name = prog->pg_name; - serv->sv_program = prog; + serv->sv_programs = prog; + serv->sv_nprogs = nprogs; serv->sv_stats = stats; if (bufsize > RPCSVC_MAXPAYLOAD) bufsize = RPCSVC_MAXPAYLOAD; @@ -499,17 +501,18 @@ __svc_create(struct svc_program *prog, struct svc_stat *stats, serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE); serv->sv_threadfn = threadfn; xdrsize = 0; - while (prog) { - prog->pg_lovers = prog->pg_nvers-1; - for (vers=0; verspg_nvers ; vers++) - if (prog->pg_vers[vers]) { - prog->pg_hivers = vers; - if (prog->pg_lovers > vers) - prog->pg_lovers = vers; - if (prog->pg_vers[vers]->vs_xdrsize > xdrsize) - xdrsize = prog->pg_vers[vers]->vs_xdrsize; + for (i = 0; i < nprogs; i++) { + struct svc_program *progp = &prog[i]; + + progp->pg_lovers = progp->pg_nvers-1; + for (vers = 0; vers < progp->pg_nvers ; vers++) + if (progp->pg_vers[vers]) { + progp->pg_hivers = vers; + if (progp->pg_lovers > vers) + progp->pg_lovers = vers; + if (progp->pg_vers[vers]->vs_xdrsize > xdrsize) + xdrsize = progp->pg_vers[vers]->vs_xdrsize; } - prog = prog->pg_next; } serv->sv_xdrsize = xdrsize; INIT_LIST_HEAD(&serv->sv_tempsocks); @@ -558,13 +561,14 @@ __svc_create(struct svc_program *prog, struct svc_stat *stats, struct svc_serv *svc_create(struct svc_program *prog, unsigned int bufsize, int (*threadfn)(void *data)) { - return __svc_create(prog, NULL, bufsize, 1, threadfn); + return __svc_create(prog, 1, NULL, bufsize, 1, threadfn); } EXPORT_SYMBOL_GPL(svc_create); /** * svc_create_pooled - Create an RPC service with pooled threads - * @prog: the RPC program the new service will handle + * @prog: Array of RPC programs the new service will handle + * @nprogs: Number of programs in the array * @stats: the stats struct if desired * @bufsize: maximum message size for @prog * @threadfn: a function to service RPC requests for @prog @@ -572,6 +576,7 @@ EXPORT_SYMBOL_GPL(svc_create); * Returns an instantiated struct svc_serv object or NULL. */ struct svc_serv *svc_create_pooled(struct svc_program *prog, + unsigned int nprogs, struct svc_stat *stats, unsigned int bufsize, int (*threadfn)(void *data)) @@ -579,7 +584,7 @@ struct svc_serv *svc_create_pooled(struct svc_program *prog, struct svc_serv *serv; unsigned int npools = svc_pool_map_get(); - serv = __svc_create(prog, stats, bufsize, npools, threadfn); + serv = __svc_create(prog, nprogs, stats, bufsize, npools, threadfn); if (!serv) goto out_err; serv->sv_is_pooled = true; @@ -602,16 +607,16 @@ svc_destroy(struct svc_serv **servp) *servp = NULL; - dprintk("svc: svc_destroy(%s)\n", serv->sv_program->pg_name); + dprintk("svc: svc_destroy(%s)\n", serv->sv_programs->pg_name); timer_shutdown_sync(&serv->sv_temptimer); /* * Remaining transports at this point are not expected. */ WARN_ONCE(!list_empty(&serv->sv_permsocks), - "SVC: permsocks remain for %s\n", serv->sv_program->pg_name); + "SVC: permsocks remain for %s\n", serv->sv_programs->pg_name); WARN_ONCE(!list_empty(&serv->sv_tempsocks), - "SVC: tempsocks remain for %s\n", serv->sv_program->pg_name); + "SVC: tempsocks remain for %s\n", serv->sv_programs->pg_name); cache_clean_deferred(serv); @@ -1148,15 +1153,16 @@ int svc_register(const struct svc_serv *serv, struct net *net, const int family, const unsigned short proto, const unsigned short port) { - struct svc_program *progp; - unsigned int i; + unsigned int p, i; int error = 0; WARN_ON_ONCE(proto == 0 && port == 0); if (proto == 0 && port == 0) return -EINVAL; - for (progp = serv->sv_program; progp; progp = progp->pg_next) { + for (p = 0; p < serv->sv_nprogs; p++) { + struct svc_program *progp = &serv->sv_programs[p]; + for (i = 0; i < progp->pg_nvers; i++) { error = progp->pg_rpcbind_set(net, progp, i, @@ -1208,13 +1214,14 @@ static void __svc_unregister(struct net *net, const u32 program, const u32 versi static void svc_unregister(const struct svc_serv *serv, struct net *net) { struct sighand_struct *sighand; - struct svc_program *progp; unsigned long flags; - unsigned int i; + unsigned int p, i; clear_thread_flag(TIF_SIGPENDING); - for (progp = serv->sv_program; progp; progp = progp->pg_next) { + for (p = 0; p < serv->sv_nprogs; p++) { + struct svc_program *progp = &serv->sv_programs[p]; + for (i = 0; i < progp->pg_nvers; i++) { if (progp->pg_vers[i] == NULL) continue; @@ -1320,7 +1327,7 @@ svc_process_common(struct svc_rqst *rqstp) struct svc_process_info process; enum svc_auth_status auth_res; unsigned int aoffset; - int rc; + int pr, rc; __be32 *p; /* Will be turned off only when NFSv4 Sessions are used */ @@ -1344,9 +1351,12 @@ svc_process_common(struct svc_rqst *rqstp) rqstp->rq_vers = be32_to_cpup(p++); rqstp->rq_proc = be32_to_cpup(p); - for (progp = serv->sv_program; progp; progp = progp->pg_next) + for (pr = 0; pr < serv->sv_nprogs; pr++) { + progp = &serv->sv_programs[pr]; + if (rqstp->rq_prog == progp->pg_prog) break; + } /* * Decode auth data, and add verifier to reply buffer. diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 53ebc719ff5a..43c57124de52 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -268,7 +268,7 @@ static int _svc_xprt_create(struct svc_serv *serv, const char *xprt_name, spin_unlock(&svc_xprt_class_lock); newxprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags); if (IS_ERR(newxprt)) { - trace_svc_xprt_create_err(serv->sv_program->pg_name, + trace_svc_xprt_create_err(serv->sv_programs->pg_name, xcl->xcl_name, sap, len, newxprt); module_put(xcl->xcl_owner); diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 04b45588ae6f..8ca98b146ec8 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -697,7 +697,8 @@ svcauth_unix_set_client(struct svc_rqst *rqstp) rqstp->rq_auth_stat = rpc_autherr_badcred; ipm = ip_map_cached_get(xprt); if (ipm == NULL) - ipm = __ip_map_lookup(sn->ip_map_cache, rqstp->rq_server->sv_program->pg_class, + ipm = __ip_map_lookup(sn->ip_map_cache, + rqstp->rq_server->sv_programs->pg_class, &sin6->sin6_addr); if (ipm == NULL) From 2a33a85be45178198245e1f656e6224c899895e4 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:48 -0400 Subject: [PATCH 097/111] nfs_common: add NFS LOCALIO auxiliary protocol enablement fs/nfs_common/nfslocalio.c provides interfaces that enable an NFS client to generate a nonce (single-use UUID) and associated nfs_uuid_t struct, register it with nfs_common for subsequent lookup and verification by the NFS server and if matched the NFS server populates members in the nfs_uuid_t struct. nfs_common's nfs_uuids list is the basis for localio enablement, as such it has members that point to nfsd memory for direct use by the client (e.g. 'net' is the server's network namespace, through it the client can access nn->nfsd_serv). This commit also provides the base nfs_uuid_t interfaces to allow proper net namespace refcounting for the LOCALIO use case. CONFIG_NFS_LOCALIO controls the nfs_common, NFS server and NFS client enablement for LOCALIO. If both NFS_FS=m and NFSD=m then NFS_COMMON_LOCALIO_SUPPORT=m and nfs_localio.ko is built (and provides nfs_common's LOCALIO support). # lsmod | grep nfs_localio nfs_localio 12288 2 nfsd,nfs sunrpc 745472 35 nfs_localio,nfsd,auth_rpcgss,lockd,nfsv3,nfs Signed-off-by: Mike Snitzer Co-developed-by: NeilBrown Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/Kconfig | 23 ++++++++ fs/nfs_common/Makefile | 3 + fs/nfs_common/nfslocalio.c | 116 +++++++++++++++++++++++++++++++++++++ include/linux/nfslocalio.h | 36 ++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 fs/nfs_common/nfslocalio.c create mode 100644 include/linux/nfslocalio.h diff --git a/fs/Kconfig b/fs/Kconfig index a46b0cbc4d8f..24d4e4b419d1 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -382,6 +382,29 @@ config NFS_COMMON depends on NFSD || NFS_FS || LOCKD default y +config NFS_COMMON_LOCALIO_SUPPORT + tristate + default n + default y if NFSD=y || NFS_FS=y + default m if NFSD=m && NFS_FS=m + select SUNRPC + +config NFS_LOCALIO + bool "NFS client and server support for LOCALIO auxiliary protocol" + depends on NFSD && NFS_FS + select NFS_COMMON_LOCALIO_SUPPORT + default n + help + Some NFS servers support an auxiliary NFS LOCALIO protocol + that is not an official part of the NFS protocol. + + This option enables support for the LOCALIO protocol in the + kernel's NFS server and client. Enable this to permit local + NFS clients to bypass the network when issuing reads and + writes to the local NFS server. + + If unsure, say N. + config NFS_V4_2_SSC_HELPER bool default y if NFS_V4_2 diff --git a/fs/nfs_common/Makefile b/fs/nfs_common/Makefile index e58b01bb8dda..a5e54809701e 100644 --- a/fs/nfs_common/Makefile +++ b/fs/nfs_common/Makefile @@ -6,6 +6,9 @@ obj-$(CONFIG_NFS_ACL_SUPPORT) += nfs_acl.o nfs_acl-objs := nfsacl.o +obj-$(CONFIG_NFS_COMMON_LOCALIO_SUPPORT) += nfs_localio.o +nfs_localio-objs := nfslocalio.o + obj-$(CONFIG_GRACE_PERIOD) += grace.o obj-$(CONFIG_NFS_V4_2_SSC_HELPER) += nfs_ssc.o diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c new file mode 100644 index 000000000000..f0bff023bb5e --- /dev/null +++ b/fs/nfs_common/nfslocalio.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("NFS localio protocol bypass support"); + +static DEFINE_SPINLOCK(nfs_uuid_lock); + +/* + * Global list of nfs_uuid_t instances + * that is protected by nfs_uuid_lock. + */ +static LIST_HEAD(nfs_uuids); + +void nfs_uuid_begin(nfs_uuid_t *nfs_uuid) +{ + nfs_uuid->net = NULL; + nfs_uuid->dom = NULL; + uuid_gen(&nfs_uuid->uuid); + + spin_lock(&nfs_uuid_lock); + list_add_tail_rcu(&nfs_uuid->list, &nfs_uuids); + spin_unlock(&nfs_uuid_lock); +} +EXPORT_SYMBOL_GPL(nfs_uuid_begin); + +void nfs_uuid_end(nfs_uuid_t *nfs_uuid) +{ + if (nfs_uuid->net == NULL) { + spin_lock(&nfs_uuid_lock); + list_del_init(&nfs_uuid->list); + spin_unlock(&nfs_uuid_lock); + } +} +EXPORT_SYMBOL_GPL(nfs_uuid_end); + +static nfs_uuid_t * nfs_uuid_lookup_locked(const uuid_t *uuid) +{ + nfs_uuid_t *nfs_uuid; + + list_for_each_entry(nfs_uuid, &nfs_uuids, list) + if (uuid_equal(&nfs_uuid->uuid, uuid)) + return nfs_uuid; + + return NULL; +} + +static struct module *nfsd_mod; + +void nfs_uuid_is_local(const uuid_t *uuid, struct list_head *list, + struct net *net, struct auth_domain *dom, + struct module *mod) +{ + nfs_uuid_t *nfs_uuid; + + spin_lock(&nfs_uuid_lock); + nfs_uuid = nfs_uuid_lookup_locked(uuid); + if (nfs_uuid) { + kref_get(&dom->ref); + nfs_uuid->dom = dom; + /* + * We don't hold a ref on the net, but instead put + * ourselves on a list so the net pointer can be + * invalidated. + */ + list_move(&nfs_uuid->list, list); + nfs_uuid->net = net; + + __module_get(mod); + nfsd_mod = mod; + } + spin_unlock(&nfs_uuid_lock); +} +EXPORT_SYMBOL_GPL(nfs_uuid_is_local); + +static void nfs_uuid_put_locked(nfs_uuid_t *nfs_uuid) +{ + if (nfs_uuid->net) { + module_put(nfsd_mod); + nfs_uuid->net = NULL; + } + if (nfs_uuid->dom) { + auth_domain_put(nfs_uuid->dom); + nfs_uuid->dom = NULL; + } + list_del_init(&nfs_uuid->list); +} + +void nfs_uuid_invalidate_clients(struct list_head *list) +{ + nfs_uuid_t *nfs_uuid, *tmp; + + spin_lock(&nfs_uuid_lock); + list_for_each_entry_safe(nfs_uuid, tmp, list, list) + nfs_uuid_put_locked(nfs_uuid); + spin_unlock(&nfs_uuid_lock); +} +EXPORT_SYMBOL_GPL(nfs_uuid_invalidate_clients); + +void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid) +{ + if (nfs_uuid->net) { + spin_lock(&nfs_uuid_lock); + nfs_uuid_put_locked(nfs_uuid); + spin_unlock(&nfs_uuid_lock); + } +} +EXPORT_SYMBOL_GPL(nfs_uuid_invalidate_one_client); diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h new file mode 100644 index 000000000000..4165ff8390c1 --- /dev/null +++ b/include/linux/nfslocalio.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ +#ifndef __LINUX_NFSLOCALIO_H +#define __LINUX_NFSLOCALIO_H + +#include +#include +#include +#include +#include +#include + +/* + * Useful to allow a client to negotiate if localio + * possible with its server. + * + * See Documentation/filesystems/nfs/localio.rst for more detail. + */ +typedef struct { + uuid_t uuid; + struct list_head list; + struct net *net; /* nfsd's network namespace */ + struct auth_domain *dom; /* auth_domain for localio */ +} nfs_uuid_t; + +void nfs_uuid_begin(nfs_uuid_t *); +void nfs_uuid_end(nfs_uuid_t *); +void nfs_uuid_is_local(const uuid_t *, struct list_head *, + struct net *, struct auth_domain *, struct module *); +void nfs_uuid_invalidate_clients(struct list_head *list); +void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid); + +#endif /* __LINUX_NFSLOCALIO_H */ From a61e147e6be6e763d9c6dec8061d2893c0bb3423 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:49 -0400 Subject: [PATCH 098/111] nfs_common: prepare for the NFS client to use nfsd_file for LOCALIO The next commit will introduce nfsd_open_local_fh() which returns an nfsd_file structure. This commit exposes LOCALIO's required NFSD symbols to the NFS client: - Make nfsd_open_local_fh() symbol and other required NFSD symbols available to NFS in a global 'nfs_to' nfsd_localio_operations struct (global access suggested by Trond, nfsd_localio_operations suggested by NeilBrown). The next commit will also introduce nfsd_localio_ops_init() that init_nfsd() will call to initialize 'nfs_to'. - Introduce nfsd_file_file() that provides access to nfsd_file's backing file. Keeps nfsd_file structure opaque to NFS client (as suggested by Jeff Layton). - Introduce nfsd_file_put_local() that will put the reference to the nfsd_file's associated nn->nfsd_serv and then put the reference to the nfsd_file (as suggested by NeilBrown). Suggested-by: Trond Myklebust # nfs_to Suggested-by: NeilBrown # nfsd_localio_operations Suggested-by: Jeff Layton # nfsd_file_file Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs_common/nfslocalio.c | 25 ++++++++++++++++++++++++- fs/nfsd/filecache.c | 28 ++++++++++++++++++++++++++++ fs/nfsd/filecache.h | 2 ++ fs/nfsd/nfssvc.c | 1 + include/linux/nfslocalio.h | 27 ++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index f0bff023bb5e..f61761ad19b1 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -72,7 +72,7 @@ void nfs_uuid_is_local(const uuid_t *uuid, struct list_head *list, * invalidated. */ list_move(&nfs_uuid->list, list); - nfs_uuid->net = net; + rcu_assign_pointer(nfs_uuid->net, net); __module_get(mod); nfsd_mod = mod; @@ -114,3 +114,26 @@ void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid) } } EXPORT_SYMBOL_GPL(nfs_uuid_invalidate_one_client); + +/* + * The NFS LOCALIO code needs to call into NFSD using various symbols, + * but cannot be statically linked, because that will make the NFS + * module always depend on the NFSD module. + * + * 'nfs_to' provides NFS access to NFSD functions needed for LOCALIO, + * its lifetime is tightly coupled to the NFSD module and will always + * be available to NFS LOCALIO because any successful client<->server + * LOCALIO handshake results in a reference on the NFSD module (above), + * so NFS implicitly holds a reference to the NFSD module and its + * functions in the 'nfs_to' nfsd_localio_operations cannot disappear. + * + * If the last NFS client using LOCALIO disconnects (and its reference + * on NFSD dropped) then NFSD could be unloaded, resulting in 'nfs_to' + * functions being invalid pointers. But if NFSD isn't loaded then NFS + * will not be able to handshake with NFSD and will have no cause to + * try to call 'nfs_to' function pointers. If/when NFSD is reloaded it + * will reinitialize the 'nfs_to' function pointers and make LOCALIO + * possible. + */ +const struct nfsd_localio_operations *nfs_to; +EXPORT_SYMBOL_GPL(nfs_to); diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index caeffc43ca4b..414b5bf738df 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -390,6 +390,34 @@ nfsd_file_put(struct nfsd_file *nf) nfsd_file_free(nf); } +/** + * nfsd_file_put_local - put the reference to nfsd_file and local nfsd_serv + * @nf: nfsd_file of which to put the references + * + * First put the reference of the nfsd_file and then put the + * reference to the associated nn->nfsd_serv. + */ +void +nfsd_file_put_local(struct nfsd_file *nf) +{ + struct net *net = nf->nf_net; + + nfsd_file_put(nf); + nfsd_serv_put(net); +} + +/** + * nfsd_file_file - get the backing file of an nfsd_file + * @nf: nfsd_file of which to access the backing file. + * + * Return backing file for @nf. + */ +struct file * +nfsd_file_file(struct nfsd_file *nf) +{ + return nf->nf_file; +} + static void nfsd_file_dispose_list(struct list_head *dispose) { diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index 26ada78b8c1e..cadf3c2689c4 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -55,7 +55,9 @@ void nfsd_file_cache_shutdown(void); int nfsd_file_cache_start_net(struct net *net); void nfsd_file_cache_shutdown_net(struct net *net); void nfsd_file_put(struct nfsd_file *nf); +void nfsd_file_put_local(struct nfsd_file *nf); struct nfsd_file *nfsd_file_get(struct nfsd_file *nf); +struct file *nfsd_file_file(struct nfsd_file *nf); void nfsd_file_close_inode_sync(struct inode *inode); void nfsd_file_net_dispose(struct nfsd_net *nn); bool nfsd_file_is_cached(struct inode *inode); diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index fa34de5e59bd..f9a6d888ac9d 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 4165ff8390c1..1a39d7d727bd 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -22,7 +23,7 @@ typedef struct { uuid_t uuid; struct list_head list; - struct net *net; /* nfsd's network namespace */ + struct net __rcu *net; /* nfsd's network namespace */ struct auth_domain *dom; /* auth_domain for localio */ } nfs_uuid_t; @@ -33,4 +34,28 @@ void nfs_uuid_is_local(const uuid_t *, struct list_head *, void nfs_uuid_invalidate_clients(struct list_head *list); void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid); +struct nfsd_file; + +/* localio needs to map filehandle -> struct nfsd_file */ +extern struct nfsd_file * +nfsd_open_local_fh(struct net *, struct auth_domain *, struct rpc_clnt *, + const struct cred *, const struct nfs_fh *, + const fmode_t) __must_hold(rcu); + +struct nfsd_localio_operations { + bool (*nfsd_serv_try_get)(struct net *); + void (*nfsd_serv_put)(struct net *); + struct nfsd_file *(*nfsd_open_local_fh)(struct net *, + struct auth_domain *, + struct rpc_clnt *, + const struct cred *, + const struct nfs_fh *, + const fmode_t); + void (*nfsd_file_put_local)(struct nfsd_file *); + struct file *(*nfsd_file_file)(struct nfsd_file *); +} ____cacheline_aligned; + +extern void nfsd_localio_ops_init(void); +extern const struct nfsd_localio_operations *nfs_to; + #endif /* __LINUX_NFSLOCALIO_H */ From fa4983862e506d395acc1b8d14dbebf63acc2e82 Mon Sep 17 00:00:00 2001 From: Weston Andros Adamson Date: Thu, 5 Sep 2024 15:09:50 -0400 Subject: [PATCH 099/111] nfsd: add LOCALIO support Add server support for bypassing NFS for localhost reads, writes, and commits. This is only useful when both the client and server are running on the same host. If nfsd_open_local_fh() fails then the NFS client will both retry and fallback to normal network-based read, write and commit operations if localio is no longer supported. Care is taken to ensure the same NFS security mechanisms are used (authentication, etc) regardless of whether localio or regular NFS access is used. The auth_domain established as part of the traditional NFS client access to the NFS server is also used for localio. Store auth_domain for localio in nfsd_uuid_t and transfer it to the client if it is local to the server. Relative to containers, localio gives the client access to the network namespace the server has. This is required to allow the client to access the server's per-namespace nfsd_net struct. This commit also introduces the use of NFSD's percpu_ref to interlock nfsd_destroy_serv and nfsd_open_local_fh, to ensure nn->nfsd_serv is not destroyed while in use by nfsd_open_local_fh and other LOCALIO client code. CONFIG_NFS_LOCALIO enables NFS server support for LOCALIO. Signed-off-by: Weston Andros Adamson Signed-off-by: Trond Myklebust Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Co-developed-by: NeilBrown Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Acked-by: Chuck Lever Signed-off-by: Anna Schumaker --- fs/nfsd/Makefile | 1 + fs/nfsd/filecache.c | 2 +- fs/nfsd/localio.c | 92 ++++++++++++++++++++++++++++++++++++++ fs/nfsd/netns.h | 4 ++ fs/nfsd/nfsctl.c | 25 ++++++++++- fs/nfsd/trace.h | 3 +- fs/nfsd/vfs.h | 2 + include/linux/nfslocalio.h | 8 ++++ 8 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 fs/nfsd/localio.c diff --git a/fs/nfsd/Makefile b/fs/nfsd/Makefile index b8736a82e57c..18cbd3fa7691 100644 --- a/fs/nfsd/Makefile +++ b/fs/nfsd/Makefile @@ -23,3 +23,4 @@ nfsd-$(CONFIG_NFSD_PNFS) += nfs4layouts.o nfsd-$(CONFIG_NFSD_BLOCKLAYOUT) += blocklayout.o blocklayoutxdr.o nfsd-$(CONFIG_NFSD_SCSILAYOUT) += blocklayout.o blocklayoutxdr.o nfsd-$(CONFIG_NFSD_FLEXFILELAYOUT) += flexfilelayout.o flexfilelayoutxdr.o +nfsd-$(CONFIG_NFS_LOCALIO) += localio.o diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 414b5bf738df..19bb88c7eebd 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -52,7 +52,7 @@ #define NFSD_FILE_CACHE_UP (0) /* We only care about NFSD_MAY_READ/WRITE for this cache */ -#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE) +#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c new file mode 100644 index 000000000000..9fa10a80b2da --- /dev/null +++ b/fs/nfsd/localio.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NFS server support for local clients to bypass network stack + * + * Copyright (C) 2014 Weston Andros Adamson + * Copyright (C) 2019 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nfsd.h" +#include "vfs.h" +#include "netns.h" +#include "filecache.h" + +static const struct nfsd_localio_operations nfsd_localio_ops = { + .nfsd_serv_try_get = nfsd_serv_try_get, + .nfsd_serv_put = nfsd_serv_put, + .nfsd_open_local_fh = nfsd_open_local_fh, + .nfsd_file_put_local = nfsd_file_put_local, + .nfsd_file_file = nfsd_file_file, +}; + +void nfsd_localio_ops_init(void) +{ + nfs_to = &nfsd_localio_ops; +} + +/** + * nfsd_open_local_fh - lookup a local filehandle @nfs_fh and map to nfsd_file + * + * @net: 'struct net' to get the proper nfsd_net required for LOCALIO access + * @dom: 'struct auth_domain' required for LOCALIO access + * @rpc_clnt: rpc_clnt that the client established + * @cred: cred that the client established + * @nfs_fh: filehandle to lookup + * @fmode: fmode_t to use for open + * + * This function maps a local fh to a path on a local filesystem. + * This is useful when the nfs client has the local server mounted - it can + * avoid all the NFS overhead with reads, writes and commits. + * + * On successful return, returned nfsd_file will have its nf_net member + * set. Caller (NFS client) is responsible for calling nfsd_serv_put and + * nfsd_file_put (via nfs_to->nfsd_file_put_local). + */ +struct nfsd_file * +nfsd_open_local_fh(struct net *net, struct auth_domain *dom, + struct rpc_clnt *rpc_clnt, const struct cred *cred, + const struct nfs_fh *nfs_fh, const fmode_t fmode) +{ + int mayflags = NFSD_MAY_LOCALIO; + struct svc_cred rq_cred; + struct svc_fh fh; + struct nfsd_file *localio; + __be32 beres; + + if (nfs_fh->size > NFS4_FHSIZE) + return ERR_PTR(-EINVAL); + + /* nfs_fh -> svc_fh */ + fh_init(&fh, NFS4_FHSIZE); + fh.fh_handle.fh_size = nfs_fh->size; + memcpy(fh.fh_handle.fh_raw, nfs_fh->data, nfs_fh->size); + + if (fmode & FMODE_READ) + mayflags |= NFSD_MAY_READ; + if (fmode & FMODE_WRITE) + mayflags |= NFSD_MAY_WRITE; + + svcauth_map_clnt_to_svc_cred_local(rpc_clnt, cred, &rq_cred); + + beres = nfsd_file_acquire_local(net, &rq_cred, dom, + &fh, mayflags, &localio); + if (beres) + localio = ERR_PTR(nfs_stat_to_errno(be32_to_cpu(beres))); + + fh_put(&fh); + if (rq_cred.cr_group_info) + put_group_info(rq_cred.cr_group_info); + + return localio; +} +EXPORT_SYMBOL_GPL(nfsd_open_local_fh); diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index a228b9d6a6e1..26f7b34d1a03 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -217,6 +217,10 @@ struct nfsd_net { /* last time an admin-revoke happened for NFSv4.0 */ time64_t nfs40_last_revoke; +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + /* Local clients to be invalidated when net is shut down */ + struct list_head local_clients; +#endif }; /* Simple check to find out if a given net was properly initialized */ diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 64c1b4d649bc..3adbc05ebaac 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "idmap.h" #include "nfsd.h" @@ -2257,7 +2258,9 @@ static __net_init int nfsd_net_init(struct net *net) get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key)); seqlock_init(&nn->writeverf_lock); nfsd_proc_stat_init(net); - +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + INIT_LIST_HEAD(&nn->local_clients); +#endif return 0; out_repcache_error: @@ -2268,6 +2271,22 @@ out_export_error: return retval; } +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +/** + * nfsd_net_pre_exit - Disconnect localio clients from net namespace + * @net: a network namespace that is about to be destroyed + * + * This invalidated ->net pointers held by localio clients + * while they can still safely access nn->counter. + */ +static __net_exit void nfsd_net_pre_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs_uuid_invalidate_clients(&nn->local_clients); +} +#endif + /** * nfsd_net_exit - Release the nfsd_net portion of a net namespace * @net: a network namespace that is about to be destroyed @@ -2285,6 +2304,9 @@ static __net_exit void nfsd_net_exit(struct net *net) static struct pernet_operations nfsd_net_ops = { .init = nfsd_net_init, +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + .pre_exit = nfsd_net_pre_exit, +#endif .exit = nfsd_net_exit, .id = &nfsd_net_id, .size = sizeof(struct nfsd_net), @@ -2322,6 +2344,7 @@ static int __init init_nfsd(void) retval = genl_register_family(&nfsd_nl_family); if (retval) goto out_free_all; + nfsd_localio_ops_init(); return 0; out_free_all: diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index e339d04ff32f..c625966cfcf3 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -86,7 +86,8 @@ DEFINE_NFSD_XDR_ERR_EVENT(cant_encode); { NFSD_MAY_NOT_BREAK_LEASE, "NOT_BREAK_LEASE" }, \ { NFSD_MAY_BYPASS_GSS, "BYPASS_GSS" }, \ { NFSD_MAY_READ_IF_EXEC, "READ_IF_EXEC" }, \ - { NFSD_MAY_64BIT_COOKIE, "64BIT_COOKIE" }) + { NFSD_MAY_64BIT_COOKIE, "64BIT_COOKIE" }, \ + { NFSD_MAY_LOCALIO, "LOCALIO" }) TRACE_EVENT(nfsd_compound, TP_PROTO( diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 01947561d375..3ff146522556 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -33,6 +33,8 @@ #define NFSD_MAY_64BIT_COOKIE 0x1000 /* 64 bit readdir cookies for >= NFSv3 */ +#define NFSD_MAY_LOCALIO 0x2000 /* for tracing, reflects when localio used */ + #define NFSD_MAY_CREATE (NFSD_MAY_EXEC|NFSD_MAY_WRITE) #define NFSD_MAY_REMOVE (NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC) diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 1a39d7d727bd..546f46b4e46f 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -6,6 +6,8 @@ #ifndef __LINUX_NFSLOCALIO_H #define __LINUX_NFSLOCALIO_H +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + #include #include #include @@ -58,4 +60,10 @@ struct nfsd_localio_operations { extern void nfsd_localio_ops_init(void); extern const struct nfsd_localio_operations *nfs_to; +#else /* CONFIG_NFS_LOCALIO */ +static inline void nfsd_localio_ops_init(void) +{ +} +#endif /* CONFIG_NFS_LOCALIO */ + #endif /* __LINUX_NFSLOCALIO_H */ From 946af9b3a0e7571c01447107d5e8968401e659ba Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:51 -0400 Subject: [PATCH 100/111] nfsd: implement server support for NFS_LOCALIO_PROGRAM The LOCALIO auxiliary RPC protocol consists of a single "UUID_IS_LOCAL" RPC method that allows the Linux NFS client to verify the local Linux NFS server can see the nonce (single-use UUID) the client generated and made available in nfs_common. The server expects this protocol to use the same transport as NFS and NFSACL for its RPCs. This protocol isn't part of an IETF standard, nor does it need to be considering it is Linux-to-Linux auxiliary RPC protocol that amounts to an implementation detail. The UUID_IS_LOCAL method encodes the client generated uuid_t in terms of the fixed UUID_SIZE (16 bytes). The fixed size opaque encode and decode XDR methods are used instead of the less efficient variable sized methods. The RPC program number for the NFS_LOCALIO_PROGRAM is 400122 (as assigned by IANA, see https://www.iana.org/assignments/rpc-program-numbers/ ): Linux Kernel Organization 400122 nfslocalio Signed-off-by: Mike Snitzer [neilb: factored out and simplified single localio protocol] Co-developed-by: NeilBrown Signed-off-by: NeilBrown Acked-by: Chuck Lever Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfsd/localio.c | 77 +++++++++++++++++++++++++++++++++++++++++++++ fs/nfsd/nfsd.h | 4 +++ fs/nfsd/nfssvc.c | 23 +++++++++++++- include/linux/nfs.h | 7 +++++ 4 files changed, 110 insertions(+), 1 deletion(-) diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c index 9fa10a80b2da..291e9c69cae4 100644 --- a/fs/nfsd/localio.c +++ b/fs/nfsd/localio.c @@ -14,12 +14,15 @@ #include #include #include +#include +#include #include #include "nfsd.h" #include "vfs.h" #include "netns.h" #include "filecache.h" +#include "cache.h" static const struct nfsd_localio_operations nfsd_localio_ops = { .nfsd_serv_try_get = nfsd_serv_try_get, @@ -90,3 +93,77 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, return localio; } EXPORT_SYMBOL_GPL(nfsd_open_local_fh); + +/* + * UUID_IS_LOCAL XDR functions + */ + +static __be32 localio_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +struct localio_uuidarg { + uuid_t uuid; +}; + +static __be32 localio_proc_uuid_is_local(struct svc_rqst *rqstp) +{ + struct localio_uuidarg *argp = rqstp->rq_argp; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs_uuid_is_local(&argp->uuid, &nn->local_clients, + net, rqstp->rq_client, THIS_MODULE); + + return rpc_success; +} + +static bool localio_decode_uuidarg(struct svc_rqst *rqstp, + struct xdr_stream *xdr) +{ + struct localio_uuidarg *argp = rqstp->rq_argp; + u8 uuid[UUID_SIZE]; + + if (decode_opaque_fixed(xdr, uuid, UUID_SIZE)) + return false; + import_uuid(&argp->uuid, uuid); + + return true; +} + +static const struct svc_procedure localio_procedures1[] = { + [LOCALIOPROC_NULL] = { + .pc_func = localio_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 0, + .pc_name = "NULL", + }, + [LOCALIOPROC_UUID_IS_LOCAL] = { + .pc_func = localio_proc_uuid_is_local, + .pc_decode = localio_decode_uuidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct localio_uuidarg), + .pc_argzero = sizeof(struct localio_uuidarg), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_name = "UUID_IS_LOCAL", + }, +}; + +#define LOCALIO_NR_PROCEDURES ARRAY_SIZE(localio_procedures1) +static DEFINE_PER_CPU_ALIGNED(unsigned long, + localio_count[LOCALIO_NR_PROCEDURES]); +const struct svc_version localio_version1 = { + .vs_vers = 1, + .vs_nproc = LOCALIO_NR_PROCEDURES, + .vs_proc = localio_procedures1, + .vs_dispatch = nfsd_dispatch, + .vs_count = localio_count, + .vs_xdrsize = XDR_QUADLEN(UUID_SIZE), + .vs_hidden = true, +}; diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index b0d3e82d6dcd..4b56ba1e8e48 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -146,6 +146,10 @@ extern const struct svc_version nfsd_acl_version3; #endif #endif +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +extern const struct svc_version localio_version1; +#endif + struct nfsd_net; enum vers_op {NFSD_SET, NFSD_CLEAR, NFSD_TEST, NFSD_AVAIL }; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index f9a6d888ac9d..e236135ddc63 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -80,6 +80,15 @@ DEFINE_SPINLOCK(nfsd_drc_lock); unsigned long nfsd_drc_max_mem; unsigned long nfsd_drc_mem_used; +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +static const struct svc_version *localio_versions[] = { + [1] = &localio_version1, +}; + +#define NFSD_LOCALIO_NRVERS ARRAY_SIZE(localio_versions) + +#endif /* CONFIG_NFS_LOCALIO */ + #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) static const struct svc_version *nfsd_acl_version[] = { # if defined(CONFIG_NFSD_V2_ACL) @@ -128,6 +137,18 @@ struct svc_program nfsd_programs[] = { .pg_rpcbind_set = nfsd_acl_rpcbind_set, }, #endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + { + .pg_prog = NFS_LOCALIO_PROGRAM, + .pg_nvers = NFSD_LOCALIO_NRVERS, + .pg_vers = localio_versions, + .pg_name = "nfslocalio", + .pg_class = "nfsd", + .pg_authenticate = svc_set_client, + .pg_init_request = svc_generic_init_request, + .pg_rpcbind_set = svc_generic_rpcbind_set, + } +#endif /* CONFIG_NFS_LOCALIO */ }; bool nfsd_support_version(int vers) @@ -949,7 +970,7 @@ nfsd(void *vrqstp) } /** - * nfsd_dispatch - Process an NFS or NFSACL Request + * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request * @rqstp: incoming request * * This RPC dispatcher integrates the NFS server's duplicate reply cache. diff --git a/include/linux/nfs.h b/include/linux/nfs.h index ceb70a926b95..73da75908d95 100644 --- a/include/linux/nfs.h +++ b/include/linux/nfs.h @@ -13,6 +13,13 @@ #include #include +/* The LOCALIO program is entirely private to Linux and is + * NOT part of the uapi. + */ +#define NFS_LOCALIO_PROGRAM 400122 +#define LOCALIOPROC_NULL 0 +#define LOCALIOPROC_UUID_IS_LOCAL 1 + /* * This is the kernel NFS client file handle representation */ From df24c483e28f7f9a421afde15d0497e61bc2d3ea Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:52 -0400 Subject: [PATCH 101/111] nfs: pass struct nfsd_file to nfs_init_pgio and nfs_init_commit The nfsd_file will be passed, in future commits, by callers that enable LOCALIO support (for both regular NFS and pNFS IO). [Derived from patch authored by Weston Andros Adamson, but switched from passing struct file to struct nfsd_file] Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/filelayout/filelayout.c | 6 +++--- fs/nfs/flexfilelayout/flexfilelayout.c | 6 +++--- fs/nfs/internal.h | 7 +++++-- fs/nfs/pagelist.c | 6 ++++-- fs/nfs/pnfs_nfs.c | 2 +- fs/nfs/write.c | 5 +++-- include/linux/nfslocalio.h | 5 +++-- 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c index b6e9aeaf4ce2..d39a1f58e18d 100644 --- a/fs/nfs/filelayout/filelayout.c +++ b/fs/nfs/filelayout/filelayout.c @@ -488,7 +488,7 @@ filelayout_read_pagelist(struct nfs_pgio_header *hdr) /* Perform an asynchronous read to ds */ nfs_initiate_pgio(ds_clnt, hdr, hdr->cred, NFS_PROTO(hdr->inode), &filelayout_read_call_ops, - 0, RPC_TASK_SOFTCONN); + 0, RPC_TASK_SOFTCONN, NULL); return PNFS_ATTEMPTED; } @@ -530,7 +530,7 @@ filelayout_write_pagelist(struct nfs_pgio_header *hdr, int sync) /* Perform an asynchronous write */ nfs_initiate_pgio(ds_clnt, hdr, hdr->cred, NFS_PROTO(hdr->inode), &filelayout_write_call_ops, - sync, RPC_TASK_SOFTCONN); + sync, RPC_TASK_SOFTCONN, NULL); return PNFS_ATTEMPTED; } @@ -1011,7 +1011,7 @@ static int filelayout_initiate_commit(struct nfs_commit_data *data, int how) data->args.fh = fh; return nfs_initiate_commit(ds_clnt, data, NFS_PROTO(data->inode), &filelayout_commit_call_ops, how, - RPC_TASK_SOFTCONN); + RPC_TASK_SOFTCONN, NULL); out_err: pnfs_generic_prepare_to_resend_writes(data); pnfs_generic_commit_release(data); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index d4d551ffea7b..01ee52551a63 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -1806,7 +1806,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_read_call_ops_v3 : &ff_layout_read_call_ops_v4, - 0, RPC_TASK_SOFTCONN); + 0, RPC_TASK_SOFTCONN, NULL); put_cred(ds_cred); return PNFS_ATTEMPTED; @@ -1874,7 +1874,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_write_call_ops_v3 : &ff_layout_write_call_ops_v4, - sync, RPC_TASK_SOFTCONN); + sync, RPC_TASK_SOFTCONN, NULL); put_cred(ds_cred); return PNFS_ATTEMPTED; @@ -1949,7 +1949,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_commit_call_ops_v3 : &ff_layout_commit_call_ops_v4, - how, RPC_TASK_SOFTCONN); + how, RPC_TASK_SOFTCONN, NULL); put_cred(ds_cred); return ret; out_err: diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index b3dc7c84eef9..f0af96a665d2 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #define NFS_SB_MASK (SB_RDONLY|SB_NOSUID|SB_NODEV|SB_NOEXEC|SB_SYNCHRONOUS) @@ -308,7 +309,8 @@ void nfs_pgio_header_free(struct nfs_pgio_header *); int nfs_generic_pgio(struct nfs_pageio_descriptor *, struct nfs_pgio_header *); int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, const struct cred *cred, const struct nfs_rpc_ops *rpc_ops, - const struct rpc_call_ops *call_ops, int how, int flags); + const struct rpc_call_ops *call_ops, int how, int flags, + struct nfsd_file *localio); void nfs_free_request(struct nfs_page *req); struct nfs_pgio_mirror * nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc); @@ -527,7 +529,8 @@ extern int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, const struct nfs_rpc_ops *nfs_ops, const struct rpc_call_ops *call_ops, - int how, int flags); + int how, int flags, + struct nfsd_file *localio); extern void nfs_init_commit(struct nfs_commit_data *data, struct list_head *head, struct pnfs_layout_segment *lseg, diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 04124f226665..50f3d6c9ac2a 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -731,7 +731,8 @@ static void nfs_pgio_prepare(struct rpc_task *task, void *calldata) int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, const struct cred *cred, const struct nfs_rpc_ops *rpc_ops, - const struct rpc_call_ops *call_ops, int how, int flags) + const struct rpc_call_ops *call_ops, int how, int flags, + struct nfsd_file *localio) { struct rpc_task *task; struct rpc_message msg = { @@ -961,7 +962,8 @@ static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) NFS_PROTO(hdr->inode), desc->pg_rpc_callops, desc->pg_ioflags, - RPC_TASK_CRED_NOREF | task_flags); + RPC_TASK_CRED_NOREF | task_flags, + NULL); } return ret; } diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index a74ee69a2fa6..dbef837e871a 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -490,7 +490,7 @@ pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(data->inode), data->mds_ops, how, - RPC_TASK_CRED_NOREF); + RPC_TASK_CRED_NOREF, NULL); } else { nfs_init_commit(data, NULL, data->lseg, cinfo); initiate_commit(data, how); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 61b580bcb356..ef72922263bb 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1664,7 +1664,8 @@ EXPORT_SYMBOL_GPL(nfs_commitdata_release); int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, const struct nfs_rpc_ops *nfs_ops, const struct rpc_call_ops *call_ops, - int how, int flags) + int how, int flags, + struct nfsd_file *localio) { struct rpc_task *task; int priority = flush_task_priority(how); @@ -1810,7 +1811,7 @@ nfs_commit_list(struct inode *inode, struct list_head *head, int how, task_flags = RPC_TASK_MOVEABLE; return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), data->mds_ops, how, - RPC_TASK_CRED_NOREF | task_flags); + RPC_TASK_CRED_NOREF | task_flags, NULL); } /* diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 546f46b4e46f..2c4e0fd9da6b 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -6,6 +6,9 @@ #ifndef __LINUX_NFSLOCALIO_H #define __LINUX_NFSLOCALIO_H +/* nfsd_file structure is purposely kept opaque to NFS client */ +struct nfsd_file; + #if IS_ENABLED(CONFIG_NFS_LOCALIO) #include @@ -36,8 +39,6 @@ void nfs_uuid_is_local(const uuid_t *, struct list_head *, void nfs_uuid_invalidate_clients(struct list_head *list); void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid); -struct nfsd_file; - /* localio needs to map filehandle -> struct nfsd_file */ extern struct nfsd_file * nfsd_open_local_fh(struct net *, struct auth_domain *, struct rpc_clnt *, From 70ba381e1a431245c137ed597ec6a05991c79bd9 Mon Sep 17 00:00:00 2001 From: Weston Andros Adamson Date: Thu, 5 Sep 2024 15:09:53 -0400 Subject: [PATCH 102/111] nfs: add LOCALIO support Add client support for bypassing NFS for localhost reads, writes, and commits. This is only useful when the client and the server are running on the same host. nfs_local_probe() is stubbed out, later commits will enable client and server handshake via a Linux-only LOCALIO auxiliary RPC protocol. This has dynamic binding with the nfsd module (via nfs_localio module which is part of nfs_common). LOCALIO will only work if nfsd is already loaded. The "localio_enabled" nfs kernel module parameter can be used to disable and enable the ability to use LOCALIO support. CONFIG_NFS_LOCALIO enables NFS client support for LOCALIO. Lastly, LOCALIO uses an nfsd_file to initiate all IO. To make proper use of nfsd_file (and nfsd's filecache) its lifetime (duration before nfsd_file_put is called) must extend until after commit, read and write operations. So rather than immediately drop the nfsd_file reference in nfs_local_open_fh(), that doesn't happen until nfs_local_pgio_release() for read/write and not until nfs_local_release_commit_data() for commit. The same applies to the reference held on nfsd's nn->nfsd_serv. Both objects' lifetimes and associated references are managed through calls to nfs_to->nfsd_file_put_local(). Signed-off-by: Weston Andros Adamson Signed-off-by: Trond Myklebust Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Signed-off-by: NeilBrown # nfs_open_local_fh Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/Makefile | 1 + fs/nfs/client.c | 11 + fs/nfs/internal.h | 45 +++ fs/nfs/localio.c | 600 +++++++++++++++++++++++++++++++++++++ fs/nfs/nfstrace.h | 61 ++++ fs/nfs/pagelist.c | 4 + fs/nfs/write.c | 3 + fs/nfs_common/nfslocalio.c | 33 ++ include/linux/nfs.h | 2 + include/linux/nfs_fs_sb.h | 9 + include/linux/nfslocalio.h | 4 + 11 files changed, 773 insertions(+) create mode 100644 fs/nfs/localio.c diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile index 5f6db37f461e..9fb2f2cac87e 100644 --- a/fs/nfs/Makefile +++ b/fs/nfs/Makefile @@ -13,6 +13,7 @@ nfs-y := client.o dir.o file.o getroot.o inode.o super.o \ nfs-$(CONFIG_ROOT_NFS) += nfsroot.o nfs-$(CONFIG_SYSCTL) += sysctl.o nfs-$(CONFIG_NFS_FSCACHE) += fscache.o +nfs-$(CONFIG_NFS_LOCALIO) += localio.o obj-$(CONFIG_NFS_V2) += nfsv2.o nfsv2-y := nfs2super.o proc.o nfs2xdr.o diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 3fea7aa1366f..c09ef6088451 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -178,6 +178,14 @@ struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init) clp->cl_max_connect = cl_init->max_connect ? cl_init->max_connect : 1; clp->cl_net = get_net(cl_init->net); +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + seqlock_init(&clp->cl_boot_lock); + ktime_get_real_ts64(&clp->cl_nfssvc_boot); + clp->cl_uuid.net = NULL; + clp->cl_uuid.dom = NULL; + spin_lock_init(&clp->cl_localio_lock); +#endif /* CONFIG_NFS_LOCALIO */ + clp->cl_principal = "*"; clp->cl_xprtsec = cl_init->xprtsec; return clp; @@ -233,6 +241,8 @@ static void pnfs_init_server(struct nfs_server *server) */ void nfs_free_client(struct nfs_client *clp) { + nfs_local_disable(clp); + /* -EIO all pending I/O */ if (!IS_ERR(clp->cl_rpcclient)) rpc_shutdown_client(clp->cl_rpcclient); @@ -424,6 +434,7 @@ struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init) list_add_tail(&new->cl_share_link, &nn->nfs_client_list); spin_unlock(&nn->nfs_client_lock); + nfs_local_probe(new); return rpc_ops->init_client(new, cl_init); } diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index f0af96a665d2..534c1ac16c57 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -451,6 +451,51 @@ extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags); extern bool nfs_check_cache_invalid(struct inode *, unsigned long); extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode); +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +/* localio.c */ +extern void nfs_local_disable(struct nfs_client *); +extern void nfs_local_probe(struct nfs_client *); +extern struct nfsd_file *nfs_local_open_fh(struct nfs_client *, + const struct cred *, + struct nfs_fh *, + const fmode_t); +extern int nfs_local_doio(struct nfs_client *, + struct nfsd_file *, + struct nfs_pgio_header *, + const struct rpc_call_ops *); +extern int nfs_local_commit(struct nfsd_file *, + struct nfs_commit_data *, + const struct rpc_call_ops *, int); +extern bool nfs_server_is_local(const struct nfs_client *clp); + +#else /* CONFIG_NFS_LOCALIO */ +static inline void nfs_local_disable(struct nfs_client *clp) {} +static inline void nfs_local_probe(struct nfs_client *clp) {} +static inline struct nfsd_file * +nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, const fmode_t mode) +{ + return NULL; +} +static inline int nfs_local_doio(struct nfs_client *clp, + struct nfsd_file *localio, + struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + return -EINVAL; +} +static inline int nfs_local_commit(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops, int how) +{ + return -EINVAL; +} +static inline bool nfs_server_is_local(const struct nfs_client *clp) +{ + return false; +} +#endif /* CONFIG_NFS_LOCALIO */ + /* super.c */ extern const struct super_operations nfs_sops; bool nfs_auth_info_match(const struct nfs_auth_info *, rpc_authflavor_t); diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c new file mode 100644 index 000000000000..3cf4374de366 --- /dev/null +++ b/fs/nfs/localio.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NFS client support for local clients to bypass network stack + * + * Copyright (C) 2014 Weston Andros Adamson + * Copyright (C) 2019 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "pnfs.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +struct nfs_local_kiocb { + struct kiocb kiocb; + struct bio_vec *bvec; + struct nfs_pgio_header *hdr; + struct work_struct work; + struct nfsd_file *localio; +}; + +struct nfs_local_fsync_ctx { + struct nfsd_file *localio; + struct nfs_commit_data *data; + struct work_struct work; + struct kref kref; + struct completion *done; +}; +static void nfs_local_fsync_work(struct work_struct *work); + +static bool localio_enabled __read_mostly = true; +module_param(localio_enabled, bool, 0644); + +bool nfs_server_is_local(const struct nfs_client *clp) +{ + return test_bit(NFS_CS_LOCAL_IO, &clp->cl_flags) != 0 && + localio_enabled; +} +EXPORT_SYMBOL_GPL(nfs_server_is_local); + +/* + * nfs_local_enable - enable local i/o for an nfs_client + */ +static __maybe_unused void nfs_local_enable(struct nfs_client *clp) +{ + spin_lock(&clp->cl_localio_lock); + set_bit(NFS_CS_LOCAL_IO, &clp->cl_flags); + trace_nfs_local_enable(clp); + spin_unlock(&clp->cl_localio_lock); +} + +/* + * nfs_local_disable - disable local i/o for an nfs_client + */ +void nfs_local_disable(struct nfs_client *clp) +{ + spin_lock(&clp->cl_localio_lock); + if (test_and_clear_bit(NFS_CS_LOCAL_IO, &clp->cl_flags)) { + trace_nfs_local_disable(clp); + nfs_uuid_invalidate_one_client(&clp->cl_uuid); + } + spin_unlock(&clp->cl_localio_lock); +} + +/* + * nfs_local_probe - probe local i/o support for an nfs_server and nfs_client + */ +void nfs_local_probe(struct nfs_client *clp) +{ +} +EXPORT_SYMBOL_GPL(nfs_local_probe); + +/* + * nfs_local_open_fh - open a local filehandle in terms of nfsd_file + * + * Returns a pointer to a struct nfsd_file or NULL + */ +struct nfsd_file * +nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, const fmode_t mode) +{ + struct nfsd_file *localio; + int status; + + if (!nfs_server_is_local(clp)) + return NULL; + if (mode & ~(FMODE_READ | FMODE_WRITE)) + return NULL; + + localio = nfs_open_local_fh(&clp->cl_uuid, clp->cl_rpcclient, + cred, fh, mode); + if (IS_ERR(localio)) { + status = PTR_ERR(localio); + trace_nfs_local_open_fh(fh, mode, status); + switch (status) { + case -ENOMEM: + case -ENXIO: + case -ENOENT: + nfs_local_disable(clp); + } + return NULL; + } + return localio; +} +EXPORT_SYMBOL_GPL(nfs_local_open_fh); + +static struct bio_vec * +nfs_bvec_alloc_and_import_pagevec(struct page **pagevec, + unsigned int npages, gfp_t flags) +{ + struct bio_vec *bvec, *p; + + bvec = kmalloc_array(npages, sizeof(*bvec), flags); + if (bvec != NULL) { + for (p = bvec; npages > 0; p++, pagevec++, npages--) { + p->bv_page = *pagevec; + p->bv_len = PAGE_SIZE; + p->bv_offset = 0; + } + } + return bvec; +} + +static void +nfs_local_iocb_free(struct nfs_local_kiocb *iocb) +{ + kfree(iocb->bvec); + kfree(iocb); +} + +static struct nfs_local_kiocb * +nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, + struct nfsd_file *localio, gfp_t flags) +{ + struct nfs_local_kiocb *iocb; + + iocb = kmalloc(sizeof(*iocb), flags); + if (iocb == NULL) + return NULL; + iocb->bvec = nfs_bvec_alloc_and_import_pagevec(hdr->page_array.pagevec, + hdr->page_array.npages, flags); + if (iocb->bvec == NULL) { + kfree(iocb); + return NULL; + } + init_sync_kiocb(&iocb->kiocb, nfs_to->nfsd_file_file(localio)); + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->localio = localio; + iocb->hdr = hdr; + iocb->kiocb.ki_flags &= ~IOCB_APPEND; + return iocb; +} + +static void +nfs_local_iter_init(struct iov_iter *i, struct nfs_local_kiocb *iocb, int dir) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + + iov_iter_bvec(i, dir, iocb->bvec, hdr->page_array.npages, + hdr->args.count + hdr->args.pgbase); + if (hdr->args.pgbase != 0) + iov_iter_advance(i, hdr->args.pgbase); +} + +static void +nfs_local_hdr_release(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + call_ops->rpc_call_done(&hdr->task, hdr); + call_ops->rpc_release(hdr); +} + +static void +nfs_local_pgio_init(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + hdr->task.tk_ops = call_ops; + if (!hdr->task.tk_start) + hdr->task.tk_start = ktime_get(); +} + +static void +nfs_local_pgio_done(struct nfs_pgio_header *hdr, long status) +{ + if (status >= 0) { + hdr->res.count = status; + hdr->res.op_status = NFS4_OK; + hdr->task.tk_status = 0; + } else { + hdr->res.op_status = nfs4_stat_to_errno(status); + hdr->task.tk_status = status; + } +} + +static void +nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + + nfs_to->nfsd_file_put_local(iocb->localio); + nfs_local_iocb_free(iocb); + nfs_local_hdr_release(hdr, hdr->task.tk_ops); +} + +static void +nfs_local_read_done(struct nfs_local_kiocb *iocb, long status) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + struct file *filp = iocb->kiocb.ki_filp; + + nfs_local_pgio_done(hdr, status); + + if (hdr->res.count != hdr->args.count || + hdr->args.offset + hdr->res.count >= i_size_read(file_inode(filp))) + hdr->res.eof = true; + + dprintk("%s: read %ld bytes eof %d.\n", __func__, + status > 0 ? status : 0, hdr->res.eof); +} + +static int +nfs_do_local_read(struct nfs_pgio_header *hdr, + struct nfsd_file *localio, + const struct rpc_call_ops *call_ops) +{ + struct file *filp = nfs_to->nfsd_file_file(localio); + struct nfs_local_kiocb *iocb; + struct iov_iter iter; + ssize_t status; + + dprintk("%s: vfs_read count=%u pos=%llu\n", + __func__, hdr->args.count, hdr->args.offset); + + iocb = nfs_local_iocb_alloc(hdr, localio, GFP_KERNEL); + if (iocb == NULL) + return -ENOMEM; + nfs_local_iter_init(&iter, iocb, READ); + + nfs_local_pgio_init(hdr, call_ops); + hdr->res.eof = false; + + status = filp->f_op->read_iter(&iocb->kiocb, &iter); + WARN_ON_ONCE(status == -EIOCBQUEUED); + + nfs_local_read_done(iocb, status); + nfs_local_pgio_release(iocb); + + return 0; +} + +static void +nfs_copy_boot_verifier(struct nfs_write_verifier *verifier, struct inode *inode) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + u32 *verf = (u32 *)verifier->data; + int seq = 0; + + do { + read_seqbegin_or_lock(&clp->cl_boot_lock, &seq); + verf[0] = (u32)clp->cl_nfssvc_boot.tv_sec; + verf[1] = (u32)clp->cl_nfssvc_boot.tv_nsec; + } while (need_seqretry(&clp->cl_boot_lock, seq)); + done_seqretry(&clp->cl_boot_lock, seq); +} + +static void +nfs_reset_boot_verifier(struct inode *inode) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + + write_seqlock(&clp->cl_boot_lock); + ktime_get_real_ts64(&clp->cl_nfssvc_boot); + write_sequnlock(&clp->cl_boot_lock); +} + +static void +nfs_set_local_verifier(struct inode *inode, + struct nfs_writeverf *verf, + enum nfs3_stable_how how) +{ + nfs_copy_boot_verifier(&verf->verifier, inode); + verf->committed = how; +} + +/* Factored out from fs/nfsd/vfs.h:fh_getattr() */ +static int __vfs_getattr(struct path *p, struct kstat *stat, int version) +{ + u32 request_mask = STATX_BASIC_STATS; + + if (version == 4) + request_mask |= (STATX_BTIME | STATX_CHANGE_COOKIE); + return vfs_getattr(p, stat, request_mask, AT_STATX_SYNC_AS_STAT); +} + +/* Copied from fs/nfsd/nfsfh.c:nfsd4_change_attribute() */ +static u64 __nfsd4_change_attribute(const struct kstat *stat, + const struct inode *inode) +{ + u64 chattr; + + if (stat->result_mask & STATX_CHANGE_COOKIE) { + chattr = stat->change_cookie; + if (S_ISREG(inode->i_mode) && + !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) { + chattr += (u64)stat->ctime.tv_sec << 30; + chattr += stat->ctime.tv_nsec; + } + } else { + chattr = time_to_chattr(&stat->ctime); + } + return chattr; +} + +static void nfs_local_vfs_getattr(struct nfs_local_kiocb *iocb) +{ + struct kstat stat; + struct file *filp = iocb->kiocb.ki_filp; + struct nfs_pgio_header *hdr = iocb->hdr; + struct nfs_fattr *fattr = hdr->res.fattr; + int version = NFS_PROTO(hdr->inode)->version; + + if (unlikely(!fattr) || __vfs_getattr(&filp->f_path, &stat, version)) + return; + + fattr->valid = (NFS_ATTR_FATTR_FILEID | + NFS_ATTR_FATTR_CHANGE | + NFS_ATTR_FATTR_SIZE | + NFS_ATTR_FATTR_ATIME | + NFS_ATTR_FATTR_MTIME | + NFS_ATTR_FATTR_CTIME | + NFS_ATTR_FATTR_SPACE_USED); + + fattr->fileid = stat.ino; + fattr->size = stat.size; + fattr->atime = stat.atime; + fattr->mtime = stat.mtime; + fattr->ctime = stat.ctime; + if (version == 4) { + fattr->change_attr = + __nfsd4_change_attribute(&stat, file_inode(filp)); + } else + fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime); + fattr->du.nfs3.used = stat.blocks << 9; +} + +static void +nfs_local_write_done(struct nfs_local_kiocb *iocb, long status) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + struct inode *inode = hdr->inode; + + dprintk("%s: wrote %ld bytes.\n", __func__, status > 0 ? status : 0); + + /* Handle short writes as if they are ENOSPC */ + if (status > 0 && status < hdr->args.count) { + hdr->mds_offset += status; + hdr->args.offset += status; + hdr->args.pgbase += status; + hdr->args.count -= status; + nfs_set_pgio_error(hdr, -ENOSPC, hdr->args.offset); + status = -ENOSPC; + } + if (status < 0) + nfs_reset_boot_verifier(inode); + else if (nfs_should_remove_suid(inode)) { + /* Deal with the suid/sgid bit corner case */ + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); + spin_unlock(&inode->i_lock); + } + nfs_local_pgio_done(hdr, status); +} + +static int +nfs_do_local_write(struct nfs_pgio_header *hdr, + struct nfsd_file *localio, + const struct rpc_call_ops *call_ops) +{ + struct file *filp = nfs_to->nfsd_file_file(localio); + struct nfs_local_kiocb *iocb; + struct iov_iter iter; + ssize_t status; + + dprintk("%s: vfs_write count=%u pos=%llu %s\n", + __func__, hdr->args.count, hdr->args.offset, + (hdr->args.stable == NFS_UNSTABLE) ? "unstable" : "stable"); + + iocb = nfs_local_iocb_alloc(hdr, localio, GFP_NOIO); + if (iocb == NULL) + return -ENOMEM; + nfs_local_iter_init(&iter, iocb, WRITE); + + switch (hdr->args.stable) { + default: + break; + case NFS_DATA_SYNC: + iocb->kiocb.ki_flags |= IOCB_DSYNC; + break; + case NFS_FILE_SYNC: + iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC; + } + nfs_local_pgio_init(hdr, call_ops); + + nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable); + + file_start_write(filp); + status = filp->f_op->write_iter(&iocb->kiocb, &iter); + file_end_write(filp); + WARN_ON_ONCE(status == -EIOCBQUEUED); + + nfs_local_write_done(iocb, status); + nfs_local_vfs_getattr(iocb); + nfs_local_pgio_release(iocb); + + return 0; +} + +int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, + struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + int status = 0; + struct file *filp = nfs_to->nfsd_file_file(localio); + + if (!hdr->args.count) + return 0; + /* Don't support filesystems without read_iter/write_iter */ + if (!filp->f_op->read_iter || !filp->f_op->write_iter) { + nfs_local_disable(clp); + status = -EAGAIN; + goto out; + } + + switch (hdr->rw_mode) { + case FMODE_READ: + status = nfs_do_local_read(hdr, localio, call_ops); + break; + case FMODE_WRITE: + status = nfs_do_local_write(hdr, localio, call_ops); + break; + default: + dprintk("%s: invalid mode: %d\n", __func__, + hdr->rw_mode); + status = -EINVAL; + } +out: + if (status != 0) { + nfs_to->nfsd_file_put_local(localio); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, call_ops); + } + return status; +} + +static void +nfs_local_init_commit(struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops) +{ + data->task.tk_ops = call_ops; +} + +static int +nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) +{ + loff_t start = data->args.offset; + loff_t end = LLONG_MAX; + + if (data->args.count > 0) { + end = start + data->args.count - 1; + if (end < start) + end = LLONG_MAX; + } + + dprintk("%s: commit %llu - %llu\n", __func__, start, end); + return vfs_fsync_range(filp, start, end, 0); +} + +static void +nfs_local_commit_done(struct nfs_commit_data *data, int status) +{ + if (status >= 0) { + nfs_set_local_verifier(data->inode, + data->res.verf, + NFS_FILE_SYNC); + data->res.op_status = NFS4_OK; + data->task.tk_status = 0; + } else { + nfs_reset_boot_verifier(data->inode); + data->res.op_status = nfs4_stat_to_errno(status); + data->task.tk_status = status; + } +} + +static void +nfs_local_release_commit_data(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops) +{ + nfs_to->nfsd_file_put_local(localio); + call_ops->rpc_call_done(&data->task, data); + call_ops->rpc_release(data); +} + +static struct nfs_local_fsync_ctx * +nfs_local_fsync_ctx_alloc(struct nfs_commit_data *data, + struct nfsd_file *localio, gfp_t flags) +{ + struct nfs_local_fsync_ctx *ctx = kmalloc(sizeof(*ctx), flags); + + if (ctx != NULL) { + ctx->localio = localio; + ctx->data = data; + INIT_WORK(&ctx->work, nfs_local_fsync_work); + kref_init(&ctx->kref); + ctx->done = NULL; + } + return ctx; +} + +static void +nfs_local_fsync_ctx_kref_free(struct kref *kref) +{ + kfree(container_of(kref, struct nfs_local_fsync_ctx, kref)); +} + +static void +nfs_local_fsync_ctx_put(struct nfs_local_fsync_ctx *ctx) +{ + kref_put(&ctx->kref, nfs_local_fsync_ctx_kref_free); +} + +static void +nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) +{ + nfs_local_release_commit_data(ctx->localio, ctx->data, + ctx->data->task.tk_ops); + nfs_local_fsync_ctx_put(ctx); +} + +static void +nfs_local_fsync_work(struct work_struct *work) +{ + struct nfs_local_fsync_ctx *ctx; + int status; + + ctx = container_of(work, struct nfs_local_fsync_ctx, work); + + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), + ctx->data); + nfs_local_commit_done(ctx->data, status); + if (ctx->done != NULL) + complete(ctx->done); + nfs_local_fsync_ctx_free(ctx); +} + +int nfs_local_commit(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops, int how) +{ + struct nfs_local_fsync_ctx *ctx; + + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_KERNEL); + if (!ctx) { + nfs_local_commit_done(data, -ENOMEM); + nfs_local_release_commit_data(localio, data, call_ops); + return -ENOMEM; + } + + nfs_local_init_commit(data, call_ops); + kref_get(&ctx->kref); + if (how & FLUSH_SYNC) { + DECLARE_COMPLETION_ONSTACK(done); + ctx->done = &done; + queue_work(nfsiod_workqueue, &ctx->work); + wait_for_completion(&done); + } else + queue_work(nfsiod_workqueue, &ctx->work); + nfs_local_fsync_ctx_put(ctx); + return 0; +} diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index 352fdaed4075..1eab98c277fa 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -1685,6 +1685,67 @@ TRACE_EVENT(nfs_mount_path, TP_printk("path='%s'", __get_str(path)) ); +TRACE_EVENT(nfs_local_open_fh, + TP_PROTO( + const struct nfs_fh *fh, + fmode_t fmode, + int error + ), + + TP_ARGS(fh, fmode, error), + + TP_STRUCT__entry( + __field(int, error) + __field(u32, fhandle) + __field(unsigned int, fmode) + ), + + TP_fast_assign( + __entry->error = error; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->fmode = (__force unsigned int)fmode; + ), + + TP_printk( + "error=%d fhandle=0x%08x mode=%s", + __entry->error, + __entry->fhandle, + show_fs_fmode_flags(__entry->fmode) + ) +); + +DECLARE_EVENT_CLASS(nfs_local_client_event, + TP_PROTO( + const struct nfs_client *clp + ), + + TP_ARGS(clp), + + TP_STRUCT__entry( + __field(unsigned int, protocol) + __string(server, clp->cl_hostname) + ), + + TP_fast_assign( + __entry->protocol = clp->rpc_ops->version; + __assign_str(server); + ), + + TP_printk( + "server=%s NFSv%u", __get_str(server), __entry->protocol + ) +); + +#define DEFINE_NFS_LOCAL_CLIENT_EVENT(name) \ + DEFINE_EVENT(nfs_local_client_event, name, \ + TP_PROTO( \ + const struct nfs_client *clp \ + ), \ + TP_ARGS(clp)) + +DEFINE_NFS_LOCAL_CLIENT_EVENT(nfs_local_enable); +DEFINE_NFS_LOCAL_CLIENT_EVENT(nfs_local_disable); + DECLARE_EVENT_CLASS(nfs_xdr_event, TP_PROTO( const struct xdr_stream *xdr, diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 50f3d6c9ac2a..97d5524c379a 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -762,6 +762,10 @@ int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, hdr->args.count, (unsigned long long)hdr->args.offset); + if (localio) + return nfs_local_doio(NFS_SERVER(hdr->inode)->nfs_client, + localio, hdr, call_ops); + task = rpc_run_task(&task_setup_data); if (IS_ERR(task)) return PTR_ERR(task); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index ef72922263bb..508e279da648 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1694,6 +1694,9 @@ int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, dprintk("NFS: initiated commit call\n"); + if (localio) + return nfs_local_commit(localio, data, call_ops, how); + task = rpc_run_task(&task_setup_data); if (IS_ERR(task)) return PTR_ERR(task); diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index f61761ad19b1..42b479b9191f 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -115,6 +115,39 @@ void nfs_uuid_invalidate_one_client(nfs_uuid_t *nfs_uuid) } EXPORT_SYMBOL_GPL(nfs_uuid_invalidate_one_client); +struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, + struct rpc_clnt *rpc_clnt, const struct cred *cred, + const struct nfs_fh *nfs_fh, const fmode_t fmode) +{ + struct net *net; + struct nfsd_file *localio; + + /* + * Not running in nfsd context, so must safely get reference on nfsd_serv. + * But the server may already be shutting down, if so disallow new localio. + * uuid->net is NOT a counted reference, but rcu_read_lock() ensures that + * if uuid->net is not NULL, then calling nfsd_serv_try_get() is safe + * and if it succeeds we will have an implied reference to the net. + * + * Otherwise NFS may not have ref on NFSD and therefore cannot safely + * make 'nfs_to' calls. + */ + rcu_read_lock(); + net = rcu_dereference(uuid->net); + if (!net || !nfs_to->nfsd_serv_try_get(net)) { + rcu_read_unlock(); + return ERR_PTR(-ENXIO); + } + rcu_read_unlock(); + /* We have an implied reference to net thanks to nfsd_serv_try_get */ + localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, + cred, nfs_fh, fmode); + if (IS_ERR(localio)) + nfs_to->nfsd_serv_put(net); + return localio; +} +EXPORT_SYMBOL_GPL(nfs_open_local_fh); + /* * The NFS LOCALIO code needs to call into NFSD using various symbols, * but cannot be statically linked, because that will make the NFS diff --git a/include/linux/nfs.h b/include/linux/nfs.h index 73da75908d95..9ad727ddfedb 100644 --- a/include/linux/nfs.h +++ b/include/linux/nfs.h @@ -8,6 +8,8 @@ #ifndef _LINUX_NFS_H #define _LINUX_NFS_H +#include +#include #include #include #include diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index c49bfdded5c1..853df3fcd4c2 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,7 @@ struct nfs_client { #define NFS_CS_DS 7 /* - Server is a DS */ #define NFS_CS_REUSEPORT 8 /* - reuse src port on reconnect */ #define NFS_CS_PNFS 9 /* - Server used for pnfs */ +#define NFS_CS_LOCAL_IO 10 /* - client is local */ struct sockaddr_storage cl_addr; /* server identifier */ size_t cl_addrlen; char * cl_hostname; /* hostname of server */ @@ -125,6 +127,13 @@ struct nfs_client { struct net *cl_net; struct list_head pending_cb_stateids; struct rcu_head rcu; + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + struct timespec64 cl_nfssvc_boot; + seqlock_t cl_boot_lock; + nfs_uuid_t cl_uuid; + spinlock_t cl_localio_lock; +#endif /* CONFIG_NFS_LOCALIO */ }; /* diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 2c4e0fd9da6b..b353abe00357 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -61,6 +61,10 @@ struct nfsd_localio_operations { extern void nfsd_localio_ops_init(void); extern const struct nfsd_localio_operations *nfs_to; +struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *, + struct rpc_clnt *, const struct cred *, + const struct nfs_fh *, const fmode_t); + #else /* CONFIG_NFS_LOCALIO */ static inline void nfsd_localio_ops_init(void) { From fa88a7d6ae089c07aba872fff30a1342d3503e80 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 5 Sep 2024 15:09:54 -0400 Subject: [PATCH 103/111] nfs: enable localio for non-pNFS IO Try a local open of the file being written to, and if it succeeds, then use localio to issue IO. Signed-off-by: Trond Myklebust Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/pagelist.c | 8 +++++++- fs/nfs/write.c | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 97d5524c379a..e27c07bd8929 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -958,6 +958,12 @@ static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) nfs_pgheader_init(desc, hdr, nfs_pgio_header_free); ret = nfs_generic_pgio(desc, hdr); if (ret == 0) { + struct nfs_client *clp = NFS_SERVER(hdr->inode)->nfs_client; + + struct nfsd_file *localio = + nfs_local_open_fh(clp, hdr->cred, + hdr->args.fh, hdr->args.context->mode); + if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion) task_flags = RPC_TASK_MOVEABLE; ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), @@ -967,7 +973,7 @@ static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) desc->pg_rpc_callops, desc->pg_ioflags, RPC_TASK_CRED_NOREF | task_flags, - NULL); + localio); } return ret; } diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 508e279da648..ead2dc55952d 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1796,6 +1796,7 @@ nfs_commit_list(struct inode *inode, struct list_head *head, int how, struct nfs_commit_info *cinfo) { struct nfs_commit_data *data; + struct nfsd_file *localio; unsigned short task_flags = 0; /* another commit raced with us */ @@ -1812,9 +1813,12 @@ nfs_commit_list(struct inode *inode, struct list_head *head, int how, nfs_init_commit(data, head, NULL, cinfo); if (NFS_SERVER(inode)->nfs_client->cl_minorversion) task_flags = RPC_TASK_MOVEABLE; + + localio = nfs_local_open_fh(NFS_SERVER(inode)->nfs_client, data->cred, + data->args.fh, data->context->mode); return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), data->mds_ops, how, - RPC_TASK_CRED_NOREF | task_flags, NULL); + RPC_TASK_CRED_NOREF | task_flags, localio); } /* From d488b9d01fbc2ff5ccf15bcd47422eb156726c0d Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 5 Sep 2024 15:09:55 -0400 Subject: [PATCH 104/111] pnfs/flexfiles: enable localio support If the DS is local to this client use localio to write the data. Signed-off-by: Trond Myklebust Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/flexfilelayout/flexfilelayout.c | 50 +++++++++++++++++++++-- fs/nfs/flexfilelayout/flexfilelayoutdev.c | 6 +++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 01ee52551a63..f78115c6c2c1 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -162,6 +163,21 @@ decode_name(struct xdr_stream *xdr, u32 *id) return 0; } +static struct nfsd_file * +ff_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, fmode_t mode) +{ + if (mode & FMODE_WRITE) { + /* + * Always request read and write access since this corresponds + * to a rw layout. + */ + mode |= FMODE_READ; + } + + return nfs_local_open_fh(clp, cred, fh, mode); +} + static bool ff_mirror_match_fh(const struct nfs4_ff_layout_mirror *m1, const struct nfs4_ff_layout_mirror *m2) { @@ -237,7 +253,7 @@ static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(gfp_t gfp_flags) static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror) { - const struct cred *cred; + const struct cred *cred; ff_layout_remove_mirror(mirror); kfree(mirror->fh_versions); @@ -1756,6 +1772,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) struct pnfs_layout_segment *lseg = hdr->lseg; struct nfs4_pnfs_ds *ds; struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; const struct cred *ds_cred; loff_t offset = hdr->args.offset; @@ -1802,11 +1819,18 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) hdr->args.offset = offset; hdr->mds_offset = offset; + /* Start IO accounting for local read */ + localio = ff_local_open_fh(ds->ds_clp, ds_cred, fh, FMODE_READ); + if (localio) { + hdr->task.tk_start = ktime_get(); + ff_layout_read_record_layoutstats_start(&hdr->task, hdr); + } + /* Perform an asynchronous read to ds */ nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_read_call_ops_v3 : &ff_layout_read_call_ops_v4, - 0, RPC_TASK_SOFTCONN, NULL); + 0, RPC_TASK_SOFTCONN, localio); put_cred(ds_cred); return PNFS_ATTEMPTED; @@ -1826,6 +1850,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) struct pnfs_layout_segment *lseg = hdr->lseg; struct nfs4_pnfs_ds *ds; struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; const struct cred *ds_cred; loff_t offset = hdr->args.offset; @@ -1870,11 +1895,19 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) */ hdr->args.offset = offset; + /* Start IO accounting for local write */ + localio = ff_local_open_fh(ds->ds_clp, ds_cred, fh, + FMODE_READ|FMODE_WRITE); + if (localio) { + hdr->task.tk_start = ktime_get(); + ff_layout_write_record_layoutstats_start(&hdr->task, hdr); + } + /* Perform an asynchronous write */ nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_write_call_ops_v3 : &ff_layout_write_call_ops_v4, - sync, RPC_TASK_SOFTCONN, NULL); + sync, RPC_TASK_SOFTCONN, localio); put_cred(ds_cred); return PNFS_ATTEMPTED; @@ -1908,6 +1941,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) struct pnfs_layout_segment *lseg = data->lseg; struct nfs4_pnfs_ds *ds; struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; struct nfs4_ff_layout_mirror *mirror; const struct cred *ds_cred; u32 idx; @@ -1946,10 +1980,18 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) if (fh) data->args.fh = fh; + /* Start IO accounting for local commit */ + localio = ff_local_open_fh(ds->ds_clp, ds_cred, fh, + FMODE_READ|FMODE_WRITE); + if (localio) { + data->task.tk_start = ktime_get(); + ff_layout_commit_record_layoutstats_start(&data->task, data); + } + ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops, vers == 3 ? &ff_layout_commit_call_ops_v3 : &ff_layout_commit_call_ops_v4, - how, RPC_TASK_SOFTCONN, NULL); + how, RPC_TASK_SOFTCONN, localio); put_cred(ds_cred); return ret; out_err: diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index e028f5a0ef5f..e58bedfb1dcc 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -395,6 +395,12 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, /* connect success, check rsize/wsize limit */ if (!status) { + /* + * ds_clp is put in destroy_ds(). + * keep ds_clp even if DS is local, so that if local IO cannot + * proceed somehow, we can fall back to NFS whenever we want. + */ + nfs_local_probe(ds->ds_clp); max_payload = nfs_block_size(rpc_max_payload(ds->ds_clp->cl_rpcclient), NULL); From b9f5dd57f4a52990963eeb1f1b58d00f717ece69 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 5 Sep 2024 15:09:56 -0400 Subject: [PATCH 105/111] nfs/localio: use dedicated workqueues for filesystem read and write For localio access, don't call filesystem read() and write() routines directly. This solves two problems: 1) localio writes need to use a normal (non-memreclaim) unbound workqueue. This avoids imposing new requirements on how underlying filesystems process frontend IO, which would cause a large amount of work to update all filesystems. Without this change, when XFS starts getting low on space, XFS flushes work on a non-memreclaim work queue, which causes a priority inversion problem: 00573 workqueue: WQ_MEM_RECLAIM writeback:wb_workfn is flushing !WQ_MEM_RECLAIM xfs-sync/vdc:xfs_flush_inodes_worker 00573 WARNING: CPU: 6 PID: 8525 at kernel/workqueue.c:3706 check_flush_dependency+0x2a4/0x328 00573 Modules linked in: 00573 CPU: 6 PID: 8525 Comm: kworker/u71:5 Not tainted 6.10.0-rc3-ktest-00032-g2b0a133403ab #18502 00573 Hardware name: linux,dummy-virt (DT) 00573 Workqueue: writeback wb_workfn (flush-0:33) 00573 pstate: 400010c5 (nZcv daIF -PAN -UAO -TCO -DIT +SSBS BTYPE=--) 00573 pc : check_flush_dependency+0x2a4/0x328 00573 lr : check_flush_dependency+0x2a4/0x328 00573 sp : ffff0000c5f06bb0 00573 x29: ffff0000c5f06bb0 x28: ffff0000c998a908 x27: 1fffe00019331521 00573 x26: ffff0000d0620900 x25: ffff0000c5f06ca0 x24: ffff8000828848c0 00573 x23: 1fffe00018be0d8e x22: ffff0000c1210000 x21: ffff0000c75fde00 00573 x20: ffff800080bfd258 x19: ffff0000cad63400 x18: ffff0000cd3a4810 00573 x17: 0000000000000000 x16: 0000000000000000 x15: ffff800080508d98 00573 x14: 0000000000000000 x13: 204d49414c434552 x12: 1fffe0001b6eeab2 00573 x11: ffff60001b6eeab2 x10: dfff800000000000 x9 : ffff60001b6eeab3 00573 x8 : 0000000000000001 x7 : 00009fffe491154e x6 : ffff0000db775593 00573 x5 : ffff0000db775590 x4 : ffff0000db775590 x3 : 0000000000000000 00573 x2 : 0000000000000027 x1 : ffff600018be0d62 x0 : dfff800000000000 00573 Call trace: 00573 check_flush_dependency+0x2a4/0x328 00573 __flush_work+0x184/0x5c8 00573 flush_work+0x18/0x28 00573 xfs_flush_inodes+0x68/0x88 00573 xfs_file_buffered_write+0x128/0x6f0 00573 xfs_file_write_iter+0x358/0x448 00573 nfs_local_doio+0x854/0x1568 00573 nfs_initiate_pgio+0x214/0x418 00573 nfs_generic_pg_pgios+0x304/0x480 00573 nfs_pageio_doio+0xe8/0x240 00573 nfs_pageio_complete+0x160/0x480 00573 nfs_writepages+0x300/0x4f0 00573 do_writepages+0x12c/0x4a0 00573 __writeback_single_inode+0xd4/0xa68 00573 writeback_sb_inodes+0x470/0xcb0 00573 __writeback_inodes_wb+0xb0/0x1d0 00573 wb_writeback+0x594/0x808 00573 wb_workfn+0x5e8/0x9e0 00573 process_scheduled_works+0x53c/0xd90 00573 worker_thread+0x370/0x8c8 00573 kthread+0x258/0x2e8 00573 ret_from_fork+0x10/0x20 2) Some filesystem writeback routines can end up taking up a lot of stack space (particularly XFS). Instead of risking running over due to the extra overhead from the NFS stack, we should just call these routines from a workqueue job. Since we need to do this to address 1) above we're able to avoid possibly blowing the stack "for free". Use of dedicated workqueues improves performance over using the system_unbound_wq. Also, the creds used to open the file are used to override_creds() in both nfs_local_call_read() and nfs_local_call_write() -- otherwise the workqueue could have elevated capabilities (which the caller may not). Lastly, care is taken to set PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO in nfs_do_local_write() to avoid writeback deadlocks. The PF_LOCAL_THROTTLE flag prevents deadlocks in balance_dirty_pages() by causing writes to only be throttled against other writes to the same bdi (it keeps the throttling local). Normally all writes to bdi(s) are throttled equally (after throughput factors are allowed for). The PF_MEMALLOC_NOIO flag prevents the lower filesystem IO from causing memory reclaim to re-enter filesystems or IO devices and so prevents deadlocks from occuring where IO that cleans pages is waiting on IO to complete. Signed-off-by: Trond Myklebust Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Co-developed-by: NeilBrown Signed-off-by: NeilBrown # eliminated wait_for_completion Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/inode.c | 57 +++++++++++++++++++++++------------ fs/nfs/internal.h | 1 + fs/nfs/localio.c | 75 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 93 insertions(+), 40 deletions(-) diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index b4914a11c3c2..542c7d97b235 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -2461,35 +2461,54 @@ static void nfs_destroy_inodecache(void) kmem_cache_destroy(nfs_inode_cachep); } +struct workqueue_struct *nfslocaliod_workqueue; struct workqueue_struct *nfsiod_workqueue; EXPORT_SYMBOL_GPL(nfsiod_workqueue); /* - * start up the nfsiod workqueue - */ -static int nfsiod_start(void) -{ - struct workqueue_struct *wq; - dprintk("RPC: creating workqueue nfsiod\n"); - wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); - if (wq == NULL) - return -ENOMEM; - nfsiod_workqueue = wq; - return 0; -} - -/* - * Destroy the nfsiod workqueue + * Destroy the nfsiod workqueues */ static void nfsiod_stop(void) { struct workqueue_struct *wq; wq = nfsiod_workqueue; - if (wq == NULL) - return; - nfsiod_workqueue = NULL; - destroy_workqueue(wq); + if (wq != NULL) { + nfsiod_workqueue = NULL; + destroy_workqueue(wq); + } +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + wq = nfslocaliod_workqueue; + if (wq != NULL) { + nfslocaliod_workqueue = NULL; + destroy_workqueue(wq); + } +#endif /* CONFIG_NFS_LOCALIO */ +} + +/* + * Start the nfsiod workqueues + */ +static int nfsiod_start(void) +{ + dprintk("RPC: creating workqueue nfsiod\n"); + nfsiod_workqueue = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (nfsiod_workqueue == NULL) + return -ENOMEM; +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + /* + * localio writes need to use a normal (non-memreclaim) workqueue. + * When we start getting low on space, XFS goes and calls flush_work() on + * a non-memreclaim work queue, which causes a priority inversion problem. + */ + dprintk("RPC: creating workqueue nfslocaliod\n"); + nfslocaliod_workqueue = alloc_workqueue("nfslocaliod", WQ_UNBOUND, 0); + if (unlikely(nfslocaliod_workqueue == NULL)) { + nfsiod_stop(); + return -ENOMEM; + } +#endif /* CONFIG_NFS_LOCALIO */ + return 0; } unsigned int nfs_net_id; diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 534c1ac16c57..430733e3eff2 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -440,6 +440,7 @@ int nfs_check_flags(int); /* inode.c */ extern struct workqueue_struct *nfsiod_workqueue; +extern struct workqueue_struct *nfslocaliod_workqueue; extern struct inode *nfs_alloc_inode(struct super_block *sb); extern void nfs_free_inode(struct inode *); extern int nfs_write_inode(struct inode *, struct writeback_control *); diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 3cf4374de366..0a9f14a04eb4 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -238,15 +238,34 @@ nfs_local_read_done(struct nfs_local_kiocb *iocb, long status) status > 0 ? status : 0, hdr->res.eof); } +static void nfs_local_call_read(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + struct file *filp = iocb->kiocb.ki_filp; + const struct cred *save_cred; + struct iov_iter iter; + ssize_t status; + + save_cred = override_creds(filp->f_cred); + + nfs_local_iter_init(&iter, iocb, READ); + + status = filp->f_op->read_iter(&iocb->kiocb, &iter); + WARN_ON_ONCE(status == -EIOCBQUEUED); + + nfs_local_read_done(iocb, status); + nfs_local_pgio_release(iocb); + + revert_creds(save_cred); +} + static int nfs_do_local_read(struct nfs_pgio_header *hdr, struct nfsd_file *localio, const struct rpc_call_ops *call_ops) { - struct file *filp = nfs_to->nfsd_file_file(localio); struct nfs_local_kiocb *iocb; - struct iov_iter iter; - ssize_t status; dprintk("%s: vfs_read count=%u pos=%llu\n", __func__, hdr->args.count, hdr->args.offset); @@ -254,16 +273,12 @@ nfs_do_local_read(struct nfs_pgio_header *hdr, iocb = nfs_local_iocb_alloc(hdr, localio, GFP_KERNEL); if (iocb == NULL) return -ENOMEM; - nfs_local_iter_init(&iter, iocb, READ); nfs_local_pgio_init(hdr, call_ops); hdr->res.eof = false; - status = filp->f_op->read_iter(&iocb->kiocb, &iter); - WARN_ON_ONCE(status == -EIOCBQUEUED); - - nfs_local_read_done(iocb, status); - nfs_local_pgio_release(iocb); + INIT_WORK(&iocb->work, nfs_local_call_read); + queue_work(nfslocaliod_workqueue, &iocb->work); return 0; } @@ -391,15 +406,40 @@ nfs_local_write_done(struct nfs_local_kiocb *iocb, long status) nfs_local_pgio_done(hdr, status); } +static void nfs_local_call_write(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + struct file *filp = iocb->kiocb.ki_filp; + unsigned long old_flags = current->flags; + const struct cred *save_cred; + struct iov_iter iter; + ssize_t status; + + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + save_cred = override_creds(filp->f_cred); + + nfs_local_iter_init(&iter, iocb, WRITE); + + file_start_write(filp); + status = filp->f_op->write_iter(&iocb->kiocb, &iter); + file_end_write(filp); + WARN_ON_ONCE(status == -EIOCBQUEUED); + + nfs_local_write_done(iocb, status); + nfs_local_vfs_getattr(iocb); + nfs_local_pgio_release(iocb); + + revert_creds(save_cred); + current->flags = old_flags; +} + static int nfs_do_local_write(struct nfs_pgio_header *hdr, struct nfsd_file *localio, const struct rpc_call_ops *call_ops) { - struct file *filp = nfs_to->nfsd_file_file(localio); struct nfs_local_kiocb *iocb; - struct iov_iter iter; - ssize_t status; dprintk("%s: vfs_write count=%u pos=%llu %s\n", __func__, hdr->args.count, hdr->args.offset, @@ -408,7 +448,6 @@ nfs_do_local_write(struct nfs_pgio_header *hdr, iocb = nfs_local_iocb_alloc(hdr, localio, GFP_NOIO); if (iocb == NULL) return -ENOMEM; - nfs_local_iter_init(&iter, iocb, WRITE); switch (hdr->args.stable) { default: @@ -423,14 +462,8 @@ nfs_do_local_write(struct nfs_pgio_header *hdr, nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable); - file_start_write(filp); - status = filp->f_op->write_iter(&iocb->kiocb, &iter); - file_end_write(filp); - WARN_ON_ONCE(status == -EIOCBQUEUED); - - nfs_local_write_done(iocb, status); - nfs_local_vfs_getattr(iocb); - nfs_local_pgio_release(iocb); + INIT_WORK(&iocb->work, nfs_local_call_write); + queue_work(nfslocaliod_workqueue, &iocb->work); return 0; } From 56bcd0f07fdbf9770284bedb982236ab881ef909 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:57 -0400 Subject: [PATCH 106/111] nfs: implement client support for NFS_LOCALIO_PROGRAM The LOCALIO auxiliary RPC protocol consists of a single "UUID_IS_LOCAL" RPC method that allows the Linux NFS client to verify the local Linux NFS server can see the nonce (single-use UUID) the client generated and made available in nfs_common for subsequent lookup and verification by the NFS server. If matched, the NFS server populates members in the nfs_uuid_t struct. The NFS client then transfers these nfs_uuid_t struct member pointers to the nfs_client struct and cleans up the nfs_uuid_t struct. See: fs/nfs/localio.c:nfs_local_probe() This protocol isn't part of an IETF standard, nor does it need to be considering it is Linux-to-Linux auxiliary RPC protocol that amounts to an implementation detail. Localio is only supported when UNIX-style authentication (AUTH_UNIX, aka AUTH_SYS) is used (enforced by fs/nfs/localio.c:nfs_local_probe()). The UUID_IS_LOCAL method encodes the client generated uuid_t in terms of the fixed UUID_SIZE (16 bytes). The fixed size opaque encode and decode XDR methods are used instead of the less efficient variable sized methods. Having a nonce (single-use uuid) is better than using the same uuid for the life of the server, and sending it proactively by client rather than reactively by the server is also safer. Signed-off-by: Mike Snitzer Co-developed-by: NeilBrown Signed-off-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- fs/nfs/client.c | 6 ++- fs/nfs/localio.c | 132 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/fs/nfs/client.c b/fs/nfs/client.c index c09ef6088451..a1d21c4be0ac 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -434,8 +434,10 @@ struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init) list_add_tail(&new->cl_share_link, &nn->nfs_client_list); spin_unlock(&nn->nfs_client_lock); - nfs_local_probe(new); - return rpc_ops->init_client(new, cl_init); + new = rpc_ops->init_client(new, cl_init); + if (!IS_ERR(new)) + nfs_local_probe(new); + return new; } spin_unlock(&nn->nfs_client_lock); diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 0a9f14a04eb4..c29cdf51c458 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -51,17 +51,77 @@ static void nfs_local_fsync_work(struct work_struct *work); static bool localio_enabled __read_mostly = true; module_param(localio_enabled, bool, 0644); +static inline bool nfs_client_is_local(const struct nfs_client *clp) +{ + return !!test_bit(NFS_CS_LOCAL_IO, &clp->cl_flags); +} + bool nfs_server_is_local(const struct nfs_client *clp) { - return test_bit(NFS_CS_LOCAL_IO, &clp->cl_flags) != 0 && - localio_enabled; + return nfs_client_is_local(clp) && localio_enabled; } EXPORT_SYMBOL_GPL(nfs_server_is_local); +/* + * UUID_IS_LOCAL XDR functions + */ + +static void localio_xdr_enc_uuidargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const u8 *uuid = data; + + encode_opaque_fixed(xdr, uuid, UUID_SIZE); +} + +static int localio_xdr_dec_uuidres(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + /* void return */ + return 0; +} + +static const struct rpc_procinfo nfs_localio_procedures[] = { + [LOCALIOPROC_UUID_IS_LOCAL] = { + .p_proc = LOCALIOPROC_UUID_IS_LOCAL, + .p_encode = localio_xdr_enc_uuidargs, + .p_decode = localio_xdr_dec_uuidres, + .p_arglen = XDR_QUADLEN(UUID_SIZE), + .p_replen = 0, + .p_statidx = LOCALIOPROC_UUID_IS_LOCAL, + .p_name = "UUID_IS_LOCAL", + }, +}; + +static unsigned int nfs_localio_counts[ARRAY_SIZE(nfs_localio_procedures)]; +static const struct rpc_version nfslocalio_version1 = { + .number = 1, + .nrprocs = ARRAY_SIZE(nfs_localio_procedures), + .procs = nfs_localio_procedures, + .counts = nfs_localio_counts, +}; + +static const struct rpc_version *nfslocalio_version[] = { + [1] = &nfslocalio_version1, +}; + +extern const struct rpc_program nfslocalio_program; +static struct rpc_stat nfslocalio_rpcstat = { &nfslocalio_program }; + +const struct rpc_program nfslocalio_program = { + .name = "nfslocalio", + .number = NFS_LOCALIO_PROGRAM, + .nrvers = ARRAY_SIZE(nfslocalio_version), + .version = nfslocalio_version, + .stats = &nfslocalio_rpcstat, +}; + /* * nfs_local_enable - enable local i/o for an nfs_client */ -static __maybe_unused void nfs_local_enable(struct nfs_client *clp) +static void nfs_local_enable(struct nfs_client *clp) { spin_lock(&clp->cl_localio_lock); set_bit(NFS_CS_LOCAL_IO, &clp->cl_flags); @@ -82,11 +142,74 @@ void nfs_local_disable(struct nfs_client *clp) spin_unlock(&clp->cl_localio_lock); } +/* + * nfs_init_localioclient - Initialise an NFS localio client connection + */ +static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp) +{ + struct rpc_clnt *rpcclient_localio; + + rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient, + &nfslocalio_program, 1); + + dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n", + __func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR), + (IS_ERR(rpcclient_localio) ? "does not support" : "supports")); + + return rpcclient_localio; +} + +static bool nfs_server_uuid_is_local(struct nfs_client *clp) +{ + u8 uuid[UUID_SIZE]; + struct rpc_message msg = { + .rpc_argp = &uuid, + }; + struct rpc_clnt *rpcclient_localio; + int status; + + rpcclient_localio = nfs_init_localioclient(clp); + if (IS_ERR(rpcclient_localio)) + return false; + + export_uuid(uuid, &clp->cl_uuid.uuid); + + msg.rpc_proc = &nfs_localio_procedures[LOCALIOPROC_UUID_IS_LOCAL]; + status = rpc_call_sync(rpcclient_localio, &msg, 0); + dprintk("%s: NFS reply UUID_IS_LOCAL: status=%d\n", + __func__, status); + rpc_shutdown_client(rpcclient_localio); + + /* Server is only local if it initialized required struct members */ + if (status || !clp->cl_uuid.net || !clp->cl_uuid.dom) + return false; + + return true; +} + /* * nfs_local_probe - probe local i/o support for an nfs_server and nfs_client + * - called after alloc_client and init_client (so cl_rpcclient exists) + * - this function is idempotent, it can be called for old or new clients */ void nfs_local_probe(struct nfs_client *clp) { + /* Disallow localio if disabled via sysfs or AUTH_SYS isn't used */ + if (!localio_enabled || + clp->cl_rpcclient->cl_auth->au_flavor != RPC_AUTH_UNIX) { + nfs_local_disable(clp); + return; + } + + if (nfs_client_is_local(clp)) { + /* If already enabled, disable and re-enable */ + nfs_local_disable(clp); + } + + nfs_uuid_begin(&clp->cl_uuid); + if (nfs_server_uuid_is_local(clp)) + nfs_local_enable(clp); + nfs_uuid_end(&clp->cl_uuid); } EXPORT_SYMBOL_GPL(nfs_local_probe); @@ -116,7 +239,8 @@ nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, case -ENOMEM: case -ENXIO: case -ENOENT: - nfs_local_disable(clp); + /* Revalidate localio, will disable if unsupported */ + nfs_local_probe(clp); } return NULL; } From 92945bd81ca418ace7995bf5234ac311f6197d5d Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:09:58 -0400 Subject: [PATCH 107/111] nfs: add Documentation/filesystems/nfs/localio.rst This document gives an overview of the LOCALIO auxiliary RPC protocol added to the Linux NFS client and server to allow them to reliably handshake to determine if they are on the same host. Once an NFS client and server handshake as "local", the client will bypass the network RPC protocol for read, write and commit operations. Due to this XDR and RPC bypass, these operations will operate faster. Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- Documentation/filesystems/nfs/localio.rst | 203 ++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 Documentation/filesystems/nfs/localio.rst diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst new file mode 100644 index 000000000000..3c9bc370079b --- /dev/null +++ b/Documentation/filesystems/nfs/localio.rst @@ -0,0 +1,203 @@ +=========== +NFS LOCALIO +=========== + +Overview +======== + +The LOCALIO auxiliary RPC protocol allows the Linux NFS client and +server to reliably handshake to determine if they are on the same +host. Select "NFS client and server support for LOCALIO auxiliary +protocol" in menuconfig to enable CONFIG_NFS_LOCALIO in the kernel +config (both CONFIG_NFS_FS and CONFIG_NFSD must also be enabled). + +Once an NFS client and server handshake as "local", the client will +bypass the network RPC protocol for read, write and commit operations. +Due to this XDR and RPC bypass, these operations will operate faster. + +The LOCALIO auxiliary protocol's implementation, which uses the same +connection as NFS traffic, follows the pattern established by the NFS +ACL protocol extension. + +The LOCALIO auxiliary protocol is needed to allow robust discovery of +clients local to their servers. In a private implementation that +preceded use of this LOCALIO protocol, a fragile sockaddr network +address based match against all local network interfaces was attempted. +But unlike the LOCALIO protocol, the sockaddr-based matching didn't +handle use of iptables or containers. + +The robust handshake between local client and server is just the +beginning, the ultimate use case this locality makes possible is the +client is able to open files and issue reads, writes and commits +directly to the server without having to go over the network. The +requirement is to perform these loopback NFS operations as efficiently +as possible, this is particularly useful for container use cases +(e.g. kubernetes) where it is possible to run an IO job local to the +server. + +The performance advantage realized from LOCALIO's ability to bypass +using XDR and RPC for reads, writes and commits can be extreme, e.g.: + +fio for 20 secs with directio, qd of 8, 16 libaio threads: +- With LOCALIO: + 4K read: IOPS=979k, BW=3825MiB/s (4011MB/s)(74.7GiB/20002msec) + 4K write: IOPS=165k, BW=646MiB/s (678MB/s)(12.6GiB/20002msec) + 128K read: IOPS=402k, BW=49.1GiB/s (52.7GB/s)(982GiB/20002msec) + 128K write: IOPS=11.5k, BW=1433MiB/s (1503MB/s)(28.0GiB/20004msec) + +- Without LOCALIO: + 4K read: IOPS=79.2k, BW=309MiB/s (324MB/s)(6188MiB/20003msec) + 4K write: IOPS=59.8k, BW=234MiB/s (245MB/s)(4671MiB/20002msec) + 128K read: IOPS=33.9k, BW=4234MiB/s (4440MB/s)(82.7GiB/20004msec) + 128K write: IOPS=11.5k, BW=1434MiB/s (1504MB/s)(28.0GiB/20011msec) + +fio for 20 secs with directio, qd of 8, 1 libaio thread: +- With LOCALIO: + 4K read: IOPS=230k, BW=898MiB/s (941MB/s)(17.5GiB/20001msec) + 4K write: IOPS=22.6k, BW=88.3MiB/s (92.6MB/s)(1766MiB/20001msec) + 128K read: IOPS=38.8k, BW=4855MiB/s (5091MB/s)(94.8GiB/20001msec) + 128K write: IOPS=11.4k, BW=1428MiB/s (1497MB/s)(27.9GiB/20001msec) + +- Without LOCALIO: + 4K read: IOPS=77.1k, BW=301MiB/s (316MB/s)(6022MiB/20001msec) + 4K write: IOPS=32.8k, BW=128MiB/s (135MB/s)(2566MiB/20001msec) + 128K read: IOPS=24.4k, BW=3050MiB/s (3198MB/s)(59.6GiB/20001msec) + 128K write: IOPS=11.4k, BW=1430MiB/s (1500MB/s)(27.9GiB/20001msec) + +RPC +=== + +The LOCALIO auxiliary RPC protocol consists of a single "UUID_IS_LOCAL" +RPC method that allows the Linux NFS client to verify the local Linux +NFS server can see the nonce (single-use UUID) the client generated and +made available in nfs_common. This protocol isn't part of an IETF +standard, nor does it need to be considering it is Linux-to-Linux +auxiliary RPC protocol that amounts to an implementation detail. + +The UUID_IS_LOCAL method encodes the client generated uuid_t in terms of +the fixed UUID_SIZE (16 bytes). The fixed size opaque encode and decode +XDR methods are used instead of the less efficient variable sized +methods. + +The RPC program number for the NFS_LOCALIO_PROGRAM is 400122 (as assigned +by IANA, see https://www.iana.org/assignments/rpc-program-numbers/ ): +Linux Kernel Organization 400122 nfslocalio + +The LOCALIO protocol spec in rpcgen syntax is: + +/* raw RFC 9562 UUID */ +#define UUID_SIZE 16 +typedef u8 uuid_t; + +program NFS_LOCALIO_PROGRAM { + version LOCALIO_V1 { + void + NULL(void) = 0; + + void + UUID_IS_LOCAL(uuid_t) = 1; + } = 1; +} = 400122; + +LOCALIO uses the same transport connection as NFS traffic. As such, +LOCALIO is not registered with rpcbind. + +NFS Common and Client/Server Handshake +====================================== + +fs/nfs_common/nfslocalio.c provides interfaces that enable an NFS client +to generate a nonce (single-use UUID) and associated short-lived +nfs_uuid_t struct, register it with nfs_common for subsequent lookup and +verification by the NFS server and if matched the NFS server populates +members in the nfs_uuid_t struct. The NFS client then uses nfs_common to +transfer the nfs_uuid_t from its nfs_uuids to the nn->nfsd_serv +clients_list from the nfs_common's uuids_list. See: +fs/nfs/localio.c:nfs_local_probe() + +nfs_common's nfs_uuids list is the basis for LOCALIO enablement, as such +it has members that point to nfsd memory for direct use by the client +(e.g. 'net' is the server's network namespace, through it the client can +access nn->nfsd_serv with proper rcu read access). It is this client +and server synchronization that enables advanced usage and lifetime of +objects to span from the host kernel's nfsd to per-container knfsd +instances that are connected to nfs client's running on the same local +host. + +NFS Client issues IO instead of Server +====================================== + +Because LOCALIO is focused on protocol bypass to achieve improved IO +performance, alternatives to the traditional NFS wire protocol (SUNRPC +with XDR) must be provided to access the backing filesystem. + +See fs/nfs/localio.c:nfs_local_open_fh() and +fs/nfsd/localio.c:nfsd_open_local_fh() for the interface that makes +focused use of select nfs server objects to allow a client local to a +server to open a file pointer without needing to go over the network. + +The client's fs/nfs/localio.c:nfs_local_open_fh() will call into the +server's fs/nfsd/localio.c:nfsd_open_local_fh() and carefully access +both the associated nfsd network namespace and nn->nfsd_serv in terms of +RCU. If nfsd_open_local_fh() finds that the client no longer sees valid +nfsd objects (be it struct net or nn->nfsd_serv) it returns -ENXIO +to nfs_local_open_fh() and the client will try to reestablish the +LOCALIO resources needed by calling nfs_local_probe() again. This +recovery is needed if/when an nfsd instance running in a container were +to reboot while a LOCALIO client is connected to it. + +Once the client has an open nfsd_file pointer it will issue reads, +writes and commits directly to the underlying local filesystem (normally +done by the nfs server). As such, for these operations, the NFS client +is issuing IO to the underlying local filesystem that it is sharing with +the NFS server. See: fs/nfs/localio.c:nfs_local_doio() and +fs/nfs/localio.c:nfs_local_commit(). + +Security +======== + +Localio is only supported when UNIX-style authentication (AUTH_UNIX, aka +AUTH_SYS) is used. + +Care is taken to ensure the same NFS security mechanisms are used +(authentication, etc) regardless of whether LOCALIO or regular NFS +access is used. The auth_domain established as part of the traditional +NFS client access to the NFS server is also used for LOCALIO. + +Relative to containers, LOCALIO gives the client access to the network +namespace the server has. This is required to allow the client to access +the server's per-namespace nfsd_net struct. With traditional NFS, the +client is afforded this same level of access (albeit in terms of the NFS +protocol via SUNRPC). No other namespaces (user, mount, etc) have been +altered or purposely extended from the server to the client. + +Testing +======= + +The LOCALIO auxiliary protocol and associated NFS LOCALIO read, write +and commit access have proven stable against various test scenarios: + +- Client and server both on the same host. + +- All permutations of client and server support enablement for both + local and remote client and server. + +- Testing against NFS storage products that don't support the LOCALIO + protocol was also performed. + +- Client on host, server within a container (for both v3 and v4.2). + The container testing was in terms of podman managed containers and + includes successful container stop/restart scenario. + +- Formalizing these test scenarios in terms of existing test + infrastructure is on-going. Initial regular coverage is provided in + terms of ktest running xfstests against a LOCALIO-enabled NFS loopback + mount configuration, and includes lockdep and KASAN coverage, see: + https://evilpiepirate.org/~testdashboard/ci?user=snitzer&branch=snitm-nfs-next + https://github.com/koverstreet/ktest + +- Various kdevops testing (in terms of "Chuck's BuildBot") has been + performed to regularly verify the LOCALIO changes haven't caused any + regressions to non-LOCALIO NFS use cases. + +- All of Hammerspace's various sanity tests pass with LOCALIO enabled + (this includes numerous pNFS and flexfiles tests). From f7128262b15287e4be501e30f9e1f0258606a593 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Thu, 5 Sep 2024 15:09:59 -0400 Subject: [PATCH 108/111] nfs: add FAQ section to Documentation/filesystems/nfs/localio.rst Add a FAQ section to give answers to questions that have been raised during review of the localio feature. Signed-off-by: Trond Myklebust Co-developed-by: Mike Snitzer Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- Documentation/filesystems/nfs/localio.rst | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst index 3c9bc370079b..ef3851d48133 100644 --- a/Documentation/filesystems/nfs/localio.rst +++ b/Documentation/filesystems/nfs/localio.rst @@ -64,6 +64,92 @@ fio for 20 secs with directio, qd of 8, 1 libaio thread: 128K read: IOPS=24.4k, BW=3050MiB/s (3198MB/s)(59.6GiB/20001msec) 128K write: IOPS=11.4k, BW=1430MiB/s (1500MB/s)(27.9GiB/20001msec) +FAQ +=== + +1. What are the use cases for LOCALIO? + + a. Workloads where the NFS client and server are on the same host + realize improved IO performance. In particular, it is common when + running containerised workloads for jobs to find themselves + running on the same host as the knfsd server being used for + storage. + +2. What are the requirements for LOCALIO? + + a. Bypass use of the network RPC protocol as much as possible. This + includes bypassing XDR and RPC for open, read, write and commit + operations. + b. Allow client and server to autonomously discover if they are + running local to each other without making any assumptions about + the local network topology. + c. Support the use of containers by being compatible with relevant + namespaces (e.g. network, user, mount). + d. Support all versions of NFS. NFSv3 is of particular importance + because it has wide enterprise usage and pNFS flexfiles makes use + of it for the data path. + +3. Why doesn’t LOCALIO just compare IP addresses or hostnames when + deciding if the NFS client and server are co-located on the same + host? + + Since one of the main use cases is containerised workloads, we cannot + assume that IP addresses will be shared between the client and + server. This sets up a requirement for a handshake protocol that + needs to go over the same connection as the NFS traffic in order to + identify that the client and the server really are running on the + same host. The handshake uses a secret that is sent over the wire, + and can be verified by both parties by comparing with a value stored + in shared kernel memory if they are truly co-located. + +4. Does LOCALIO improve pNFS flexfiles? + + Yes, LOCALIO complements pNFS flexfiles by allowing it to take + advantage of NFS client and server locality. Policy that initiates + client IO as closely to the server where the data is stored naturally + benefits from the data path optimization LOCALIO provides. + +5. Why not develop a new pNFS layout to enable LOCALIO? + + A new pNFS layout could be developed, but doing so would put the + onus on the server to somehow discover that the client is co-located + when deciding to hand out the layout. + There is value in a simpler approach (as provided by LOCALIO) that + allows the NFS client to negotiate and leverage locality without + requiring more elaborate modeling and discovery of such locality in a + more centralized manner. + +6. Why is having the client perform a server-side file OPEN, without + using RPC, beneficial? Is the benefit pNFS specific? + + Avoiding the use of XDR and RPC for file opens is beneficial to + performance regardless of whether pNFS is used. Especially when + dealing with small files its best to avoid going over the wire + whenever possible, otherwise it could reduce or even negate the + benefits of avoiding the wire for doing the small file I/O itself. + Given LOCALIO's requirements the current approach of having the + client perform a server-side file open, without using RPC, is ideal. + If in the future requirements change then we can adapt accordingly. + +7. Why is LOCALIO only supported with UNIX Authentication (AUTH_UNIX)? + + Strong authentication is usually tied to the connection itself. It + works by establishing a context that is cached by the server, and + that acts as the key for discovering the authorisation token, which + can then be passed to rpc.mountd to complete the authentication + process. On the other hand, in the case of AUTH_UNIX, the credential + that was passed over the wire is used directly as the key in the + upcall to rpc.mountd. This simplifies the authentication process, and + so makes AUTH_UNIX easier to support. + +8. How do export options that translate RPC user IDs behave for LOCALIO + operations (eg. root_squash, all_squash)? + + Export options that translate user IDs are managed by nfsd_setuser() + which is called by nfsd_setuser_and_check_port() which is called by + __fh_verify(). So they get handled exactly the same way for LOCALIO + as they do for non-LOCALIO. + RPC === From 736cd2c1ae7f911df9aae0e647899b49bdbccdb8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 5 Sep 2024 15:10:00 -0400 Subject: [PATCH 109/111] nfs: add "NFS Client and Server Interlock" section to localio.rst This section answers a new FAQ entry: 9. How does LOCALIO make certain that object lifetimes are managed properly given NFSD and NFS operate in different contexts? See the detailed "NFS Client and Server Interlock" section below. The first half of the section details NeilBrown's elegant design for LOCALIO's nfs_uuid_t based interlock and is heavily based on Neil's "net namespace refcounting" description here: https://marc.info/?l=linux-nfs&m=172498546024767&w=2 The second half of the section details the per-cpu-refcount introduced to ensure NFSD's nfsd_serv isn't destroyed while in use by a LOCALIO client. Signed-off-by: Mike Snitzer Reviewed-by: NeilBrown Reviewed-by: Jeff Layton Signed-off-by: Anna Schumaker --- Documentation/filesystems/nfs/localio.rst | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst index ef3851d48133..4637c0b34753 100644 --- a/Documentation/filesystems/nfs/localio.rst +++ b/Documentation/filesystems/nfs/localio.rst @@ -150,6 +150,11 @@ FAQ __fh_verify(). So they get handled exactly the same way for LOCALIO as they do for non-LOCALIO. +9. How does LOCALIO make certain that object lifetimes are managed + properly given NFSD and NFS operate in different contexts? + + See the detailed "NFS Client and Server Interlock" section below. + RPC === @@ -209,6 +214,69 @@ objects to span from the host kernel's nfsd to per-container knfsd instances that are connected to nfs client's running on the same local host. +NFS Client and Server Interlock +=============================== + +LOCALIO provides the nfs_uuid_t object and associated interfaces to +allow proper network namespace (net-ns) and NFSD object refcounting: + + We don't want to keep a long-term counted reference on each NFSD's + net-ns in the client because that prevents a server container from + completely shutting down. + + So we avoid taking a reference at all and rely on the per-cpu + reference to the server (detailed below) being sufficient to keep + the net-ns active. This involves allowing the NFSD's net-ns exit + code to iterate all active clients and clear their ->net pointers + (which are needed to find the per-cpu-refcount for the nfsd_serv). + + Details: + + - Embed nfs_uuid_t in nfs_client. nfs_uuid_t provides a list_head + that can be used to find the client. It does add the 16-byte + uuid_t to nfs_client so it is bigger than needed (given that + uuid_t is only used during the initial NFS client and server + LOCALIO handshake to determine if they are local to each other). + If that is really a problem we can find a fix. + + - When the nfs server confirms that the uuid_t is local, it moves + the nfs_uuid_t onto a per-net-ns list in NFSD's nfsd_net. + + - When each server's net-ns is shutting down - in a "pre_exit" + handler, all these nfs_uuid_t have their ->net cleared. There is + an rcu_synchronize() call between pre_exit() handlers and exit() + handlers so any caller that sees nfs_uuid_t ->net as not NULL can + safely manage the per-cpu-refcount for nfsd_serv. + + - The client's nfs_uuid_t is passed to nfsd_open_local_fh() so it + can safely dereference ->net in a private rcu_read_lock() section + to allow safe access to the associated nfsd_net and nfsd_serv. + +So LOCALIO required the introduction and use of NFSD's percpu_ref to +interlock nfsd_destroy_serv() and nfsd_open_local_fh(), to ensure each +nn->nfsd_serv is not destroyed while in use by nfsd_open_local_fh(), and +warrants a more detailed explanation: + + nfsd_open_local_fh() uses nfsd_serv_try_get() before opening its + nfsd_file handle and then the caller (NFS client) must drop the + reference for the nfsd_file and associated nn->nfsd_serv using + nfs_file_put_local() once it has completed its IO. + + This interlock working relies heavily on nfsd_open_local_fh() being + afforded the ability to safely deal with the possibility that the + NFSD's net-ns (and nfsd_net by association) may have been destroyed + by nfsd_destroy_serv() via nfsd_shutdown_net() -- which is only + possible given the nfs_uuid_t ->net pointer managemenet detailed + above. + +All told, this elaborate interlock of the NFS client and server has been +verified to fix an easy to hit crash that would occur if an NFSD +instance running in a container, with a LOCALIO client mounted, is +shutdown. Upon restart of the container and associated NFSD the client +would go on to crash due to NULL pointer dereference that occurred due +to the LOCALIO client's attempting to nfsd_open_local_fh(), using +nn->nfsd_serv, without having a proper reference on nn->nfsd_serv. + NFS Client issues IO instead of Server ====================================== From 68898131d2df70d1a9ad5c2f93f0f54dd6d5c336 Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Tue, 24 Sep 2024 11:16:34 -0400 Subject: [PATCH 110/111] nfs: Fix `make htmldocs` warnings in the localio documentation Reported-by: Stephen Rothwell Fixes: 92945bd81ca4 ("nfs: add Documentation/filesystems/nfs/localio.rst") Signed-off-by: Anna Schumaker --- Documentation/filesystems/nfs/index.rst | 1 + Documentation/filesystems/nfs/localio.rst | 64 +++++++++++------------ 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/Documentation/filesystems/nfs/index.rst b/Documentation/filesystems/nfs/index.rst index 8536134f31fd..95c2c009874c 100644 --- a/Documentation/filesystems/nfs/index.rst +++ b/Documentation/filesystems/nfs/index.rst @@ -8,6 +8,7 @@ NFS client-identifier exporting + localio pnfs rpc-cache rpc-server-gss diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst index 4637c0b34753..bd1967e2eab3 100644 --- a/Documentation/filesystems/nfs/localio.rst +++ b/Documentation/filesystems/nfs/localio.rst @@ -39,30 +39,30 @@ The performance advantage realized from LOCALIO's ability to bypass using XDR and RPC for reads, writes and commits can be extreme, e.g.: fio for 20 secs with directio, qd of 8, 16 libaio threads: -- With LOCALIO: - 4K read: IOPS=979k, BW=3825MiB/s (4011MB/s)(74.7GiB/20002msec) - 4K write: IOPS=165k, BW=646MiB/s (678MB/s)(12.6GiB/20002msec) - 128K read: IOPS=402k, BW=49.1GiB/s (52.7GB/s)(982GiB/20002msec) - 128K write: IOPS=11.5k, BW=1433MiB/s (1503MB/s)(28.0GiB/20004msec) + - With LOCALIO: + 4K read: IOPS=979k, BW=3825MiB/s (4011MB/s)(74.7GiB/20002msec) + 4K write: IOPS=165k, BW=646MiB/s (678MB/s)(12.6GiB/20002msec) + 128K read: IOPS=402k, BW=49.1GiB/s (52.7GB/s)(982GiB/20002msec) + 128K write: IOPS=11.5k, BW=1433MiB/s (1503MB/s)(28.0GiB/20004msec) -- Without LOCALIO: - 4K read: IOPS=79.2k, BW=309MiB/s (324MB/s)(6188MiB/20003msec) - 4K write: IOPS=59.8k, BW=234MiB/s (245MB/s)(4671MiB/20002msec) - 128K read: IOPS=33.9k, BW=4234MiB/s (4440MB/s)(82.7GiB/20004msec) - 128K write: IOPS=11.5k, BW=1434MiB/s (1504MB/s)(28.0GiB/20011msec) + - Without LOCALIO: + 4K read: IOPS=79.2k, BW=309MiB/s (324MB/s)(6188MiB/20003msec) + 4K write: IOPS=59.8k, BW=234MiB/s (245MB/s)(4671MiB/20002msec) + 128K read: IOPS=33.9k, BW=4234MiB/s (4440MB/s)(82.7GiB/20004msec) + 128K write: IOPS=11.5k, BW=1434MiB/s (1504MB/s)(28.0GiB/20011msec) fio for 20 secs with directio, qd of 8, 1 libaio thread: -- With LOCALIO: - 4K read: IOPS=230k, BW=898MiB/s (941MB/s)(17.5GiB/20001msec) - 4K write: IOPS=22.6k, BW=88.3MiB/s (92.6MB/s)(1766MiB/20001msec) - 128K read: IOPS=38.8k, BW=4855MiB/s (5091MB/s)(94.8GiB/20001msec) - 128K write: IOPS=11.4k, BW=1428MiB/s (1497MB/s)(27.9GiB/20001msec) + - With LOCALIO: + 4K read: IOPS=230k, BW=898MiB/s (941MB/s)(17.5GiB/20001msec) + 4K write: IOPS=22.6k, BW=88.3MiB/s (92.6MB/s)(1766MiB/20001msec) + 128K read: IOPS=38.8k, BW=4855MiB/s (5091MB/s)(94.8GiB/20001msec) + 128K write: IOPS=11.4k, BW=1428MiB/s (1497MB/s)(27.9GiB/20001msec) -- Without LOCALIO: - 4K read: IOPS=77.1k, BW=301MiB/s (316MB/s)(6022MiB/20001msec) - 4K write: IOPS=32.8k, BW=128MiB/s (135MB/s)(2566MiB/20001msec) - 128K read: IOPS=24.4k, BW=3050MiB/s (3198MB/s)(59.6GiB/20001msec) - 128K write: IOPS=11.4k, BW=1430MiB/s (1500MB/s)(27.9GiB/20001msec) + - Without LOCALIO: + 4K read: IOPS=77.1k, BW=301MiB/s (316MB/s)(6022MiB/20001msec) + 4K write: IOPS=32.8k, BW=128MiB/s (135MB/s)(2566MiB/20001msec) + 128K read: IOPS=24.4k, BW=3050MiB/s (3198MB/s)(59.6GiB/20001msec) + 128K write: IOPS=11.4k, BW=1430MiB/s (1500MB/s)(27.9GiB/20001msec) FAQ === @@ -174,21 +174,21 @@ The RPC program number for the NFS_LOCALIO_PROGRAM is 400122 (as assigned by IANA, see https://www.iana.org/assignments/rpc-program-numbers/ ): Linux Kernel Organization 400122 nfslocalio -The LOCALIO protocol spec in rpcgen syntax is: +The LOCALIO protocol spec in rpcgen syntax is:: -/* raw RFC 9562 UUID */ -#define UUID_SIZE 16 -typedef u8 uuid_t; + /* raw RFC 9562 UUID */ + #define UUID_SIZE 16 + typedef u8 uuid_t; -program NFS_LOCALIO_PROGRAM { - version LOCALIO_V1 { - void - NULL(void) = 0; + program NFS_LOCALIO_PROGRAM { + version LOCALIO_V1 { + void + NULL(void) = 0; - void - UUID_IS_LOCAL(uuid_t) = 1; - } = 1; -} = 400122; + void + UUID_IS_LOCAL(uuid_t) = 1; + } = 1; + } = 400122; LOCALIO uses the same transport connection as NFS traffic. As such, LOCALIO is not registered with rpcbind. From 1f9c4a996756867d678833c0513eabe4e8f1ed60 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Wed, 25 Sep 2024 11:08:28 -0700 Subject: [PATCH 111/111] Kbuild: make MODVERSIONS support depend on not being a compile test build Currently the Rust support is gated on not having MODVERSIONS enabled, and as a result an "allmodconfig" build will disable Rust build tests. While MODVERSIONS configurations are worth build testing, the feature is not actually meaningful unless you run the result, and I'd rather get build coverage of Rust than MODVERSIONS. So let's disable MODVERSIONS for build testing until the Rust side clears up. Signed-off-by: Linus Torvalds --- kernel/module/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/module/Kconfig b/kernel/module/Kconfig index 4047b6d48255..05a9a06a140c 100644 --- a/kernel/module/Kconfig +++ b/kernel/module/Kconfig @@ -160,6 +160,7 @@ config MODULE_UNLOAD_TAINT_TRACKING config MODVERSIONS bool "Module versioning support" + depends on !COMPILE_TEST help Usually, you have to use modules compiled with your kernel. Saying Y here makes it sometimes possible to use modules