From fda1dd3c54ef3c4200c2e77634a91610da973950 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 18 Jul 2024 17:50:37 -0700 Subject: [PATCH 001/104] find: Switch from inline to __always_inline 'inline' keyword is only a recommendation for compiler. If it decides to not inline find_bit nodemask functions, the whole small_const_nbits() machinery doesn't work. This is how a standard GCC 11.3.0 does for my x86_64 build now. This patch replaces 'inline' directive with unconditional '__always_inline' to make sure that there's always a chance for compile-time optimization. It doesn't change size of kernel image, according to bloat-o-meter. [[ Brian: split out from: Subject: [PATCH 1/3] bitmap: switch from inline to __always_inline https://lore.kernel.org/all/20221027043810.350460-2-yury.norov@gmail.com/ But rewritten, as there were too many conflicts. ]] Co-developed-by: Brian Norris Signed-off-by: Brian Norris Reviewed-by: Kees Cook Reviewed-by: Nathan Chancellor Signed-off-by: Yury Norov --- include/linux/find.h | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/include/linux/find.h b/include/linux/find.h index 5dfca4225fef..68685714bc18 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -52,7 +52,7 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned * Returns the bit number for the next set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { @@ -81,7 +81,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, * Returns the bit number for the next set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long offset) @@ -112,7 +112,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, * Returns the bit number for the next set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long offset) @@ -142,7 +142,7 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1, * Returns the bit number for the next set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long offset) @@ -171,7 +171,7 @@ unsigned long find_next_or_bit(const unsigned long *addr1, * Returns the bit number of the next zero bit * If no bits are zero, returns @size. */ -static inline +static __always_inline unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset) { @@ -198,7 +198,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, * Returns the bit number of the first set bit. * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_first_bit(const unsigned long *addr, unsigned long size) { if (small_const_nbits(size)) { @@ -224,7 +224,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) * Returns the bit number of the N'th set bit. * If no such, returns >= @size. */ -static inline +static __always_inline unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) { if (n >= size) @@ -249,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign * Returns the bit number of the N'th set bit. * If no such, returns @size. */ -static inline +static __always_inline unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long n) { @@ -276,7 +276,7 @@ unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long * * Returns the bit number of the N'th set bit. * If no such, returns @size. */ -static inline +static __always_inline unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long n) { @@ -332,7 +332,7 @@ unsigned long find_nth_and_andnot_bit(const unsigned long *addr1, * Returns the bit number for the next set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size) @@ -357,7 +357,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1, * Returns the bit number for the first set bit * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_first_and_and_bit(const unsigned long *addr1, const unsigned long *addr2, const unsigned long *addr3, @@ -381,7 +381,7 @@ unsigned long find_first_and_and_bit(const unsigned long *addr1, * Returns the bit number of the first cleared bit. * If no bits are zero, returns @size. */ -static inline +static __always_inline unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) { if (small_const_nbits(size)) { @@ -402,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) * * Returns the bit number of the last set bit, or size. */ -static inline +static __always_inline unsigned long find_last_bit(const unsigned long *addr, unsigned long size) { if (small_const_nbits(size)) { @@ -425,7 +425,7 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) * Returns the bit number for the next set bit, or first set bit up to @offset * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_and_bit_wrap(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long offset) @@ -448,7 +448,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1, * Returns the bit number for the next set bit, or first set bit up to @offset * If no bits are set, returns @size. */ -static inline +static __always_inline unsigned long find_next_bit_wrap(const unsigned long *addr, unsigned long size, unsigned long offset) { @@ -465,7 +465,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr, * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing * before using it alone. */ -static inline +static __always_inline unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size, unsigned long start, unsigned long n) { @@ -506,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump, #if defined(__LITTLE_ENDIAN) -static inline unsigned long find_next_zero_bit_le(const void *addr, - unsigned long size, unsigned long offset) +static __always_inline +unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset) { return find_next_zero_bit(addr, size, offset); } -static inline unsigned long find_next_bit_le(const void *addr, - unsigned long size, unsigned long offset) +static __always_inline +unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset) { return find_next_bit(addr, size, offset); } -static inline unsigned long find_first_zero_bit_le(const void *addr, - unsigned long size) +static __always_inline +unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) { return find_first_zero_bit(addr, size); } @@ -527,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr, #elif defined(__BIG_ENDIAN) #ifndef find_next_zero_bit_le -static inline +static __always_inline unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset) { @@ -546,7 +546,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned #endif #ifndef find_first_zero_bit_le -static inline +static __always_inline unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) { if (small_const_nbits(size)) { @@ -560,7 +560,7 @@ unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) #endif #ifndef find_next_bit_le -static inline +static __always_inline unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset) { From ed8cd2b3bd9f070120528fc5403a2d0b5afe07f8 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 18 Jul 2024 17:50:38 -0700 Subject: [PATCH 002/104] bitmap: Switch from inline to __always_inline 'inline' keyword is only a recommendation for compiler. If it decides to not inline bitmap functions, the whole small_const_nbits() machinery doesn't work. This is how a standard GCC 11.3.0 does for my x86_64 build now. This patch replaces 'inline' directive with unconditional '__always_inline' to make sure that there's always a chance for compile-time optimization. It doesn't change size of kernel image, according to bloat-o-meter. [[ Brian: split out from: Subject: [PATCH 1/3] bitmap: switch from inline to __always_inline https://lore.kernel.org/all/20221027043810.350460-2-yury.norov@gmail.com/ But rewritten, as there were too many conflicts. ]] Co-developed-by: Brian Norris Signed-off-by: Brian Norris Reviewed-by: Kees Cook Reviewed-by: Nathan Chancellor Signed-off-by: Yury Norov --- include/linux/bitmap.h | 140 ++++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 64 deletions(-) diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 8c4768c44a01..0406f48f0a6a 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, * the bit offset of all zero areas this function finds is multiples of that * power of 2. A @align_mask of 0 means no alignment is required. */ -static inline unsigned long -bitmap_find_next_zero_area(unsigned long *map, - unsigned long size, - unsigned long start, - unsigned int nr, - unsigned long align_mask) +static __always_inline +unsigned long bitmap_find_next_zero_area(unsigned long *map, + unsigned long size, + unsigned long start, + unsigned int nr, + unsigned long align_mask) { return bitmap_find_next_zero_area_off(map, size, start, nr, align_mask, 0); @@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, #define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE) -static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) +static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits) { unsigned int len = bitmap_size(nbits); @@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits) memset(dst, 0, len); } -static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) +static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits) { unsigned int len = bitmap_size(nbits); @@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) memset(dst, 0xff, len); } -static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, - unsigned int nbits) +static __always_inline +void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits) { unsigned int len = bitmap_size(nbits); @@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, /* * Copy bitmap and clear tail bits in last word. */ -static inline void bitmap_copy_clear_tail(unsigned long *dst, - const unsigned long *src, unsigned int nbits) +static __always_inline +void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits) { bitmap_copy(dst, src, nbits); if (nbits % BITS_PER_LONG) @@ -306,16 +306,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits); bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits)) #endif -static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +bool bitmap_and(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; return __bitmap_and(dst, src1, src2, nbits); } -static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +void bitmap_or(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) *dst = *src1 | *src2; @@ -323,8 +325,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, __bitmap_or(dst, src1, src2, nbits); } -static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +void bitmap_xor(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) *dst = *src1 ^ *src2; @@ -332,16 +335,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, __bitmap_xor(dst, src1, src2, nbits); } -static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +bool bitmap_andnot(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; return __bitmap_andnot(dst, src1, src2, nbits); } -static inline void bitmap_complement(unsigned long *dst, const unsigned long *src, - unsigned int nbits) +static __always_inline +void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) *dst = ~(*src); @@ -356,8 +360,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr #endif #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1) -static inline bool bitmap_equal(const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); @@ -376,10 +380,9 @@ static inline bool bitmap_equal(const unsigned long *src1, * * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise */ -static inline bool bitmap_or_equal(const unsigned long *src1, - const unsigned long *src2, - const unsigned long *src3, - unsigned int nbits) +static __always_inline +bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2, + const unsigned long *src3, unsigned int nbits) { if (!small_const_nbits(nbits)) return __bitmap_or_equal(src1, src2, src3, nbits); @@ -387,9 +390,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1, return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits)); } -static inline bool bitmap_intersects(const unsigned long *src1, - const unsigned long *src2, - unsigned int nbits) +static __always_inline +bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; @@ -397,8 +399,8 @@ static inline bool bitmap_intersects(const unsigned long *src1, return __bitmap_intersects(src1, src2, nbits); } -static inline bool bitmap_subset(const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static __always_inline +bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); @@ -406,7 +408,8 @@ static inline bool bitmap_subset(const unsigned long *src1, return __bitmap_subset(src1, src2, nbits); } -static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) +static __always_inline +bool bitmap_empty(const unsigned long *src, unsigned nbits) { if (small_const_nbits(nbits)) return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -414,7 +417,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits) return find_first_bit(src, nbits) == nbits; } -static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) +static __always_inline +bool bitmap_full(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); @@ -448,8 +452,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, return __bitmap_weight_andnot(src1, src2, nbits); } -static __always_inline void bitmap_set(unsigned long *map, unsigned int start, - unsigned int nbits) +static __always_inline +void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) { if (__builtin_constant_p(nbits) && nbits == 1) __set_bit(start, map); @@ -464,8 +468,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start, __bitmap_set(map, start, nbits); } -static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, - unsigned int nbits) +static __always_inline +void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits) { if (__builtin_constant_p(nbits) && nbits == 1) __clear_bit(start, map); @@ -480,8 +484,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start, __bitmap_clear(map, start, nbits); } -static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src, - unsigned int shift, unsigned int nbits) +static __always_inline +void bitmap_shift_right(unsigned long *dst, const unsigned long *src, + unsigned int shift, unsigned int nbits) { if (small_const_nbits(nbits)) *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift; @@ -489,8 +494,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s __bitmap_shift_right(dst, src, shift, nbits); } -static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src, - unsigned int shift, unsigned int nbits) +static __always_inline +void bitmap_shift_left(unsigned long *dst, const unsigned long *src, + unsigned int shift, unsigned int nbits) { if (small_const_nbits(nbits)) *dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits); @@ -498,11 +504,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr __bitmap_shift_left(dst, src, shift, nbits); } -static inline void bitmap_replace(unsigned long *dst, - const unsigned long *old, - const unsigned long *new, - const unsigned long *mask, - unsigned int nbits) +static __always_inline +void bitmap_replace(unsigned long *dst, + const unsigned long *old, + const unsigned long *new, + const unsigned long *mask, + unsigned int nbits) { if (small_const_nbits(nbits)) *dst = (*old & ~(*mask)) | (*new & *mask); @@ -545,8 +552,9 @@ static inline void bitmap_replace(unsigned long *dst, * bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation. * See bitmap_scatter() for details related to this relationship. */ -static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, - const unsigned long *mask, unsigned int nbits) +static __always_inline +void bitmap_scatter(unsigned long *dst, const unsigned long *src, + const unsigned long *mask, unsigned int nbits) { unsigned int n = 0; unsigned int bit; @@ -599,8 +607,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src, * bitmap_scatter(res, src, mask, n) and a call to * bitmap_scatter(res, result, mask, n) will lead to the same res value. */ -static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, - const unsigned long *mask, unsigned int nbits) +static __always_inline +void bitmap_gather(unsigned long *dst, const unsigned long *src, + const unsigned long *mask, unsigned int nbits) { unsigned int n = 0; unsigned int bit; @@ -611,9 +620,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src, __assign_bit(n++, dst, test_bit(bit, src)); } -static inline void bitmap_next_set_region(unsigned long *bitmap, - unsigned int *rs, unsigned int *re, - unsigned int end) +static __always_inline +void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs, + unsigned int *re, unsigned int end) { *rs = find_next_bit(bitmap, end, *rs); *re = find_next_zero_bit(bitmap, end, *rs + 1); @@ -628,7 +637,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, * This is the complement to __bitmap_find_free_region() and releases * the found region (by clearing it in the bitmap). */ -static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) +static __always_inline +void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) { bitmap_clear(bitmap, pos, BIT(order)); } @@ -644,7 +654,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos * Returns: 0 on success, or %-EBUSY if specified region wasn't * free (not all bits were zero). */ -static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) +static __always_inline +int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) { unsigned int len = BIT(order); @@ -668,7 +679,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos * Returns: the bit offset in bitmap of the allocated region, * or -errno on failure. */ -static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) +static __always_inline +int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) { unsigned int pos, end; /* scans bitmap by regions of size order */ @@ -722,7 +734,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits, * but we expect the lower 32-bits of u64. */ -static inline void bitmap_from_u64(unsigned long *dst, u64 mask) +static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask) { bitmap_from_arr64(dst, &mask, 64); } @@ -737,9 +749,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) * @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return * value is undefined. */ -static inline unsigned long bitmap_read(const unsigned long *map, - unsigned long start, - unsigned long nbits) +static __always_inline +unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits) { size_t index = BIT_WORD(start); unsigned long offset = start % BITS_PER_LONG; @@ -772,8 +783,9 @@ static inline unsigned long bitmap_read(const unsigned long *map, * * For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed. */ -static inline void bitmap_write(unsigned long *map, unsigned long value, - unsigned long start, unsigned long nbits) +static __always_inline +void bitmap_write(unsigned long *map, unsigned long value, + unsigned long start, unsigned long nbits) { size_t index; unsigned long offset; From ab6b1010dab68f6d4bf063517db4ce2d63554bc6 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Thu, 18 Jul 2024 17:50:39 -0700 Subject: [PATCH 003/104] cpumask: Switch from inline to __always_inline On recent (v6.6+) builds with Clang (based on Clang 18.0.0) and certain configurations [0], I'm finding that (lack of) inlining decisions may lead to section mismatch warnings like the following: WARNING: modpost: vmlinux.o: section mismatch in reference: cpumask_andnot (section: .text) -> cpuhp_bringup_cpus_parallel.tmp_mask (section: .init.data) ERROR: modpost: Section mismatches detected. or more confusingly: WARNING: modpost: vmlinux: section mismatch in reference: cpumask_andnot+0x5f (section: .text) -> efi_systab_phys (section: .init.data) The first warning makes a little sense, because cpuhp_bringup_cpus_parallel() (an __init function) calls cpumask_andnot() on tmp_mask (an __initdata symbol). If the compiler doesn't inline cpumask_andnot(), this may appear like a mismatch. The second warning makes less sense, but might be because efi_systab_phys and cpuhp_bringup_cpus_parallel.tmp_mask are laid out near each other, and the latter isn't a proper C symbol definition. In any case, it seems a reasonable solution to suggest more strongly to the compiler that these cpumask macros *must* be inlined, as 'inline' is just a recommendation. This change has been previously proposed in the past as: Subject: [PATCH 1/3] bitmap: switch from inline to __always_inline https://lore.kernel.org/all/20221027043810.350460-2-yury.norov@gmail.com/ But the change has been split up, to separately justify the cpumask changes (which drive my work) and the bitmap/const optimizations (that Yury separately proposed for other reasons). This ends up as somewhere between a "rebase" and "rewrite" -- I had to rewrite most of the patch. According to bloat-o-meter, vmlinux decreases minimally in size (-0.00% to -0.01%, depending on the version of GCC or Clang and .config in question) with this series of changes: gcc 13.2.0, x86_64_defconfig -3005 bytes, Before=21944501, After=21941496, chg -0.01% clang 16.0.6, x86_64_defconfig -105 bytes, Before=22571692, After=22571587, chg -0.00% gcc 9.5.0, x86_64_defconfig -1771 bytes, Before=21557598, After=21555827, chg -0.01% clang 18.0_pre516547 (ChromiumOS toolchain), x86_64_defconfig -191 bytes, Before=22615339, After=22615148, chg -0.00% clang 18.0_pre516547 (ChromiumOS toolchain), based on ChromiumOS config + gcov -979 bytes, Before=76294783, After=76293804, chg -0.00% [0] CONFIG_HOTPLUG_PARALLEL=y ('select'ed for x86 as of [1]) and CONFIG_GCOV_PROFILE_ALL. [1] commit 0c7ffa32dbd6 ("x86/smpboot/64: Implement arch_cpuhp_init_parallel_bringup() and enable it") Co-developed-by: Brian Norris Signed-off-by: Brian Norris Reviewed-by: Kees Cook Reviewed-by: Nathan Chancellor Signed-off-by: Yury Norov --- include/linux/cpumask.h | 212 +++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 100 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 801a7e524113..9e3f3c96607d 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -30,7 +30,7 @@ extern unsigned int nr_cpu_ids; #endif -static inline void set_nr_cpu_ids(unsigned int nr) +static __always_inline void set_nr_cpu_ids(unsigned int nr) { #if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) WARN_ON(nr != nr_cpu_ids); @@ -149,7 +149,7 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) * * Return: >= nr_cpu_ids if no cpus set. */ -static inline unsigned int cpumask_first(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_first(const struct cpumask *srcp) { return find_first_bit(cpumask_bits(srcp), small_cpumask_bits); } @@ -160,7 +160,7 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) * * Return: >= nr_cpu_ids if all cpus are set. */ -static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp) { return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits); } @@ -172,7 +172,7 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) * * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). */ -static inline +static __always_inline unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) { return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); @@ -186,7 +186,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask * * Return: >= nr_cpu_ids if no cpus set in all. */ -static inline +static __always_inline unsigned int cpumask_first_and_and(const struct cpumask *srcp1, const struct cpumask *srcp2, const struct cpumask *srcp3) @@ -201,7 +201,7 @@ unsigned int cpumask_first_and_and(const struct cpumask *srcp1, * * Return: >= nr_cpumask_bits if no CPUs set. */ -static inline unsigned int cpumask_last(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_last(const struct cpumask *srcp) { return find_last_bit(cpumask_bits(srcp), small_cpumask_bits); } @@ -213,7 +213,7 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) * * Return: >= nr_cpu_ids if no further cpus set. */ -static inline +static __always_inline unsigned int cpumask_next(int n, const struct cpumask *srcp) { /* -1 is a legal arg here. */ @@ -229,7 +229,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) * * Return: >= nr_cpu_ids if no further cpus unset. */ -static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) { /* -1 is a legal arg here. */ if (n != -1) @@ -239,18 +240,21 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) #if NR_CPUS == 1 /* Uniprocessor: there is only one valid CPU */ -static inline unsigned int cpumask_local_spread(unsigned int i, int node) +static __always_inline +unsigned int cpumask_local_spread(unsigned int i, int node) { return 0; } -static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, + const struct cpumask *src2p) { return cpumask_first_and(src1p, src2p); } -static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_any_distribute(const struct cpumask *srcp) { return cpumask_first(srcp); } @@ -269,9 +273,9 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp); * * Return: >= nr_cpu_ids if no further cpus set in both. */ -static inline +static __always_inline unsigned int cpumask_next_and(int n, const struct cpumask *src1p, - const struct cpumask *src2p) + const struct cpumask *src2p) { /* -1 is a legal arg here. */ if (n != -1) @@ -291,7 +295,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits) #if NR_CPUS == 1 -static inline +static __always_inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap) { cpumask_check(start); @@ -394,7 +398,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta * Often used to find any cpu but smp_processor_id() in a mask. * Return: >= nr_cpu_ids if no cpus set. */ -static inline +static __always_inline unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) { unsigned int i; @@ -414,7 +418,7 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) * * Returns >= nr_cpu_ids if no cpus set. */ -static inline +static __always_inline unsigned int cpumask_any_and_but(const struct cpumask *mask1, const struct cpumask *mask2, unsigned int cpu) @@ -436,7 +440,8 @@ unsigned int cpumask_any_and_but(const struct cpumask *mask1, * * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) +static __always_inline +unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) { return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu)); } @@ -449,7 +454,7 @@ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *s * * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline +static __always_inline unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, const struct cpumask *srcp2) { @@ -465,7 +470,7 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, * * Return: >= nr_cpu_ids if such cpu doesn't exist. */ -static inline +static __always_inline unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, const struct cpumask *srcp2) { @@ -508,12 +513,14 @@ unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp * @cpu: cpu number (< nr_cpu_ids) * @dstp: the cpumask pointer */ -static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +static __always_inline +void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) { set_bit(cpumask_check(cpu), cpumask_bits(dstp)); } -static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) +static __always_inline +void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) { __set_bit(cpumask_check(cpu), cpumask_bits(dstp)); } @@ -557,7 +564,8 @@ static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp, * * Return: true if @cpu is set in @cpumask, else returns false */ -static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) +static __always_inline +bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) { return test_bit(cpumask_check(cpu), cpumask_bits((cpumask))); } @@ -571,7 +579,8 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum * * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ -static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) +static __always_inline +bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -585,7 +594,8 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp * * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ -static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) +static __always_inline +bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask)); } @@ -594,7 +604,7 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask * * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask * @dstp: the cpumask pointer */ -static inline void cpumask_setall(struct cpumask *dstp) +static __always_inline void cpumask_setall(struct cpumask *dstp) { if (small_const_nbits(small_cpumask_bits)) { cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits); @@ -607,7 +617,7 @@ static inline void cpumask_setall(struct cpumask *dstp) * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask * @dstp: the cpumask pointer */ -static inline void cpumask_clear(struct cpumask *dstp) +static __always_inline void cpumask_clear(struct cpumask *dstp) { bitmap_zero(cpumask_bits(dstp), large_cpumask_bits); } @@ -620,9 +630,9 @@ static inline void cpumask_clear(struct cpumask *dstp) * * Return: false if *@dstp is empty, else returns true */ -static inline bool cpumask_and(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -634,8 +644,9 @@ static inline bool cpumask_and(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input */ -static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -647,9 +658,9 @@ static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input */ -static inline void cpumask_xor(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -663,9 +674,9 @@ static inline void cpumask_xor(struct cpumask *dstp, * * Return: false if *@dstp is empty, else returns true */ -static inline bool cpumask_andnot(struct cpumask *dstp, - const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, + const struct cpumask *src2p) { return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -678,8 +689,8 @@ static inline bool cpumask_andnot(struct cpumask *dstp, * * Return: true if the cpumasks are equal, false if not */ -static inline bool cpumask_equal(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -694,9 +705,9 @@ static inline bool cpumask_equal(const struct cpumask *src1p, * Return: true if first cpumask ORed with second cpumask == third cpumask, * otherwise false */ -static inline bool cpumask_or_equal(const struct cpumask *src1p, - const struct cpumask *src2p, - const struct cpumask *src3p) +static __always_inline +bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p, + const struct cpumask *src3p) { return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), cpumask_bits(src3p), small_cpumask_bits); @@ -710,8 +721,8 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p, * Return: true if first cpumask ANDed with second cpumask is non-empty, * otherwise false */ -static inline bool cpumask_intersects(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -724,8 +735,8 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, * * Return: true if *@src1p is a subset of *@src2p, else returns false */ -static inline bool cpumask_subset(const struct cpumask *src1p, - const struct cpumask *src2p) +static __always_inline +bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p), small_cpumask_bits); @@ -737,7 +748,7 @@ static inline bool cpumask_subset(const struct cpumask *src1p, * * Return: true if srcp is empty (has no bits set), else false */ -static inline bool cpumask_empty(const struct cpumask *srcp) +static __always_inline bool cpumask_empty(const struct cpumask *srcp) { return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits); } @@ -748,7 +759,7 @@ static inline bool cpumask_empty(const struct cpumask *srcp) * * Return: true if srcp is full (has all bits set), else false */ -static inline bool cpumask_full(const struct cpumask *srcp) +static __always_inline bool cpumask_full(const struct cpumask *srcp) { return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits); } @@ -759,7 +770,7 @@ static inline bool cpumask_full(const struct cpumask *srcp) * * Return: count of bits set in *srcp */ -static inline unsigned int cpumask_weight(const struct cpumask *srcp) +static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp) { return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits); } @@ -771,8 +782,8 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) * * Return: count of bits set in both *srcp1 and *srcp2 */ -static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, - const struct cpumask *srcp2) +static __always_inline +unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2) { return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); } @@ -784,8 +795,9 @@ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, * * Return: count of bits set in both *srcp1 and *srcp2 */ -static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, - const struct cpumask *srcp2) +static __always_inline +unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, + const struct cpumask *srcp2) { return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits); } @@ -796,8 +808,8 @@ static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1, * @srcp: the input to shift * @n: the number of bits to shift by */ -static inline void cpumask_shift_right(struct cpumask *dstp, - const struct cpumask *srcp, int n) +static __always_inline +void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n, small_cpumask_bits); @@ -809,8 +821,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp, * @srcp: the input to shift * @n: the number of bits to shift by */ -static inline void cpumask_shift_left(struct cpumask *dstp, - const struct cpumask *srcp, int n) +static __always_inline +void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n, nr_cpumask_bits); @@ -821,8 +833,8 @@ static inline void cpumask_shift_left(struct cpumask *dstp, * @dstp: the result * @srcp: the input cpumask */ -static inline void cpumask_copy(struct cpumask *dstp, - const struct cpumask *srcp) +static __always_inline +void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) { bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits); } @@ -858,8 +870,8 @@ static inline void cpumask_copy(struct cpumask *dstp, * * Return: -errno, or 0 for success. */ -static inline int cpumask_parse_user(const char __user *buf, int len, - struct cpumask *dstp) +static __always_inline +int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) { return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); } @@ -872,8 +884,8 @@ static inline int cpumask_parse_user(const char __user *buf, int len, * * Return: -errno, or 0 for success. */ -static inline int cpumask_parselist_user(const char __user *buf, int len, - struct cpumask *dstp) +static __always_inline +int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) { return bitmap_parselist_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits); @@ -886,7 +898,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, * * Return: -errno, or 0 for success. */ -static inline int cpumask_parse(const char *buf, struct cpumask *dstp) +static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp) { return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits); } @@ -898,7 +910,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) * * Return: -errno, or 0 for success. */ -static inline int cpulist_parse(const char *buf, struct cpumask *dstp) +static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp) { return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits); } @@ -908,7 +920,7 @@ static inline int cpulist_parse(const char *buf, struct cpumask *dstp) * * Return: size to allocate for a &struct cpumask in bytes */ -static inline unsigned int cpumask_size(void) +static __always_inline unsigned int cpumask_size(void) { return bitmap_size(large_cpumask_bits); } @@ -920,7 +932,7 @@ static inline unsigned int cpumask_size(void) bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node); -static inline +static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); @@ -938,13 +950,13 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) * * Return: %true if allocation succeeded, %false if not */ -static inline +static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); } -static inline +static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return alloc_cpumask_var(mask, flags | __GFP_ZERO); @@ -954,7 +966,7 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask); void free_cpumask_var(cpumask_var_t mask); void free_bootmem_cpumask_var(cpumask_var_t mask); -static inline bool cpumask_available(cpumask_var_t mask) +static __always_inline bool cpumask_available(cpumask_var_t mask) { return mask != NULL; } @@ -964,43 +976,43 @@ static inline bool cpumask_available(cpumask_var_t mask) #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x) #define __cpumask_var_read_mostly -static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return true; } -static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, +static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { return true; } -static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) +static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { cpumask_clear(*mask); return true; } -static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, +static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { cpumask_clear(*mask); return true; } -static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) +static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask) { } -static inline void free_cpumask_var(cpumask_var_t mask) +static __always_inline void free_cpumask_var(cpumask_var_t mask) { } -static inline void free_bootmem_cpumask_var(cpumask_var_t mask) +static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask) { } -static inline bool cpumask_available(cpumask_var_t mask) +static __always_inline bool cpumask_available(cpumask_var_t mask) { return true; } @@ -1058,7 +1070,7 @@ void set_cpu_online(unsigned int cpu, bool online); ((struct cpumask *)(1 ? (bitmap) \ : (void *)sizeof(__check_is_bitmap(bitmap)))) -static inline int __check_is_bitmap(const unsigned long *bitmap) +static __always_inline int __check_is_bitmap(const unsigned long *bitmap) { return 1; } @@ -1073,7 +1085,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap) extern const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; -static inline const struct cpumask *get_cpu_mask(unsigned int cpu) +static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu) { const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; p -= cpu / BITS_PER_LONG; @@ -1100,32 +1112,32 @@ static __always_inline unsigned int num_online_cpus(void) #define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask) -static inline bool cpu_online(unsigned int cpu) +static __always_inline bool cpu_online(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_online_mask); } -static inline bool cpu_enabled(unsigned int cpu) +static __always_inline bool cpu_enabled(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_enabled_mask); } -static inline bool cpu_possible(unsigned int cpu) +static __always_inline bool cpu_possible(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_possible_mask); } -static inline bool cpu_present(unsigned int cpu) +static __always_inline bool cpu_present(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_present_mask); } -static inline bool cpu_active(unsigned int cpu) +static __always_inline bool cpu_active(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_active_mask); } -static inline bool cpu_dying(unsigned int cpu) +static __always_inline bool cpu_dying(unsigned int cpu) { return cpumask_test_cpu(cpu, cpu_dying_mask); } @@ -1138,32 +1150,32 @@ static inline bool cpu_dying(unsigned int cpu) #define num_present_cpus() 1U #define num_active_cpus() 1U -static inline bool cpu_online(unsigned int cpu) +static __always_inline bool cpu_online(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_possible(unsigned int cpu) +static __always_inline bool cpu_possible(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_enabled(unsigned int cpu) +static __always_inline bool cpu_enabled(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_present(unsigned int cpu) +static __always_inline bool cpu_present(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_active(unsigned int cpu) +static __always_inline bool cpu_active(unsigned int cpu) { return cpu == 0; } -static inline bool cpu_dying(unsigned int cpu) +static __always_inline bool cpu_dying(unsigned int cpu) { return false; } @@ -1197,7 +1209,7 @@ static inline bool cpu_dying(unsigned int cpu) * Return: the length of the (null-terminated) @buf string, zero if * nothing is copied. */ -static inline ssize_t +static __always_inline ssize_t cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) { return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), @@ -1220,9 +1232,9 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) * Return: the length of how many bytes have been copied, excluding * terminating '\0'. */ -static inline ssize_t -cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, - loff_t off, size_t count) +static __always_inline +ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, + loff_t off, size_t count) { return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask), nr_cpu_ids, off, count) - 1; @@ -1242,9 +1254,9 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, * Return: the length of how many bytes have been copied, excluding * terminating '\0'. */ -static inline ssize_t -cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, - loff_t off, size_t count) +static __always_inline +ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, + loff_t off, size_t count) { return bitmap_print_list_to_buf(buf, cpumask_bits(mask), nr_cpu_ids, off, count) - 1; From 54c9e0085bd1015e524d8f4d3c4e78a7bc77ffca Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 18 Jul 2024 17:50:40 -0700 Subject: [PATCH 004/104] nodemask: Switch from inline to __always_inline 'inline' keyword is only a recommendation for compiler. If it decides to not inline nodemask functions, the whole small_const_nbits() machinery doesn't work. This is how a standard GCC 11.3.0 does for my x86_64 build now. This patch replaces 'inline' directive with unconditional '__always_inline' to make sure that there's always a chance for compile-time optimization. It doesn't change size of kernel image, according to bloat-o-meter. [[ Brian: split out from: Subject: [PATCH 1/3] bitmap: switch from inline to __always_inline https://lore.kernel.org/all/20221027043810.350460-2-yury.norov@gmail.com/ But rewritten, as there were too many conflicts. ]] Co-developed-by: Brian Norris Signed-off-by: Brian Norris Reviewed-by: Kees Cook Reviewed-by: Nathan Chancellor Signed-off-by: Yury Norov --- include/linux/nodemask.h | 86 ++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index b61438313a73..9fd7a0ce9c1a 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -107,11 +107,11 @@ extern nodemask_t _unused_nodemask_arg_; */ #define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \ __nodemask_pr_bits(maskp) -static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m) +static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m) { return m ? MAX_NUMNODES : 0; } -static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m) +static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m) { return m ? m->bits : NULL; } @@ -132,19 +132,19 @@ static __always_inline void __node_set(int node, volatile nodemask_t *dstp) } #define node_clear(node, dst) __node_clear((node), &(dst)) -static inline void __node_clear(int node, volatile nodemask_t *dstp) +static __always_inline void __node_clear(int node, volatile nodemask_t *dstp) { clear_bit(node, dstp->bits); } #define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES) -static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits) +static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits) { bitmap_fill(dstp->bits, nbits); } #define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES) -static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) +static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) { bitmap_zero(dstp->bits, nbits); } @@ -154,14 +154,14 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits) #define node_test_and_set(node, nodemask) \ __node_test_and_set((node), &(nodemask)) -static inline bool __node_test_and_set(int node, nodemask_t *addr) +static __always_inline bool __node_test_and_set(int node, nodemask_t *addr) { return test_and_set_bit(node, addr->bits); } #define nodes_and(dst, src1, src2) \ __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES) -static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, +static __always_inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits); @@ -169,7 +169,7 @@ static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, #define nodes_or(dst, src1, src2) \ __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES) -static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, +static __always_inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits); @@ -177,7 +177,7 @@ static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, #define nodes_xor(dst, src1, src2) \ __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES) -static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, +static __always_inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits); @@ -185,7 +185,7 @@ static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, #define nodes_andnot(dst, src1, src2) \ __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES) -static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, +static __always_inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits); @@ -193,7 +193,7 @@ static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, #define nodes_complement(dst, src) \ __nodes_complement(&(dst), &(src), MAX_NUMNODES) -static inline void __nodes_complement(nodemask_t *dstp, +static __always_inline void __nodes_complement(nodemask_t *dstp, const nodemask_t *srcp, unsigned int nbits) { bitmap_complement(dstp->bits, srcp->bits, nbits); @@ -201,7 +201,7 @@ static inline void __nodes_complement(nodemask_t *dstp, #define nodes_equal(src1, src2) \ __nodes_equal(&(src1), &(src2), MAX_NUMNODES) -static inline bool __nodes_equal(const nodemask_t *src1p, +static __always_inline bool __nodes_equal(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_equal(src1p->bits, src2p->bits, nbits); @@ -209,7 +209,7 @@ static inline bool __nodes_equal(const nodemask_t *src1p, #define nodes_intersects(src1, src2) \ __nodes_intersects(&(src1), &(src2), MAX_NUMNODES) -static inline bool __nodes_intersects(const nodemask_t *src1p, +static __always_inline bool __nodes_intersects(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_intersects(src1p->bits, src2p->bits, nbits); @@ -217,33 +217,33 @@ static inline bool __nodes_intersects(const nodemask_t *src1p, #define nodes_subset(src1, src2) \ __nodes_subset(&(src1), &(src2), MAX_NUMNODES) -static inline bool __nodes_subset(const nodemask_t *src1p, +static __always_inline bool __nodes_subset(const nodemask_t *src1p, const nodemask_t *src2p, unsigned int nbits) { return bitmap_subset(src1p->bits, src2p->bits, nbits); } #define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES) -static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits) +static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits) { return bitmap_empty(srcp->bits, nbits); } #define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES) -static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits) +static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits) { return bitmap_full(srcp->bits, nbits); } #define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES) -static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits) +static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits) { return bitmap_weight(srcp->bits, nbits); } #define nodes_shift_right(dst, src, n) \ __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES) -static inline void __nodes_shift_right(nodemask_t *dstp, +static __always_inline void __nodes_shift_right(nodemask_t *dstp, const nodemask_t *srcp, int n, int nbits) { bitmap_shift_right(dstp->bits, srcp->bits, n, nbits); @@ -251,7 +251,7 @@ static inline void __nodes_shift_right(nodemask_t *dstp, #define nodes_shift_left(dst, src, n) \ __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES) -static inline void __nodes_shift_left(nodemask_t *dstp, +static __always_inline void __nodes_shift_left(nodemask_t *dstp, const nodemask_t *srcp, int n, int nbits) { bitmap_shift_left(dstp->bits, srcp->bits, n, nbits); @@ -261,13 +261,13 @@ static inline void __nodes_shift_left(nodemask_t *dstp, > MAX_NUMNODES, then the silly min_ts could be dropped. */ #define first_node(src) __first_node(&(src)) -static inline unsigned int __first_node(const nodemask_t *srcp) +static __always_inline unsigned int __first_node(const nodemask_t *srcp) { return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); } #define next_node(n, src) __next_node((n), &(src)) -static inline unsigned int __next_node(int n, const nodemask_t *srcp) +static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp) { return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); } @@ -277,7 +277,7 @@ static inline unsigned int __next_node(int n, const nodemask_t *srcp) * the first node in src if needed. Returns MAX_NUMNODES if src is empty. */ #define next_node_in(n, src) __next_node_in((n), &(src)) -static inline unsigned int __next_node_in(int node, const nodemask_t *srcp) +static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp) { unsigned int ret = __next_node(node, srcp); @@ -286,7 +286,7 @@ static inline unsigned int __next_node_in(int node, const nodemask_t *srcp) return ret; } -static inline void init_nodemask_of_node(nodemask_t *mask, int node) +static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node) { nodes_clear(*mask); node_set(node, *mask); @@ -304,7 +304,7 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node) }) #define first_unset_node(mask) __first_unset_node(&(mask)) -static inline unsigned int __first_unset_node(const nodemask_t *maskp) +static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp) { return min_t(unsigned int, MAX_NUMNODES, find_first_zero_bit(maskp->bits, MAX_NUMNODES)); @@ -338,21 +338,21 @@ static inline unsigned int __first_unset_node(const nodemask_t *maskp) #define nodemask_parse_user(ubuf, ulen, dst) \ __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES) -static inline int __nodemask_parse_user(const char __user *buf, int len, +static __always_inline int __nodemask_parse_user(const char __user *buf, int len, nodemask_t *dstp, int nbits) { return bitmap_parse_user(buf, len, dstp->bits, nbits); } #define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES) -static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits) +static __always_inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits) { return bitmap_parselist(buf, dstp->bits, nbits); } #define node_remap(oldbit, old, new) \ __node_remap((oldbit), &(old), &(new), MAX_NUMNODES) -static inline int __node_remap(int oldbit, +static __always_inline int __node_remap(int oldbit, const nodemask_t *oldp, const nodemask_t *newp, int nbits) { return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits); @@ -360,7 +360,7 @@ static inline int __node_remap(int oldbit, #define nodes_remap(dst, src, old, new) \ __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES) -static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, +static __always_inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, const nodemask_t *oldp, const nodemask_t *newp, int nbits) { bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits); @@ -368,7 +368,7 @@ static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, #define nodes_onto(dst, orig, relmap) \ __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES) -static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, +static __always_inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, const nodemask_t *relmapp, int nbits) { bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits); @@ -376,7 +376,7 @@ static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, #define nodes_fold(dst, orig, sz) \ __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES) -static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp, +static __always_inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp, int sz, int nbits) { bitmap_fold(dstp->bits, origp->bits, sz, nbits); @@ -418,22 +418,22 @@ enum node_states { extern nodemask_t node_states[NR_NODE_STATES]; #if MAX_NUMNODES > 1 -static inline int node_state(int node, enum node_states state) +static __always_inline int node_state(int node, enum node_states state) { return node_isset(node, node_states[state]); } -static inline void node_set_state(int node, enum node_states state) +static __always_inline void node_set_state(int node, enum node_states state) { __node_set(node, &node_states[state]); } -static inline void node_clear_state(int node, enum node_states state) +static __always_inline void node_clear_state(int node, enum node_states state) { __node_clear(node, &node_states[state]); } -static inline int num_node_state(enum node_states state) +static __always_inline int num_node_state(enum node_states state) { return nodes_weight(node_states[state]); } @@ -443,11 +443,11 @@ static inline int num_node_state(enum node_states state) #define first_online_node first_node(node_states[N_ONLINE]) #define first_memory_node first_node(node_states[N_MEMORY]) -static inline unsigned int next_online_node(int nid) +static __always_inline unsigned int next_online_node(int nid) { return next_node(nid, node_states[N_ONLINE]); } -static inline unsigned int next_memory_node(int nid) +static __always_inline unsigned int next_memory_node(int nid) { return next_node(nid, node_states[N_MEMORY]); } @@ -455,13 +455,13 @@ static inline unsigned int next_memory_node(int nid) extern unsigned int nr_node_ids; extern unsigned int nr_online_nodes; -static inline void node_set_online(int nid) +static __always_inline void node_set_online(int nid) { node_set_state(nid, N_ONLINE); nr_online_nodes = num_node_state(N_ONLINE); } -static inline void node_set_offline(int nid) +static __always_inline void node_set_offline(int nid) { node_clear_state(nid, N_ONLINE); nr_online_nodes = num_node_state(N_ONLINE); @@ -469,20 +469,20 @@ static inline void node_set_offline(int nid) #else -static inline int node_state(int node, enum node_states state) +static __always_inline int node_state(int node, enum node_states state) { return node == 0; } -static inline void node_set_state(int node, enum node_states state) +static __always_inline void node_set_state(int node, enum node_states state) { } -static inline void node_clear_state(int node, enum node_states state) +static __always_inline void node_clear_state(int node, enum node_states state) { } -static inline int num_node_state(enum node_states state) +static __always_inline int num_node_state(enum node_states state) { return 1; } @@ -502,7 +502,7 @@ static inline int num_node_state(enum node_states state) #endif -static inline int node_random(const nodemask_t *maskp) +static __always_inline int node_random(const nodemask_t *maskp) { #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) int w, bit; From 0808ebf2f80b962e75741a41ced372a7116f1e26 Mon Sep 17 00:00:00 2001 From: Ken Raeburn Date: Fri, 26 Jul 2024 15:08:53 -0400 Subject: [PATCH 005/104] dm vdo: don't refer to dedupe_context after releasing it Clear the dedupe_context pointer in a data_vio whenever ownership of the context is lost, so that vdo can't examine it accidentally. Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/dedupe.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/md/dm-vdo/dedupe.c b/drivers/md/dm-vdo/dedupe.c index 39ac68614419..80628ae93fba 100644 --- a/drivers/md/dm-vdo/dedupe.c +++ b/drivers/md/dm-vdo/dedupe.c @@ -729,6 +729,7 @@ static void process_update_result(struct data_vio *agent) !change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) return; + agent->dedupe_context = NULL; release_context(context); } @@ -1648,6 +1649,7 @@ static void process_query_result(struct data_vio *agent) if (change_context_state(context, DEDUPE_CONTEXT_COMPLETE, DEDUPE_CONTEXT_IDLE)) { agent->is_duplicate = decode_uds_advice(context); + agent->dedupe_context = NULL; release_context(context); } } @@ -2321,6 +2323,7 @@ static void timeout_index_operations_callback(struct vdo_completion *completion) * send its requestor on its way. */ list_del_init(&context->list_entry); + context->requestor->dedupe_context = NULL; continue_data_vio(context->requestor); timed_out++; } From 3a59b2ec2400017f7fcc5d794802ebc7f4187d22 Mon Sep 17 00:00:00 2001 From: Ken Raeburn Date: Tue, 6 Aug 2024 05:13:19 -0400 Subject: [PATCH 006/104] dm vdo: remove bad check of bi_next field Remove this check to prevent spurious warning messages due to the behavior of other storage layers. This check was intended to make sure dm-vdo does not chain metadata bios together. However, vdo has no control over underlying storage layers, so the assertion is not always true. Signed-off-by: Ken Raeburn Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/io-submitter.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/md/dm-vdo/io-submitter.c b/drivers/md/dm-vdo/io-submitter.c index 9a3716bb3c05..ab62abe18827 100644 --- a/drivers/md/dm-vdo/io-submitter.c +++ b/drivers/md/dm-vdo/io-submitter.c @@ -346,7 +346,6 @@ void __submit_metadata_vio(struct vio *vio, physical_block_number_t physical, VDO_ASSERT_LOG_ONLY(!code->quiescent, "I/O not allowed in state %s", code->name); - VDO_ASSERT_LOG_ONLY(vio->bio->bi_next == NULL, "metadata bio has no next bio"); vdo_reset_completion(completion); completion->error_handler = error_handler; From 47874c98dc0873aad65259d09f9b9029e97429e3 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Thu, 18 Jul 2024 14:02:38 -0400 Subject: [PATCH 007/104] dm vdo: add dmsetup message for returning configuration info Add a new dmsetup message called config, which will return useful configuration information for the vdo volume and the uds index associated with it. The output is a YAML string, and contains a version number to allow future additions to the content. Signed-off-by: Bruce Johnston Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- .../admin-guide/device-mapper/vdo.rst | 7 ++- drivers/md/dm-vdo/dm-vdo-target.c | 5 +- drivers/md/dm-vdo/message-stats.c | 48 +++++++++++++++++++ drivers/md/dm-vdo/message-stats.h | 1 + 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/device-mapper/vdo.rst b/Documentation/admin-guide/device-mapper/vdo.rst index c69ac186863a..a14e6d3e787c 100644 --- a/Documentation/admin-guide/device-mapper/vdo.rst +++ b/Documentation/admin-guide/device-mapper/vdo.rst @@ -251,7 +251,12 @@ The messages are: by the vdostats userspace program to interpret the output buffer. - dump: + config: + Outputs useful vdo configuration information. Mostly used + by users who want to recreate a similar VDO volume and + want to know the creation configuration used. + + dump: Dumps many internal structures to the system log. This is not always safe to run, so it should only be used to debug a hung vdo. Optional parameters to specify structures to diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index dd05691e4097..4b94f5718ed7 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -1105,6 +1105,9 @@ static int vdo_message(struct dm_target *ti, unsigned int argc, char **argv, if ((argc == 1) && (strcasecmp(argv[0], "stats") == 0)) { vdo_write_stats(vdo, result_buffer, maxlen); result = 1; + } else if ((argc == 1) && (strcasecmp(argv[0], "config") == 0)) { + vdo_write_config(vdo, &result_buffer, &maxlen); + result = 1; } else { result = vdo_status_to_errno(process_vdo_message(vdo, argc, argv)); } @@ -2832,7 +2835,7 @@ static void vdo_resume(struct dm_target *ti) static struct target_type vdo_target_bio = { .features = DM_TARGET_SINGLETON, .name = "vdo", - .version = { 9, 0, 0 }, + .version = { 9, 1, 0 }, .module = THIS_MODULE, .ctr = vdo_ctr, .dtr = vdo_dtr, diff --git a/drivers/md/dm-vdo/message-stats.c b/drivers/md/dm-vdo/message-stats.c index 2802cf92922b..75dfcd7c5f63 100644 --- a/drivers/md/dm-vdo/message-stats.c +++ b/drivers/md/dm-vdo/message-stats.c @@ -4,6 +4,7 @@ */ #include "dedupe.h" +#include "indexer.h" #include "logger.h" #include "memory-alloc.h" #include "message-stats.h" @@ -430,3 +431,50 @@ int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen) vdo_free(stats); return VDO_SUCCESS; } + +static void write_index_memory(u32 mem, char **buf, unsigned int *maxlen) +{ + char *prefix = "memorySize : "; + + /* Convert index memory to fractional value */ + if (mem == (u32)UDS_MEMORY_CONFIG_256MB) + write_string(prefix, "0.25, ", NULL, buf, maxlen); + else if (mem == (u32)UDS_MEMORY_CONFIG_512MB) + write_string(prefix, "0.50, ", NULL, buf, maxlen); + else if (mem == (u32)UDS_MEMORY_CONFIG_768MB) + write_string(prefix, "0.75, ", NULL, buf, maxlen); + else + write_u32(prefix, mem, ", ", buf, maxlen); +} + +static void write_index_config(struct index_config *config, char **buf, + unsigned int *maxlen) +{ + write_string("index : ", "{ ", NULL, buf, maxlen); + /* index mem size */ + write_index_memory(config->mem, buf, maxlen); + /* whether the index is sparse or not */ + write_bool("isSparse : ", config->sparse, ", ", buf, maxlen); + write_string(NULL, "}", ", ", buf, maxlen); +} + +int vdo_write_config(struct vdo *vdo, char **buf, unsigned int *maxlen) +{ + struct vdo_config *config = &vdo->states.vdo.config; + + write_string(NULL, "{ ", NULL, buf, maxlen); + /* version */ + write_u32("version : ", 1, ", ", buf, maxlen); + /* physical size */ + write_block_count_t("physicalSize : ", config->physical_blocks * VDO_BLOCK_SIZE, ", ", + buf, maxlen); + /* logical size */ + write_block_count_t("logicalSize : ", config->logical_blocks * VDO_BLOCK_SIZE, ", ", + buf, maxlen); + /* slab size */ + write_block_count_t("slabSize : ", config->slab_size, ", ", buf, maxlen); + /* index config */ + write_index_config(&vdo->geometry.index_config, buf, maxlen); + write_string(NULL, "}", NULL, buf, maxlen); + return VDO_SUCCESS; +} diff --git a/drivers/md/dm-vdo/message-stats.h b/drivers/md/dm-vdo/message-stats.h index f7fceca9acab..f9c95eff569d 100644 --- a/drivers/md/dm-vdo/message-stats.h +++ b/drivers/md/dm-vdo/message-stats.h @@ -8,6 +8,7 @@ #include "types.h" +int vdo_write_config(struct vdo *vdo, char **buf, unsigned int *maxlen); int vdo_write_stats(struct vdo *vdo, char *buf, unsigned int maxlen); #endif /* VDO_MESSAGE_STATS_H */ From f3ff668352c59db2c0ab47f2e86488a70694ed57 Mon Sep 17 00:00:00 2001 From: Susan LeGendre-McGhee Date: Fri, 19 Jul 2024 17:52:32 -0400 Subject: [PATCH 008/104] dm vdo: abort loading dirty VDO with the old recovery journal format Abort the load process with status code VDO_UNSUPPORTED_VERSION without forcing read-only mode when a journal block with the old format version is detected. Forcing the VDO volume into read-only mode and thus requiring a read-only rebuild should only be done when absolutely necessary. Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/dm-vdo-target.c | 24 +++++++++++++++++++++++- drivers/md/dm-vdo/repair.c | 4 +--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c index 4b94f5718ed7..0e04c2021682 100644 --- a/drivers/md/dm-vdo/dm-vdo-target.c +++ b/drivers/md/dm-vdo/dm-vdo-target.c @@ -2296,6 +2296,14 @@ static void handle_load_error(struct vdo_completion *completion) return; } + if ((completion->result == VDO_UNSUPPORTED_VERSION) && + (vdo->admin.phase == LOAD_PHASE_MAKE_DIRTY)) { + vdo_log_error("Aborting load due to unsupported version"); + vdo->admin.phase = LOAD_PHASE_FINISHED; + load_callback(completion); + return; + } + vdo_log_error_strerror(completion->result, "Entering read-only mode due to load error"); vdo->admin.phase = LOAD_PHASE_WAIT_FOR_READ_ONLY; @@ -2740,6 +2748,19 @@ static int vdo_preresume_registered(struct dm_target *ti, struct vdo *vdo) vdo_log_info("starting device '%s'", device_name); result = perform_admin_operation(vdo, LOAD_PHASE_START, load_callback, handle_load_error, "load"); + if (result == VDO_UNSUPPORTED_VERSION) { + /* + * A component version is not supported. This can happen when the + * recovery journal metadata is in an old version format. Abort the + * load without saving the state. + */ + vdo->suspend_type = VDO_ADMIN_STATE_SUSPENDING; + perform_admin_operation(vdo, SUSPEND_PHASE_START, + suspend_callback, suspend_callback, + "suspend"); + return result; + } + if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { /* * Something has gone very wrong. Make sure everything has drained and @@ -2811,7 +2832,8 @@ static int vdo_preresume(struct dm_target *ti) vdo_register_thread_device_id(&instance_thread, &vdo->instance); result = vdo_preresume_registered(ti, vdo); - if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE)) + if ((result == VDO_PARAMETER_MISMATCH) || (result == VDO_INVALID_ADMIN_STATE) || + (result == VDO_UNSUPPORTED_VERSION)) result = -EINVAL; vdo_unregister_thread_device_id(); return vdo_status_to_errno(result); diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index 7e0009d2f67d..8aae78bb8a05 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -1575,9 +1575,7 @@ static int parse_journal_for_recovery(struct repair_completion *repair) if (header.metadata_type == VDO_METADATA_RECOVERY_JOURNAL) { /* This is an old format block, so we need to upgrade */ vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, - "Recovery journal is in the old format, a read-only rebuild is required."); - vdo_enter_read_only_mode(repair->completion.vdo, - VDO_UNSUPPORTED_VERSION); + "Recovery journal is in the old format. Downgrade and complete recovery, then upgrade with a clean volume"); return VDO_UNSUPPORTED_VERSION; } From 448c4e4eb1901543259c038ed08266c250f5b079 Mon Sep 17 00:00:00 2001 From: Susan LeGendre-McGhee Date: Tue, 30 Jul 2024 17:18:07 -0400 Subject: [PATCH 009/104] dm vdo: force read-only mode for a corrupt recovery journal Ensure the recovery journal does not attempt recovery when blocks with mismatched metadata versions are detected. This check is performed after determining that the blocks are otherwise valid so that it does not interfere with normal recovery. Signed-off-by: Susan LeGendre-McGhee Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/repair.c | 39 ++++++++++++++++++-------------- drivers/md/dm-vdo/status-codes.c | 2 +- drivers/md/dm-vdo/status-codes.h | 2 +- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/md/dm-vdo/repair.c b/drivers/md/dm-vdo/repair.c index 8aae78bb8a05..ffff2c999518 100644 --- a/drivers/md/dm-vdo/repair.c +++ b/drivers/md/dm-vdo/repair.c @@ -1202,17 +1202,14 @@ static bool __must_check is_valid_recovery_journal_block(const struct recovery_j * @journal: The journal to use. * @header: The unpacked block header to check. * @sequence: The expected sequence number. - * @type: The expected metadata type. * * Return: True if the block matches. */ static bool __must_check is_exact_recovery_journal_block(const struct recovery_journal *journal, const struct recovery_block_header *header, - sequence_number_t sequence, - enum vdo_metadata_type type) + sequence_number_t sequence) { - return ((header->metadata_type == type) && - (header->sequence_number == sequence) && + return ((header->sequence_number == sequence) && (is_valid_recovery_journal_block(journal, header, true))); } @@ -1371,7 +1368,8 @@ static void extract_entries_from_block(struct repair_completion *repair, get_recovery_journal_block_header(journal, repair->journal_data, sequence); - if (!is_exact_recovery_journal_block(journal, &header, sequence, format)) { + if (!is_exact_recovery_journal_block(journal, &header, sequence) || + (header.metadata_type != format)) { /* This block is invalid, so skip it. */ return; } @@ -1557,10 +1555,13 @@ static int parse_journal_for_recovery(struct repair_completion *repair) sequence_number_t i, head; bool found_entries = false; struct recovery_journal *journal = repair->completion.vdo->recovery_journal; + struct recovery_block_header header; + enum vdo_metadata_type expected_format; head = min(repair->block_map_head, repair->slab_journal_head); + header = get_recovery_journal_block_header(journal, repair->journal_data, head); + expected_format = header.metadata_type; for (i = head; i <= repair->highest_tail; i++) { - struct recovery_block_header header; journal_entry_count_t block_entries; u8 j; @@ -1572,17 +1573,15 @@ static int parse_journal_for_recovery(struct repair_completion *repair) }; header = get_recovery_journal_block_header(journal, repair->journal_data, i); - if (header.metadata_type == VDO_METADATA_RECOVERY_JOURNAL) { - /* This is an old format block, so we need to upgrade */ - vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, - "Recovery journal is in the old format. Downgrade and complete recovery, then upgrade with a clean volume"); - return VDO_UNSUPPORTED_VERSION; - } - - if (!is_exact_recovery_journal_block(journal, &header, i, - VDO_METADATA_RECOVERY_JOURNAL_2)) { + if (!is_exact_recovery_journal_block(journal, &header, i)) { /* A bad block header was found so this must be the end of the journal. */ break; + } else if (header.metadata_type != expected_format) { + /* There is a mix of old and new format blocks, so we need to rebuild. */ + vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, + "Recovery journal is in an invalid format, a read-only rebuild is required."); + vdo_enter_read_only_mode(repair->completion.vdo, VDO_CORRUPT_JOURNAL); + return VDO_CORRUPT_JOURNAL; } block_entries = header.entry_count; @@ -1618,8 +1617,14 @@ static int parse_journal_for_recovery(struct repair_completion *repair) break; } - if (!found_entries) + if (!found_entries) { return validate_heads(repair); + } else if (expected_format == VDO_METADATA_RECOVERY_JOURNAL) { + /* All journal blocks have the old format, so we need to upgrade. */ + vdo_log_error_strerror(VDO_UNSUPPORTED_VERSION, + "Recovery journal is in the old format. Downgrade and complete recovery, then upgrade with a clean volume"); + return VDO_UNSUPPORTED_VERSION; + } /* Set the tail to the last valid tail block, if there is one. */ if (repair->tail_recovery_point.sector_count == 0) diff --git a/drivers/md/dm-vdo/status-codes.c b/drivers/md/dm-vdo/status-codes.c index d3493450b169..dd252d660b6d 100644 --- a/drivers/md/dm-vdo/status-codes.c +++ b/drivers/md/dm-vdo/status-codes.c @@ -28,7 +28,7 @@ const struct error_info vdo_status_list[] = { { "VDO_LOCK_ERROR", "A lock is held incorrectly" }, { "VDO_READ_ONLY", "The device is in read-only mode" }, { "VDO_SHUTTING_DOWN", "The device is shutting down" }, - { "VDO_CORRUPT_JOURNAL", "Recovery journal entries corrupted" }, + { "VDO_CORRUPT_JOURNAL", "Recovery journal corrupted" }, { "VDO_TOO_MANY_SLABS", "Exceeds maximum number of slabs supported" }, { "VDO_INVALID_FRAGMENT", "Compressed block fragment is invalid" }, { "VDO_RETRY_AFTER_REBUILD", "Retry operation after rebuilding finishes" }, diff --git a/drivers/md/dm-vdo/status-codes.h b/drivers/md/dm-vdo/status-codes.h index 72da04159f88..426dc8e2ca5d 100644 --- a/drivers/md/dm-vdo/status-codes.h +++ b/drivers/md/dm-vdo/status-codes.h @@ -52,7 +52,7 @@ enum vdo_status_codes { VDO_READ_ONLY, /* the VDO is shutting down */ VDO_SHUTTING_DOWN, - /* the recovery journal has corrupt entries */ + /* the recovery journal has corrupt entries or corrupt metadata */ VDO_CORRUPT_JOURNAL, /* exceeds maximum number of slabs supported */ VDO_TOO_MANY_SLABS, From 013f510d1fce563f37f420bf231b065885a16f21 Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Thu, 1 Aug 2024 21:31:21 +0800 Subject: [PATCH 010/104] dm: Remove unused declaration dm_get_rq_mapinfo() Commit ae6ad75e5c3c ("dm: remove unused dm_get_rq_mapinfo()") removed the implementation but leave declaration. Signed-off-by: Yue Haibing Reviewed-by: Damien Le Moal Signed-off-by: Mikulas Patocka --- include/linux/device-mapper.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 53ca3a913d06..8321f65897f3 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -524,7 +524,6 @@ int dm_post_suspending(struct dm_target *ti); int dm_noflush_suspending(struct dm_target *ti); void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors); void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone); -union map_info *dm_get_rq_mapinfo(struct request *rq); #ifdef CONFIG_BLK_DEV_ZONED struct dm_report_zones_args { From 5d3691a8266e77fd2d695064532d0926af974331 Mon Sep 17 00:00:00 2001 From: Heinz Mauelshagen Date: Thu, 8 Aug 2024 13:50:36 +0200 Subject: [PATCH 011/104] dm delay: enhance kernel documentation This commit improves documentation of the dm-delay target. Signed-off-by: Heinz Mauelshagen Signed-off-by: Mikulas Patocka --- .../admin-guide/device-mapper/delay.rst | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/Documentation/admin-guide/device-mapper/delay.rst b/Documentation/admin-guide/device-mapper/delay.rst index 917ba8c33359..4d667228e744 100644 --- a/Documentation/admin-guide/device-mapper/delay.rst +++ b/Documentation/admin-guide/device-mapper/delay.rst @@ -3,29 +3,52 @@ dm-delay ======== Device-Mapper's "delay" target delays reads and/or writes -and maps them to different devices. +and/or flushs and optionally maps them to different devices. -Parameters:: +Arguments:: [ [ ]] -With separate write parameters, the first set is only used for reads. +Table line has to either have 3, 6 or 9 arguments: + +3: apply offset and delay to read, write and flush operations on device + +6: apply offset and delay to device, also apply write_offset and write_delay + to write and flush operations on optionally different write_device with + optionally different sector offset + +9: same as 6 arguments plus define flush_offset and flush_delay explicitely + on/with optionally different flush_device/flush_offset. + Offsets are specified in sectors. + Delays are specified in milliseconds. + Example scripts =============== :: - #!/bin/sh - # Create device delaying rw operation for 500ms - echo "0 `blockdev --getsz $1` delay $1 0 500" | dmsetup create delayed + # + # Create mapped device named "delayed" delaying read, write and flush operations for 500ms. + # + dmsetup create delayed --table "0 `blockdev --getsz $1` delay $1 0 500" :: - #!/bin/sh - # Create device delaying only write operation for 500ms and - # splitting reads and writes to different devices $1 $2 - echo "0 `blockdev --getsz $1` delay $1 0 0 $2 0 500" | dmsetup create delayed + # + # Create mapped device delaying write and flush operations for 400ms and + # splitting reads to device $1 but writes and flushs to different device $2 + # to different offsets of 2048 and 4096 sectors respectively. + # + dmsetup create delayed --table "0 `blockdev --getsz $1` delay $1 2048 0 $2 4096 400" + +:: + #!/bin/sh + # + # Create mapped device delaying reads for 50ms, writes for 100ms and flushs for 333ms + # onto the same backing device at offset 0 sectors. + # + dmsetup create delayed --table "0 `blockdev --getsz $1` delay $1 0 50 $2 0 100 $1 0 333" From f3631ae11d4694e2befff9dd10dab8cd56033f6c Mon Sep 17 00:00:00 2001 From: Zhang Zekun Date: Mon, 12 Aug 2024 19:53:09 +0800 Subject: [PATCH 012/104] dm: Remove unused declaration and empty definition "dm_zone_map_bio" dm_zone_map_bio() has beed removed since commit f211268ed1f9 ("dm: Use the block layer zone append emulation"), remain the declaration unused in header files. So, let's remove this unused declaration and empty definition. Signed-off-by: Zhang Zekun Reviewed-by: Damien Le Moal Signed-off-by: Mikulas Patocka --- drivers/md/dm.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/md/dm.h b/drivers/md/dm.h index cc466ad5cb1d..8ad782249af8 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -109,7 +109,6 @@ void dm_zone_endio(struct dm_io *io, struct bio *clone); int dm_blk_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); bool dm_is_zone_write(struct mapped_device *md, struct bio *bio); -int dm_zone_map_bio(struct dm_target_io *io); int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t, sector_t sector, unsigned int nr_zones, unsigned long *need_reset); @@ -119,10 +118,6 @@ static inline bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) { return false; } -static inline int dm_zone_map_bio(struct dm_target_io *tio) -{ - return DM_MAPIO_KILL; -} #endif /* From 4441686b24a1d7acf9834ca95864d67e3f97666a Mon Sep 17 00:00:00 2001 From: Ingo Franzki Date: Fri, 16 Aug 2024 13:21:33 +0200 Subject: [PATCH 013/104] dm-crypt: Allow to specify the integrity key size as option For the MAC based integrity operation, the integrity key size (i.e. key_mac_size) is currently set to the digest size of the used digest. For wrapped key HMAC algorithms, the key size is independent of the cryptographic key size. So there is no known size of the mac key in such cases. The desired key size can optionally be specified as argument when the dm-crypt device is configured via 'integrity_key_size:%u'. If no integrity_key_size argument is specified, the mac key size is still set to the digest size, as before. Increase version number to 1.28.0 so that support for the new argument can be detected by user space (i.e. cryptsetup). Signed-off-by: Ingo Franzki Reviewed-by: Milan Broz Signed-off-by: Mikulas Patocka --- .../admin-guide/device-mapper/dm-crypt.rst | 4 ++++ drivers/md/dm-crypt.c | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Documentation/admin-guide/device-mapper/dm-crypt.rst b/Documentation/admin-guide/device-mapper/dm-crypt.rst index 552c9155165d..cde1d5594b0d 100644 --- a/Documentation/admin-guide/device-mapper/dm-crypt.rst +++ b/Documentation/admin-guide/device-mapper/dm-crypt.rst @@ -160,6 +160,10 @@ iv_large_sectors The must be multiple of (in 512 bytes units) if this flag is specified. +integrity_key_size: + Use an integrity key of size instead of using an integrity key size + of the digest size of the used HMAC algorithm. + Module parameters:: diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 348b4b26c272..d5533b43054e 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -147,6 +147,7 @@ enum cipher_flags { CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */ CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */ CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */ + CRYPT_KEY_MAC_SIZE_SET, /* The integrity_key_size option was used */ }; /* @@ -2937,7 +2938,8 @@ static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api) if (IS_ERR(mac)) return PTR_ERR(mac); - cc->key_mac_size = crypto_ahash_digestsize(mac); + if (!test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags)) + cc->key_mac_size = crypto_ahash_digestsize(mac); crypto_free_ahash(mac); cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL); @@ -3219,6 +3221,13 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar cc->cipher_auth = kstrdup(sval, GFP_KERNEL); if (!cc->cipher_auth) return -ENOMEM; + } else if (sscanf(opt_string, "integrity_key_size:%u%c", &val, &dummy) == 1) { + if (!val) { + ti->error = "Invalid integrity_key_size argument"; + return -EINVAL; + } + cc->key_mac_size = val; + set_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags); } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) { if (cc->sector_size < (1 << SECTOR_SHIFT) || cc->sector_size > 4096 || @@ -3607,10 +3616,10 @@ static void crypt_status(struct dm_target *ti, status_type_t type, num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags); num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags); + num_feature_args += !!cc->used_tag_size; num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT); num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags); - if (cc->used_tag_size) - num_feature_args++; + num_feature_args += test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags); if (num_feature_args) { DMEMIT(" %d", num_feature_args); if (ti->num_discard_bios) @@ -3631,6 +3640,8 @@ static void crypt_status(struct dm_target *ti, status_type_t type, DMEMIT(" sector_size:%d", cc->sector_size); if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags)) DMEMIT(" iv_large_sectors"); + if (test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags)) + DMEMIT(" integrity_key_size:%u", cc->key_mac_size); } break; @@ -3758,7 +3769,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) static struct target_type crypt_target = { .name = "crypt", - .version = {1, 27, 0}, + .version = {1, 28, 0}, .module = THIS_MODULE, .ctr = crypt_ctr, .dtr = crypt_dtr, From 02c0207ecdcc9abdf3442676154f8fd2be0f649a Mon Sep 17 00:00:00 2001 From: Yuesong Li Date: Thu, 22 Aug 2024 10:14:00 +0800 Subject: [PATCH 014/104] dm bufio: Remove NULL check of list_entry() list_entry() will never return a NULL pointer, thus remove the check. Signed-off-by: Yuesong Li Signed-off-by: Mikulas Patocka --- drivers/md/dm-bufio.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 098bf526136c..d478aafa02c9 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -529,9 +529,6 @@ static struct dm_buffer *list_to_buffer(struct list_head *l) { struct lru_entry *le = list_entry(l, struct lru_entry, list); - if (!le) - return NULL; - return le_to_buffer(le); } From 947697c6f0f75f9866f2f891a102dece7a09a064 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Thu, 22 Aug 2024 10:18:52 +0530 Subject: [PATCH 015/104] uapi: Define GENMASK_U128 This adds GENMASK_U128() and __GENMASK_U128() macros using __BITS_PER_U128 and __int128 data types. These macros will be used in providing support for generating 128 bit masks. The macros wouldn't work in all assembler flavors for reasons described in the comments on top of declarations. Enforce it for more by adding !__ASSEMBLY__ guard. Cc: Yury Norov Cc: Rasmus Villemoes Cc: Arnd Bergmann > Cc: linux-kernel@vger.kernel.org Cc: linux-arch@vger.kernel.org Reviewed-by: Arnd Bergmann Signed-off-by: Anshuman Khandual Signed-off-by: Yury Norov --- include/linux/bits.h | 15 +++++++++++++++ include/uapi/linux/bits.h | 3 +++ include/uapi/linux/const.h | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/include/linux/bits.h b/include/linux/bits.h index 0eb24d21aac2..60044b608817 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -36,4 +36,19 @@ #define GENMASK_ULL(h, l) \ (GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l)) +#if !defined(__ASSEMBLY__) +/* + * Missing asm support + * + * __GENMASK_U128() depends on _BIT128() which would not work + * in the asm code, as it shifts an 'unsigned __init128' data + * type instead of direct representation of 128 bit constants + * such as long and unsigned long. The fundamental problem is + * that a 128 bit constant will get silently truncated by the + * gcc compiler. + */ +#define GENMASK_U128(h, l) \ + (GENMASK_INPUT_CHECK(h, l) + __GENMASK_U128(h, l)) +#endif + #endif /* __LINUX_BITS_H */ diff --git a/include/uapi/linux/bits.h b/include/uapi/linux/bits.h index 3c2a101986a3..5ee30f882736 100644 --- a/include/uapi/linux/bits.h +++ b/include/uapi/linux/bits.h @@ -12,4 +12,7 @@ (((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \ (~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h)))) +#define __GENMASK_U128(h, l) \ + ((_BIT128((h)) << 1) - (_BIT128(l))) + #endif /* _UAPI_LINUX_BITS_H */ diff --git a/include/uapi/linux/const.h b/include/uapi/linux/const.h index a429381e7ca5..e16be0d37746 100644 --- a/include/uapi/linux/const.h +++ b/include/uapi/linux/const.h @@ -28,6 +28,23 @@ #define _BITUL(x) (_UL(1) << (x)) #define _BITULL(x) (_ULL(1) << (x)) +#if !defined(__ASSEMBLY__) +/* + * Missing asm support + * + * __BIT128() would not work in the asm code, as it shifts an + * 'unsigned __init128' data type as direct representation of + * 128 bit constants is not supported in the gcc compiler, as + * they get silently truncated. + * + * TODO: Please revisit this implementation when gcc compiler + * starts representing 128 bit constants directly like long + * and unsigned long etc. Subsequently drop the comment for + * GENMASK_U128() which would then start supporting asm code. + */ +#define _BIT128(x) ((unsigned __int128)(1) << (x)) +#endif + #define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (__typeof__(x))(a) - 1) #define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask)) From d7bcc37436c7d373a715d48a7cd74cd3b1724a68 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Thu, 22 Aug 2024 10:18:53 +0530 Subject: [PATCH 016/104] lib/test_bits.c: Add tests for GENMASK_U128() This adds GENMASK_U128() tests although currently only 64 bit wide masks are being tested. Cc: Andrew Morton Cc: linux-kernel@vger.kernel.org Reviewed-by: Arnd Bergmann Signed-off-by: Anshuman Khandual Signed-off-by: Yury Norov --- lib/test_bits.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/test_bits.c b/lib/test_bits.c index 01313980f175..c7b38d91e1f1 100644 --- a/lib/test_bits.c +++ b/lib/test_bits.c @@ -39,6 +39,36 @@ static void genmask_ull_test(struct kunit *test) #endif } +static void genmask_u128_test(struct kunit *test) +{ +#ifdef CONFIG_ARCH_SUPPORTS_INT128 + /* Below 64 bit masks */ + KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(0, 0)); + KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(1, 0)); + KUNIT_EXPECT_EQ(test, 0x0000000000000006ull, GENMASK_U128(2, 1)); + KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(31, 0)); + KUNIT_EXPECT_EQ(test, 0x000000ffffe00000ull, GENMASK_U128(39, 21)); + KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(63, 0)); + + /* Above 64 bit masks - only 64 bit portion can be validated once */ + KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(64, 0) >> 1); + KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(81, 50) >> 50); + KUNIT_EXPECT_EQ(test, 0x0000000000ffffffull, GENMASK_U128(87, 64) >> 64); + KUNIT_EXPECT_EQ(test, 0x0000000000ff0000ull, GENMASK_U128(87, 80) >> 64); + + KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(127, 0) >> 64); + KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, (u64)GENMASK_U128(127, 0)); + KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(127, 126) >> 126); + KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(127, 127) >> 127); +#ifdef TEST_GENMASK_FAILURES + /* these should fail compilation */ + GENMASK_U128(0, 1); + GENMASK_U128(0, 10); + GENMASK_U128(9, 10); +#endif /* TEST_GENMASK_FAILURES */ +#endif /* CONFIG_ARCH_SUPPORTS_INT128 */ +} + static void genmask_input_check_test(struct kunit *test) { unsigned int x, y; @@ -56,12 +86,16 @@ static void genmask_input_check_test(struct kunit *test) /* Valid input */ KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1)); KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(100, 80)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(110, 65)); + KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(127, 0)); } static struct kunit_case bits_test_cases[] = { KUNIT_CASE(genmask_test), KUNIT_CASE(genmask_ull_test), + KUNIT_CASE(genmask_u128_test), KUNIT_CASE(genmask_input_check_test), {} }; From 00565cff01262c888512bffe9d41e4831a6f2cc7 Mon Sep 17 00:00:00 2001 From: Yuesong Li Date: Fri, 30 Aug 2024 11:16:42 +0800 Subject: [PATCH 017/104] dm: Convert to use ERR_CAST() Use ERR_CAST() as it is designed for casting an error pointer to another type. This macro utilizes the __force and __must_check modifiers, which instruct the compiler to verify for errors at the locations where it is employed. Signed-off-by: Yuesong Li Signed-off-by: Mikulas Patocka --- drivers/md/dm-thin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index a0c1620e90c8..89632ce97760 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -2948,7 +2948,7 @@ static struct pool *pool_create(struct mapped_device *pool_md, pmd = dm_pool_metadata_open(metadata_dev, block_size, format_device); if (IS_ERR(pmd)) { *error = "Error creating metadata object"; - return (struct pool *)pmd; + return ERR_CAST(pmd); } pool = kzalloc(sizeof(*pool), GFP_KERNEL); From 35c9f09b5691d6dff1f3e62fe1825fa10b644b45 Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Sat, 31 Aug 2024 17:48:01 +0800 Subject: [PATCH 018/104] dm integrity: Remove extra unlikely helper In IS_ERR, the unlikely is used for the input parameter, so these is no need to use it again outside. Signed-off-by: Hongbo Li Signed-off-by: Kunwu Chan Signed-off-by: Mikulas Patocka --- drivers/md/dm-integrity.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 51e6964c1305..8306f8511078 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2447,7 +2447,7 @@ retry: bio->bi_iter.bi_sector += ic->start + SB_SECTORS; bip = bio_integrity_alloc(bio, GFP_NOIO, 1); - if (unlikely(IS_ERR(bip))) { + if (IS_ERR(bip)) { bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); bio_endio(bio); return DM_MAPIO_SUBMITTED; @@ -2520,7 +2520,7 @@ static void dm_integrity_inline_recheck(struct work_struct *w) } bip = bio_integrity_alloc(outgoing_bio, GFP_NOIO, 1); - if (unlikely(IS_ERR(bip))) { + if (IS_ERR(bip)) { bio_put(outgoing_bio); bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); bio_endio(bio); From 26207c6332e83583a74228da9e5278d4fe5d26cf Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Mon, 2 Sep 2024 21:11:23 +0800 Subject: [PATCH 019/104] dm: Make use of __assign_bit() API We have for some time the __assign_bit() API to replace open coded if (foo) __set_bit(n, bar); else __clear_bit(n, bar); Use this API to simplify the code. No functional change intended. Signed-off-by: Hongbo Li Signed-off-by: Mikulas Patocka --- drivers/md/dm-clone-metadata.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/md/dm-clone-metadata.c b/drivers/md/dm-clone-metadata.c index 2db84cd2202b..14c5c28d938b 100644 --- a/drivers/md/dm-clone-metadata.c +++ b/drivers/md/dm-clone-metadata.c @@ -530,10 +530,7 @@ static int __load_bitset_in_core(struct dm_clone_metadata *cmd) return r; for (i = 0; ; i++) { - if (dm_bitset_cursor_get_value(&c)) - __set_bit(i, cmd->region_map); - else - __clear_bit(i, cmd->region_map); + __assign_bit(i, cmd->region_map, dm_bitset_cursor_get_value(&c)); if (i >= (cmd->nr_regions - 1)) break; From a8fa6483b40943a6c8feea803a2dc8e9982cc766 Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Tue, 3 Sep 2024 11:50:11 +0200 Subject: [PATCH 020/104] dm integrity: fix gcc 5 warning This commit fixes gcc 5 warning "logical not is only applied to the left hand side of comparison" Reported-by: Geert Uytterhoeven Fixes: fb0987682c62 ("dm-integrity: introduce the Inline mode") Signed-off-by: Mikulas Patocka --- drivers/md/dm-integrity.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 8306f8511078..b3489d3fe7db 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -4715,13 +4715,18 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ti->error = "Block size doesn't match the information in superblock"; goto bad; } - if (!le32_to_cpu(ic->sb->journal_sections) != (ic->mode == 'I')) { - r = -EINVAL; - if (ic->mode != 'I') + if (ic->mode != 'I') { + if (!le32_to_cpu(ic->sb->journal_sections)) { + r = -EINVAL; ti->error = "Corrupted superblock, journal_sections is 0"; - else + goto bad; + } + } else { + if (le32_to_cpu(ic->sb->journal_sections)) { + r = -EINVAL; ti->error = "Corrupted superblock, journal_sections is not 0"; - goto bad; + goto bad; + } } /* make sure that ti->max_io_len doesn't overflow */ if (!ic->meta_dev) { From 1f9651bfc51326fbed10df820c4e80f885f921d8 Mon Sep 17 00:00:00 2001 From: Yuesong Li Date: Thu, 29 Aug 2024 20:52:35 +0800 Subject: [PATCH 021/104] cxl/port: Convert to use ERR_CAST() Use ERR_CAST() as it is designed for casting an error pointer to another type. This macro utilizes the __force and __must_check modifiers, which instruct the compiler to verify for errors at the locations where it is employed. Signed-off-by: Yuesong Li Reviewed-by: Jonathan Cameron Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240829125235.3266865-1-liyuesong@vivo.com Signed-off-by: Dave Jiang --- drivers/cxl/core/port.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1d5007e3795a..bdd3275ad417 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -941,7 +941,7 @@ struct cxl_root *devm_cxl_add_root(struct device *host, port = devm_cxl_add_port(host, host, CXL_RESOURCE_NONE, NULL); if (IS_ERR(port)) - return (struct cxl_root *)port; + return ERR_CAST(port); cxl_root = to_cxl_root(port); cxl_root->ops = ops; From 1c1d348b08ab7c106dd96bafb120f0888827174b Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Tue, 27 Aug 2024 17:51:23 +0800 Subject: [PATCH 022/104] tools/testing/cxl: Use dev_is_platform() Use dev_is_platform() instead of checking bus type directly. Signed-off-by: Kunwu Chan Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240827095123.168696-1-kunwu.chan@linux.dev Signed-off-by: Dave Jiang --- tools/testing/cxl/mock_acpi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/cxl/mock_acpi.c b/tools/testing/cxl/mock_acpi.c index 55813de26d46..8da94378ccec 100644 --- a/tools/testing/cxl/mock_acpi.c +++ b/tools/testing/cxl/mock_acpi.c @@ -18,7 +18,7 @@ struct acpi_device *to_cxl_host_bridge(struct device *host, struct device *dev) goto out; } - if (dev->bus == &platform_bus_type) + if (dev_is_platform(dev)) goto out; adev = to_acpi_device(dev); From fa724cd747cdc347cde0d2ef6ba4ee34344f72cb Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Fri, 30 Aug 2024 16:00:16 +0800 Subject: [PATCH 023/104] cxl: Remove duplicate included header file core.h The header file core.h is included twice. Remove the last one. The compilation test has passed. Signed-off-by: Hongbo Li Acked-by: Jonathan Cameron Reviewed-by: Jonathan Cameron Reviewed-by: Dave Jiang Link: https://patch.msgid.link/20240830080016.3542184-1-lihongbo22@huawei.com Signed-off-by: Dave Jiang --- drivers/cxl/core/cdat.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index bb83867d9fec..e2e1ccda88fa 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -9,7 +9,6 @@ #include "cxlmem.h" #include "core.h" #include "cxl.h" -#include "core.h" struct dsmas_entry { struct range dpa_range; From dd2617ebd2a69c012001a29274557199409eff39 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 01:31:36 +0000 Subject: [PATCH 024/104] cxl/port: Use __free() to drop put_device() for cxl_port Using scope-based resource management __free() marco with a new helper called put_cxl_port() to drop open coded the put_device() used to dereference the 'struct device' in cxl_port. Suggested-by: Dan Williams Signed-off-by: Li Ming Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830013138.2256244-1-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 6 ++---- drivers/cxl/core/port.c | 30 +++++++++++++----------------- drivers/cxl/cxl.h | 1 + drivers/cxl/mem.c | 5 ++--- drivers/cxl/pci.c | 7 ++----- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 51132a575b27..4725e37d90fb 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -915,15 +915,13 @@ static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) struct pci_dev *pdev = to_pci_dev(cxlds->dev); struct aer_capability_regs aer_regs; struct cxl_dport *dport; - struct cxl_port *port; int severity; - port = cxl_pci_find_port(pdev, &dport); + struct cxl_port *port __free(put_cxl_port) = + cxl_pci_find_port(pdev, &dport); if (!port) return; - put_device(&port->dev); - if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) return; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index bdd3275ad417..e1e762076476 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1477,12 +1477,11 @@ static void cxl_detach_ep(void *data) .cxlmd = cxlmd, .depth = i, }; - struct device *dev; struct cxl_ep *ep; bool died = false; - dev = bus_find_device(&cxl_bus_type, NULL, &ctx, - port_has_memdev); + struct device *dev __free(put_device) = + bus_find_device(&cxl_bus_type, NULL, &ctx, port_has_memdev); if (!dev) continue; port = to_cxl_port(dev); @@ -1512,7 +1511,6 @@ static void cxl_detach_ep(void *data) dev_name(&port->dev)); delete_switch_port(port); } - put_device(&port->dev); device_unlock(&parent_port->dev); } } @@ -1540,7 +1538,6 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, struct device *dport_dev) { struct device *dparent = grandparent(dport_dev); - struct cxl_port *port, *parent_port = NULL; struct cxl_dport *dport, *parent_dport; resource_size_t component_reg_phys; int rc; @@ -1556,12 +1553,18 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, return -ENXIO; } - parent_port = find_cxl_port(dparent, &parent_dport); + struct cxl_port *parent_port __free(put_cxl_port) = + find_cxl_port(dparent, &parent_dport); if (!parent_port) { /* iterate to create this parent_port */ return -EAGAIN; } + /* + * Definition with __free() here to keep the sequence of + * dereferencing the device of the port before the parent_port releasing. + */ + struct cxl_port *port __free(put_cxl_port) = NULL; device_lock(&parent_port->dev); if (!parent_port->dev.driver) { dev_warn(&cxlmd->dev, @@ -1596,10 +1599,8 @@ out: */ rc = -ENXIO; } - put_device(&port->dev); } - put_device(&parent_port->dev); return rc; } @@ -1630,7 +1631,6 @@ retry: struct device *dport_dev = grandparent(iter); struct device *uport_dev; struct cxl_dport *dport; - struct cxl_port *port; /* * The terminal "grandparent" in PCI is NULL and @platform_bus @@ -1649,7 +1649,8 @@ retry: dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n", dev_name(iter), dev_name(dport_dev), dev_name(uport_dev)); - port = find_cxl_port(dport_dev, &dport); + struct cxl_port *port __free(put_cxl_port) = + find_cxl_port(dport_dev, &dport); if (port) { dev_dbg(&cxlmd->dev, "found already registered port %s:%s\n", @@ -1664,18 +1665,13 @@ retry: * the parent_port lock as the current port may be being * reaped. */ - if (rc && rc != -EBUSY) { - put_device(&port->dev); + if (rc && rc != -EBUSY) return rc; - } /* Any more ports to add between this one and the root? */ - if (!dev_is_cxl_root_child(&port->dev)) { - put_device(&port->dev); + if (!dev_is_cxl_root_child(&port->dev)) continue; - } - put_device(&port->dev); return 0; } diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 9afb407d438f..84cf3b4d60a1 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -744,6 +744,7 @@ struct cxl_root *find_cxl_root(struct cxl_port *port); void put_cxl_root(struct cxl_root *cxl_root); DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_cxl_root(_T)) +DEFINE_FREE(put_cxl_port, struct cxl_port *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev)) int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); void cxl_bus_rescan(void); void cxl_bus_drain(void); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 7de232eaeb17..ab9b8ab8df44 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -109,7 +109,6 @@ static int cxl_mem_probe(struct device *dev) struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct device *endpoint_parent; - struct cxl_port *parent_port; struct cxl_dport *dport; struct dentry *dentry; int rc; @@ -146,7 +145,8 @@ static int cxl_mem_probe(struct device *dev) if (rc) return rc; - parent_port = cxl_mem_find_port(cxlmd, &dport); + struct cxl_port *parent_port __free(put_cxl_port) = + cxl_mem_find_port(cxlmd, &dport); if (!parent_port) { dev_err(dev, "CXL port topology not found\n"); return -ENXIO; @@ -179,7 +179,6 @@ static int cxl_mem_probe(struct device *dev) rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport); unlock: device_unlock(endpoint_parent); - put_device(&parent_port->dev); if (rc) return rc; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 4be35dc22202..26e75499abdd 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -473,7 +473,6 @@ static bool is_cxl_restricted(struct pci_dev *pdev) static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev, struct cxl_register_map *map) { - struct cxl_port *port; struct cxl_dport *dport; resource_size_t component_reg_phys; @@ -482,14 +481,12 @@ static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev, .resource = CXL_RESOURCE_NONE, }; - port = cxl_pci_find_port(pdev, &dport); + struct cxl_port *port __free(put_cxl_port) = + cxl_pci_find_port(pdev, &dport); if (!port) return -EPROBE_DEFER; component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport); - - put_device(&port->dev); - if (component_reg_phys == CXL_RESOURCE_NONE) return -ENXIO; From 7f569e917b7866b6760e2cfc3a9549cabc890071 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 01:31:37 +0000 Subject: [PATCH 025/104] cxl/port: Use scoped_guard()/guard() to drop device_lock() for cxl_port A device_lock() and device_unlock() pair can be replaced by a cleanup helper scoped_guard() or guard(), that can enhance code readability. In CXL subsystem, still use device_lock() and device_unlock() pairs for cxl port resource protection, most of them can be replaced by a scoped_guard() or a guard() simply. Suggested-by: Dan Williams Signed-off-by: Li Ming Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830013138.2256244-2-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/mbox.c | 3 +- drivers/cxl/core/port.c | 99 +++++++++++++++++---------------------- drivers/cxl/core/region.c | 25 +++++----- drivers/cxl/mem.c | 22 ++++----- drivers/cxl/pmem.c | 16 +++---- 5 files changed, 75 insertions(+), 90 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index e5cdeafdf76e..0a913b30c7fc 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -1214,7 +1214,7 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) int rc; /* synchronize with cxl_mem_probe() and decoder write operations */ - device_lock(&cxlmd->dev); + guard(device)(&cxlmd->dev); endpoint = cxlmd->endpoint; down_read(&cxl_region_rwsem); /* @@ -1226,7 +1226,6 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) else rc = -EBUSY; up_read(&cxl_region_rwsem); - device_unlock(&cxlmd->dev); return rc; } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index e1e762076476..ebd7ddadbb54 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1258,18 +1258,13 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL); static int add_ep(struct cxl_ep *new) { struct cxl_port *port = new->dport->port; - int rc; - device_lock(&port->dev); - if (port->dead) { - device_unlock(&port->dev); + guard(device)(&port->dev); + if (port->dead) return -ENXIO; - } - rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new, - GFP_KERNEL); - device_unlock(&port->dev); - return rc; + return xa_insert(&port->endpoints, (unsigned long)new->ep, + new, GFP_KERNEL); } /** @@ -1393,14 +1388,14 @@ static void delete_endpoint(void *data) struct cxl_port *endpoint = cxlmd->endpoint; struct device *host = endpoint_host(endpoint); - device_lock(host); - if (host->driver && !endpoint->dead) { - devm_release_action(host, cxl_unlink_parent_dport, endpoint); - devm_release_action(host, cxl_unlink_uport, endpoint); - devm_release_action(host, unregister_port, endpoint); + scoped_guard(device, host) { + if (host->driver && !endpoint->dead) { + devm_release_action(host, cxl_unlink_parent_dport, endpoint); + devm_release_action(host, cxl_unlink_uport, endpoint); + devm_release_action(host, unregister_port, endpoint); + } + cxlmd->endpoint = NULL; } - cxlmd->endpoint = NULL; - device_unlock(host); put_device(&endpoint->dev); put_device(host); } @@ -1565,40 +1560,38 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, * dereferencing the device of the port before the parent_port releasing. */ struct cxl_port *port __free(put_cxl_port) = NULL; - device_lock(&parent_port->dev); - if (!parent_port->dev.driver) { - dev_warn(&cxlmd->dev, - "port %s:%s disabled, failed to enumerate CXL.mem\n", - dev_name(&parent_port->dev), dev_name(uport_dev)); - port = ERR_PTR(-ENXIO); - goto out; - } - - port = find_cxl_port_at(parent_port, dport_dev, &dport); - if (!port) { - component_reg_phys = find_component_registers(uport_dev); - port = devm_cxl_add_port(&parent_port->dev, uport_dev, - component_reg_phys, parent_dport); - /* retry find to pick up the new dport information */ - if (!IS_ERR(port)) - port = find_cxl_port_at(parent_port, dport_dev, &dport); - } -out: - device_unlock(&parent_port->dev); - - if (IS_ERR(port)) - rc = PTR_ERR(port); - else { - dev_dbg(&cxlmd->dev, "add to new port %s:%s\n", - dev_name(&port->dev), dev_name(port->uport_dev)); - rc = cxl_add_ep(dport, &cxlmd->dev); - if (rc == -EBUSY) { - /* - * "can't" happen, but this error code means - * something to the caller, so translate it. - */ - rc = -ENXIO; + scoped_guard(device, &parent_port->dev) { + if (!parent_port->dev.driver) { + dev_warn(&cxlmd->dev, + "port %s:%s disabled, failed to enumerate CXL.mem\n", + dev_name(&parent_port->dev), dev_name(uport_dev)); + return -ENXIO; } + + port = find_cxl_port_at(parent_port, dport_dev, &dport); + if (!port) { + component_reg_phys = find_component_registers(uport_dev); + port = devm_cxl_add_port(&parent_port->dev, uport_dev, + component_reg_phys, parent_dport); + if (IS_ERR(port)) + return PTR_ERR(port); + + /* retry find to pick up the new dport information */ + port = find_cxl_port_at(parent_port, dport_dev, &dport); + if (!port) + return -ENXIO; + } + } + + dev_dbg(&cxlmd->dev, "add to new port %s:%s\n", + dev_name(&port->dev), dev_name(port->uport_dev)); + rc = cxl_add_ep(dport, &cxlmd->dev); + if (rc == -EBUSY) { + /* + * "can't" happen, but this error code means + * something to the caller, so translate it. + */ + rc = -ENXIO; } return rc; @@ -1979,7 +1972,6 @@ EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL); int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map) { struct cxl_port *port; - int rc; if (WARN_ON_ONCE(!cxld)) return -EINVAL; @@ -1989,11 +1981,8 @@ int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map) port = to_cxl_port(cxld->dev.parent); - device_lock(&port->dev); - rc = cxl_decoder_add_locked(cxld, target_map); - device_unlock(&port->dev); - - return rc; + guard(device)(&port->dev); + return cxl_decoder_add_locked(cxld, target_map); } EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL); diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 21ad5f242875..9a7c001eff1e 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -3094,11 +3094,11 @@ static void cxlr_release_nvdimm(void *_cxlr) struct cxl_region *cxlr = _cxlr; struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; - device_lock(&cxl_nvb->dev); - if (cxlr->cxlr_pmem) - devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, - cxlr->cxlr_pmem); - device_unlock(&cxl_nvb->dev); + scoped_guard(device, &cxl_nvb->dev) { + if (cxlr->cxlr_pmem) + devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, + cxlr->cxlr_pmem); + } cxlr->cxl_nvb = NULL; put_device(&cxl_nvb->dev); } @@ -3134,13 +3134,14 @@ static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), dev_name(dev)); - device_lock(&cxl_nvb->dev); - if (cxl_nvb->dev.driver) - rc = devm_add_action_or_reset(&cxl_nvb->dev, - cxlr_pmem_unregister, cxlr_pmem); - else - rc = -ENXIO; - device_unlock(&cxl_nvb->dev); + scoped_guard(device, &cxl_nvb->dev) { + if (cxl_nvb->dev.driver) + rc = devm_add_action_or_reset(&cxl_nvb->dev, + cxlr_pmem_unregister, + cxlr_pmem); + else + rc = -ENXIO; + } if (rc) goto err_bridge; diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index ab9b8ab8df44..ae94018a01bd 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -168,19 +168,17 @@ static int cxl_mem_probe(struct device *dev) cxl_setup_parent_dport(dev, dport); - device_lock(endpoint_parent); - if (!endpoint_parent->driver) { - dev_err(dev, "CXL port topology %s not enabled\n", - dev_name(endpoint_parent)); - rc = -ENXIO; - goto unlock; - } + scoped_guard(device, endpoint_parent) { + if (!endpoint_parent->driver) { + dev_err(dev, "CXL port topology %s not enabled\n", + dev_name(endpoint_parent)); + return -ENXIO; + } - rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport); -unlock: - device_unlock(endpoint_parent); - if (rc) - return rc; + rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport); + if (rc) + return rc; + } /* * The kernel may be operating out of CXL memory on this device, diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 4ef93da22335..647c7e25ef3a 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -233,15 +233,13 @@ static int detach_nvdimm(struct device *dev, void *data) if (!is_cxl_nvdimm(dev)) return 0; - device_lock(dev); - if (!dev->driver) - goto out; - - cxl_nvd = to_cxl_nvdimm(dev); - if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) - release = true; -out: - device_unlock(dev); + scoped_guard(device, dev) { + if (dev->driver) { + cxl_nvd = to_cxl_nvdimm(dev); + if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) + release = true; + } + } if (release) device_release_driver(dev); return 0; From 91c0e9d6a205ab0e318dc0dc6e72cbbdb21f8094 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 01:31:38 +0000 Subject: [PATCH 026/104] cxl/port: Refactor __devm_cxl_add_port() to drop goto pattern In __devm_cxl_add_port(), there is a 'goto' to call put_device() for the error cases between device_initialize() and device_add() to dereference the 'struct device' of a new cxl_port. The 'goto' pattern in the case can be removed by refactoring. Introducing a new function called cxl_port_add() which is used to add the 'struct device' of a new cxl_port to device hierarchy, moving the functions needing the help of the 'goto' into cxl_port_add(), and using a scoped-based resource management __free() to drop the open coded put_device() and the 'goto' for the error cases. Suggested-by: Dan Williams Signed-off-by: Li Ming Reviewed-by: Ira Weiny Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830013138.2256244-3-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/port.c | 59 ++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index ebd7ddadbb54..7ca57285a7b7 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -828,27 +828,20 @@ static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) &cxl_einj_inject_fops); } -static struct cxl_port *__devm_cxl_add_port(struct device *host, - struct device *uport_dev, - resource_size_t component_reg_phys, - struct cxl_dport *parent_dport) +static int cxl_port_add(struct cxl_port *port, + resource_size_t component_reg_phys, + struct cxl_dport *parent_dport) { - struct cxl_port *port; - struct device *dev; + struct device *dev __free(put_device) = &port->dev; int rc; - port = cxl_port_alloc(uport_dev, parent_dport); - if (IS_ERR(port)) - return port; - - dev = &port->dev; - if (is_cxl_memdev(uport_dev)) { - struct cxl_memdev *cxlmd = to_cxl_memdev(uport_dev); + if (is_cxl_memdev(port->uport_dev)) { + struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; rc = dev_set_name(dev, "endpoint%d", port->id); if (rc) - goto err; + return rc; /* * The endpoint driver already enumerated the component and RAS @@ -861,19 +854,41 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, } else if (parent_dport) { rc = dev_set_name(dev, "port%d", port->id); if (rc) - goto err; + return rc; rc = cxl_port_setup_regs(port, component_reg_phys); if (rc) - goto err; - } else + return rc; + } else { rc = dev_set_name(dev, "root%d", port->id); - if (rc) - goto err; + if (rc) + return rc; + } rc = device_add(dev); if (rc) - goto err; + return rc; + + /* Inhibit the cleanup function invoked */ + dev = NULL; + return 0; +} + +static struct cxl_port *__devm_cxl_add_port(struct device *host, + struct device *uport_dev, + resource_size_t component_reg_phys, + struct cxl_dport *parent_dport) +{ + struct cxl_port *port; + int rc; + + port = cxl_port_alloc(uport_dev, parent_dport); + if (IS_ERR(port)) + return port; + + rc = cxl_port_add(port, component_reg_phys, parent_dport); + if (rc) + return ERR_PTR(rc); rc = devm_add_action_or_reset(host, unregister_port, port); if (rc) @@ -891,10 +906,6 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host, port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev)); return port; - -err: - put_device(dev); - return ERR_PTR(rc); } /** From 577a67662ff529f617981fe9692ff277b5756402 Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 06:13:06 +0000 Subject: [PATCH 027/104] cxl/pci: Rename cxl_setup_parent_dport() and cxl_dport_map_regs() The name of cxl_setup_parent_dport() function is not clear, the function is used to initialize AER and RAS capabilities on a dport, therefore, rename the function to cxl_dport_init_ras_reporting(), it is easier for user to understand what the function does. Besides, adjust the order of the function parameters, the subject of cxl_dport_init_ras_reporting() is a cxl dport, so a struct cxl_dport as the first parameter of the function should be better. cxl_dport_map_regs() is used to map CXL RAS capability on a cxl dport, using cxl_dport_map_ras() as the function name. Signed-off-by: Li Ming Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830061308.2327065-1-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 13 +++++++++---- drivers/cxl/cxl.h | 5 +++-- drivers/cxl/mem.c | 2 +- tools/testing/cxl/Kbuild | 2 +- tools/testing/cxl/test/mock.c | 6 +++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 4725e37d90fb..cca078068a05 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -787,7 +787,7 @@ static void cxl_dport_map_rch_aer(struct cxl_dport *dport) dport->regs.dport_aer = dport_aer; } -static void cxl_dport_map_regs(struct cxl_dport *dport) +static void cxl_dport_map_ras(struct cxl_dport *dport) { struct cxl_register_map *map = &dport->reg_map; struct device *dev = dport->dport_dev; @@ -831,7 +831,12 @@ static void cxl_disable_rch_root_ints(struct cxl_dport *dport) } } -void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +/** + * cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport + * @dport: the cxl_dport that needs to be initialized + * @host: host device for devm operations + */ +void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host) { struct device *dport_dev = dport->dport_dev; @@ -843,12 +848,12 @@ void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) } dport->reg_map.host = host; - cxl_dport_map_regs(dport); + cxl_dport_map_ras(dport); if (dport->rch) cxl_disable_rch_root_ints(dport); } -EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); +EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, CXL); static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds, struct cxl_dport *dport) diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 84cf3b4d60a1..2890f7e2aad5 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -763,9 +763,10 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, #ifdef CONFIG_PCIEAER_CXL void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport); +void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host); #else -static inline void cxl_setup_parent_dport(struct device *host, - struct cxl_dport *dport) { } +static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport, + struct device *host) { } #endif struct cxl_decoder *to_cxl_decoder(struct device *dev); diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index ae94018a01bd..a9fd5cd5a0d2 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -166,7 +166,7 @@ static int cxl_mem_probe(struct device *dev) else endpoint_parent = &parent_port->dev; - cxl_setup_parent_dport(dev, dport); + cxl_dport_init_ras_reporting(dport, dev); scoped_guard(device, endpoint_parent) { if (!endpoint_parent->driver) { diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 3d1ca9e38b1f..b1256fee3567 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -14,7 +14,7 @@ ldflags-y += --wrap=cxl_dvsec_rr_decode ldflags-y += --wrap=devm_cxl_add_rch_dport ldflags-y += --wrap=cxl_rcd_component_reg_phys ldflags-y += --wrap=cxl_endpoint_parse_cdat -ldflags-y += --wrap=cxl_setup_parent_dport +ldflags-y += --wrap=cxl_dport_init_ras_reporting DRIVERS := ../../../drivers CXL_SRC := $(DRIVERS)/cxl diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index d619672faa49..bbd7d938156d 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -299,17 +299,17 @@ void __wrap_cxl_endpoint_parse_cdat(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_endpoint_parse_cdat, CXL); -void __wrap_cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) +void __wrap_cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host) { int index; struct cxl_mock_ops *ops = get_cxl_mock_ops(&index); if (!ops || !ops->is_mock_port(dport->dport_dev)) - cxl_setup_parent_dport(host, dport); + cxl_dport_init_ras_reporting(dport, host); put_cxl_mock_ops(index); } -EXPORT_SYMBOL_NS_GPL(__wrap_cxl_setup_parent_dport, CXL); +EXPORT_SYMBOL_NS_GPL(__wrap_cxl_dport_init_ras_reporting, CXL); MODULE_LICENSE("GPL v2"); MODULE_IMPORT_NS(ACPI); From c8706cc15a5814becacff778017bbcc5c031490a Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 06:13:07 +0000 Subject: [PATCH 028/104] cxl/pci: cxl_dport_map_rch_aer() cleanup cxl_dport_map_ras() is used to map CXL RAS capability, the RCH AER capability should not be mapped in the function but should mapped in cxl_dport_init_ras_reporting(). Moving cxl_dport_map_ras() out of cxl_dport_map_ras() and into cxl_dport_init_ras_reporting(). In cxl_dport_init_ras_reporting(), the AER capability position in RCRB will be located but the position is only used in cxl_dport_map_rch_aer(), getting the position in cxl_dport_map_rch_aer() rather than cxl_dport_init_ras_reporting() is more reasonable and makes the code clearer. Besides, some local variables in cxl_dport_map_rch_aer() are unnecessary, remove them to make the function more concise. Signed-off-by: Li Ming Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830061308.2327065-2-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index cca078068a05..ccd46843e1e7 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -772,19 +772,17 @@ static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) static void cxl_dport_map_rch_aer(struct cxl_dport *dport) { - struct cxl_rcrb_info *ri = &dport->rcrb; - void __iomem *dport_aer = NULL; resource_size_t aer_phys; struct device *host; + u16 aer_cap; - if (dport->rch && ri->aer_cap) { + aer_cap = cxl_rcrb_to_aer(dport->dport_dev, dport->rcrb.base); + if (aer_cap) { host = dport->reg_map.host; - aer_phys = ri->aer_cap + ri->base; - dport_aer = devm_cxl_iomap_block(host, aer_phys, - sizeof(struct aer_capability_regs)); + aer_phys = aer_cap + dport->rcrb.base; + dport->regs.dport_aer = devm_cxl_iomap_block(host, aer_phys, + sizeof(struct aer_capability_regs)); } - - dport->regs.dport_aer = dport_aer; } static void cxl_dport_map_ras(struct cxl_dport *dport) @@ -797,9 +795,6 @@ static void cxl_dport_map_ras(struct cxl_dport *dport) else if (cxl_map_component_regs(map, &dport->regs.component, BIT(CXL_CM_CAP_CAP_ID_RAS))) dev_dbg(dev, "Failed to map RAS capability.\n"); - - if (dport->rch) - cxl_dport_map_rch_aer(dport); } static void cxl_disable_rch_root_ints(struct cxl_dport *dport) @@ -838,20 +833,18 @@ static void cxl_disable_rch_root_ints(struct cxl_dport *dport) */ void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host) { - struct device *dport_dev = dport->dport_dev; - - if (dport->rch) { - struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport_dev); - - if (host_bridge->native_aer) - dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); - } - dport->reg_map.host = host; cxl_dport_map_ras(dport); - if (dport->rch) + if (dport->rch) { + struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev); + + if (!host_bridge->native_aer) + return; + + cxl_dport_map_rch_aer(dport); cxl_disable_rch_root_ints(dport); + } } EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, CXL); From d75ccd4f2ea2aa2679e6c807d76953dcd3f9d56d Mon Sep 17 00:00:00 2001 From: Li Ming Date: Fri, 30 Aug 2024 06:13:08 +0000 Subject: [PATCH 029/104] cxl/pci: Remove duplicate host_bridge->native_aer checking cxl_dport_init_ras_reporting() already checks host_bridge->native_aer before invoking cxl_disable_rch_root_ints(), so cxl_disable_rch_root_ints() does not need to check it again. Signed-off-by: Li Ming Reviewed-by: Jonathan Cameron Link: https://patch.msgid.link/20240830061308.2327065-3-ming4.li@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index ccd46843e1e7..5e5c24eed545 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -800,14 +800,11 @@ static void cxl_dport_map_ras(struct cxl_dport *dport) static void cxl_disable_rch_root_ints(struct cxl_dport *dport) { void __iomem *aer_base = dport->regs.dport_aer; - struct pci_host_bridge *bridge; u32 aer_cmd_mask, aer_cmd; if (!aer_base) return; - bridge = to_pci_host_bridge(dport->dport_dev); - /* * Disable RCH root port command interrupts. * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors @@ -816,14 +813,12 @@ static void cxl_disable_rch_root_ints(struct cxl_dport *dport) * the root cmd register's interrupts is required. But, PCI spec * shows these are disabled by default on reset. */ - if (bridge->native_aer) { - aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | - PCI_ERR_ROOT_CMD_NONFATAL_EN | - PCI_ERR_ROOT_CMD_FATAL_EN); - aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); - aer_cmd &= ~aer_cmd_mask; - writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); - } + aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | + PCI_ERR_ROOT_CMD_NONFATAL_EN | + PCI_ERR_ROOT_CMD_FATAL_EN); + aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); + aer_cmd &= ~aer_cmd_mask; + writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); } /** From a7722b82c2ca9ca771d6c698643883be76f61a7e Mon Sep 17 00:00:00 2001 From: Chen Ni Date: Thu, 5 Sep 2024 10:28:32 +0800 Subject: [PATCH 030/104] dm integrity: Convert comma to semicolon Replace comma between expressions with semicolons. Using a ',' in place of a ';' can have unintended side effects. Although that is not the case here, it is seems best to use ';' unless ',' is intended. Found by inspection. No functional change intended. Compile tested only. Signed-off-by: Chen Ni Reviewed-by: Mike Snitzer Signed-off-by: Mikulas Patocka --- drivers/md/dm-integrity.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index b3489d3fe7db..34fa98efa9fa 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -1500,15 +1500,15 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat if (!ic->meta_dev) flush_data = false; if (flush_data) { - fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC, - fr.io_req.mem.type = DM_IO_KMEM, - fr.io_req.mem.ptr.addr = NULL, - fr.io_req.notify.fn = flush_notify, + fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; + fr.io_req.mem.type = DM_IO_KMEM; + fr.io_req.mem.ptr.addr = NULL; + fr.io_req.notify.fn = flush_notify; fr.io_req.notify.context = &fr; - fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio), - fr.io_reg.bdev = ic->dev->bdev, - fr.io_reg.sector = 0, - fr.io_reg.count = 0, + fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio); + fr.io_reg.bdev = ic->dev->bdev; + fr.io_reg.sector = 0; + fr.io_reg.count = 0; fr.ic = ic; init_completion(&fr.comp); r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); From 90da77987dd59c8f6ec6d508d23d5a77c7af64f1 Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Thu, 5 Sep 2024 21:01:52 +0200 Subject: [PATCH 031/104] dm-integrity: support recalculation in the 'I' mode In the kernel 6.11, dm-integrity was enhanced with an inline ('I') mode. This mode uses devices with non-power-of-2 sector size. The extra metadata after each sector are used to hold the integrity hash. This commit enhances the inline mode, so that there is automatic recalculation of the integrity hashes when the 'reclaculate' parameter is used. It allows us to activate the device instantly, and the recalculation is done on background. If the device is deactivated while recalculation is in progress, it will remember the point where it stopped and it will continue from this point when activated again. Signed-off-by: Mikulas Patocka --- drivers/md/dm-integrity.c | 288 ++++++++++++++++++++++++++++++++------ 1 file changed, 246 insertions(+), 42 deletions(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 34fa98efa9fa..c40df05e0521 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -284,6 +284,7 @@ struct dm_integrity_c { mempool_t recheck_pool; struct bio_set recheck_bios; + struct bio_set recalc_bios; struct notifier_block reboot_notifier; }; @@ -321,7 +322,9 @@ struct dm_integrity_io { struct dm_bio_details bio_details; char *integrity_payload; + unsigned payload_len; bool integrity_payload_from_mempool; + bool integrity_range_locked; }; struct journal_completion { @@ -359,7 +362,7 @@ static struct kmem_cache *journal_io_cache; #endif static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); -static int dm_integrity_map_inline(struct dm_integrity_io *dio); +static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map); static void integrity_bio_wait(struct work_struct *w); static void dm_integrity_dtr(struct dm_target *ti); @@ -1946,8 +1949,13 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) dio->bi_status = 0; dio->op = bio_op(bio); - if (ic->mode == 'I') - return dm_integrity_map_inline(dio); + if (ic->mode == 'I') { + bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector); + dio->integrity_payload = NULL; + dio->integrity_payload_from_mempool = false; + dio->integrity_range_locked = false; + return dm_integrity_map_inline(dio, true); + } if (unlikely(dio->op == REQ_OP_DISCARD)) { if (ti->max_io_len) { @@ -2395,15 +2403,13 @@ journal_read_write: do_endio_flush(ic, dio); } -static int dm_integrity_map_inline(struct dm_integrity_io *dio) +static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) { struct dm_integrity_c *ic = dio->ic; struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); struct bio_integrity_payload *bip; - unsigned payload_len, digest_size, extra_size, ret; - - dio->integrity_payload = NULL; - dio->integrity_payload_from_mempool = false; + unsigned ret; + sector_t recalc_sector; if (unlikely(bio_integrity(bio))) { bio->bi_status = BLK_STS_NOTSUPP; @@ -2416,28 +2422,67 @@ static int dm_integrity_map_inline(struct dm_integrity_io *dio) return DM_MAPIO_REMAPPED; retry: - payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block); - digest_size = crypto_shash_digestsize(ic->internal_hash); - extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; - payload_len += extra_size; - dio->integrity_payload = kmalloc(payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); - if (unlikely(!dio->integrity_payload)) { - const unsigned x_size = PAGE_SIZE << 1; - if (payload_len > x_size) { - unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block; - if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - return DM_MAPIO_SUBMITTED; + if (!dio->integrity_payload) { + unsigned digest_size, extra_size; + dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block); + digest_size = crypto_shash_digestsize(ic->internal_hash); + extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; + dio->payload_len += extra_size; + dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); + if (unlikely(!dio->integrity_payload)) { + const unsigned x_size = PAGE_SIZE << 1; + if (dio->payload_len > x_size) { + unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block; + if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) { + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + return DM_MAPIO_SUBMITTED; + } + dm_accept_partial_bio(bio, sectors); + goto retry; } - dm_accept_partial_bio(bio, sectors); - goto retry; } + } + + dio->range.logical_sector = bio->bi_iter.bi_sector; + dio->range.n_sectors = bio_sectors(bio); + + if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) + goto skip_spinlock; +#ifdef CONFIG_64BIT + /* + * On 64-bit CPUs we can optimize the lock away (so that it won't cause + * cache line bouncing) and use acquire/release barriers instead. + * + * Paired with smp_store_release in integrity_recalc_inline. + */ + recalc_sector = le64_to_cpu(smp_load_acquire(&ic->sb->recalc_sector)); + if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)) + goto skip_spinlock; +#endif + spin_lock_irq(&ic->endio_wait.lock); + recalc_sector = le64_to_cpu(ic->sb->recalc_sector); + if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector) + goto skip_unlock; + if (unlikely(!add_new_range(ic, &dio->range, true))) { + if (from_map) { + spin_unlock_irq(&ic->endio_wait.lock); + INIT_WORK(&dio->work, integrity_bio_wait); + queue_work(ic->wait_wq, &dio->work); + return DM_MAPIO_SUBMITTED; + } + wait_and_add_new_range(ic, &dio->range); + } + dio->integrity_range_locked = true; +skip_unlock: + spin_unlock_irq(&ic->endio_wait.lock); +skip_spinlock: + + if (unlikely(!dio->integrity_payload)) { dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO)); dio->integrity_payload_from_mempool = true; } - bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector); dio->bio_details.bi_iter = bio->bi_iter; if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { @@ -2468,8 +2513,8 @@ retry: } ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload), - payload_len, offset_in_page(dio->integrity_payload)); - if (unlikely(ret != payload_len)) { + dio->payload_len, offset_in_page(dio->integrity_payload)); + if (unlikely(ret != dio->payload_len)) { bio->bi_status = BLK_STS_RESOURCE; bio_endio(bio); return DM_MAPIO_SUBMITTED; @@ -2577,6 +2622,9 @@ static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) { unsigned pos = 0; + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + unlikely(dio->integrity_range_locked)) + goto skip_check; while (dio->bio_details.bi_iter.bi_size) { char digest[HASH_MAX_DIGESTSIZE]; struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); @@ -2596,9 +2644,10 @@ static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); } } - if (likely(dio->op == REQ_OP_READ) || likely(dio->op == REQ_OP_WRITE)) { - dm_integrity_free_payload(dio); - } +skip_check: + dm_integrity_free_payload(dio); + if (unlikely(dio->integrity_range_locked)) + remove_range(ic, &dio->range); } return DM_ENDIO_DONE; } @@ -2606,8 +2655,26 @@ static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status static void integrity_bio_wait(struct work_struct *w) { struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); + struct dm_integrity_c *ic = dio->ic; - dm_integrity_map_continue(dio, false); + if (ic->mode == 'I') { + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + int r = dm_integrity_map_inline(dio, false); + switch (r) { + case DM_MAPIO_KILL: + bio->bi_status = BLK_STS_IOERR; + fallthrough; + case DM_MAPIO_REMAPPED: + submit_bio_noacct(bio); + fallthrough; + case DM_MAPIO_SUBMITTED: + return; + default: + BUG(); + } + } else { + dm_integrity_map_continue(dio, false); + } } static void pad_uncommitted(struct dm_integrity_c *ic) @@ -3079,6 +3146,133 @@ free_ret: kvfree(recalc_tags); } +static void integrity_recalc_inline(struct work_struct *w) +{ + struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); + size_t recalc_tags_size; + u8 *recalc_buffer = NULL; + u8 *recalc_tags = NULL; + struct dm_integrity_range range; + struct bio *bio; + struct bio_integrity_payload *bip; + __u8 *t; + unsigned int i; + int r; + unsigned ret; + unsigned int super_counter = 0; + unsigned recalc_sectors = RECALC_SECTORS; + +retry: + recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN); + if (!recalc_buffer) { +oom: + recalc_sectors >>= 1; + if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block) + goto retry; + DMCRIT("out of memory for recalculate buffer - recalculation disabled"); + goto free_ret; + } + + recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size; + if (crypto_shash_digestsize(ic->internal_hash) > ic->tuple_size) + recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tuple_size; + recalc_tags = kmalloc(recalc_tags_size, GFP_NOIO | __GFP_NOWARN); + if (!recalc_tags) { + kfree(recalc_buffer); + recalc_buffer = NULL; + goto oom; + } + + spin_lock_irq(&ic->endio_wait.lock); + +next_chunk: + if (unlikely(dm_post_suspending(ic->ti))) + goto unlock_ret; + + range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); + if (unlikely(range.logical_sector >= ic->provided_data_sectors)) + goto unlock_ret; + range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector); + + add_new_range_and_wait(ic, &range); + spin_unlock_irq(&ic->endio_wait.lock); + + if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { + recalc_write_super(ic); + super_counter = 0; + } + + if (unlikely(dm_integrity_failed(ic))) + goto err; + + DEBUG_print("recalculating: %llx - %llx\n", range.logical_sector, range.n_sectors); + + bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recalc_bios); + bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; + __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer)); + r = submit_bio_wait(bio); + bio_put(bio); + if (unlikely(r)) { + dm_integrity_io_error(ic, "reading data", r); + goto err; + } + + t = recalc_tags; + for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { + memset(t, 0, ic->tuple_size); + integrity_sector_checksum(ic, range.logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t); + t += ic->tuple_size; + } + + bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_WRITE, GFP_NOIO, &ic->recalc_bios); + bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; + __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer)); + + bip = bio_integrity_alloc(bio, GFP_NOIO, 1); + if (unlikely(IS_ERR(bip))) { + bio_put(bio); + DMCRIT("out of memory for bio integrity payload - recalculation disabled"); + goto err; + } + ret = bio_integrity_add_page(bio, virt_to_page(recalc_tags), t - recalc_tags, offset_in_page(recalc_tags)); + if (unlikely(ret != t - recalc_tags)) { + bio_put(bio); + dm_integrity_io_error(ic, "attaching integrity tags", -ENOMEM); + goto err; + } + + r = submit_bio_wait(bio); + bio_put(bio); + if (unlikely(r)) { + dm_integrity_io_error(ic, "writing data", r); + goto err; + } + + cond_resched(); + spin_lock_irq(&ic->endio_wait.lock); + remove_range_unlocked(ic, &range); +#ifdef CONFIG_64BIT + /* Paired with smp_load_acquire in dm_integrity_map_inline. */ + smp_store_release(&ic->sb->recalc_sector, cpu_to_le64(range.logical_sector + range.n_sectors)); +#else + ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); +#endif + goto next_chunk; + +err: + remove_range(ic, &range); + goto free_ret; + +unlock_ret: + spin_unlock_irq(&ic->endio_wait.lock); + + recalc_write_super(ic); + +free_ret: + kfree(recalc_buffer); + kfree(recalc_tags); +} + static void bitmap_block_work(struct work_struct *w) { struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work); @@ -4617,6 +4811,17 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv r = -ENOMEM; goto bad; } + r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS); + if (r) { + ti->error = "Cannot allocate bio set"; + goto bad; + } + r = bioset_integrity_create(&ic->recalc_bios, 1); + if (r) { + ti->error = "Cannot allocate bio integrity set"; + r = -ENOMEM; + goto bad; + } } ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", @@ -4833,7 +5038,7 @@ try_smaller_buffer: r = -ENOMEM; goto bad; } - INIT_WORK(&ic->recalc_work, integrity_recalc); + INIT_WORK(&ic->recalc_work, ic->mode == 'I' ? integrity_recalc_inline : integrity_recalc); } else { if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { ti->error = "Recalculate can only be specified with internal_hash"; @@ -4850,17 +5055,15 @@ try_smaller_buffer: goto bad; } - if (ic->mode != 'I') { - ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, - 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0); - if (IS_ERR(ic->bufio)) { - r = PTR_ERR(ic->bufio); - ti->error = "Cannot initialize dm-bufio"; - ic->bufio = NULL; - goto bad; - } - dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); + ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, + 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0); + if (IS_ERR(ic->bufio)) { + r = PTR_ERR(ic->bufio); + ti->error = "Cannot initialize dm-bufio"; + ic->bufio = NULL; + goto bad; } + dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); if (ic->mode != 'R' && ic->mode != 'I') { r = create_journal(ic, &ti->error); @@ -4982,6 +5185,7 @@ static void dm_integrity_dtr(struct dm_target *ti) kvfree(ic->bbs); if (ic->bufio) dm_bufio_client_destroy(ic->bufio); + bioset_exit(&ic->recalc_bios); bioset_exit(&ic->recheck_bios); mempool_exit(&ic->recheck_pool); mempool_exit(&ic->journal_io_mempool); @@ -5036,7 +5240,7 @@ static void dm_integrity_dtr(struct dm_target *ti) static struct target_type integrity_target = { .name = "integrity", - .version = {1, 12, 0}, + .version = {1, 13, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, From 55e268694e8b07026c88191f9b6949b6887d9ce3 Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Wed, 28 Aug 2024 16:42:28 +0800 Subject: [PATCH 032/104] cxl/pci: Fix to record only non-zero ranges The function cxl_dvsec_rr_decode() retrieves and records DVSEC ranges into info->dvsec_range[], regardless of whether it is non-zero range, and the variable info->ranges indicates the number of non-zero ranges. However, in cxl_hdm_decode_init(), the validation for info->dvsec_range[] occurs in a for loop that iterates based on info->ranges. It may result in zero range to be validated but non-zero range not be validated, in turn, the number of allowed ranges is to be 0. Address it by only record non-zero ranges. This fix is not urgent as it requires a configuration that zeroes out the first dvsec range while populating the second. This has not been observed, but it is theoretically possible. If this gets picked up for -stable, no harm done, but there is no urgency to backport. Fixes: 560f78559006 ("cxl/pci: Retrieve CXL DVSEC memory info") Reviewed-by: Jonathan Cameron Signed-off-by: Yanfei Xu Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240828084231.1378789-2-yanfei.xu@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 5e5c24eed545..075a2b2d2ea0 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -390,10 +390,6 @@ int cxl_dvsec_rr_decode(struct device *dev, int d, size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK; if (!size) { - info->dvsec_range[i] = (struct range) { - .start = 0, - .end = CXL_RESOURCE_NONE, - }; continue; } @@ -411,12 +407,10 @@ int cxl_dvsec_rr_decode(struct device *dev, int d, base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK; - info->dvsec_range[i] = (struct range) { + info->dvsec_range[ranges++] = (struct range) { .start = base, .end = base + size - 1 }; - - ranges++; } info->ranges = ranges; From 5c6e3d5a5da118be2ae074bd70d111994147c708 Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Wed, 28 Aug 2024 16:42:29 +0800 Subject: [PATCH 033/104] cxl/pci: Remove duplicated implementation of waiting for memory_info_valid commit ce17ad0d5498 ("cxl: Wait Memory_Info_Valid before access memory related info") added another implementation, which is cxl_dvsec_mem_range_valid(), of waiting for memory_info_valid without realizing it duplicated wait_for_valid(). Remove wait_for_valid() and retain cxl_dvsec_mem_range_valid() as the former is hardcoded to check only the Memory_Info_Valid bit of DVSEC range 1, while the latter allows for selection between DVSEC range 1 or 2 via parameter. Suggested-by: Dan Williams Reviewed-by: Jonathan Cameron Signed-off-by: Yanfei Xu Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240828084231.1378789-3-yanfei.xu@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 41 +++++------------------------------ drivers/cxl/cxl.h | 2 +- drivers/cxl/port.c | 2 +- tools/testing/cxl/test/mock.c | 4 ++-- 4 files changed, 9 insertions(+), 40 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 075a2b2d2ea0..37e537e50b34 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -211,37 +211,6 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds) } EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL); -static int wait_for_valid(struct pci_dev *pdev, int d) -{ - u32 val; - int rc; - - /* - * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high - * and Size Low registers are valid. Must be set within 1 second of - * deassertion of reset to CXL device. Likely it is already set by the - * time this runs, but otherwise give a 1.5 second timeout in case of - * clock skew. - */ - rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); - if (rc) - return rc; - - if (val & CXL_DVSEC_MEM_INFO_VALID) - return 0; - - msleep(1500); - - rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); - if (rc) - return rc; - - if (val & CXL_DVSEC_MEM_INFO_VALID) - return 0; - - return -ETIMEDOUT; -} - static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) { struct pci_dev *pdev = to_pci_dev(cxlds->dev); @@ -322,11 +291,13 @@ static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) return devm_add_action_or_reset(host, disable_hdm, cxlhdm); } -int cxl_dvsec_rr_decode(struct device *dev, int d, +int cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port, struct cxl_endpoint_dvsec_info *info) { struct pci_dev *pdev = to_pci_dev(dev); + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); int hdm_count, rc, i, ranges = 0; + int d = cxlds->cxl_dvsec; u16 cap, ctrl; if (!d) { @@ -353,11 +324,9 @@ int cxl_dvsec_rr_decode(struct device *dev, int d, if (!hdm_count || hdm_count > 2) return -EINVAL; - rc = wait_for_valid(pdev, d); - if (rc) { - dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc); + rc = cxl_dvsec_mem_range_valid(cxlds, 0); + if (rc) return rc; - } /* * The current DVSEC values are moot if the memory capability is diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 2890f7e2aad5..0fc96f8bf15c 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -811,7 +811,7 @@ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, struct cxl_endpoint_dvsec_info *info); int devm_cxl_add_passthrough_decoder(struct cxl_port *port); -int cxl_dvsec_rr_decode(struct device *dev, int dvsec, +int cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port, struct cxl_endpoint_dvsec_info *info); bool is_cxl_region(struct device *dev); diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index d7d5d982ce69..861dde65768f 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -98,7 +98,7 @@ static int cxl_endpoint_port_probe(struct cxl_port *port) struct cxl_port *root; int rc; - rc = cxl_dvsec_rr_decode(cxlds->dev, cxlds->cxl_dvsec, &info); + rc = cxl_dvsec_rr_decode(cxlds->dev, port, &info); if (rc < 0) return rc; diff --git a/tools/testing/cxl/test/mock.c b/tools/testing/cxl/test/mock.c index bbd7d938156d..f4ce96cc11d4 100644 --- a/tools/testing/cxl/test/mock.c +++ b/tools/testing/cxl/test/mock.c @@ -228,7 +228,7 @@ int __wrap_cxl_hdm_decode_init(struct cxl_dev_state *cxlds, } EXPORT_SYMBOL_NS_GPL(__wrap_cxl_hdm_decode_init, CXL); -int __wrap_cxl_dvsec_rr_decode(struct device *dev, int dvsec, +int __wrap_cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port, struct cxl_endpoint_dvsec_info *info) { int rc = 0, index; @@ -237,7 +237,7 @@ int __wrap_cxl_dvsec_rr_decode(struct device *dev, int dvsec, if (ops && ops->is_mock_dev(dev)) rc = 0; else - rc = cxl_dvsec_rr_decode(dev, dvsec, info); + rc = cxl_dvsec_rr_decode(dev, port, info); put_cxl_mock_ops(index); return rc; From 99bf0eebc75c9085440d6dca014724e7e49b5116 Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Wed, 28 Aug 2024 16:42:30 +0800 Subject: [PATCH 034/104] cxl/pci: Check Mem_info_valid bit for each applicable DVSEC In theory a device might set the mem_info_valid bit for a first range after it is ready but before as second range has reached that state. Therefore, the correct approach is to check the Mem_info_valid bit for each applicable DVSEC range against HDM_COUNT, rather than only for the DVSEC range 1. Consequently, let's move the check into the "for loop" that handles each DVSEC range. Reviewed-by: Jonathan Cameron Signed-off-by: Yanfei Xu Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240828084231.1378789-4-yanfei.xu@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 37e537e50b34..043775f3f9a5 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -324,10 +324,6 @@ int cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port, if (!hdm_count || hdm_count > 2) return -EINVAL; - rc = cxl_dvsec_mem_range_valid(cxlds, 0); - if (rc) - return rc; - /* * The current DVSEC values are moot if the memory capability is * disabled, and they will remain moot after the HDM Decoder @@ -345,6 +341,10 @@ int cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port, u64 base, size; u32 temp; + rc = cxl_dvsec_mem_range_valid(cxlds, i); + if (rc) + return rc; + rc = pci_read_config_dword( pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp); if (rc) From 3f9e07531778ce66e0100d93f482e9a299d10d8d Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Wed, 28 Aug 2024 16:42:31 +0800 Subject: [PATCH 035/104] cxl/pci: simplify the check of mem_enabled in cxl_hdm_decode_init() Cases can be divided into two categories which are DVSEC range enabled and not enabled when HDM decoders exist but is not enabled. To avoid checking info->mem_enabled, which indicates the enablement of DVSEC range, every time, we can check !info->mem_enabled once in advance. This simplification can make the code clearer. No functional change intended. Reviewed-by: Jonathan Cameron Signed-off-by: Yanfei Xu Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240828084231.1378789-5-yanfei.xu@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/pci.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 043775f3f9a5..9d396ef0aeed 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -426,7 +426,15 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, return -ENODEV; } - for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) { + if (!info->mem_enabled) { + rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); + if (rc) + return rc; + + return devm_cxl_enable_mem(&port->dev, cxlds); + } + + for (i = 0, allowed = 0; i < info->ranges; i++) { struct device *cxld_dev; cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i], @@ -440,7 +448,7 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, allowed++; } - if (!allowed && info->mem_enabled) { + if (!allowed) { dev_err(dev, "Range register decodes outside platform defined CXL ranges.\n"); return -ENXIO; } @@ -454,14 +462,7 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, * match. If at least one DVSEC range is enabled and allowed, skip HDM * Decoder Capability Enable. */ - if (info->mem_enabled) - return 0; - - rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); - if (rc) - return rc; - - return devm_cxl_enable_mem(&port->dev, cxlds); + return 0; } EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); From d9a476c837fab38856c6b6ff9f794c33907a9f81 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 4 Sep 2024 09:47:54 -0500 Subject: [PATCH 036/104] cxl/region: Remove lock from memory notifier callback In testing Dynamic Capacity Device (DCD) support, a lockdep splat revealed an ABBA issue between the memory notifiers and the DCD extent processing code.[0] Changing the lock ordering within DCD proved difficult because regions must be stable while searching for the proper region and then the device lock must be held to properly notify the DAX region driver of memory changes. Dan points out in the thread that notifiers should be able to trust that it is safe to access static data. Region data is static once the device is realized and until it's destruction. Thus it is better to manage the notifiers within the region driver. Remove the need for a lock by ensuring the notifiers are active only during the region's lifetime. Furthermore, remove cxl_region_nid() because resource can't be NULL while the region is stable. Link: https://lore.kernel.org/all/66b4cf539a79b_a36e829416@iweiny-mobl.notmuch/ [0] Cc: Ying Huang Suggested-by: Dan Williams Reviewed-by: Davidlohr Bueso Reviewed-by: Jonathan Cameron Reviewed-by: Ying Huang Signed-off-by: Ira Weiny Link: https://patch.msgid.link/20240904-fix-notifiers-v3-1-576b4e950266@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/region.c | 54 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 9a7c001eff1e..7bb79f3f318c 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2313,8 +2313,6 @@ static void unregister_region(void *_cxlr) struct cxl_region_params *p = &cxlr->params; int i; - unregister_memory_notifier(&cxlr->memory_notifier); - unregister_mt_adistance_algorithm(&cxlr->adist_notifier); device_del(&cxlr->dev); /* @@ -2391,18 +2389,6 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) return true; } -static int cxl_region_nid(struct cxl_region *cxlr) -{ - struct cxl_region_params *p = &cxlr->params; - struct resource *res; - - guard(rwsem_read)(&cxl_region_rwsem); - res = p->res; - if (!res) - return NUMA_NO_NODE; - return phys_to_target_node(res->start); -} - static int cxl_region_perf_attrs_callback(struct notifier_block *nb, unsigned long action, void *arg) { @@ -2415,7 +2401,11 @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb, if (nid == NUMA_NO_NODE || action != MEM_ONLINE) return NOTIFY_DONE; - region_nid = cxl_region_nid(cxlr); + /* + * No need to hold cxl_region_rwsem; region parameters are stable + * within the cxl_region driver. + */ + region_nid = phys_to_target_node(cxlr->params.res->start); if (nid != region_nid) return NOTIFY_DONE; @@ -2434,7 +2424,11 @@ static int cxl_region_calculate_adistance(struct notifier_block *nb, int *adist = data; int region_nid; - region_nid = cxl_region_nid(cxlr); + /* + * No need to hold cxl_region_rwsem; region parameters are stable + * within the cxl_region driver. + */ + region_nid = phys_to_target_node(cxlr->params.res->start); if (nid != region_nid) return NOTIFY_OK; @@ -2484,14 +2478,6 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; - cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; - cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; - register_memory_notifier(&cxlr->memory_notifier); - - cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance; - cxlr->adist_notifier.priority = 100; - register_mt_adistance_algorithm(&cxlr->adist_notifier); - rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); @@ -3387,6 +3373,14 @@ static int is_system_ram(struct resource *res, void *arg) return 1; } +static void shutdown_notifiers(void *_cxlr) +{ + struct cxl_region *cxlr = _cxlr; + + unregister_memory_notifier(&cxlr->memory_notifier); + unregister_mt_adistance_algorithm(&cxlr->adist_notifier); +} + static int cxl_region_probe(struct device *dev) { struct cxl_region *cxlr = to_cxl_region(dev); @@ -3419,6 +3413,18 @@ static int cxl_region_probe(struct device *dev) out: up_read(&cxl_region_rwsem); + if (rc) + return rc; + + cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; + cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; + register_memory_notifier(&cxlr->memory_notifier); + + cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance; + cxlr->adist_notifier.priority = 100; + register_mt_adistance_algorithm(&cxlr->adist_notifier); + + rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr); if (rc) return rc; From 40a895fd9a358eea16901026360bd2cd3ca691a7 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 5 Sep 2024 15:35:45 -0700 Subject: [PATCH 037/104] cxl: move cxl headers to new include/cxl/ directory Group all cxl related kernel headers into include/cxl/ directory. Reviewed-by: Alison Schofield Reviewed-by: Ira Weiny Link: https://patch.msgid.link/20240905223711.1990186-2-dave.jiang@intel.com Signed-off-by: Dave Jiang --- MAINTAINERS | 3 +-- drivers/acpi/apei/einj-cxl.c | 2 +- drivers/acpi/apei/ghes.c | 2 +- drivers/cxl/core/port.c | 2 +- drivers/cxl/cxlmem.h | 2 +- include/{linux/einj-cxl.h => cxl/einj.h} | 0 include/{linux/cxl-event.h => cxl/event.h} | 0 7 files changed, 5 insertions(+), 6 deletions(-) rename include/{linux/einj-cxl.h => cxl/einj.h} (100%) rename include/{linux/cxl-event.h => cxl/event.h} (100%) diff --git a/MAINTAINERS b/MAINTAINERS index fe83ba7194ea..df59354af438 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5620,8 +5620,7 @@ L: linux-cxl@vger.kernel.org S: Maintained F: Documentation/driver-api/cxl F: drivers/cxl/ -F: include/linux/einj-cxl.h -F: include/linux/cxl-event.h +F: include/cxl/ F: include/uapi/linux/cxl_mem.h F: tools/testing/cxl/ diff --git a/drivers/acpi/apei/einj-cxl.c b/drivers/acpi/apei/einj-cxl.c index 8b8be0c90709..4f81a119ec08 100644 --- a/drivers/acpi/apei/einj-cxl.c +++ b/drivers/acpi/apei/einj-cxl.c @@ -7,9 +7,9 @@ * * Author: Ben Cheatham */ -#include #include #include +#include #include "apei-internal.h" diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 623cc0cb4a65..ada93cfde9ba 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -50,6 +49,7 @@ #include #include #include +#include #include #include "apei-internal.h" diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 7ca57285a7b7..a5e6f3d23cfb 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index afb53d058d62..a81a8982bf93 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -6,8 +6,8 @@ #include #include #include -#include #include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ diff --git a/include/linux/einj-cxl.h b/include/cxl/einj.h similarity index 100% rename from include/linux/einj-cxl.h rename to include/cxl/einj.h diff --git a/include/linux/cxl-event.h b/include/cxl/event.h similarity index 100% rename from include/linux/cxl-event.h rename to include/cxl/event.h From 9c2010bccc0ce012f52de18ebd0c3add241f75b8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 10 Sep 2024 10:52:59 -0700 Subject: [PATCH 038/104] dm-integrity: check mac_size against HASH_MAX_DIGESTSIZE in sb_mac() sb_mac() verifies that the superblock + MAC don't exceed 512 bytes. Because the superblock is currently 64 bytes, this really verifies mac_size <= 448. This confuses smatch into thinking that mac_size may be as large as 448, which is inconsistent with the later code that assumes the MAC fits in a buffer of size HASH_MAX_DIGESTSIZE (64). In fact mac_size <= HASH_MAX_DIGESTSIZE is guaranteed by the crypto API, as that is the whole point of HASH_MAX_DIGESTSIZE. But, let's be defensive and explicitly check for this. This suppresses the false positive smatch warning. It does not fix an actual bug. Reported-by: kernel test robot Reported-by: Dan Carpenter Closes: https://lore.kernel.org/r/202409061401.44rtN1bh-lkp@intel.com/ Signed-off-by: Eric Biggers Signed-off-by: Mikulas Patocka --- drivers/md/dm-integrity.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index c40df05e0521..42c9dc2ee0c0 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -494,7 +494,8 @@ static int sb_mac(struct dm_integrity_c *ic, bool wr) __u8 *sb = (__u8 *)ic->sb; __u8 *mac = sb + (1 << SECTOR_SHIFT) - mac_size; - if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT) { + if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT || + mac_size > HASH_MAX_DIGESTSIZE) { dm_integrity_io_error(ic, "digest is too long", -EINVAL); return -EINVAL; } From 8d8081cecfb9940beeb4a8a700db34e615a96056 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 5 Sep 2024 15:35:46 -0700 Subject: [PATCH 039/104] cxl: Move mailbox related bits to the same context Create a new 'struct cxl_mailbox' and move all mailbox related bits to it. This allows isolation of all CXL mailbox data in order to export some of the calls to external kernel callers and avoid exporting of CXL driver specific bits such has device states. The allocation of 'struct cxl_mailbox' is also split out with cxl_mailbox_init() so the mailbox can be created independently. Reviewed-by: Jonathan Cameron Reviewed-by: Alejandro Lucero Reviewed-by: Fan Ni Reviewed-by: Alison Schofield Reviewed-by: Ira Weiny Link: https://patch.msgid.link/20240905223711.1990186-3-dave.jiang@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/mbox.c | 57 ++++++++++++++++++-------- drivers/cxl/core/memdev.c | 18 +++++---- drivers/cxl/cxlmem.h | 21 +++++----- drivers/cxl/pci.c | 78 ++++++++++++++++++++++++------------ drivers/cxl/pmem.c | 4 +- include/cxl/mailbox.h | 28 +++++++++++++ tools/testing/cxl/test/mem.c | 44 +++++++++++++++----- 7 files changed, 177 insertions(+), 73 deletions(-) create mode 100644 include/cxl/mailbox.h diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 0a913b30c7fc..3d7ee6a12915 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -244,16 +244,17 @@ static const char *cxl_mem_opcode_to_name(u16 opcode) int cxl_internal_send_cmd(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *mbox_cmd) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; size_t out_size, min_out; int rc; - if (mbox_cmd->size_in > mds->payload_size || - mbox_cmd->size_out > mds->payload_size) + if (mbox_cmd->size_in > cxl_mbox->payload_size || + mbox_cmd->size_out > cxl_mbox->payload_size) return -E2BIG; out_size = mbox_cmd->size_out; min_out = mbox_cmd->min_out; - rc = mds->mbox_send(mds, mbox_cmd); + rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd); /* * EIO is reserved for a payload size mismatch and mbox_send() * may not return this error. @@ -353,6 +354,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox, struct cxl_memdev_state *mds, u16 opcode, size_t in_size, size_t out_size, u64 in_payload) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; *mbox = (struct cxl_mbox_cmd) { .opcode = opcode, .size_in = in_size, @@ -374,7 +376,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox, /* Prepare to handle a full payload for variable sized output */ if (out_size == CXL_VARIABLE_PAYLOAD) - mbox->size_out = mds->payload_size; + mbox->size_out = cxl_mbox->payload_size; else mbox->size_out = out_size; @@ -398,6 +400,8 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd, const struct cxl_send_command *send_cmd, struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; + if (send_cmd->raw.rsvd) return -EINVAL; @@ -406,7 +410,7 @@ static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd, * gets passed along without further checking, so it must be * validated here. */ - if (send_cmd->out.size > mds->payload_size) + if (send_cmd->out.size > cxl_mbox->payload_size) return -EINVAL; if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) @@ -494,6 +498,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd, struct cxl_memdev_state *mds, const struct cxl_send_command *send_cmd) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mem_command mem_cmd; int rc; @@ -505,7 +510,7 @@ static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd, * supports, but output can be arbitrarily large (simply write out as * much data as the hardware provides). */ - if (send_cmd->in.size > mds->payload_size) + if (send_cmd->in.size > cxl_mbox->payload_size) return -EINVAL; /* Sanitize and construct a cxl_mem_command */ @@ -591,6 +596,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds, u64 out_payload, s32 *size_out, u32 *retval) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct device *dev = mds->cxlds.dev; int rc; @@ -601,7 +607,7 @@ static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds, cxl_mem_opcode_to_name(mbox_cmd->opcode), mbox_cmd->opcode, mbox_cmd->size_in); - rc = mds->mbox_send(mds, mbox_cmd); + rc = cxl_mbox->mbox_send(cxl_mbox, mbox_cmd); if (rc) goto out; @@ -659,11 +665,12 @@ int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s) static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid, u32 *size, u8 *out) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; u32 remaining = *size; u32 offset = 0; while (remaining) { - u32 xfer_size = min_t(u32, remaining, mds->payload_size); + u32 xfer_size = min_t(u32, remaining, cxl_mbox->payload_size); struct cxl_mbox_cmd mbox_cmd; struct cxl_mbox_get_log log; int rc; @@ -752,17 +759,18 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel) static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_supported_logs *ret; struct cxl_mbox_cmd mbox_cmd; int rc; - ret = kvmalloc(mds->payload_size, GFP_KERNEL); + ret = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL); if (!ret) return ERR_PTR(-ENOMEM); mbox_cmd = (struct cxl_mbox_cmd) { .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS, - .size_out = mds->payload_size, + .size_out = cxl_mbox->payload_size, .payload_out = ret, /* At least the record number field must be valid */ .min_out = 2, @@ -910,6 +918,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, enum cxl_event_log_type log, struct cxl_get_event_payload *get_pl) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_clear_event_payload *payload; u16 total = le16_to_cpu(get_pl->record_count); u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES; @@ -920,8 +929,8 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, int i; /* Payload size may limit the max handles */ - if (pl_size > mds->payload_size) { - max_handles = (mds->payload_size - sizeof(*payload)) / + if (pl_size > cxl_mbox->payload_size) { + max_handles = (cxl_mbox->payload_size - sizeof(*payload)) / sizeof(__le16); pl_size = struct_size(payload, handles, max_handles); } @@ -979,6 +988,7 @@ free_pl: static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, enum cxl_event_log_type type) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; struct device *dev = mds->cxlds.dev; struct cxl_get_event_payload *payload; @@ -995,7 +1005,7 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, .payload_in = &log_type, .size_in = sizeof(log_type), .payload_out = payload, - .size_out = mds->payload_size, + .size_out = cxl_mbox->payload_size, .min_out = struct_size(payload, records, 0), }; @@ -1327,6 +1337,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, struct cxl_region *cxlr) { struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_poison_out *po; struct cxl_mbox_poison_in pi; int nr_records = 0; @@ -1345,7 +1356,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, .opcode = CXL_MBOX_OP_GET_POISON, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = mds->payload_size, + .size_out = cxl_mbox->payload_size, .payload_out = po, .min_out = struct_size(po, record, 0), }; @@ -1381,7 +1392,9 @@ static void free_poison_buf(void *buf) /* Get Poison List output buffer is protected by mds->poison.lock */ static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds) { - mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL); + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; + + mds->poison.list_out = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL); if (!mds->poison.list_out) return -ENOMEM; @@ -1407,6 +1420,19 @@ int cxl_poison_state_init(struct cxl_memdev_state *mds) } EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL); +int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host) +{ + if (!cxl_mbox || !host) + return -EINVAL; + + cxl_mbox->host = host; + mutex_init(&cxl_mbox->mbox_mutex); + rcuwait_init(&cxl_mbox->mbox_wait); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, CXL); + struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) { struct cxl_memdev_state *mds; @@ -1417,7 +1443,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) return ERR_PTR(-ENOMEM); } - mutex_init(&mds->mbox_mutex); mutex_init(&mds->event.log_lock); mds->cxlds.dev = dev; mds->cxlds.reg_map.host = dev; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 0277726afd04..05bb84cb1274 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -58,7 +58,7 @@ static ssize_t payload_max_show(struct device *dev, if (!mds) return sysfs_emit(buf, "\n"); - return sysfs_emit(buf, "%zu\n", mds->payload_size); + return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); } static DEVICE_ATTR_RO(payload_max); @@ -124,15 +124,16 @@ static ssize_t security_state_show(struct device *dev, { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); unsigned long state = mds->security.state; int rc = 0; /* sync with latest submission state */ - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_active) rc = sysfs_emit(buf, "sanitize\n"); - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); if (rc) return rc; @@ -829,12 +830,13 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, { struct cxl_memdev_state *mds = fwl->dd_handle; struct cxl_mbox_transfer_fw *transfer; + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; if (!size) return FW_UPLOAD_ERR_INVALID_SIZE; mds->fw.oneshot = struct_size(transfer, data, size) < - mds->payload_size; + cxl_mbox->payload_size; if (cxl_mem_get_fw_info(mds)) return FW_UPLOAD_ERR_HW_ERROR; @@ -854,6 +856,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, { struct cxl_memdev_state *mds = fwl->dd_handle; struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev *cxlmd = cxlds->cxlmd; struct cxl_mbox_transfer_fw *transfer; struct cxl_mbox_cmd mbox_cmd; @@ -877,7 +880,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, * sizeof(*transfer) is 128. These constraints imply that @cur_size * will always be 128b aligned. */ - cur_size = min_t(size_t, size, mds->payload_size - sizeof(*transfer)); + cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); remaining = size - cur_size; size_in = struct_size(transfer, data, cur_size); @@ -1059,16 +1062,17 @@ EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL); static void sanitize_teardown_notifier(void *data) { struct cxl_memdev_state *mds = data; + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct kernfs_node *state; /* * Prevent new irq triggered invocations of the workqueue and * flush inflight invocations. */ - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); state = mds->security.sanitize_node; mds->security.sanitize_node = NULL; - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); cancel_delayed_work_sync(&mds->security.poll_dwork); sysfs_put(state); diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index a81a8982bf93..4ead415f8971 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -3,11 +3,12 @@ #ifndef __CXL_MEM_H__ #define __CXL_MEM_H__ #include +#include #include #include -#include #include #include +#include #include "cxl.h" /* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ @@ -424,6 +425,7 @@ struct cxl_dpa_perf { * @ram_res: Active Volatile memory capacity configuration * @serial: PCIe Device Serial Number * @type: Generic Memory Class device or Vendor Specific Memory device + * @cxl_mbox: CXL mailbox context */ struct cxl_dev_state { struct device *dev; @@ -438,8 +440,14 @@ struct cxl_dev_state { struct resource ram_res; u64 serial; enum cxl_devtype type; + struct cxl_mailbox cxl_mbox; }; +static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) +{ + return dev_get_drvdata(cxl_mbox->host); +} + /** * struct cxl_memdev_state - Generic Type-3 Memory Device Class driver data * @@ -448,11 +456,8 @@ struct cxl_dev_state { * the functionality related to that like Identify Memory Device and Get * Partition Info * @cxlds: Core driver state common across Type-2 and Type-3 devices - * @payload_size: Size of space for payload - * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) * @lsa_size: Size of Label Storage Area * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device) - * @mbox_mutex: Mutex to synchronize mailbox access. * @firmware_version: Firmware version for the memory device. * @enabled_cmds: Hardware commands found enabled in CEL. * @exclusive_cmds: Commands that are kernel-internal only @@ -470,17 +475,13 @@ struct cxl_dev_state { * @poison: poison driver state info * @security: security driver state info * @fw: firmware upload / activation state - * @mbox_wait: RCU wait for mbox send completely - * @mbox_send: @dev specific transport for transmitting mailbox commands * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. */ struct cxl_memdev_state { struct cxl_dev_state cxlds; - size_t payload_size; size_t lsa_size; - struct mutex mbox_mutex; /* Protects device mailbox and firmware */ char firmware_version[0x10]; DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); @@ -500,10 +501,6 @@ struct cxl_memdev_state { struct cxl_poison_state poison; struct cxl_security_state security; struct cxl_fw_state fw; - - struct rcuwait mbox_wait; - int (*mbox_send)(struct cxl_memdev_state *mds, - struct cxl_mbox_cmd *cmd); }; static inline struct cxl_memdev_state * diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 26e75499abdd..2fc7fd252c2b 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "cxlmem.h" #include "cxlpci.h" #include "cxl.h" @@ -124,6 +125,7 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id) u16 opcode; struct cxl_dev_id *dev_id = id; struct cxl_dev_state *cxlds = dev_id->cxlds; + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); if (!cxl_mbox_background_complete(cxlds)) @@ -132,13 +134,13 @@ static irqreturn_t cxl_pci_mbox_irq(int irq, void *id) reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET); opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg); if (opcode == CXL_MBOX_OP_SANITIZE) { - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_node) mod_delayed_work(system_wq, &mds->security.poll_dwork, 0); - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); } else { /* short-circuit the wait in __cxl_pci_mbox_send_cmd() */ - rcuwait_wake_up(&mds->mbox_wait); + rcuwait_wake_up(&cxl_mbox->mbox_wait); } return IRQ_HANDLED; @@ -152,8 +154,9 @@ static void cxl_mbox_sanitize_work(struct work_struct *work) struct cxl_memdev_state *mds = container_of(work, typeof(*mds), security.poll_dwork.work); struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); if (cxl_mbox_background_complete(cxlds)) { mds->security.poll_tmo_secs = 0; if (mds->security.sanitize_node) @@ -167,12 +170,12 @@ static void cxl_mbox_sanitize_work(struct work_struct *work) mds->security.poll_tmo_secs = min(15 * 60, timeout); schedule_delayed_work(&mds->security.poll_dwork, timeout * HZ); } - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); } /** * __cxl_pci_mbox_send_cmd() - Execute a mailbox command - * @mds: The memory device driver data + * @cxl_mbox: CXL mailbox context * @mbox_cmd: Command to send to the memory device. * * Context: Any context. Expects mbox_mutex to be held. @@ -192,17 +195,18 @@ static void cxl_mbox_sanitize_work(struct work_struct *work) * not need to coordinate with each other. The driver only uses the primary * mailbox. */ -static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds, +static int __cxl_pci_mbox_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *mbox_cmd) { - struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_dev_state *cxlds = mbox_to_cxlds(cxl_mbox); + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET; struct device *dev = cxlds->dev; u64 cmd_reg, status_reg; size_t out_len; int rc; - lockdep_assert_held(&mds->mbox_mutex); + lockdep_assert_held(&cxl_mbox->mbox_mutex); /* * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. @@ -315,10 +319,10 @@ static int __cxl_pci_mbox_send_cmd(struct cxl_memdev_state *mds, timeout = mbox_cmd->poll_interval_ms; for (i = 0; i < mbox_cmd->poll_count; i++) { - if (rcuwait_wait_event_timeout(&mds->mbox_wait, - cxl_mbox_background_complete(cxlds), - TASK_UNINTERRUPTIBLE, - msecs_to_jiffies(timeout)) > 0) + if (rcuwait_wait_event_timeout(&cxl_mbox->mbox_wait, + cxl_mbox_background_complete(cxlds), + TASK_UNINTERRUPTIBLE, + msecs_to_jiffies(timeout)) > 0) break; } @@ -360,7 +364,7 @@ success: */ size_t n; - n = min3(mbox_cmd->size_out, mds->payload_size, out_len); + n = min3(mbox_cmd->size_out, cxl_mbox->payload_size, out_len); memcpy_fromio(mbox_cmd->payload_out, payload, n); mbox_cmd->size_out = n; } else { @@ -370,14 +374,14 @@ success: return 0; } -static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, +static int cxl_pci_mbox_send(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd) { int rc; - mutex_lock_io(&mds->mbox_mutex); - rc = __cxl_pci_mbox_send_cmd(mds, cmd); - mutex_unlock(&mds->mbox_mutex); + mutex_lock_io(&cxl_mbox->mbox_mutex); + rc = __cxl_pci_mbox_send_cmd(cxl_mbox, cmd); + mutex_unlock(&cxl_mbox->mbox_mutex); return rc; } @@ -385,6 +389,7 @@ static int cxl_pci_mbox_send(struct cxl_memdev_state *mds, static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail) { struct cxl_dev_state *cxlds = &mds->cxlds; + struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET); struct device *dev = cxlds->dev; unsigned long timeout; @@ -417,8 +422,8 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail) return -ETIMEDOUT; } - mds->mbox_send = cxl_pci_mbox_send; - mds->payload_size = + cxl_mbox->mbox_send = cxl_pci_mbox_send; + cxl_mbox->payload_size = 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); /* @@ -428,16 +433,15 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail) * there's no point in going forward. If the size is too large, there's * no harm is soft limiting it. */ - mds->payload_size = min_t(size_t, mds->payload_size, SZ_1M); - if (mds->payload_size < 256) { + cxl_mbox->payload_size = min_t(size_t, cxl_mbox->payload_size, SZ_1M); + if (cxl_mbox->payload_size < 256) { dev_err(dev, "Mailbox is too small (%zub)", - mds->payload_size); + cxl_mbox->payload_size); return -ENXIO; } - dev_dbg(dev, "Mailbox payload sized %zu", mds->payload_size); + dev_dbg(dev, "Mailbox payload sized %zu", cxl_mbox->payload_size); - rcuwait_init(&mds->mbox_wait); INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mbox_sanitize_work); /* background command interrupts are optional */ @@ -575,9 +579,10 @@ static void free_event_buf(void *buf) */ static int cxl_mem_alloc_event_buf(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_get_event_payload *buf; - buf = kvmalloc(mds->payload_size, GFP_KERNEL); + buf = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL); if (!buf) return -ENOMEM; mds->event.buf = buf; @@ -783,6 +788,23 @@ static int cxl_event_config(struct pci_host_bridge *host_bridge, return 0; } +static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) +{ + int rc; + + /* + * Fail the init if there's no mailbox. For a type3 this is out of spec. + */ + if (!cxlds->reg_map.device_map.mbox.valid) + return -ENODEV; + + rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); + if (rc) + return rc; + + return 0; +} + static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct pci_host_bridge *host_bridge = pci_find_host_bridge(pdev->bus); @@ -843,6 +865,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) dev_dbg(&pdev->dev, "Failed to map RAS capability.\n"); + rc = cxl_pci_type3_init_mailbox(cxlds); + if (rc) + return rc; + rc = cxl_await_media_ready(cxlds); if (rc == 0) cxlds->media_ready = true; diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 647c7e25ef3a..50c6d25ede89 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -102,13 +102,15 @@ static int cxl_pmem_get_config_size(struct cxl_memdev_state *mds, struct nd_cmd_get_config_size *cmd, unsigned int buf_len) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; + if (sizeof(*cmd) > buf_len) return -EINVAL; *cmd = (struct nd_cmd_get_config_size){ .config_size = mds->lsa_size, .max_xfer = - mds->payload_size - sizeof(struct cxl_mbox_set_lsa), + cxl_mbox->payload_size - sizeof(struct cxl_mbox_set_lsa), }; return 0; diff --git a/include/cxl/mailbox.h b/include/cxl/mailbox.h new file mode 100644 index 000000000000..bacd111e75f1 --- /dev/null +++ b/include/cxl/mailbox.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2024 Intel Corporation. */ +#ifndef __CXL_MBOX_H__ +#define __CXL_MBOX_H__ +#include + +struct cxl_mbox_cmd; + +/** + * struct cxl_mailbox - context for CXL mailbox operations + * @host: device that hosts the mailbox + * @payload_size: Size of space for payload + * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register) + * @mbox_mutex: mutex protects device mailbox and firmware + * @mbox_wait: rcuwait for mailbox + * @mbox_send: @dev specific transport for transmitting mailbox commands + */ +struct cxl_mailbox { + struct device *host; + size_t payload_size; + struct mutex mbox_mutex; /* lock to protect mailbox context */ + struct rcuwait mbox_wait; + int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); +}; + +int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host); + +#endif diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 129f179b0ac5..ccdd6a504222 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -534,6 +535,7 @@ static int mock_gsl(struct cxl_mbox_cmd *cmd) static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_log *gl = cmd->payload_in; u32 offset = le32_to_cpu(gl->offset); u32 length = le32_to_cpu(gl->length); @@ -542,7 +544,7 @@ static int mock_get_log(struct cxl_memdev_state *mds, struct cxl_mbox_cmd *cmd) if (cmd->size_in < sizeof(*gl)) return -EINVAL; - if (length > mds->payload_size) + if (length > cxl_mbox->payload_size) return -EINVAL; if (offset + length > sizeof(mock_cel)) return -EINVAL; @@ -617,12 +619,13 @@ void cxl_mockmem_sanitize_work(struct work_struct *work) { struct cxl_memdev_state *mds = container_of(work, typeof(*mds), security.poll_dwork.work); + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); if (mds->security.sanitize_node) sysfs_notify_dirent(mds->security.sanitize_node); mds->security.sanitize_active = false; - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); dev_dbg(mds->cxlds.dev, "sanitize complete\n"); } @@ -631,6 +634,7 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata, struct cxl_mbox_cmd *cmd) { struct cxl_memdev_state *mds = mdata->mds; + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; int rc = 0; if (cmd->size_in != 0) @@ -648,14 +652,14 @@ static int mock_sanitize(struct cxl_mockmem_data *mdata, return -ENXIO; } - mutex_lock(&mds->mbox_mutex); + mutex_lock(&cxl_mbox->mbox_mutex); if (schedule_delayed_work(&mds->security.poll_dwork, msecs_to_jiffies(mdata->sanitize_timeout))) { mds->security.sanitize_active = true; dev_dbg(mds->cxlds.dev, "sanitize issued\n"); } else rc = -EBUSY; - mutex_unlock(&mds->mbox_mutex); + mutex_unlock(&cxl_mbox->mbox_mutex); return rc; } @@ -1333,12 +1337,13 @@ static int mock_activate_fw(struct cxl_mockmem_data *mdata, return -EINVAL; } -static int cxl_mock_mbox_send(struct cxl_memdev_state *mds, +static int cxl_mock_mbox_send(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd) { - struct cxl_dev_state *cxlds = &mds->cxlds; - struct device *dev = cxlds->dev; + struct device *dev = cxl_mbox->host; struct cxl_mockmem_data *mdata = dev_get_drvdata(dev); + struct cxl_memdev_state *mds = mdata->mds; + struct cxl_dev_state *cxlds = &mds->cxlds; int rc = -EIO; switch (cmd->opcode) { @@ -1453,6 +1458,17 @@ static ssize_t event_trigger_store(struct device *dev, } static DEVICE_ATTR_WO(event_trigger); +static int cxl_mock_mailbox_create(struct cxl_dev_state *cxlds) +{ + int rc; + + rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); + if (rc) + return rc; + + return 0; +} + static int cxl_mock_mem_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1460,6 +1476,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) struct cxl_memdev_state *mds; struct cxl_dev_state *cxlds; struct cxl_mockmem_data *mdata; + struct cxl_mailbox *cxl_mbox; int rc; mdata = devm_kzalloc(dev, sizeof(*mdata), GFP_KERNEL); @@ -1487,13 +1504,18 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (IS_ERR(mds)) return PTR_ERR(mds); + cxlds = &mds->cxlds; + rc = cxl_mock_mailbox_create(cxlds); + if (rc) + return rc; + + cxl_mbox = &mds->cxlds.cxl_mbox; mdata->mds = mds; - mds->mbox_send = cxl_mock_mbox_send; - mds->payload_size = SZ_4K; + cxl_mbox->mbox_send = cxl_mock_mbox_send; + cxl_mbox->payload_size = SZ_4K; mds->event.buf = (struct cxl_get_event_payload *) mdata->event_buf; INIT_DELAYED_WORK(&mds->security.poll_dwork, cxl_mockmem_sanitize_work); - cxlds = &mds->cxlds; cxlds->serial = pdev->id; if (is_rcd(pdev)) cxlds->rcd = true; From b5209da36b19b573cf25fe7e698e3a45b0f40a75 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Thu, 5 Sep 2024 15:35:47 -0700 Subject: [PATCH 040/104] cxl: Convert cxl_internal_send_cmd() to use 'struct cxl_mailbox' as input With the CXL mailbox context split out, cxl_internal_send_cmd() can take 'struct cxl_mailbox' as an input parameter rather than 'struct memdev_dev_state'. Change input parameter for cxl_internal_send_cmd() and fixup all impacted call sites. Reviewed-by: Fan Ni Reviewed-by: Jonathan Cameron Reviewed-by: Alison Schofield Reviewed-by: Ira Weiny Link: https://patch.msgid.link/20240905223711.1990186-4-dave.jiang@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/mbox.c | 38 ++++++++++++++++++++------------------ drivers/cxl/core/memdev.c | 23 +++++++++++++---------- drivers/cxl/cxlmem.h | 2 +- drivers/cxl/pci.c | 6 ++++-- drivers/cxl/pmem.c | 6 ++++-- drivers/cxl/security.c | 23 ++++++++++++----------- 6 files changed, 54 insertions(+), 44 deletions(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 3d7ee6a12915..915a93ced78f 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -225,7 +225,7 @@ static const char *cxl_mem_opcode_to_name(u16 opcode) /** * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command - * @mds: The driver data for the operation + * @cxl_mbox: CXL mailbox context * @mbox_cmd: initialized command to execute * * Context: Any context. @@ -241,10 +241,9 @@ static const char *cxl_mem_opcode_to_name(u16 opcode) * error. While this distinction can be useful for commands from userspace, the * kernel will only be able to use results when both are successful. */ -int cxl_internal_send_cmd(struct cxl_memdev_state *mds, +int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *mbox_cmd) { - struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; size_t out_size, min_out; int rc; @@ -689,7 +688,7 @@ static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid, .payload_out = out, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); /* * The output payload length that indicates the number @@ -775,7 +774,7 @@ static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state * /* At least the record number field must be valid */ .min_out = 2, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) { kvfree(ret); return ERR_PTR(rc); @@ -964,7 +963,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, if (i == max_handles) { payload->nr_recs = i; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) goto free_pl; i = 0; @@ -975,7 +974,7 @@ static int cxl_clear_event_record(struct cxl_memdev_state *mds, if (i) { payload->nr_recs = i; mbox_cmd.size_in = struct_size(payload, handles, i); - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) goto free_pl; } @@ -1009,7 +1008,7 @@ static void cxl_mem_get_records_log(struct cxl_memdev_state *mds, .min_out = struct_size(payload, records, 0), }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) { dev_err_ratelimited(dev, "Event log '%d': Failed to query event records : %d", @@ -1080,6 +1079,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL); */ static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_partition_info pi; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -1089,7 +1089,7 @@ static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) .size_out = sizeof(pi), .payload_out = &pi, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) return rc; @@ -1116,6 +1116,7 @@ static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) */ int cxl_dev_state_identify(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ struct cxl_mbox_identify id; struct cxl_mbox_cmd mbox_cmd; @@ -1130,7 +1131,7 @@ int cxl_dev_state_identify(struct cxl_memdev_state *mds) .size_out = sizeof(id), .payload_out = &id, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return rc; @@ -1158,6 +1159,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL); static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; int rc; u32 sec_out = 0; struct cxl_get_security_output { @@ -1169,14 +1171,13 @@ static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd) .size_out = sizeof(out), }; struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd }; - struct cxl_dev_state *cxlds = &mds->cxlds; if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE) return -EINVAL; - rc = cxl_internal_send_cmd(mds, &sec_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &sec_cmd); if (rc < 0) { - dev_err(cxlds->dev, "Failed to get security state : %d", rc); + dev_err(cxl_mbox->host, "Failed to get security state : %d", rc); return rc; } @@ -1193,9 +1194,9 @@ static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd) sec_out & CXL_PMEM_SEC_STATE_LOCKED) return -EINVAL; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) { - dev_err(cxlds->dev, "Failed to sanitize device : %d", rc); + dev_err(cxl_mbox->host, "Failed to sanitize device : %d", rc); return rc; } @@ -1309,6 +1310,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL); int cxl_set_timestamp(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_cmd mbox_cmd; struct cxl_mbox_set_timestamp_in pi; int rc; @@ -1320,7 +1322,7 @@ int cxl_set_timestamp(struct cxl_memdev_state *mds) .payload_in = &pi, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); /* * Command is optional. Devices may have another way of providing * a timestamp, or may return all 0s in timestamp fields. @@ -1337,7 +1339,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, struct cxl_region *cxlr) { struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); - struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_poison_out *po; struct cxl_mbox_poison_in pi; int nr_records = 0; @@ -1361,7 +1363,7 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, .min_out = struct_size(po, record, 0), }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) break; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 05bb84cb1274..84fefb76dafa 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -278,7 +278,7 @@ static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) { - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_inject_poison inject; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; @@ -308,13 +308,13 @@ int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) .size_in = sizeof(inject), .payload_in = &inject, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) goto out; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) - dev_warn_once(mds->cxlds.dev, + dev_warn_once(cxl_mbox->host, "poison inject dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); @@ -333,7 +333,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, CXL); int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) { - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_clear_poison clear; struct cxl_poison_record record; struct cxl_mbox_cmd mbox_cmd; @@ -372,13 +372,13 @@ int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) .payload_in = &clear, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc) goto out; cxlr = cxl_dpa_to_region(cxlmd, dpa); if (cxlr) - dev_warn_once(mds->cxlds.dev, + dev_warn_once(cxl_mbox->host, "poison clear dpa:%#llx region: %s\n", dpa, dev_name(&cxlr->dev)); @@ -715,6 +715,7 @@ static int cxl_memdev_release_file(struct inode *inode, struct file *file) */ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_fw_info info; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -725,7 +726,7 @@ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) .payload_out = &info, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return rc; @@ -749,6 +750,7 @@ static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) */ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_activate_fw activate; struct cxl_mbox_cmd mbox_cmd; @@ -765,7 +767,7 @@ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) activate.action = CXL_FW_ACTIVATE_OFFLINE; activate.slot = slot; - return cxl_internal_send_cmd(mds, &mbox_cmd); + return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); } /** @@ -780,6 +782,7 @@ static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) */ static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_transfer_fw *transfer; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -799,7 +802,7 @@ static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); kfree(transfer); return rc; } @@ -924,7 +927,7 @@ static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, .poll_count = 30, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) { rc = FW_UPLOAD_ERR_RW_ERROR; goto out_free; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index 4ead415f8971..c7c423ae16ab 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -811,7 +811,7 @@ enum { CXL_PMEM_SEC_PASS_USER, }; -int cxl_internal_send_cmd(struct cxl_memdev_state *mds, +int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); int cxl_dev_state_identify(struct cxl_memdev_state *mds); int cxl_await_media_ready(struct cxl_dev_state *cxlds); diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 2fc7fd252c2b..37164174b5fb 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -655,6 +655,7 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, struct cxl_event_interrupt_policy *policy) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_cmd mbox_cmd = { .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY, .payload_out = policy, @@ -662,7 +663,7 @@ static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, }; int rc; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) dev_err(mds->cxlds.dev, "Failed to get event interrupt policy : %d", rc); @@ -673,6 +674,7 @@ static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, static int cxl_event_config_msgnums(struct cxl_memdev_state *mds, struct cxl_event_interrupt_policy *policy) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -689,7 +691,7 @@ static int cxl_event_config_msgnums(struct cxl_memdev_state *mds, .size_in = sizeof(*policy), }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) { dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", rc); diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 50c6d25ede89..a6538a5f5c9f 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -120,6 +120,7 @@ static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds, struct nd_cmd_get_config_data_hdr *cmd, unsigned int buf_len) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_lsa get_lsa; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -141,7 +142,7 @@ static int cxl_pmem_get_config_data(struct cxl_memdev_state *mds, .payload_out = cmd->out_buf, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); cmd->status = 0; return rc; @@ -151,6 +152,7 @@ static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds, struct nd_cmd_set_config_hdr *cmd, unsigned int buf_len) { + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_set_lsa *set_lsa; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -177,7 +179,7 @@ static int cxl_pmem_set_config_data(struct cxl_memdev_state *mds, .size_in = struct_size(set_lsa, data, cmd->in_length), }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); /* * Set "firmware" status (4-packed bytes at the end of the input diff --git a/drivers/cxl/security.c b/drivers/cxl/security.c index 21856a3f408e..452d1a9b9148 100644 --- a/drivers/cxl/security.c +++ b/drivers/cxl/security.c @@ -14,6 +14,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm, { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); unsigned long security_flags = 0; struct cxl_get_security_output { @@ -29,7 +30,7 @@ static unsigned long cxl_pmem_get_security_flags(struct nvdimm *nvdimm, .payload_out = &out, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return 0; @@ -70,7 +71,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm, { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_cmd mbox_cmd; struct cxl_set_pass set_pass; @@ -87,7 +88,7 @@ static int cxl_pmem_security_change_key(struct nvdimm *nvdimm, .payload_in = &set_pass, }; - return cxl_internal_send_cmd(mds, &mbox_cmd); + return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); } static int __cxl_pmem_security_disable(struct nvdimm *nvdimm, @@ -96,7 +97,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm, { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_disable_pass dis_pass; struct cxl_mbox_cmd mbox_cmd; @@ -112,7 +113,7 @@ static int __cxl_pmem_security_disable(struct nvdimm *nvdimm, .payload_in = &dis_pass, }; - return cxl_internal_send_cmd(mds, &mbox_cmd); + return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); } static int cxl_pmem_security_disable(struct nvdimm *nvdimm, @@ -131,12 +132,12 @@ static int cxl_pmem_security_freeze(struct nvdimm *nvdimm) { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_cmd mbox_cmd = { .opcode = CXL_MBOX_OP_FREEZE_SECURITY, }; - return cxl_internal_send_cmd(mds, &mbox_cmd); + return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); } static int cxl_pmem_security_unlock(struct nvdimm *nvdimm, @@ -144,7 +145,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm, { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; u8 pass[NVDIMM_PASSPHRASE_LEN]; struct cxl_mbox_cmd mbox_cmd; int rc; @@ -156,7 +157,7 @@ static int cxl_pmem_security_unlock(struct nvdimm *nvdimm, .payload_in = pass, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return rc; @@ -169,7 +170,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm, { struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); + struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_mbox_cmd mbox_cmd; struct cxl_pass_erase erase; int rc; @@ -185,7 +186,7 @@ static int cxl_pmem_security_passphrase_erase(struct nvdimm *nvdimm, .payload_in = &erase, }; - rc = cxl_internal_send_cmd(mds, &mbox_cmd); + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) return rc; From c8691cd0fc11197515ed148de0780d927bfca38b Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Fri, 13 Sep 2024 15:05:18 +0200 Subject: [PATCH 041/104] Revert "dm: requeue IO if mapping table not yet available" This reverts commit fa247089de9936a46e290d4724cb5f0b845600f5. The following sequence of commands causes a livelock - there will be workqueue process looping and consuming 100% CPU: dmsetup create --notable test truncate -s 1MiB testdata losetup /dev/loop0 testdata dmsetup load test --table '0 2048 linear /dev/loop0 0' dd if=/dev/zero of=/dev/dm-0 bs=16k count=1 conv=fdatasync The livelock is caused by the commit fa247089de99. The commit claims that it fixes a race condition, however, it is unknown what the actual race condition is and what program is involved in the race condition. When the inactive table is loaded, the nodes /dev/dm-0 and /sys/block/dm-0 are created. /dev/dm-0 has zero size at this point. When the device is suspended and resumed, the nodes /dev/mapper/test and /dev/disk/* are created. If some program opens a block device before it is created by dmsetup or lvm, the program is buggy, so dm could just report an error as it used to do before. Reported-by: Zdenek Kabelac Signed-off-by: Mikulas Patocka Fixes: fa247089de99 ("dm: requeue IO if mapping table not yet available") --- drivers/md/dm-rq.c | 4 +++- drivers/md/dm.c | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index f7e9a3632eb3..499f8cc8a39f 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -496,8 +496,10 @@ static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, map = dm_get_live_table(md, &srcu_idx); if (unlikely(!map)) { + DMERR_LIMIT("%s: mapping table unavailable, erroring io", + dm_device_name(md)); dm_put_live_table(md, srcu_idx); - return BLK_STS_RESOURCE; + return BLK_STS_IOERR; } ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 87bb90303435..ff4a6b570b76 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -2030,10 +2030,15 @@ static void dm_submit_bio(struct bio *bio) struct dm_table *map; map = dm_get_live_table(md, &srcu_idx); + if (unlikely(!map)) { + DMERR_LIMIT("%s: mapping table unavailable, erroring io", + dm_device_name(md)); + bio_io_error(bio); + goto out; + } - /* If suspended, or map not yet available, queue this IO for later */ - if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) || - unlikely(!map)) { + /* If suspended, queue this IO for later */ + if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); else if (bio->bi_opf & REQ_RAHEAD) From da5b2ad1c2f18834cb1ce429e2e5a5cf5cbdf21b Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: [PATCH 042/104] objtool: Handle frame pointer related instructions After commit a0f7085f6a63 ("LoongArch: Add RANDOMIZE_KSTACK_OFFSET support"), there are three new instructions "addi.d $fp, $sp, 32", "sub.d $sp, $sp, $t0" and "addi.d $sp, $fp, -32" for the secondary stack in do_syscall(), then there is a objtool warning "return with modified stack frame" and no handle_syscall() which is the previous frame of do_syscall() in the call trace when executing the command "echo l > /proc/sysrq-trigger". objdump shows something like this: 0000000000000000 : 0: 02ff8063 addi.d $sp, $sp, -32 4: 29c04076 st.d $fp, $sp, 16 8: 29c02077 st.d $s0, $sp, 8 c: 29c06061 st.d $ra, $sp, 24 10: 02c08076 addi.d $fp, $sp, 32 ... 74: 0011b063 sub.d $sp, $sp, $t0 ... a8: 4c000181 jirl $ra, $t0, 0 ... dc: 02ff82c3 addi.d $sp, $fp, -32 e0: 28c06061 ld.d $ra, $sp, 24 e4: 28c04076 ld.d $fp, $sp, 16 e8: 28c02077 ld.d $s0, $sp, 8 ec: 02c08063 addi.d $sp, $sp, 32 f0: 4c000020 jirl $zero, $ra, 0 The instruction "sub.d $sp, $sp, $t0" changes the stack bottom and the new stack size is a random value, in order to find the return address of do_syscall() which is stored in the original stack frame after executing "jirl $ra, $t0, 0", it should use fp which points to the original stack top. At the beginning, the thought is tended to decode the secondary stack instruction "sub.d $sp, $sp, $t0" and set it as a label, then check this label for the two frame pointer instructions to change the cfa base and cfa offset during the period of secondary stack in update_cfi_state(). This is valid for GCC but invalid for Clang due to there are different secondary stack instructions for ClangBuiltLinux on LoongArch, something like this: 0000000000000000 : ... 88: 00119064 sub.d $a0, $sp, $a0 8c: 00150083 or $sp, $a0, $zero ... Actually, it equals to a single instruction "sub.d $sp, $sp, $a0", but there is no proper condition to check it as a label like GCC, and so the beginning thought is not a good way. Essentially, there are two special frame pointer instructions which are "addi.d $fp, $sp, imm" and "addi.d $sp, $fp, imm", the first one points fp to the original stack top and the second one restores the original stack bottom from fp. Based on the above analysis, in order to avoid adding an arch-specific update_cfi_state(), we just add a member "frame_pointer" in the "struct symbol" as a label to avoid affecting the current normal case, then set it as true only if there is "addi.d $sp, $fp, imm". The last is to check this label for the two frame pointer instructions to change the cfa base and cfa offset in update_cfi_state(). Tested with the following two configs: (1) CONFIG_RANDOMIZE_KSTACK_OFFSET=y && CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=n (2) CONFIG_RANDOMIZE_KSTACK_OFFSET=y && CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT=y By the way, there is no effect for x86 with this patch, tested on the x86 machine with Fedora 40 system. Cc: stable@vger.kernel.org # 6.9+ Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- tools/objtool/arch/loongarch/decode.c | 11 ++++++++++- tools/objtool/check.c | 23 ++++++++++++++++++++--- tools/objtool/include/objtool/elf.h | 1 + 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/tools/objtool/arch/loongarch/decode.c b/tools/objtool/arch/loongarch/decode.c index aee479d2191c..69b66994f2a1 100644 --- a/tools/objtool/arch/loongarch/decode.c +++ b/tools/objtool/arch/loongarch/decode.c @@ -122,7 +122,7 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, switch (inst.reg2i12_format.opcode) { case addid_op: if ((inst.reg2i12_format.rd == CFI_SP) || (inst.reg2i12_format.rj == CFI_SP)) { - /* addi.d sp,sp,si12 or addi.d fp,sp,si12 */ + /* addi.d sp,sp,si12 or addi.d fp,sp,si12 or addi.d sp,fp,si12 */ insn->immediate = sign_extend64(inst.reg2i12_format.immediate, 11); ADD_OP(op) { op->src.type = OP_SRC_ADD; @@ -132,6 +132,15 @@ static bool decode_insn_reg2i12_fomat(union loongarch_instruction inst, op->dest.reg = inst.reg2i12_format.rd; } } + if ((inst.reg2i12_format.rd == CFI_SP) && (inst.reg2i12_format.rj == CFI_FP)) { + /* addi.d sp,fp,si12 */ + struct symbol *func = find_func_containing(insn->sec, insn->offset); + + if (!func) + return false; + + func->frame_pointer = true; + } break; case ldd_op: if (inst.reg2i12_format.rj == CFI_SP) { diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 01237d167223..af9cfed7f4ec 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -2993,10 +2993,27 @@ static int update_cfi_state(struct instruction *insn, break; } - if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + if (op->dest.reg == CFI_BP && op->src.reg == CFI_SP && + insn->sym->frame_pointer) { + /* addi.d fp,sp,imm on LoongArch */ + if (cfa->base == CFI_SP && cfa->offset == op->src.offset) { + cfa->base = CFI_BP; + cfa->offset = 0; + } + break; + } - /* lea disp(%rbp), %rsp */ - cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) { + /* addi.d sp,fp,imm on LoongArch */ + if (cfa->base == CFI_BP && cfa->offset == 0) { + if (insn->sym->frame_pointer) { + cfa->base = CFI_SP; + cfa->offset = -op->src.offset; + } + } else { + /* lea disp(%rbp), %rsp */ + cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset); + } break; } diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index 2b8a69de4db8..d7e815c2fd15 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -68,6 +68,7 @@ struct symbol { u8 warned : 1; u8 embedded_insn : 1; u8 local_label : 1; + u8 frame_pointer : 1; struct list_head pv_target; struct reloc *relocs; }; From b8468bd92ae19939d4844899fa05147888732519 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: [PATCH 043/104] LoongArch: Enable objtool for Clang For now, it can enable objtool for Clang, just remove !CC_IS_CLANG for HAVE_OBJTOOL in arch/loongarch/Kconfig. Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 0e3abf7b0bd3..8dff6ddaf9c5 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -147,7 +147,7 @@ config LOONGARCH select HAVE_LIVEPATCH select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI - select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS && AS_HAS_THIN_ADD_SUB && !CC_IS_CLANG + select HAVE_OBJTOOL if AS_HAS_EXPLICIT_RELOCS && AS_HAS_THIN_ADD_SUB select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS From a7e0837724562ea8c1d869dd1a5cb1119ef651c3 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:09 +0800 Subject: [PATCH 044/104] LoongArch: Set AS_HAS_THIN_ADD_SUB as y if AS_IS_LLVM When building kernel with "make CC=clang defconfig", LLVM Assembler is used due to LLVM_IAS=0 is not specified, then AS_HAS_THIN_ADD_SUB is not set, thus objtool can not be built after enable it for Clang. config AS_HAS_THIN_ADD_SUB is to check whether -mthin-add-sub option is available to know R_LARCH_{32,64}_PCREL are supported for GNU Assembler, there is no such an option for LLVM Assembler. The minimal version of Clang is 18 for building LoongArch kernel, and Clang >= 17 has already supported R_LARCH_{32,64}_PCREL, that is to say, there is no need to depend on AS_HAS_THIN_ADD_SUB for Clang, so just set AS_HAS_THIN_ADD_SUB as y if AS_IS_LLVM. Fixes: 120dd4118e58 ("LoongArch: Only allow OBJTOOL & ORC unwinder if toolchain supports -mthin-add-sub") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 8dff6ddaf9c5..34531dc8a61f 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -266,7 +266,7 @@ config AS_HAS_FCSR_CLASS def_bool $(as-instr,movfcsr2gr \$t0$(comma)\$fcsr0) config AS_HAS_THIN_ADD_SUB - def_bool $(cc-option,-Wa$(comma)-mthin-add-sub) + def_bool $(cc-option,-Wa$(comma)-mthin-add-sub) || AS_IS_LLVM config AS_HAS_LSX_EXTENSION def_bool $(as-instr,vld \$vr0$(comma)\$a0$(comma)0) From 0eb0bd21e8382d10be8108952a0bb819915e1e2d Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 17 Sep 2024 22:23:10 +0800 Subject: [PATCH 045/104] LoongArch: Remove STACK_FRAME_NON_STANDARD(do_syscall) For now, we can remove STACK_FRAME_NON_STANDARD(do_syscall) because there is no objtool warning "do_syscall+0x11c: return with modified stack frame", then there is handle_syscall() which is the previous frame of do_syscall() in the call trace when executing the command "echo l > /proc/sysrq-trigger". Fixes: a0f7085f6a63 ("LoongArch: Add RANDOMIZE_KSTACK_OFFSET support") Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/kernel/syscall.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c index ba5d0930a74f..168bd97540f8 100644 --- a/arch/loongarch/kernel/syscall.c +++ b/arch/loongarch/kernel/syscall.c @@ -79,7 +79,3 @@ void noinstr __no_stack_protector do_syscall(struct pt_regs *regs) syscall_exit_to_user_mode(regs); } - -#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET -STACK_FRAME_NON_STANDARD(do_syscall); -#endif From 6ff2c290147a65027fb04b154a52723a6efabced Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 16 Sep 2024 15:28:00 +0200 Subject: [PATCH 046/104] MAINTAINERS: make vDSO getrandom matches more generic Rather than specifying matches for every architecture, what we actually want is an extglob-like thing, such as: F: **/vdso/*getrandom* Unfortunately, this isn't possible, but get_maintainers.pl does support a "N:" directive that uses regexes, so just encode that extglob statement as an ordinary regex. Signed-off-by: Jason A. Donenfeld --- MAINTAINERS | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 8a76c256229c..cba8bfedeadd 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19217,10 +19217,7 @@ F: drivers/char/random.c F: include/linux/random.h F: include/uapi/linux/random.h F: drivers/virt/vmgenid.c -F: include/vdso/getrandom.h -F: lib/vdso/getrandom.c -F: arch/x86/entry/vdso/vgetrandom* -F: arch/x86/include/asm/vdso/getrandom* +N: ^.*/vdso/[^/]*getrandom[^/]+$ RAPIDIO SUBSYSTEM M: Matt Porter From c5391c0e04f1b6ede3623962192b08a4eb224491 Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Wed, 18 Sep 2024 15:05:29 +0200 Subject: [PATCH 047/104] dm-crypt: Use up_read() together with key_put() only once in crypt_set_keyring_key() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The combination of the calls “up_read(&key->sem)” and “key_put(key)” was immediately used after a return code check for a set_key() call in this function implementation. Thus use such a function call pair only once instead directly before the check. This issue was transformed by using the Coccinelle software. Signed-off-by: Markus Elfring Signed-off-by: Mikulas Patocka --- drivers/md/dm-crypt.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index d5533b43054e..dae2fe3cb182 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2621,16 +2621,13 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string down_read(&key->sem); ret = set_key(cc, key); + up_read(&key->sem); + key_put(key); if (ret < 0) { - up_read(&key->sem); - key_put(key); kfree_sensitive(new_key_string); return ret; } - up_read(&key->sem); - key_put(key); - /* clear the flag since following operations may invalidate previously valid key */ clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); From 5d49054ef616095d160c1072ba458e16e2f825de Mon Sep 17 00:00:00 2001 From: Markus Elfring Date: Wed, 18 Sep 2024 15:34:45 +0200 Subject: [PATCH 048/104] dm-crypt: Use common error handling code in crypt_set_keyring_key() Add a jump target so that a bit of exception handling can be better reused at the end of this function implementation. Signed-off-by: Markus Elfring Signed-off-by: Mikulas Patocka --- drivers/md/dm-crypt.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index dae2fe3cb182..5228b03b6fe0 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2614,32 +2614,31 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string key = request_key(type, key_desc + 1, NULL); if (IS_ERR(key)) { - kfree_sensitive(new_key_string); - return PTR_ERR(key); + ret = PTR_ERR(key); + goto free_new_key_string; } down_read(&key->sem); - ret = set_key(cc, key); up_read(&key->sem); key_put(key); - if (ret < 0) { - kfree_sensitive(new_key_string); - return ret; - } + if (ret < 0) + goto free_new_key_string; /* clear the flag since following operations may invalidate previously valid key */ clear_bit(DM_CRYPT_KEY_VALID, &cc->flags); ret = crypt_setkey(cc); + if (ret) + goto free_new_key_string; - if (!ret) { - set_bit(DM_CRYPT_KEY_VALID, &cc->flags); - kfree_sensitive(cc->key_string); - cc->key_string = new_key_string; - } else - kfree_sensitive(new_key_string); + set_bit(DM_CRYPT_KEY_VALID, &cc->flags); + kfree_sensitive(cc->key_string); + cc->key_string = new_key_string; + return 0; +free_new_key_string: + kfree_sensitive(new_key_string); return ret; } From 9bcd923952078e08cd4570e15f751a6c9ec7633f Mon Sep 17 00:00:00 2001 From: Shen Lichuan Date: Sat, 14 Sep 2024 14:38:08 +0800 Subject: [PATCH 049/104] dm vdo indexer: Convert comma to semicolon To ensure code clarity and prevent potential errors, it's advisable to employ the ';' as a statement separator, except when ',' are intentionally used for specific purposes. Signed-off-by: Shen Lichuan Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/indexer/chapter-index.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c index 7e32a25d3f2f..fb1db41c794b 100644 --- a/drivers/md/dm-vdo/indexer/chapter-index.c +++ b/drivers/md/dm-vdo/indexer/chapter-index.c @@ -177,7 +177,7 @@ int uds_pack_open_chapter_index_page(struct open_chapter_index *chapter_index, if (list_number < 0) return UDS_OVERFLOW; - next_list = first_list + list_number--, + next_list = first_list + list_number--; result = uds_start_delta_index_search(delta_index, next_list, 0, &entry); if (result != UDS_SUCCESS) From 423c9baae4c7fe73d5812e28610418fd2e8050bd Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 13 Sep 2024 15:32:16 -0700 Subject: [PATCH 050/104] cxl: Fix comment regarding cxl_query_cmd() return data The code indicates that the min of n_commands and total commands is returned. The comment incorrectly says it's the max(). Correct comment to min(). Reviewed-by: Alison Schofield Link: https://patch.msgid.link/20240913223216.3234173-1-dave.jiang@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/mbox.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 915a93ced78f..946f8e44455f 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -546,7 +546,7 @@ int cxl_query_cmd(struct cxl_memdev *cxlmd, return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands); /* - * otherwise, return max(n_commands, total commands) cxl_command_info + * otherwise, return min(n_commands, total commands) cxl_command_info * structures. */ cxl_for_each_cmd(cmd) { From e91be3ed30d79ccd3e87e3970a26dea844c04919 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Sep 2024 17:11:50 -0700 Subject: [PATCH 051/104] cxl: Preserve the CDAT access_coordinate for an endpoint Keep the access_coordinate from the CDAT tables for region perf calculations. The region perf calculation requires all participating endpoints to have arrived in order to determine if there are limitations of bandwidth data due to shared uplink. Reviewed-by: Jonathan Cameron Reviewed-by: Ira Weiny Acked-by: Dan Williams Link: https://patch.msgid.link/20240904001316.1688225-2-dave.jiang@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/cdat.c | 10 ++++++---- drivers/cxl/cxlmem.h | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index e2e1ccda88fa..7bbf266a1612 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -14,7 +14,7 @@ struct dsmas_entry { struct range dpa_range; u8 handle; struct access_coordinate coord[ACCESS_COORDINATE_MAX]; - + struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX]; int entries; int qos_class; }; @@ -162,7 +162,7 @@ static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base), dslbis->data_type); - cxl_access_coordinate_set(dent->coord, dslbis->data_type, val); + cxl_access_coordinate_set(dent->cdat_coord, dslbis->data_type, val); return 0; } @@ -219,7 +219,7 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, xa_for_each(dsmas_xa, index, dent) { int qos_class; - cxl_coordinates_combine(dent->coord, dent->coord, ep_c); + cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c); dent->entries = 1; rc = cxl_root->ops->qos_class(cxl_root, &dent->coord[ACCESS_COORDINATE_CPU], @@ -240,8 +240,10 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, struct cxl_dpa_perf *dpa_perf) { - for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { dpa_perf->coord[i] = dent->coord[i]; + dpa_perf->cdat_coord[i] = dent->cdat_coord[i]; + } dpa_perf->dpa_range = dent->dpa_range; dpa_perf->qos_class = dent->qos_class; dev_dbg(dev, diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index c7c423ae16ab..2a25d1957ddb 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -398,11 +398,13 @@ enum cxl_devtype { * struct cxl_dpa_perf - DPA performance property entry * @dpa_range: range for DPA address * @coord: QoS performance data (i.e. latency, bandwidth) + * @cdat_coord: raw QoS performance data from CDAT * @qos_class: QoS Class cookies */ struct cxl_dpa_perf { struct range dpa_range; struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX]; int qos_class; }; From a5ab0de0ebaa65e0a75ec0761a2745c66a9d17dc Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Sep 2024 17:11:51 -0700 Subject: [PATCH 052/104] cxl: Calculate region bandwidth of targets with shared upstream link The current bandwidth calculation aggregates all the targets. This simple method does not take into account where multiple targets sharing under a switch or a root port where the aggregated bandwidth can be greater than the upstream link of the switch. To accurately account for the shared upstream uplink cases, a new update function is introduced by walking from the leaves to the root of the hierarchy and clamp the bandwidth in the process as needed. This process is done when all the targets for a region are present but before the final values are send to the HMAT handling code cached access_coordinate targets. The original perf calculation path was kept to calculate the latency performance data that does not require the shared link consideration. The shared upstream link calculation is done as a second pass when all the endpoints have arrived. Testing is done via qemu with CXL hierarchy. run_qemu[1] is modified to support several CXL hierarchy layouts. The following layouts are tested: HB: Host Bridge RP: Root Port SW: Switch EP: End Point 2 HB 2 RP 2 EP: resulting bandwidth: 624 1 HB 2 RP 2 EP: resulting bandwidth: 624 2 HB 2 RP 2 SW 4 EP: resulting bandwidth: 624 Current testing, perf number from SRAT/HMAT is hacked into the kernel code. However with new QEMU support of Generic Target Port that's incoming, the perf data injection is no longer needed. [1]: https://github.com/pmem/run_qemu Suggested-by: Jonathan Cameron Link: https://lore.kernel.org/linux-cxl/20240501152503.00002e60@Huawei.com/ Reviewed-by: Jonathan Cameron Reviewed-by: Alison Schofield Acked-by: Dan Williams Link: https://patch.msgid.link/20240904001316.1688225-3-dave.jiang@intel.com Signed-off-by: Dave Jiang --- drivers/cxl/core/cdat.c | 501 +++++++++++++++++++++++++++++++++++++- drivers/cxl/core/core.h | 4 +- drivers/cxl/core/pci.c | 23 ++ drivers/cxl/core/port.c | 20 ++ drivers/cxl/core/region.c | 2 + drivers/cxl/cxl.h | 1 + 6 files changed, 539 insertions(+), 12 deletions(-) diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 7bbf266a1612..ef1621d40f05 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -547,19 +547,37 @@ void cxl_coordinates_combine(struct access_coordinate *out, MODULE_IMPORT_NS(CXL); -void cxl_region_perf_data_calculate(struct cxl_region *cxlr, - struct cxl_endpoint_decoder *cxled) +static void cxl_bandwidth_add(struct access_coordinate *coord, + struct access_coordinate *c1, + struct access_coordinate *c2) +{ + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + coord[i].read_bandwidth = c1[i].read_bandwidth + + c2[i].read_bandwidth; + coord[i].write_bandwidth = c1[i].write_bandwidth + + c2[i].write_bandwidth; + } +} + +static bool dpa_perf_contains(struct cxl_dpa_perf *perf, + struct resource *dpa_res) +{ + struct range dpa = { + .start = dpa_res->start, + .end = dpa_res->end, + }; + + return range_contains(&perf->dpa_range, &dpa); +} + +static struct cxl_dpa_perf *cxled_get_dpa_perf(struct cxl_endpoint_decoder *cxled, + enum cxl_decoder_mode mode) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); - struct cxl_dev_state *cxlds = cxlmd->cxlds; - struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); - struct range dpa = { - .start = cxled->dpa_res->start, - .end = cxled->dpa_res->end, - }; + struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); struct cxl_dpa_perf *perf; - switch (cxlr->mode) { + switch (mode) { case CXL_DECODER_RAM: perf = &mds->ram_perf; break; @@ -567,12 +585,473 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, perf = &mds->pmem_perf; break; default: - return; + return ERR_PTR(-EINVAL); } + if (!dpa_perf_contains(perf, cxled->dpa_res)) + return ERR_PTR(-EINVAL); + + return perf; +} + +/* + * Transient context for containing the current calculation of bandwidth when + * doing walking the port hierarchy to deal with shared upstream link. + */ +struct cxl_perf_ctx { + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct cxl_port *port; +}; + +/** + * cxl_endpoint_gather_bandwidth - collect all the endpoint bandwidth in an xarray + * @cxlr: CXL region for the bandwidth calculation + * @cxled: endpoint decoder to start on + * @usp_xa: (output) the xarray that collects all the bandwidth coordinates + * indexed by the upstream device with data of 'struct cxl_perf_ctx'. + * @gp_is_root: (output) bool of whether the grandparent is cxl root. + * + * Return: 0 for success or -errno + * + * Collects aggregated endpoint bandwidth and store the bandwidth in + * an xarray indexed by the upstream device of the switch or the RP + * device. Each endpoint consists the minimum of the bandwidth from DSLBIS + * from the endpoint CDAT, the endpoint upstream link bandwidth, and the + * bandwidth from the SSLBIS of the switch CDAT for the switch upstream port to + * the downstream port that's associated with the endpoint. If the + * device is directly connected to a RP, then no SSLBIS is involved. + */ +static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled, + struct xarray *usp_xa, + bool *gp_is_root) +{ + struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent); + struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent); + struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent); + struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX]; + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + struct cxl_dev_state *cxlds = cxlmd->cxlds; + struct pci_dev *pdev = to_pci_dev(cxlds->dev); + struct cxl_perf_ctx *perf_ctx; + struct cxl_dpa_perf *perf; + unsigned long index; + void *ptr; + int rc; + + if (cxlds->rcd) + return -ENODEV; + + perf = cxled_get_dpa_perf(cxled, cxlr->mode); + if (IS_ERR(perf)) + return PTR_ERR(perf); + + gp_port = to_cxl_port(parent_port->dev.parent); + *gp_is_root = is_cxl_root(gp_port); + + /* + * If the grandparent is cxl root, then index is the root port, + * otherwise it's the parent switch upstream device. + */ + if (*gp_is_root) + index = (unsigned long)endpoint->parent_dport->dport_dev; + else + index = (unsigned long)parent_port->uport_dev; + + perf_ctx = xa_load(usp_xa, index); + if (!perf_ctx) { + struct cxl_perf_ctx *c __free(kfree) = + kzalloc(sizeof(*perf_ctx), GFP_KERNEL); + + if (!c) + return -ENOMEM; + ptr = xa_store(usp_xa, index, c, GFP_KERNEL); + if (xa_is_err(ptr)) + return xa_err(ptr); + perf_ctx = no_free_ptr(c); + perf_ctx->port = parent_port; + } + + /* Direct upstream link from EP bandwidth */ + rc = cxl_pci_get_bandwidth(pdev, pci_coord); + if (rc < 0) + return rc; + + /* + * Min of upstream link bandwidth and Endpoint CDAT bandwidth from + * DSLBIS. + */ + cxl_coordinates_combine(ep_coord, pci_coord, perf->cdat_coord); + + /* + * If grandparent port is root, then there's no switch involved and + * the endpoint is connected to a root port. + */ + if (!*gp_is_root) { + /* + * Retrieve the switch SSLBIS for switch downstream port + * associated with the endpoint bandwidth. + */ + rc = cxl_port_get_switch_dport_bandwidth(endpoint, sw_coord); + if (rc) + return rc; + + /* + * Min of the earlier coordinates with the switch SSLBIS + * bandwidth + */ + cxl_coordinates_combine(ep_coord, ep_coord, sw_coord); + } + + /* + * Aggregate the computed bandwidth with the current aggregated bandwidth + * of the endpoints with the same switch upstream device or RP. + */ + cxl_bandwidth_add(perf_ctx->coord, perf_ctx->coord, ep_coord); + + return 0; +} + +static void free_perf_xa(struct xarray *xa) +{ + struct cxl_perf_ctx *ctx; + unsigned long index; + + if (!xa) + return; + + xa_for_each(xa, index, ctx) + kfree(ctx); + xa_destroy(xa); + kfree(xa); +} +DEFINE_FREE(free_perf_xa, struct xarray *, if (_T) free_perf_xa(_T)) + +/** + * cxl_switch_gather_bandwidth - collect all the bandwidth at switch level in an xarray + * @cxlr: The region being operated on + * @input_xa: xarray indexed by upstream device of a switch with data of 'struct + * cxl_perf_ctx' + * @gp_is_root: (output) bool of whether the grandparent is cxl root. + * + * Return: a xarray of resulting cxl_perf_ctx per parent switch or root port + * or ERR_PTR(-errno) + * + * Iterate through the xarray. Take the minimum of the downstream calculated + * bandwidth, the upstream link bandwidth, and the SSLBIS of the upstream + * switch if exists. Sum the resulting bandwidth under the switch upstream + * device or a RP device. The function can be iterated over multiple switches + * if the switches are present. + */ +static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr, + struct xarray *input_xa, + bool *gp_is_root) +{ + struct xarray *res_xa __free(free_perf_xa) = + kzalloc(sizeof(*res_xa), GFP_KERNEL); + struct access_coordinate coords[ACCESS_COORDINATE_MAX]; + struct cxl_perf_ctx *ctx, *us_ctx; + unsigned long index, us_index; + int dev_count = 0; + int gp_count = 0; + void *ptr; + int rc; + + if (!res_xa) + return ERR_PTR(-ENOMEM); + xa_init(res_xa); + + xa_for_each(input_xa, index, ctx) { + struct device *dev = (struct device *)index; + struct cxl_port *port = ctx->port; + struct cxl_port *parent_port = to_cxl_port(port->dev.parent); + struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent); + struct cxl_dport *dport = port->parent_dport; + bool is_root = false; + + dev_count++; + if (is_cxl_root(gp_port)) { + is_root = true; + gp_count++; + } + + /* + * If the grandparent is cxl root, then index is the root port, + * otherwise it's the parent switch upstream device. + */ + if (is_root) + us_index = (unsigned long)port->parent_dport->dport_dev; + else + us_index = (unsigned long)parent_port->uport_dev; + + us_ctx = xa_load(res_xa, us_index); + if (!us_ctx) { + struct cxl_perf_ctx *n __free(kfree) = + kzalloc(sizeof(*n), GFP_KERNEL); + + if (!n) + return ERR_PTR(-ENOMEM); + + ptr = xa_store(res_xa, us_index, n, GFP_KERNEL); + if (xa_is_err(ptr)) + return ERR_PTR(xa_err(ptr)); + us_ctx = no_free_ptr(n); + us_ctx->port = parent_port; + } + + /* + * If the device isn't an upstream PCIe port, there's something + * wrong with the topology. + */ + if (!dev_is_pci(dev)) + return ERR_PTR(-EINVAL); + + /* Retrieve the upstream link bandwidth */ + rc = cxl_pci_get_bandwidth(to_pci_dev(dev), coords); + if (rc) + return ERR_PTR(-ENXIO); + + /* + * Take the min of downstream bandwidth and the upstream link + * bandwidth. + */ + cxl_coordinates_combine(coords, coords, ctx->coord); + + /* + * Take the min of the calculated bandwdith and the upstream + * switch SSLBIS bandwidth if there's a parent switch + */ + if (!is_root) + cxl_coordinates_combine(coords, coords, dport->coord); + + /* + * Aggregate the calculated bandwidth common to an upstream + * switch. + */ + cxl_bandwidth_add(us_ctx->coord, us_ctx->coord, coords); + } + + /* Asymmetric topology detected. */ + if (gp_count) { + if (gp_count != dev_count) { + dev_dbg(&cxlr->dev, + "Asymmetric hierarchy detected, bandwidth not updated\n"); + return ERR_PTR(-EOPNOTSUPP); + } + *gp_is_root = true; + } + + return no_free_ptr(res_xa); +} + +/** + * cxl_rp_gather_bandwidth - handle the root port level bandwidth collection + * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated + * below each root port device. + * + * Return: xarray that holds cxl_perf_ctx per host bridge or ERR_PTR(-errno) + */ +static struct xarray *cxl_rp_gather_bandwidth(struct xarray *xa) +{ + struct xarray *hb_xa __free(free_perf_xa) = + kzalloc(sizeof(*hb_xa), GFP_KERNEL); + struct cxl_perf_ctx *ctx; + unsigned long index; + + if (!hb_xa) + return ERR_PTR(-ENOMEM); + xa_init(hb_xa); + + xa_for_each(xa, index, ctx) { + struct cxl_port *port = ctx->port; + unsigned long hb_index = (unsigned long)port->uport_dev; + struct cxl_perf_ctx *hb_ctx; + void *ptr; + + hb_ctx = xa_load(hb_xa, hb_index); + if (!hb_ctx) { + struct cxl_perf_ctx *n __free(kfree) = + kzalloc(sizeof(*n), GFP_KERNEL); + + if (!n) + return ERR_PTR(-ENOMEM); + ptr = xa_store(hb_xa, hb_index, n, GFP_KERNEL); + if (xa_is_err(ptr)) + return ERR_PTR(xa_err(ptr)); + hb_ctx = no_free_ptr(n); + hb_ctx->port = port; + } + + cxl_bandwidth_add(hb_ctx->coord, hb_ctx->coord, ctx->coord); + } + + return no_free_ptr(hb_xa); +} + +/** + * cxl_hb_gather_bandwidth - handle the host bridge level bandwidth collection + * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated + * below each host bridge. + * + * Return: xarray that holds cxl_perf_ctx per ACPI0017 device or ERR_PTR(-errno) + */ +static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa) +{ + struct xarray *mw_xa __free(free_perf_xa) = + kzalloc(sizeof(*mw_xa), GFP_KERNEL); + struct cxl_perf_ctx *ctx; + unsigned long index; + + if (!mw_xa) + return ERR_PTR(-ENOMEM); + xa_init(mw_xa); + + xa_for_each(xa, index, ctx) { + struct cxl_port *port = ctx->port; + struct cxl_port *parent_port; + struct cxl_perf_ctx *mw_ctx; + struct cxl_dport *dport; + unsigned long mw_index; + void *ptr; + + parent_port = to_cxl_port(port->dev.parent); + mw_index = (unsigned long)parent_port->uport_dev; + + mw_ctx = xa_load(mw_xa, mw_index); + if (!mw_ctx) { + struct cxl_perf_ctx *n __free(kfree) = + kzalloc(sizeof(*n), GFP_KERNEL); + + if (!n) + return ERR_PTR(-ENOMEM); + ptr = xa_store(mw_xa, mw_index, n, GFP_KERNEL); + if (xa_is_err(ptr)) + return ERR_PTR(xa_err(ptr)); + mw_ctx = no_free_ptr(n); + } + + dport = port->parent_dport; + cxl_coordinates_combine(ctx->coord, ctx->coord, dport->coord); + cxl_bandwidth_add(mw_ctx->coord, mw_ctx->coord, ctx->coord); + } + + return no_free_ptr(mw_xa); +} + +/** + * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region + * @cxlr: The region being operated on + * @input_xa: xarray holds cxl_perf_ctx wht calculated bandwidth per ACPI0017 instance + */ +static void cxl_region_update_bandwidth(struct cxl_region *cxlr, + struct xarray *input_xa) +{ + struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct cxl_perf_ctx *ctx; + unsigned long index; + + memset(coord, 0, sizeof(coord)); + xa_for_each(input_xa, index, ctx) + cxl_bandwidth_add(coord, coord, ctx->coord); + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + cxlr->coord[i].read_bandwidth = coord[i].read_bandwidth; + cxlr->coord[i].write_bandwidth = coord[i].write_bandwidth; + } +} + +/** + * cxl_region_shared_upstream_bandwidth_update - Recalculate the bandwidth for + * the region + * @cxlr: the cxl region to recalculate + * + * The function walks the topology from bottom up and calculates the bandwidth. It + * starts at the endpoints, processes at the switches if any, processes at the rootport + * level, at the host bridge level, and finally aggregates at the region. + */ +void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr) +{ + struct xarray *working_xa; + int root_count = 0; + bool is_root; + int rc; + lockdep_assert_held(&cxl_dpa_rwsem); - if (!range_contains(&perf->dpa_range, &dpa)) + struct xarray *usp_xa __free(free_perf_xa) = + kzalloc(sizeof(*usp_xa), GFP_KERNEL); + + if (!usp_xa) + return; + + xa_init(usp_xa); + + /* Collect bandwidth data from all the endpoints. */ + for (int i = 0; i < cxlr->params.nr_targets; i++) { + struct cxl_endpoint_decoder *cxled = cxlr->params.targets[i]; + + is_root = false; + rc = cxl_endpoint_gather_bandwidth(cxlr, cxled, usp_xa, &is_root); + if (rc) + return; + root_count += is_root; + } + + /* Detect asymmetric hierarchy with some direct attached endpoints. */ + if (root_count && root_count != cxlr->params.nr_targets) { + dev_dbg(&cxlr->dev, + "Asymmetric hierarchy detected, bandwidth not updated\n"); + return; + } + + /* + * Walk up one or more switches to deal with the bandwidth of the + * switches if they exist. Endpoints directly attached to RPs skip + * over this part. + */ + if (!root_count) { + do { + working_xa = cxl_switch_gather_bandwidth(cxlr, usp_xa, + &is_root); + if (IS_ERR(working_xa)) + return; + free_perf_xa(usp_xa); + usp_xa = working_xa; + } while (!is_root); + } + + /* Handle the bandwidth at the root port of the hierarchy */ + working_xa = cxl_rp_gather_bandwidth(usp_xa); + if (IS_ERR(working_xa)) + return; + free_perf_xa(usp_xa); + usp_xa = working_xa; + + /* Handle the bandwidth at the host bridge of the hierarchy */ + working_xa = cxl_hb_gather_bandwidth(usp_xa); + if (IS_ERR(working_xa)) + return; + free_perf_xa(usp_xa); + usp_xa = working_xa; + + /* + * Aggregate all the bandwidth collected per CFMWS (ACPI0017) and + * update the region bandwidth with the final calculated values. + */ + cxl_region_update_bandwidth(cxlr, usp_xa); +} + +void cxl_region_perf_data_calculate(struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled) +{ + struct cxl_dpa_perf *perf; + + lockdep_assert_held(&cxl_dpa_rwsem); + + perf = cxled_get_dpa_perf(cxled, cxlr->mode); + if (IS_ERR(perf)) return; for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 72a506c9dbd0..0c62b4069ba0 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -103,9 +103,11 @@ enum cxl_poison_trace_type { }; long cxl_pci_get_latency(struct pci_dev *pdev); - +int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); bool cxl_need_node_perf_attrs_update(int nid); +int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port, + struct access_coordinate *c); #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 9d396ef0aeed..5b46bc46aaa9 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -1031,3 +1031,26 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port) __cxl_endpoint_decoder_reset_detected); } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL); + +int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c) +{ + int speed, bw; + u16 lnksta; + u32 width; + + speed = pcie_link_speed_mbps(pdev); + if (speed < 0) + return speed; + speed /= BITS_PER_BYTE; + + pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + bw = speed * width; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].read_bandwidth = bw; + c[i].write_bandwidth = bw; + } + + return 0; +} diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index a5e6f3d23cfb..e666ec6a9085 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -2237,6 +2237,26 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, } EXPORT_SYMBOL_NS_GPL(cxl_endpoint_get_perf_coordinates, CXL); +int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port, + struct access_coordinate *c) +{ + struct cxl_dport *dport = port->parent_dport; + + /* Check this port is connected to a switch DSP and not an RP */ + if (parent_port_is_cxl_root(to_cxl_port(port->dev.parent))) + return -ENODEV; + + if (!coordinates_valid(dport->coord)) + return -EINVAL; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + c[i].read_bandwidth = dport->coord[i].read_bandwidth; + c[i].write_bandwidth = dport->coord[i].write_bandwidth; + } + + return 0; +} + /* for user tooling to ensure port disable work has completed */ static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count) { diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 7bb79f3f318c..e701e4b04032 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1983,6 +1983,7 @@ static int cxl_region_attach(struct cxl_region *cxlr, * then the region is already committed. */ p->state = CXL_CONFIG_COMMIT; + cxl_region_shared_upstream_bandwidth_update(cxlr); return 0; } @@ -2004,6 +2005,7 @@ static int cxl_region_attach(struct cxl_region *cxlr, if (rc) return rc; p->state = CXL_CONFIG_ACTIVE; + cxl_region_shared_upstream_bandwidth_update(cxlr); } cxled->cxld.interleave_ways = p->interleave_ways; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 0fc96f8bf15c..0d8b810a51f0 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -891,6 +891,7 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, struct access_coordinate *coord); void cxl_region_perf_data_calculate(struct cxl_region *cxlr, struct cxl_endpoint_decoder *cxled); +void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr); void cxl_memdev_update_perf(struct cxl_memdev *cxlmd); From 2c70677dabb5e326467160e28915b804b925b53b Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Tue, 3 Sep 2024 17:11:52 -0700 Subject: [PATCH 053/104] cxl: Add documentation to explain the shared link bandwidth calculation Create a kernel documentation to describe how the CXL shared upstream link bandwidth is calculated. Suggested-by: Dan Williams Reviewed-by: Alison Schofield Acked-by: Dan Williams Link: https://patch.msgid.link/20240904001316.1688225-4-dave.jiang@intel.com Signed-off-by: Dave Jiang --- .../driver-api/cxl/access-coordinates.rst | 91 +++++++++++++++++++ Documentation/driver-api/cxl/index.rst | 1 + 2 files changed, 92 insertions(+) create mode 100644 Documentation/driver-api/cxl/access-coordinates.rst diff --git a/Documentation/driver-api/cxl/access-coordinates.rst b/Documentation/driver-api/cxl/access-coordinates.rst new file mode 100644 index 000000000000..b07950ea30c9 --- /dev/null +++ b/Documentation/driver-api/cxl/access-coordinates.rst @@ -0,0 +1,91 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: + +================================== +CXL Access Coordinates Computation +================================== + +Shared Upstream Link Calculation +================================ +For certain CXL region construction with endpoints behind CXL switches (SW) or +Root Ports (RP), there is the possibility of the total bandwidth for all +the endpoints behind a switch being more than the switch upstream link. +A similar situation can occur within the host, upstream of the root ports. +The CXL driver performs an additional pass after all the targets have +arrived for a region in order to recalculate the bandwidths with possible +upstream link being a limiting factor in mind. + +The algorithm assumes the configuration is a symmetric topology as that +maximizes performance. When asymmetric topology is detected, the calculation +is aborted. An asymmetric topology is detected during topology walk where the +number of RPs detected as a grandparent is not equal to the number of devices +iterated in the same iteration loop. The assumption is made that subtle +asymmetry in properties does not happen and all paths to EPs are equal. + +There can be multiple switches under an RP. There can be multiple RPs under +a CXL Host Bridge (HB). There can be multiple HBs under a CXL Fixed Memory +Window Structure (CFMWS). + +An example hierarchy: + +> CFMWS 0 +> | +> _________|_________ +> | | +> ACPI0017-0 ACPI0017-1 +> GP0/HB0/ACPI0016-0 GP1/HB1/ACPI0016-1 +> | | | | +> RP0 RP1 RP2 RP3 +> | | | | +> SW 0 SW 1 SW 2 SW 3 +> | | | | | | | | +> EP0 EP1 EP2 EP3 EP4 EP5 EP6 EP7 + +Computation for the example hierarchy: + +Min (GP0 to CPU BW, + Min(SW 0 Upstream Link to RP0 BW, + Min(SW0SSLBIS for SW0DSP0 (EP0), EP0 DSLBIS, EP0 Upstream Link) + + Min(SW0SSLBIS for SW0DSP1 (EP1), EP1 DSLBIS, EP1 Upstream link)) + + Min(SW 1 Upstream Link to RP1 BW, + Min(SW1SSLBIS for SW1DSP0 (EP2), EP2 DSLBIS, EP2 Upstream Link) + + Min(SW1SSLBIS for SW1DSP1 (EP3), EP3 DSLBIS, EP3 Upstream link))) + +Min (GP1 to CPU BW, + Min(SW 2 Upstream Link to RP2 BW, + Min(SW2SSLBIS for SW2DSP0 (EP4), EP4 DSLBIS, EP4 Upstream Link) + + Min(SW2SSLBIS for SW2DSP1 (EP5), EP5 DSLBIS, EP5 Upstream link)) + + Min(SW 3 Upstream Link to RP3 BW, + Min(SW3SSLBIS for SW3DSP0 (EP6), EP6 DSLBIS, EP6 Upstream Link) + + Min(SW3SSLBIS for SW3DSP1 (EP7), EP7 DSLBIS, EP7 Upstream link)))) + +The calculation starts at cxl_region_shared_upstream_perf_update(). A xarray +is created to collect all the endpoint bandwidths via the +cxl_endpoint_gather_bandwidth() function. The min() of bandwidth from the +endpoint CDAT and the upstream link bandwidth is calculated. If the endpoint +has a CXL switch as a parent, then min() of calculated bandwidth and the +bandwidth from the SSLBIS for the switch downstream port that is associated +with the endpoint is calculated. The final bandwidth is stored in a +'struct cxl_perf_ctx' in the xarray indexed by a device pointer. If the +endpoint is direct attached to a root port (RP), the device pointer would be an +RP device. If the endpoint is behind a switch, the device pointer would be the +upstream device of the parent switch. + +At the next stage, the code walks through one or more switches if they exist +in the topology. For endpoints directly attached to RPs, this step is skipped. +If there is another switch upstream, the code takes the min() of the current +gathered bandwidth and the upstream link bandwidth. If there's a switch +upstream, then the SSLBIS of the upstream switch. + +Once the topology walk reaches the RP, whether it's direct attached endpoints +or walking through the switch(es), cxl_rp_gather_bandwidth() is called. At +this point all the bandwidths are aggregated per each host bridge, which is +also the index for the resulting xarray. + +The next step is to take the min() of the per host bridge bandwidth and the +bandwidth from the Generic Port (GP). The bandwidths for the GP is retrieved +via ACPI tables SRAT/HMAT. The min bandwidth are aggregated under the same +ACPI0017 device to form a new xarray. + +Finally, the cxl_region_update_bandwidth() is called and the aggregated +bandwidth from all the members of the last xarray is updated for the +access coordinates residing in the cxl region (cxlr) context. diff --git a/Documentation/driver-api/cxl/index.rst b/Documentation/driver-api/cxl/index.rst index 12b82725d322..965ba90e8fb7 100644 --- a/Documentation/driver-api/cxl/index.rst +++ b/Documentation/driver-api/cxl/index.rst @@ -8,6 +8,7 @@ Compute Express Link :maxdepth: 1 memory-devices + access-coordinates maturity-map From 268225a1de1a021bac4884e7d61fe047345cc9be Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Mon, 23 Sep 2024 19:00:21 +0900 Subject: [PATCH 054/104] tomoyo: preparation step for building as a loadable LSM module In order to allow Makefile to generate tomoyo.ko as output, rename tomoyo.c to hooks.h and cut out LSM hook registration part that will be built into vmlinux from hooks.h to init.c . Also, update comments and relocate some variables. No behavior changes. Signed-off-by: Tetsuo Handa --- security/tomoyo/Makefile | 2 +- security/tomoyo/gc.c | 3 + security/tomoyo/{tomoyo.c => hooks.h} | 110 +------------------------- security/tomoyo/init.c | 104 ++++++++++++++++++++++++ security/tomoyo/securityfs_if.c | 6 +- security/tomoyo/util.c | 3 - 6 files changed, 112 insertions(+), 116 deletions(-) rename security/tomoyo/{tomoyo.c => hooks.h} (79%) create mode 100644 security/tomoyo/init.c diff --git a/security/tomoyo/Makefile b/security/tomoyo/Makefile index 55c67b9846a9..e573a5cad732 100644 --- a/security/tomoyo/Makefile +++ b/security/tomoyo/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y = audit.o common.o condition.o domain.o environ.o file.o gc.o group.o load_policy.o memory.o mount.o network.o realpath.o securityfs_if.o tomoyo.o util.o +obj-y = audit.o common.o condition.o domain.o environ.o file.o gc.o group.o init.o load_policy.o memory.o mount.o network.o realpath.o securityfs_if.o util.o targets += builtin-policy.h diff --git a/security/tomoyo/gc.c b/security/tomoyo/gc.c index 026e29ea3796..6eccca150839 100644 --- a/security/tomoyo/gc.c +++ b/security/tomoyo/gc.c @@ -9,6 +9,9 @@ #include #include +/* Lock for GC. */ +DEFINE_SRCU(tomoyo_ss); + /** * tomoyo_memory_free - Free memory for elements. * diff --git a/security/tomoyo/tomoyo.c b/security/tomoyo/hooks.h similarity index 79% rename from security/tomoyo/tomoyo.c rename to security/tomoyo/hooks.h index 04a92c3d65d4..58929bb71477 100644 --- a/security/tomoyo/tomoyo.c +++ b/security/tomoyo/hooks.h @@ -1,12 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* - * security/tomoyo/tomoyo.c + * security/tomoyo/hooks.h * * Copyright (C) 2005-2011 NTT DATA CORPORATION */ -#include -#include #include "common.h" /** @@ -18,10 +16,6 @@ struct tomoyo_domain_info *tomoyo_domain(void) { struct tomoyo_task *s = tomoyo_task(current); - if (s->old_domain_info && !current->in_execve) { - atomic_dec(&s->old_domain_info->users); - s->old_domain_info = NULL; - } return s->domain_info; } @@ -62,26 +56,6 @@ static void tomoyo_bprm_committed_creds(const struct linux_binprm *bprm) s->old_domain_info = NULL; } -#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER -/** - * tomoyo_bprm_creds_for_exec - Target for security_bprm_creds_for_exec(). - * - * @bprm: Pointer to "struct linux_binprm". - * - * Returns 0. - */ -static int tomoyo_bprm_creds_for_exec(struct linux_binprm *bprm) -{ - /* - * Load policy if /sbin/tomoyo-init exists and /sbin/init is requested - * for the first time. - */ - if (!tomoyo_policy_loaded) - tomoyo_load_policy(bprm->filename); - return 0; -} -#endif - /** * tomoyo_bprm_check_security - Target for security_bprm_check(). * @@ -501,10 +475,6 @@ static int tomoyo_socket_sendmsg(struct socket *sock, struct msghdr *msg, return tomoyo_socket_sendmsg_permission(sock, msg, size); } -struct lsm_blob_sizes tomoyo_blob_sizes __ro_after_init = { - .lbs_task = sizeof(struct tomoyo_task), -}; - /** * tomoyo_task_alloc - Target for security_task_alloc(). * @@ -543,81 +513,3 @@ static void tomoyo_task_free(struct task_struct *task) s->old_domain_info = NULL; } } - -static const struct lsm_id tomoyo_lsmid = { - .name = "tomoyo", - .id = LSM_ID_TOMOYO, -}; - -/* - * tomoyo_security_ops is a "struct security_operations" which is used for - * registering TOMOYO. - */ -static struct security_hook_list tomoyo_hooks[] __ro_after_init = { - LSM_HOOK_INIT(cred_prepare, tomoyo_cred_prepare), - LSM_HOOK_INIT(bprm_committed_creds, tomoyo_bprm_committed_creds), - LSM_HOOK_INIT(task_alloc, tomoyo_task_alloc), - LSM_HOOK_INIT(task_free, tomoyo_task_free), -#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER - LSM_HOOK_INIT(bprm_creds_for_exec, tomoyo_bprm_creds_for_exec), -#endif - LSM_HOOK_INIT(bprm_check_security, tomoyo_bprm_check_security), - LSM_HOOK_INIT(file_fcntl, tomoyo_file_fcntl), - LSM_HOOK_INIT(file_open, tomoyo_file_open), - LSM_HOOK_INIT(file_truncate, tomoyo_file_truncate), - LSM_HOOK_INIT(path_truncate, tomoyo_path_truncate), - LSM_HOOK_INIT(path_unlink, tomoyo_path_unlink), - LSM_HOOK_INIT(path_mkdir, tomoyo_path_mkdir), - LSM_HOOK_INIT(path_rmdir, tomoyo_path_rmdir), - LSM_HOOK_INIT(path_symlink, tomoyo_path_symlink), - LSM_HOOK_INIT(path_mknod, tomoyo_path_mknod), - LSM_HOOK_INIT(path_link, tomoyo_path_link), - LSM_HOOK_INIT(path_rename, tomoyo_path_rename), - LSM_HOOK_INIT(inode_getattr, tomoyo_inode_getattr), - LSM_HOOK_INIT(file_ioctl, tomoyo_file_ioctl), - LSM_HOOK_INIT(file_ioctl_compat, tomoyo_file_ioctl), - LSM_HOOK_INIT(path_chmod, tomoyo_path_chmod), - LSM_HOOK_INIT(path_chown, tomoyo_path_chown), - LSM_HOOK_INIT(path_chroot, tomoyo_path_chroot), - LSM_HOOK_INIT(sb_mount, tomoyo_sb_mount), - LSM_HOOK_INIT(sb_umount, tomoyo_sb_umount), - LSM_HOOK_INIT(sb_pivotroot, tomoyo_sb_pivotroot), - LSM_HOOK_INIT(socket_bind, tomoyo_socket_bind), - LSM_HOOK_INIT(socket_connect, tomoyo_socket_connect), - LSM_HOOK_INIT(socket_listen, tomoyo_socket_listen), - LSM_HOOK_INIT(socket_sendmsg, tomoyo_socket_sendmsg), -}; - -/* Lock for GC. */ -DEFINE_SRCU(tomoyo_ss); - -int tomoyo_enabled __ro_after_init = 1; - -/** - * tomoyo_init - Register TOMOYO Linux as a LSM module. - * - * Returns 0. - */ -static int __init tomoyo_init(void) -{ - struct tomoyo_task *s = tomoyo_task(current); - - /* register ourselves with the security framework */ - security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks), - &tomoyo_lsmid); - pr_info("TOMOYO Linux initialized\n"); - s->domain_info = &tomoyo_kernel_domain; - atomic_inc(&tomoyo_kernel_domain.users); - s->old_domain_info = NULL; - tomoyo_mm_init(); - - return 0; -} - -DEFINE_LSM(tomoyo) = { - .name = "tomoyo", - .enabled = &tomoyo_enabled, - .flags = LSM_FLAG_LEGACY_MAJOR, - .blobs = &tomoyo_blob_sizes, - .init = tomoyo_init, -}; diff --git a/security/tomoyo/init.c b/security/tomoyo/init.c new file mode 100644 index 000000000000..2b38dbc7a559 --- /dev/null +++ b/security/tomoyo/init.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * security/tomoyo/init.c + * + * Copyright (C) 2005-2011 NTT DATA CORPORATION + */ + +#include +#include +#include "common.h" + +#include "hooks.h" + +#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER +static int tomoyo_bprm_creds_for_exec(struct linux_binprm *bprm) +{ + /* + * Load policy if /sbin/tomoyo-init exists and /sbin/init is requested + * for the first time. + */ + if (!tomoyo_policy_loaded) + tomoyo_load_policy(bprm->filename); + return 0; +} +#endif + +struct lsm_blob_sizes tomoyo_blob_sizes __ro_after_init = { + .lbs_task = sizeof(struct tomoyo_task), +}; + +static const struct lsm_id tomoyo_lsmid = { + .name = "tomoyo", + .id = LSM_ID_TOMOYO, +}; + +/* tomoyo_hooks is used for registering TOMOYO. */ +static struct security_hook_list tomoyo_hooks[] __ro_after_init = { + LSM_HOOK_INIT(cred_prepare, tomoyo_cred_prepare), + LSM_HOOK_INIT(bprm_committed_creds, tomoyo_bprm_committed_creds), + LSM_HOOK_INIT(task_alloc, tomoyo_task_alloc), + LSM_HOOK_INIT(task_free, tomoyo_task_free), +#ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER + LSM_HOOK_INIT(bprm_creds_for_exec, tomoyo_bprm_creds_for_exec), +#endif + LSM_HOOK_INIT(bprm_check_security, tomoyo_bprm_check_security), + LSM_HOOK_INIT(file_fcntl, tomoyo_file_fcntl), + LSM_HOOK_INIT(file_open, tomoyo_file_open), + LSM_HOOK_INIT(file_truncate, tomoyo_file_truncate), + LSM_HOOK_INIT(path_truncate, tomoyo_path_truncate), + LSM_HOOK_INIT(path_unlink, tomoyo_path_unlink), + LSM_HOOK_INIT(path_mkdir, tomoyo_path_mkdir), + LSM_HOOK_INIT(path_rmdir, tomoyo_path_rmdir), + LSM_HOOK_INIT(path_symlink, tomoyo_path_symlink), + LSM_HOOK_INIT(path_mknod, tomoyo_path_mknod), + LSM_HOOK_INIT(path_link, tomoyo_path_link), + LSM_HOOK_INIT(path_rename, tomoyo_path_rename), + LSM_HOOK_INIT(inode_getattr, tomoyo_inode_getattr), + LSM_HOOK_INIT(file_ioctl, tomoyo_file_ioctl), + LSM_HOOK_INIT(file_ioctl_compat, tomoyo_file_ioctl), + LSM_HOOK_INIT(path_chmod, tomoyo_path_chmod), + LSM_HOOK_INIT(path_chown, tomoyo_path_chown), + LSM_HOOK_INIT(path_chroot, tomoyo_path_chroot), + LSM_HOOK_INIT(sb_mount, tomoyo_sb_mount), + LSM_HOOK_INIT(sb_umount, tomoyo_sb_umount), + LSM_HOOK_INIT(sb_pivotroot, tomoyo_sb_pivotroot), + LSM_HOOK_INIT(socket_bind, tomoyo_socket_bind), + LSM_HOOK_INIT(socket_connect, tomoyo_socket_connect), + LSM_HOOK_INIT(socket_listen, tomoyo_socket_listen), + LSM_HOOK_INIT(socket_sendmsg, tomoyo_socket_sendmsg), +}; + +int tomoyo_enabled __ro_after_init = 1; + +/* Has /sbin/init started? */ +bool tomoyo_policy_loaded; + +/** + * tomoyo_init - Register TOMOYO Linux as a LSM module. + * + * Returns 0. + */ +static int __init tomoyo_init(void) +{ + struct tomoyo_task *s = tomoyo_task(current); + + /* register ourselves with the security framework */ + security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks), + &tomoyo_lsmid); + pr_info("TOMOYO Linux initialized\n"); + s->domain_info = &tomoyo_kernel_domain; + atomic_inc(&tomoyo_kernel_domain.users); + s->old_domain_info = NULL; + tomoyo_mm_init(); + + return 0; +} + +DEFINE_LSM(tomoyo) = { + .name = "tomoyo", + .enabled = &tomoyo_enabled, + .flags = LSM_FLAG_LEGACY_MAJOR, + .blobs = &tomoyo_blob_sizes, + .init = tomoyo_init, +}; diff --git a/security/tomoyo/securityfs_if.c b/security/tomoyo/securityfs_if.c index a2705798476f..7e69747b2f77 100644 --- a/security/tomoyo/securityfs_if.c +++ b/security/tomoyo/securityfs_if.c @@ -229,11 +229,11 @@ static void __init tomoyo_create_entry(const char *name, const umode_t mode, } /** - * tomoyo_initerface_init - Initialize /sys/kernel/security/tomoyo/ interface. + * tomoyo_interface_init - Initialize /sys/kernel/security/tomoyo/ interface. * * Returns 0. */ -static int __init tomoyo_initerface_init(void) +static int __init tomoyo_interface_init(void) { struct tomoyo_domain_info *domain; struct dentry *tomoyo_dir; @@ -270,4 +270,4 @@ static int __init tomoyo_initerface_init(void) return 0; } -fs_initcall(tomoyo_initerface_init); +fs_initcall(tomoyo_interface_init); diff --git a/security/tomoyo/util.c b/security/tomoyo/util.c index 6799b1122c9d..b851ff377382 100644 --- a/security/tomoyo/util.c +++ b/security/tomoyo/util.c @@ -13,9 +13,6 @@ /* Lock for protecting policy. */ DEFINE_MUTEX(tomoyo_policy_lock); -/* Has /sbin/init started? */ -bool tomoyo_policy_loaded; - /* * Mapping table from "enum tomoyo_mac_index" to * "enum tomoyo_mac_category_index". From 66cac80698cd1e31ae9bc5c271e83209903d4861 Mon Sep 17 00:00:00 2001 From: Matthew Sakai Date: Tue, 17 Sep 2024 23:24:37 -0400 Subject: [PATCH 055/104] dm vdo: handle unaligned discards correctly Reset the data_vio properly for each discard block, and delay acknowledgement and cleanup until all discard blocks are complete. Signed-off-by: Matthew Sakai Signed-off-by: Mikulas Patocka --- drivers/md/dm-vdo/data-vio.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/md/dm-vdo/data-vio.c b/drivers/md/dm-vdo/data-vio.c index ab3ea8337809..0d502f6a86ad 100644 --- a/drivers/md/dm-vdo/data-vio.c +++ b/drivers/md/dm-vdo/data-vio.c @@ -501,6 +501,7 @@ static void launch_data_vio(struct data_vio *data_vio, logical_block_number_t lb memset(&data_vio->record_name, 0, sizeof(data_vio->record_name)); memset(&data_vio->duplicate, 0, sizeof(data_vio->duplicate)); + vdo_reset_completion(&data_vio->decrement_completion); vdo_reset_completion(completion); completion->error_handler = handle_data_vio_error; set_data_vio_logical_callback(data_vio, attempt_logical_block_lock); @@ -1273,12 +1274,14 @@ static void clean_hash_lock(struct vdo_completion *completion) static void finish_cleanup(struct data_vio *data_vio) { struct vdo_completion *completion = &data_vio->vio.completion; + u32 discard_size = min_t(u32, data_vio->remaining_discard, + VDO_BLOCK_SIZE - data_vio->offset); VDO_ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL, "complete data_vio has no allocation lock"); VDO_ASSERT_LOG_ONLY(data_vio->hash_lock == NULL, "complete data_vio has no hash lock"); - if ((data_vio->remaining_discard <= VDO_BLOCK_SIZE) || + if ((data_vio->remaining_discard <= discard_size) || (completion->result != VDO_SUCCESS)) { struct data_vio_pool *pool = completion->vdo->data_vio_pool; @@ -1287,12 +1290,12 @@ static void finish_cleanup(struct data_vio *data_vio) return; } - data_vio->remaining_discard -= min_t(u32, data_vio->remaining_discard, - VDO_BLOCK_SIZE - data_vio->offset); + data_vio->remaining_discard -= discard_size; data_vio->is_partial = (data_vio->remaining_discard < VDO_BLOCK_SIZE); data_vio->read = data_vio->is_partial; data_vio->offset = 0; completion->requeue = true; + data_vio->first_reference_operation_complete = false; launch_data_vio(data_vio, data_vio->logical.lbn + 1); } @@ -1965,7 +1968,8 @@ static void allocate_block(struct vdo_completion *completion) .state = VDO_MAPPING_STATE_UNCOMPRESSED, }; - if (data_vio->fua) { + if (data_vio->fua || + data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) { prepare_for_dedupe(data_vio); return; } @@ -2042,7 +2046,6 @@ void continue_data_vio_with_block_map_slot(struct vdo_completion *completion) return; } - /* * We don't need to write any data, so skip allocation and just update the block map and * reference counts (via the journal). @@ -2051,7 +2054,7 @@ void continue_data_vio_with_block_map_slot(struct vdo_completion *completion) if (data_vio->is_zero) data_vio->new_mapped.state = VDO_MAPPING_STATE_UNCOMPRESSED; - if (data_vio->remaining_discard > VDO_BLOCK_SIZE) { + if (data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) { /* This is not the final block of a discard so we can't acknowledge it yet. */ update_metadata_for_data_vio_write(data_vio, NULL); return; From e8dd556c74325f29597665c2c557a8ea699a0686 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 24 Sep 2024 15:31:51 +0800 Subject: [PATCH 056/104] LoongArch: Enable generic CPU vulnerabilites support Currently, many architectures support generic CPU vulnerabilites, such as x86, arm64 and riscv: commit 61dc0f555b5c ("x86/cpu: Implement CPU vulnerabilites sysfs functions") commit 61ae1321f06c ("arm64: enable generic CPU vulnerabilites support") commit 0e3f3649d44b ("riscv: Enable generic CPU vulnerabilites support") All LoongArch CPUs (since Loongson-3A5000) implement a special mechanism in the processor core to prevent "Meltdown" and "Spectre" attacks, so it can enable generic CPU vulnerabilites support for LoongArch too. Without this patch, there are no user interfaces of vulnerabilities to check on LoongArch. The output of those files reflects the state of the CPUs in the system, the output value "Not affected" means "CPU is not affected by the vulnerability". Before: # cat /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow cat: /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow: No such file or directory # cat /sys/devices/system/cpu/vulnerabilities/spec_store_bypass cat: /sys/devices/system/cpu/vulnerabilities/spec_store_bypass: No such file or directory # cat /sys/devices/system/cpu/vulnerabilities/meltdown cat: /sys/devices/system/cpu/vulnerabilities/meltdown: No such file or directory After: # cat /sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow Not affected # cat /sys/devices/system/cpu/vulnerabilities/spec_store_bypass Not affected # cat /sys/devices/system/cpu/vulnerabilities/meltdown Not affected Link: https://www.loongson.cn/EN/news/show?id=633 Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 34531dc8a61f..b9d3c98efe18 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -82,6 +82,7 @@ config LOONGARCH select GENERIC_CMOS_UPDATE select GENERIC_CPU_AUTOPROBE select GENERIC_CPU_DEVICES + select GENERIC_CPU_VULNERABILITIES select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_IOREMAP if !ARCH_IOREMAP From d0bb0b600081bc7c246b97b2901abbb6c357ff7b Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Tue, 24 Sep 2024 15:32:06 +0800 Subject: [PATCH 057/104] LoongArch: Enable ACPI BGRT handling Add ACPI BGRT support on LoongArch so it can display image provied by acpi table at boot stage and switch to graphical UI smoothly. Signed-off-by: Bibo Mao Signed-off-by: Huacai Chen --- arch/loongarch/kernel/acpi.c | 4 ++++ drivers/acpi/Kconfig | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 929a497c987e..f1a74b80f22c 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -212,6 +213,9 @@ void __init acpi_boot_table_init(void) /* Do not enable ACPI SPCR console by default */ acpi_parse_spcr(earlycon_acpi_spcr_enable, false); + if (IS_ENABLED(CONFIG_ACPI_BGRT)) + acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt); + return; fdt_earlycon: diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index e3a7c2aedd5f..d67f63d93b2a 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -451,7 +451,7 @@ config ACPI_HED config ACPI_BGRT bool "Boottime Graphics Resource Table support" - depends on EFI && (X86 || ARM64) + depends on EFI && (X86 || ARM64 || LOONGARCH) help This driver adds support for exposing the ACPI Boottime Graphics Resource Table, which allows the operating system to obtain From 34e3c4500cdc06094b37a41b622598098308ba8f Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 058/104] LoongArch: Rework CPU feature probe from CPUCFG/IOCSR Probe ISA level, TLB, IOCSR information from CPUCFG to improve kernel resilience to different core implementations. BTW, IOCSR register definition appears to be a platform-specific spec instead of an architecture spec, even for the Loongson CPUs there is no guarantee that IOCSR will always present. Thus it's dangerous to perform IOCSR probing without checking CPU type and instruction availability. Signed-off-by: Jiaxun Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/cpu-features.h | 2 + arch/loongarch/include/asm/cpu.h | 30 ++--- arch/loongarch/include/asm/loongarch.h | 1 + arch/loongarch/include/uapi/asm/hwcap.h | 1 + arch/loongarch/kernel/cpu-probe.c | 134 +++++++++++++--------- arch/loongarch/kernel/proc.c | 10 +- 6 files changed, 107 insertions(+), 71 deletions(-) diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index 16a716f88a5c..fc83bb32f9f0 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -51,6 +51,7 @@ #define cpu_has_lbt_mips cpu_opt(LOONGARCH_CPU_LBT_MIPS) #define cpu_has_lbt (cpu_has_lbt_x86|cpu_has_lbt_arm|cpu_has_lbt_mips) #define cpu_has_csr cpu_opt(LOONGARCH_CPU_CSR) +#define cpu_has_iocsr cpu_opt(LOONGARCH_CPU_IOCSR) #define cpu_has_tlb cpu_opt(LOONGARCH_CPU_TLB) #define cpu_has_watch cpu_opt(LOONGARCH_CPU_WATCH) #define cpu_has_vint cpu_opt(LOONGARCH_CPU_VINT) @@ -65,6 +66,7 @@ #define cpu_has_guestid cpu_opt(LOONGARCH_CPU_GUESTID) #define cpu_has_hypervisor cpu_opt(LOONGARCH_CPU_HYPERVISOR) #define cpu_has_ptw cpu_opt(LOONGARCH_CPU_PTW) +#define cpu_has_lspw cpu_opt(LOONGARCH_CPU_LSPW) #define cpu_has_avecint cpu_opt(LOONGARCH_CPU_AVECINT) #endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/loongarch/include/asm/cpu.h b/arch/loongarch/include/asm/cpu.h index 843f9c4ec980..98cf4d7b4b0a 100644 --- a/arch/loongarch/include/asm/cpu.h +++ b/arch/loongarch/include/asm/cpu.h @@ -87,19 +87,21 @@ enum cpu_type_enum { #define CPU_FEATURE_LBT_MIPS 12 /* CPU has MIPS Binary Translation */ #define CPU_FEATURE_TLB 13 /* CPU has TLB */ #define CPU_FEATURE_CSR 14 /* CPU has CSR */ -#define CPU_FEATURE_WATCH 15 /* CPU has watchpoint registers */ -#define CPU_FEATURE_VINT 16 /* CPU has vectored interrupts */ -#define CPU_FEATURE_CSRIPI 17 /* CPU has CSR-IPI */ -#define CPU_FEATURE_EXTIOI 18 /* CPU has EXT-IOI */ -#define CPU_FEATURE_PREFETCH 19 /* CPU has prefetch instructions */ -#define CPU_FEATURE_PMP 20 /* CPU has perfermance counter */ -#define CPU_FEATURE_SCALEFREQ 21 /* CPU supports cpufreq scaling */ -#define CPU_FEATURE_FLATMODE 22 /* CPU has flat mode */ -#define CPU_FEATURE_EIODECODE 23 /* CPU has EXTIOI interrupt pin decode mode */ -#define CPU_FEATURE_GUESTID 24 /* CPU has GuestID feature */ -#define CPU_FEATURE_HYPERVISOR 25 /* CPU has hypervisor (running in VM) */ -#define CPU_FEATURE_PTW 26 /* CPU has hardware page table walker */ -#define CPU_FEATURE_AVECINT 27 /* CPU has avec interrupt */ +#define CPU_FEATURE_IOCSR 15 /* CPU has IOCSR */ +#define CPU_FEATURE_WATCH 16 /* CPU has watchpoint registers */ +#define CPU_FEATURE_VINT 17 /* CPU has vectored interrupts */ +#define CPU_FEATURE_CSRIPI 18 /* CPU has CSR-IPI */ +#define CPU_FEATURE_EXTIOI 19 /* CPU has EXT-IOI */ +#define CPU_FEATURE_PREFETCH 20 /* CPU has prefetch instructions */ +#define CPU_FEATURE_PMP 21 /* CPU has perfermance counter */ +#define CPU_FEATURE_SCALEFREQ 22 /* CPU supports cpufreq scaling */ +#define CPU_FEATURE_FLATMODE 23 /* CPU has flat mode */ +#define CPU_FEATURE_EIODECODE 24 /* CPU has EXTIOI interrupt pin decode mode */ +#define CPU_FEATURE_GUESTID 25 /* CPU has GuestID feature */ +#define CPU_FEATURE_HYPERVISOR 26 /* CPU has hypervisor (running in VM) */ +#define CPU_FEATURE_PTW 27 /* CPU has hardware page table walker */ +#define CPU_FEATURE_LSPW 28 /* CPU has LSPW (lddir/ldpte instructions) */ +#define CPU_FEATURE_AVECINT 29 /* CPU has AVEC interrupt */ #define LOONGARCH_CPU_CPUCFG BIT_ULL(CPU_FEATURE_CPUCFG) #define LOONGARCH_CPU_LAM BIT_ULL(CPU_FEATURE_LAM) @@ -115,6 +117,7 @@ enum cpu_type_enum { #define LOONGARCH_CPU_LBT_ARM BIT_ULL(CPU_FEATURE_LBT_ARM) #define LOONGARCH_CPU_LBT_MIPS BIT_ULL(CPU_FEATURE_LBT_MIPS) #define LOONGARCH_CPU_TLB BIT_ULL(CPU_FEATURE_TLB) +#define LOONGARCH_CPU_IOCSR BIT_ULL(CPU_FEATURE_IOCSR) #define LOONGARCH_CPU_CSR BIT_ULL(CPU_FEATURE_CSR) #define LOONGARCH_CPU_WATCH BIT_ULL(CPU_FEATURE_WATCH) #define LOONGARCH_CPU_VINT BIT_ULL(CPU_FEATURE_VINT) @@ -128,6 +131,7 @@ enum cpu_type_enum { #define LOONGARCH_CPU_GUESTID BIT_ULL(CPU_FEATURE_GUESTID) #define LOONGARCH_CPU_HYPERVISOR BIT_ULL(CPU_FEATURE_HYPERVISOR) #define LOONGARCH_CPU_PTW BIT_ULL(CPU_FEATURE_PTW) +#define LOONGARCH_CPU_LSPW BIT_ULL(CPU_FEATURE_LSPW) #define LOONGARCH_CPU_AVECINT BIT_ULL(CPU_FEATURE_AVECINT) #endif /* _ASM_CPU_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 631d249b3ef2..4084cb64ba06 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -62,6 +62,7 @@ #define LOONGARCH_CPUCFG1 0x1 #define CPUCFG1_ISGR32 BIT(0) #define CPUCFG1_ISGR64 BIT(1) +#define CPUCFG1_ISA GENMASK(1, 0) #define CPUCFG1_PAGING BIT(2) #define CPUCFG1_IOCSR BIT(3) #define CPUCFG1_PABITS GENMASK(11, 4) diff --git a/arch/loongarch/include/uapi/asm/hwcap.h b/arch/loongarch/include/uapi/asm/hwcap.h index 6955a7cb2c65..2b34e56cfa9e 100644 --- a/arch/loongarch/include/uapi/asm/hwcap.h +++ b/arch/loongarch/include/uapi/asm/hwcap.h @@ -17,5 +17,6 @@ #define HWCAP_LOONGARCH_LBT_ARM (1 << 11) #define HWCAP_LOONGARCH_LBT_MIPS (1 << 12) #define HWCAP_LOONGARCH_PTW (1 << 13) +#define HWCAP_LOONGARCH_LSPW (1 << 14) #endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 14f0449f5452..cbce099037b2 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -91,12 +91,30 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) unsigned int config; unsigned long asid_mask; - c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | - LOONGARCH_CPU_TLB | LOONGARCH_CPU_VINT | LOONGARCH_CPU_WATCH; + c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | LOONGARCH_CPU_VINT; elf_hwcap = HWCAP_LOONGARCH_CPUCFG; config = read_cpucfg(LOONGARCH_CPUCFG1); + + switch (config & CPUCFG1_ISA) { + case 0: + set_isa(c, LOONGARCH_CPU_ISA_LA32R); + break; + case 1: + set_isa(c, LOONGARCH_CPU_ISA_LA32S); + break; + case 2: + set_isa(c, LOONGARCH_CPU_ISA_LA64); + break; + default: + pr_warn("Warning: unknown ISA level\n"); + } + + if (config & CPUCFG1_PAGING) + c->options |= LOONGARCH_CPU_TLB; + if (config & CPUCFG1_IOCSR) + c->options |= LOONGARCH_CPU_IOCSR; if (config & CPUCFG1_UAL) { c->options |= LOONGARCH_CPU_UAL; elf_hwcap |= HWCAP_LOONGARCH_UAL; @@ -139,6 +157,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) c->options |= LOONGARCH_CPU_PTW; elf_hwcap |= HWCAP_LOONGARCH_PTW; } + if (config & CPUCFG2_LSPW) { + c->options |= LOONGARCH_CPU_LSPW; + elf_hwcap |= HWCAP_LOONGARCH_LSPW; + } if (config & CPUCFG2_LVZP) { c->options |= LOONGARCH_CPU_LVZ; elf_hwcap |= HWCAP_LOONGARCH_LVZ; @@ -162,22 +184,6 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) if (config & CPUCFG6_PMP) c->options |= LOONGARCH_CPU_PMP; - config = iocsr_read32(LOONGARCH_IOCSR_FEATURES); - if (config & IOCSRF_CSRIPI) - c->options |= LOONGARCH_CPU_CSRIPI; - if (config & IOCSRF_EXTIOI) - c->options |= LOONGARCH_CPU_EXTIOI; - if (config & IOCSRF_FREQSCALE) - c->options |= LOONGARCH_CPU_SCALEFREQ; - if (config & IOCSRF_FLATMODE) - c->options |= LOONGARCH_CPU_FLATMODE; - if (config & IOCSRF_EIODECODE) - c->options |= LOONGARCH_CPU_EIODECODE; - if (config & IOCSRF_AVEC) - c->options |= LOONGARCH_CPU_AVECINT; - if (config & IOCSRF_VM) - c->options |= LOONGARCH_CPU_HYPERVISOR; - config = csr_read32(LOONGARCH_CSR_ASID); config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; asid_mask = GENMASK(config - 1, 0); @@ -210,6 +216,9 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) default: pr_warn("Warning: unknown TLB type\n"); } + + if (get_num_brps() + get_num_wrps()) + c->options |= LOONGARCH_CPU_WATCH; } #define MAX_NAME_LEN 32 @@ -220,8 +229,45 @@ static char cpu_full_name[MAX_NAME_LEN] = " - "; static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int cpu) { + uint32_t config; uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]); uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]); + const char *core_name = "Unknown"; + + switch (BIT(fls(c->isa_level) - 1)) { + case LOONGARCH_CPU_ISA_LA32R: + case LOONGARCH_CPU_ISA_LA32S: + c->cputype = CPU_LOONGSON32; + __cpu_family[cpu] = "Loongson-32bit"; + break; + case LOONGARCH_CPU_ISA_LA64: + c->cputype = CPU_LOONGSON64; + __cpu_family[cpu] = "Loongson-64bit"; + break; + } + + switch (c->processor_id & PRID_SERIES_MASK) { + case PRID_SERIES_LA132: + core_name = "LA132"; + break; + case PRID_SERIES_LA264: + core_name = "LA264"; + break; + case PRID_SERIES_LA364: + core_name = "LA364"; + break; + case PRID_SERIES_LA464: + core_name = "LA464"; + break; + case PRID_SERIES_LA664: + core_name = "LA664"; + break; + } + + pr_info("%s Processor probed (%s Core)\n", __cpu_family[cpu], core_name); + + if (!cpu_has_iocsr) + return; if (!__cpu_full_name[cpu]) __cpu_full_name[cpu] = cpu_full_name; @@ -229,43 +275,21 @@ static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR); *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); - switch (c->processor_id & PRID_SERIES_MASK) { - case PRID_SERIES_LA132: - c->cputype = CPU_LOONGSON32; - set_isa(c, LOONGARCH_CPU_ISA_LA32S); - __cpu_family[cpu] = "Loongson-32bit"; - pr_info("32-bit Loongson Processor probed (LA132 Core)\n"); - break; - case PRID_SERIES_LA264: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA264 Core)\n"); - break; - case PRID_SERIES_LA364: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA364 Core)\n"); - break; - case PRID_SERIES_LA464: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA464 Core)\n"); - break; - case PRID_SERIES_LA664: - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (LA664 Core)\n"); - break; - default: /* Default to 64 bit */ - c->cputype = CPU_LOONGSON64; - set_isa(c, LOONGARCH_CPU_ISA_LA64); - __cpu_family[cpu] = "Loongson-64bit"; - pr_info("64-bit Loongson Processor probed (Unknown Core)\n"); - } + config = iocsr_read32(LOONGARCH_IOCSR_FEATURES); + if (config & IOCSRF_CSRIPI) + c->options |= LOONGARCH_CPU_CSRIPI; + if (config & IOCSRF_EXTIOI) + c->options |= LOONGARCH_CPU_EXTIOI; + if (config & IOCSRF_FREQSCALE) + c->options |= LOONGARCH_CPU_SCALEFREQ; + if (config & IOCSRF_FLATMODE) + c->options |= LOONGARCH_CPU_FLATMODE; + if (config & IOCSRF_EIODECODE) + c->options |= LOONGARCH_CPU_EIODECODE; + if (config & IOCSRF_AVEC) + c->options |= LOONGARCH_CPU_AVECINT; + if (config & IOCSRF_VM) + c->options |= LOONGARCH_CPU_HYPERVISOR; } #ifdef CONFIG_64BIT diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c index 0d33cbc47e51..6ce46d92f1f1 100644 --- a/arch/loongarch/kernel/proc.c +++ b/arch/loongarch/kernel/proc.c @@ -31,6 +31,7 @@ int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v) static int show_cpuinfo(struct seq_file *m, void *v) { unsigned long n = (unsigned long) v - 1; + unsigned int isa = cpu_data[n].isa_level; unsigned int version = cpu_data[n].processor_id & 0xff; unsigned int fp_version = cpu_data[n].fpu_vers; struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; @@ -64,9 +65,11 @@ static int show_cpuinfo(struct seq_file *m, void *v) cpu_pabits + 1, cpu_vabits + 1); seq_printf(m, "ISA\t\t\t:"); - if (cpu_has_loongarch32) - seq_printf(m, " loongarch32"); - if (cpu_has_loongarch64) + if (isa & LOONGARCH_CPU_ISA_LA32R) + seq_printf(m, " loongarch32r"); + if (isa & LOONGARCH_CPU_ISA_LA32S) + seq_printf(m, " loongarch32s"); + if (isa & LOONGARCH_CPU_ISA_LA64) seq_printf(m, " loongarch64"); seq_printf(m, "\n"); @@ -81,6 +84,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) if (cpu_has_complex) seq_printf(m, " complex"); if (cpu_has_crypto) seq_printf(m, " crypto"); if (cpu_has_ptw) seq_printf(m, " ptw"); + if (cpu_has_lspw) seq_printf(m, " lspw"); if (cpu_has_lvz) seq_printf(m, " lvz"); if (cpu_has_lbt_x86) seq_printf(m, " lbt_x86"); if (cpu_has_lbt_arm) seq_printf(m, " lbt_arm"); From e86935f705fa732d8c7c3ecf0c50ea461ffab76f Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 059/104] LoongArch: Add ARCH_HAS_SET_MEMORY support Add set_memory_ro/rw/x/nx architecture hooks to change the page attribution. Use own set_memory.h rather than generic set_memory.h (i.e. include/asm-generic/set_memory.h), because we want to add other function prototypes here. Note: We can only set attributes for KVRANGE/XKVRANGE kernel addresses. Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/set_memory.h | 17 +++ arch/loongarch/mm/Makefile | 3 +- arch/loongarch/mm/pageattr.c | 158 ++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 arch/loongarch/include/asm/set_memory.h create mode 100644 arch/loongarch/mm/pageattr.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index b9d3c98efe18..46fa58bde685 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -25,6 +25,7 @@ config LOONGARCH select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SET_MEMORY select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h new file mode 100644 index 000000000000..64c7f942e8ec --- /dev/null +++ b/arch/loongarch/include/asm/set_memory.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_LOONGARCH_SET_MEMORY_H +#define _ASM_LOONGARCH_SET_MEMORY_H + +/* + * Functions to change memory attributes. + */ +int set_memory_x(unsigned long addr, int numpages); +int set_memory_nx(unsigned long addr, int numpages); +int set_memory_ro(unsigned long addr, int numpages); +int set_memory_rw(unsigned long addr, int numpages); + +#endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/Makefile b/arch/loongarch/mm/Makefile index e4d1e581dbae..278be2c8fc36 100644 --- a/arch/loongarch/mm/Makefile +++ b/arch/loongarch/mm/Makefile @@ -4,7 +4,8 @@ # obj-y += init.o cache.o tlb.o tlbex.o extable.o \ - fault.o ioremap.o maccess.o mmap.o pgtable.o page.o + fault.o ioremap.o maccess.o mmap.o pgtable.o \ + page.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c new file mode 100644 index 000000000000..e3db1c38b5fe --- /dev/null +++ b/arch/loongarch/mm/pageattr.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include + +struct pageattr_masks { + pgprot_t set_mask; + pgprot_t clear_mask; +}; + +static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) +{ + unsigned long new_val = val; + struct pageattr_masks *masks = walk->private; + + new_val &= ~(pgprot_val(masks->clear_mask)); + new_val |= (pgprot_val(masks->set_mask)); + + return new_val; +} + +static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pgd_t val = pgdp_get(pgd); + + if (pgd_leaf(val)) { + val = __pgd(set_pageattr_masks(pgd_val(val), walk)); + set_pgd(pgd, val); + } + + return 0; +} + +static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + p4d_t val = p4dp_get(p4d); + + if (p4d_leaf(val)) { + val = __p4d(set_pageattr_masks(p4d_val(val), walk)); + set_p4d(p4d, val); + } + + return 0; +} + +static int pageattr_pud_entry(pud_t *pud, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pud_t val = pudp_get(pud); + + if (pud_leaf(val)) { + val = __pud(set_pageattr_masks(pud_val(val), walk)); + set_pud(pud, val); + } + + return 0; +} + +static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pmd_t val = pmdp_get(pmd); + + if (pmd_leaf(val)) { + val = __pmd(set_pageattr_masks(pmd_val(val), walk)); + set_pmd(pmd, val); + } + + return 0; +} + +static int pageattr_pte_entry(pte_t *pte, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t val = ptep_get(pte); + + val = __pte(set_pageattr_masks(pte_val(val), walk)); + set_pte(pte, val); + + return 0; +} + +static int pageattr_pte_hole(unsigned long addr, unsigned long next, + int depth, struct mm_walk *walk) +{ + return 0; +} + +static const struct mm_walk_ops pageattr_ops = { + .pgd_entry = pageattr_pgd_entry, + .p4d_entry = pageattr_p4d_entry, + .pud_entry = pageattr_pud_entry, + .pmd_entry = pageattr_pmd_entry, + .pte_entry = pageattr_pte_entry, + .pte_hole = pageattr_pte_hole, + .walk_lock = PGWALK_RDLOCK, +}; + +static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask) +{ + int ret; + unsigned long start = addr; + unsigned long end = start + PAGE_SIZE * numpages; + struct pageattr_masks masks = { + .set_mask = set_mask, + .clear_mask = clear_mask + }; + + if (!numpages) + return 0; + + mmap_write_lock(&init_mm); + ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, &masks); + mmap_write_unlock(&init_mm); + + flush_tlb_kernel_range(start, end); + + return ret; +} + +int set_memory_x(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_NO_EXEC)); +} + +int set_memory_nx(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(_PAGE_NO_EXEC), __pgprot(0)); +} + +int set_memory_ro(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_WRITE | _PAGE_DIRTY)); +} + +int set_memory_rw(unsigned long addr, int numpages) +{ + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0)); +} From f04de6d8f252ec6434b846895474cc205527b8b8 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 060/104] LoongArch: Add ARCH_HAS_SET_DIRECT_MAP support Add set_direct_map_*() functions for setting the direct map alias for the page to its default permissions and to an invalid state that cannot be cached in a TLB. (See d253ca0c3 ("x86/mm/cpa: Add set_direct_map_*() functions")) Add a similar implementation for LoongArch. This fixes the KFENCE warnings during hibernation: ================================================================== BUG: KFENCE: invalid read in swsusp_save+0x368/0x4d8 Invalid read at 0x00000000f7b89a3c: swsusp_save+0x368/0x4d8 hibernation_snapshot+0x3f0/0x4e0 hibernate+0x20c/0x440 state_store+0x128/0x140 kernfs_fop_write_iter+0x160/0x260 vfs_write+0x2c0/0x520 ksys_write+0x74/0x160 do_syscall+0xb0/0x160 CPU: 0 UID: 0 PID: 812 Comm: bash Tainted: G B 6.11.0-rc1+ #1566 Tainted: [B]=BAD_PAGE Hardware name: Loongson-LS3A5000-7A1000-1w-CRB, BIOS vUDK2018-LoongArch-V2.0.0 10/21/2022 ================================================================== Note: We can only set permissions for KVRANGE/XKVRANGE kernel addresses. Signed-off-by: Huacai Chen --- arch/loongarch/Kconfig | 1 + arch/loongarch/include/asm/set_memory.h | 4 ++ arch/loongarch/mm/pageattr.c | 60 +++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 46fa58bde685..4b02b0492c3b 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -26,6 +26,7 @@ config LOONGARCH select ARCH_HAS_PTE_DEVMAP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SET_MEMORY + select ARCH_HAS_SET_DIRECT_MAP select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h index 64c7f942e8ec..d70505b6676c 100644 --- a/arch/loongarch/include/asm/set_memory.h +++ b/arch/loongarch/include/asm/set_memory.h @@ -14,4 +14,8 @@ int set_memory_nx(unsigned long addr, int numpages); int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); +bool kernel_page_present(struct page *page); +int set_direct_map_default_noflush(struct page *page); +int set_direct_map_invalid_noflush(struct page *page); + #endif /* _ASM_LOONGARCH_SET_MEMORY_H */ diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c index e3db1c38b5fe..ffd8d76021d4 100644 --- a/arch/loongarch/mm/pageattr.c +++ b/arch/loongarch/mm/pageattr.c @@ -156,3 +156,63 @@ int set_memory_rw(unsigned long addr, int numpages) return __set_memory(addr, numpages, __pgprot(_PAGE_WRITE | _PAGE_DIRTY), __pgprot(0)); } + +bool kernel_page_present(struct page *page) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return true; + + pgd = pgd_offset_k(addr); + if (pgd_none(pgdp_get(pgd))) + return false; + if (pgd_leaf(pgdp_get(pgd))) + return true; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(p4dp_get(p4d))) + return false; + if (p4d_leaf(p4dp_get(p4d))) + return true; + + pud = pud_offset(p4d, addr); + if (pud_none(pudp_get(pud))) + return false; + if (pud_leaf(pudp_get(pud))) + return true; + + pmd = pmd_offset(pud, addr); + if (pmd_none(pmdp_get(pmd))) + return false; + if (pmd_leaf(pmdp_get(pmd))) + return true; + + pte = pte_offset_kernel(pmd, addr); + return pte_present(ptep_get(pte)); +} + +int set_direct_map_default_noflush(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0)); +} + +int set_direct_map_invalid_noflush(struct page *page) +{ + unsigned long addr = (unsigned long)page_address(page); + + if (addr < vm_map_base) + return 0; + + return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID)); +} From f93f67d06b1023313ef1662eac490e29c025c030 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 061/104] LoongArch: Improve hardware page table walker LoongArch has similar problems explained in commit 7f0b1bf04511348995d6 ("arm64: Fix barriers used for page table modifications"), when hardware page table walker (PTW) enabled, speculative accesses may cause spurious page fault in kernel space. Theoretically, in order to completely avoid spurious page fault we need a "dbar + ibar" pair between the page table modifications and the subsequent memory accesses using the corresponding virtual address. But "ibar" is too heavy for performace, so we only use a "dbar 0b11000" in set_pte(). And let spurious_fault() filter the rest rare spurious page faults which should be avoided by "ibar". Besides, we replace the llsc loop with amo in set_pte() which has better performace, and refactor mmu_context.h to 1) avoid any load/store/branch instructions between the writing of CSR.ASID & CSR.PGDL, 2) ensure flush tlb operation is after updating ASID. Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/atomic.h | 2 ++ arch/loongarch/include/asm/mmu_context.h | 35 +++++++++++++++----- arch/loongarch/include/asm/pgtable.h | 30 +++++++---------- arch/loongarch/mm/fault.c | 41 ++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h index 99af8b3160a8..c86f0ab922ec 100644 --- a/arch/loongarch/include/asm/atomic.h +++ b/arch/loongarch/include/asm/atomic.h @@ -15,6 +15,7 @@ #define __LL "ll.w " #define __SC "sc.w " #define __AMADD "amadd.w " +#define __AMOR "amor.w " #define __AMAND_DB "amand_db.w " #define __AMOR_DB "amor_db.w " #define __AMXOR_DB "amxor_db.w " @@ -22,6 +23,7 @@ #define __LL "ll.d " #define __SC "sc.d " #define __AMADD "amadd.d " +#define __AMOR "amor.d " #define __AMAND_DB "amand_db.d " #define __AMOR_DB "amor_db.d " #define __AMXOR_DB "amxor_db.d " diff --git a/arch/loongarch/include/asm/mmu_context.h b/arch/loongarch/include/asm/mmu_context.h index 9f97c3453b9c..304363bd3935 100644 --- a/arch/loongarch/include/asm/mmu_context.h +++ b/arch/loongarch/include/asm/mmu_context.h @@ -49,12 +49,12 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) /* Normal, classic get_new_mmu_context */ static inline void -get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) +get_new_mmu_context(struct mm_struct *mm, unsigned long cpu, bool *need_flush) { u64 asid = asid_cache(cpu); if (!((++asid) & cpu_asid_mask(&cpu_data[cpu]))) - local_flush_tlb_user(); /* start new asid cycle */ + *need_flush = true; /* start new asid cycle */ cpu_context(cpu, mm) = asid_cache(cpu) = asid; } @@ -74,21 +74,34 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +static inline void atomic_update_pgd_asid(unsigned long asid, unsigned long pgdl) +{ + __asm__ __volatile__( + "csrwr %[pgdl_val], %[pgdl_reg] \n\t" + "csrwr %[asid_val], %[asid_reg] \n\t" + : [asid_val] "+r" (asid), [pgdl_val] "+r" (pgdl) + : [asid_reg] "i" (LOONGARCH_CSR_ASID), [pgdl_reg] "i" (LOONGARCH_CSR_PGDL) + : "memory" + ); +} + static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { + bool need_flush = false; unsigned int cpu = smp_processor_id(); /* Check if our ASID is of an older version and thus invalid */ if (!asid_valid(next, cpu)) - get_new_mmu_context(next, cpu); - - write_csr_asid(cpu_asid(cpu, next)); + get_new_mmu_context(next, cpu, &need_flush); if (next != &init_mm) - csr_write64((unsigned long)next->pgd, LOONGARCH_CSR_PGDL); + atomic_update_pgd_asid(cpu_asid(cpu, next), (unsigned long)next->pgd); else - csr_write64((unsigned long)invalid_pg_dir, LOONGARCH_CSR_PGDL); + atomic_update_pgd_asid(cpu_asid(cpu, next), (unsigned long)invalid_pg_dir); + + if (need_flush) + local_flush_tlb_user(); /* Flush tlb after update ASID */ /* * Mark current->active_mm as not "active" anymore. @@ -135,9 +148,15 @@ drop_mmu_context(struct mm_struct *mm, unsigned int cpu) asid = read_csr_asid() & cpu_asid_mask(¤t_cpu_data); if (asid == cpu_asid(cpu, mm)) { + bool need_flush = false; + if (!current->mm || (current->mm == mm)) { - get_new_mmu_context(mm, cpu); + get_new_mmu_context(mm, cpu, &need_flush); + write_csr_asid(cpu_asid(cpu, mm)); + if (need_flush) + local_flush_tlb_user(); /* Flush tlb after update ASID */ + goto out; } } diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 85431f20a14d..9965f52ef65b 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -331,29 +331,23 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) * Make sure the buddy is global too (if it's !none, * it better already be global) */ + if (pte_none(ptep_get(buddy))) { #ifdef CONFIG_SMP - /* - * For SMP, multiple CPUs can race, so we need to do - * this atomically. - */ - unsigned long page_global = _PAGE_GLOBAL; - unsigned long tmp; + /* + * For SMP, multiple CPUs can race, so we need + * to do this atomically. + */ + __asm__ __volatile__( + __AMOR "$zero, %[global], %[buddy] \n" + : [buddy] "+ZB" (buddy->pte) + : [global] "r" (_PAGE_GLOBAL) + : "memory"); - __asm__ __volatile__ ( - "1:" __LL "%[tmp], %[buddy] \n" - " bnez %[tmp], 2f \n" - " or %[tmp], %[tmp], %[global] \n" - __SC "%[tmp], %[buddy] \n" - " beqz %[tmp], 1b \n" - " nop \n" - "2: \n" - __WEAK_LLSC_MB - : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp) - : [global] "r" (page_global)); + DBAR(0b11000); /* o_wrw = 0b11000 */ #else /* !CONFIG_SMP */ - if (pte_none(ptep_get(buddy))) WRITE_ONCE(*buddy, __pte(pte_val(ptep_get(buddy)) | _PAGE_GLOBAL)); #endif /* CONFIG_SMP */ + } } } diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c index 97b40defde06..deefd9617d00 100644 --- a/arch/loongarch/mm/fault.c +++ b/arch/loongarch/mm/fault.c @@ -31,11 +31,52 @@ int show_unhandled_signals = 1; +static int __kprobes spurious_fault(unsigned long write, unsigned long address) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + if (!(address & __UA_LIMIT)) + return 0; + + pgd = pgd_offset_k(address); + if (!pgd_present(pgdp_get(pgd))) + return 0; + + p4d = p4d_offset(pgd, address); + if (!p4d_present(p4dp_get(p4d))) + return 0; + + pud = pud_offset(p4d, address); + if (!pud_present(pudp_get(pud))) + return 0; + + pmd = pmd_offset(pud, address); + if (!pmd_present(pmdp_get(pmd))) + return 0; + + if (pmd_leaf(*pmd)) { + return write ? pmd_write(pmdp_get(pmd)) : 1; + } else { + pte = pte_offset_kernel(pmd, address); + if (!pte_present(ptep_get(pte))) + return 0; + + return write ? pte_write(ptep_get(pte)) : 1; + } +} + static void __kprobes no_context(struct pt_regs *regs, unsigned long write, unsigned long address) { const int field = sizeof(unsigned long) * 2; + if (spurious_fault(write, address)) + return; + /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; From d4f31acf1302088a5b16d1e4de890729acfa9638 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 062/104] LoongArch: Simplify _percpu_read() and _percpu_write() Now _percpu_read() and _percpu_write() macros call __percpu_read() and __percpu_write() static inline functions that result in a single assembly instruction. However, percpu infrastructure expects its leaf definitions to encode the size of their percpu variable, so the patch merges all the asm clauses from the static inline function into the corresponding leaf macros. The secondary effect of this change is to avoid explicit __percpu annotations for function arguments. Currently, __percpu macro is defined in include/linux/compiler_types.h, but with proposed patch [1], __percpu definition will need macros from include/asm-generic/percpu.h, creating forward dependency loop. The proposed solution is the same as x86 architecture uses. [1] https://lore.kernel.org/lkml/20240812115945.484051-4-ubizjak@gmail.com/ Tested-by: Xi Ruoyao Signed-off-by: Uros Bizjak Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/percpu.h | 124 ++++++++-------------------- 1 file changed, 35 insertions(+), 89 deletions(-) diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h index 8f290e5546cf..87be9b14e9da 100644 --- a/arch/loongarch/include/asm/percpu.h +++ b/arch/loongarch/include/asm/percpu.h @@ -68,75 +68,6 @@ PERCPU_OP(and, and, &) PERCPU_OP(or, or, |) #undef PERCPU_OP -static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size) -{ - unsigned long ret; - - switch (size) { - case 1: - __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 2: - __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 4: - __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - case 8: - __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" - : [ret] "=&r"(ret) - : [ptr] "r"(ptr) - : "memory"); - break; - default: - ret = 0; - BUILD_BUG(); - } - - return ret; -} - -static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size) -{ - switch (size) { - case 1: - __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 2: - __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 4: - __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - case 8: - __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" - : - : [val] "r" (val), [ptr] "r" (ptr) - : "memory"); - break; - default: - BUILD_BUG(); - } -} - static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size) { switch (size) { @@ -157,6 +88,33 @@ static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, return 0; } +#define __pcpu_op_1(op) op ".b " +#define __pcpu_op_2(op) op ".h " +#define __pcpu_op_4(op) op ".w " +#define __pcpu_op_8(op) op ".d " + +#define _percpu_read(size, _pcp) \ +({ \ + typeof(_pcp) __pcp_ret; \ + \ + __asm__ __volatile__( \ + __pcpu_op_##size("ldx") "%[ret], $r21, %[ptr] \n" \ + : [ret] "=&r"(__pcp_ret) \ + : [ptr] "r"(&(_pcp)) \ + : "memory"); \ + \ + __pcp_ret; \ +}) + +#define _percpu_write(size, _pcp, _val) \ +do { \ + __asm__ __volatile__( \ + __pcpu_op_##size("stx") "%[val], $r21, %[ptr] \n" \ + : \ + : [val] "r"(_val), [ptr] "r"(&(_pcp)) \ + : "memory"); \ +} while (0) + /* this_cpu_cmpxchg */ #define _protect_cmpxchg_local(pcp, o, n) \ ({ \ @@ -167,18 +125,6 @@ static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, __ret; \ }) -#define _percpu_read(pcp) \ -({ \ - typeof(pcp) __retval; \ - __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ - __retval; \ -}) - -#define _percpu_write(pcp, val) \ -do { \ - __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ -} while (0) \ - #define _pcp_protect(operation, pcp, val) \ ({ \ typeof(pcp) __retval; \ @@ -215,15 +161,15 @@ do { \ #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) -#define this_cpu_read_1(pcp) _percpu_read(pcp) -#define this_cpu_read_2(pcp) _percpu_read(pcp) -#define this_cpu_read_4(pcp) _percpu_read(pcp) -#define this_cpu_read_8(pcp) _percpu_read(pcp) +#define this_cpu_read_1(pcp) _percpu_read(1, pcp) +#define this_cpu_read_2(pcp) _percpu_read(2, pcp) +#define this_cpu_read_4(pcp) _percpu_read(4, pcp) +#define this_cpu_read_8(pcp) _percpu_read(8, pcp) -#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) -#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_1(pcp, val) _percpu_write(1, pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val) #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) From 5016c3a31a6d74eaf2fdfdec673eae8fcf90379e Mon Sep 17 00:00:00 2001 From: Wentao Guan Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 063/104] LoongArch: Fix memleak in pci_acpi_scan_root() Add kfree(root_ops) in this case to avoid memleak of root_ops, leaks when pci_find_bus() != 0. Signed-off-by: Yuli Wang Signed-off-by: Wentao Guan Signed-off-by: Huacai Chen --- arch/loongarch/pci/acpi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c index 365f7de771cb..1da4dc46df43 100644 --- a/arch/loongarch/pci/acpi.c +++ b/arch/loongarch/pci/acpi.c @@ -225,6 +225,7 @@ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) if (bus) { memcpy(bus->sysdata, info->cfg, sizeof(struct pci_config_window)); kfree(info); + kfree(root_ops); } else { struct pci_bus *child; From 64c35d6c0ff95e9507f5fb3cce4936c7c62f3d3a Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 064/104] LoongArch: Remove posix_types.h include from sigcontext.h Nothing in sigcontext.h seems to require anything from linux/posix_types.h. This include seems a MIPS relic originated from an error in Linux 2.6.11-rc2 (in 2005). The unneeded include was found debugging some vDSO self test build failure (it's not the root cause though). Link: https://lore.kernel.org/linux-mips/20240828030413.143930-2-xry111@xry111.site/ Link: https://lore.kernel.org/loongarch/0b540679ec8cfccec75aeb3463810924f6ff71e6.camel@xry111.site/ Signed-off-by: Xi Ruoyao Signed-off-by: Huacai Chen --- arch/loongarch/include/uapi/asm/sigcontext.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/loongarch/include/uapi/asm/sigcontext.h b/arch/loongarch/include/uapi/asm/sigcontext.h index 6c22f616b8f1..5cd121275bac 100644 --- a/arch/loongarch/include/uapi/asm/sigcontext.h +++ b/arch/loongarch/include/uapi/asm/sigcontext.h @@ -9,7 +9,6 @@ #define _UAPI_ASM_SIGCONTEXT_H #include -#include /* FP context was used */ #define SC_USED_FP (1 << 0) From f339bd3b51dac675fbbc08b861d2371ae3df0c0b Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Tue, 24 Sep 2024 15:32:20 +0800 Subject: [PATCH 065/104] Docs/LoongArch: Add advanced extended IRQ model description Introduce the advanced extended interrupt controllers (AVECINTC). This feature will allow each core to have 256 independent interrupt vectors and MSI interrupts can be independently routed to any vector on any CPU. The whole topology of irqchips in LoongArch machines looks like this if AVECINTC is supported: +-----+ +-----------------------+ +-------+ | IPI | --> | CPUINTC | <-- | Timer | +-----+ +-----------------------+ +-------+ ^ ^ ^ | | | +---------+ +----------+ +---------+ +-------+ | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | +---------+ +----------+ +---------+ +-------+ ^ ^ | | +---------+ +---------+ | PCH-PIC | | PCH-MSI | +---------+ +---------+ ^ ^ ^ | | | +---------+ +---------+ +---------+ | Devices | | PCH-LPC | | Devices | +---------+ +---------+ +---------+ ^ | +---------+ | Devices | +---------+ Signed-off-by: Huacai Chen Signed-off-by: Tianyang Zhang --- .../arch/loongarch/irq-chip-model.rst | 32 +++++++++++++++++++ .../zh_CN/arch/loongarch/irq-chip-model.rst | 32 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/Documentation/arch/loongarch/irq-chip-model.rst b/Documentation/arch/loongarch/irq-chip-model.rst index 7988f4192363..6dd48256e39f 100644 --- a/Documentation/arch/loongarch/irq-chip-model.rst +++ b/Documentation/arch/loongarch/irq-chip-model.rst @@ -85,6 +85,38 @@ to CPUINTC directly:: | Devices | +---------+ +Advanced Extended IRQ model +=========================== + +In this model, IPI (Inter-Processor Interrupt) and CPU Local Timer interrupt go +to CPUINTC directly, CPU UARTS interrupts go to LIOINTC, PCH-MSI interrupts go +to AVECINTC, and then go to CPUINTC directly, while all other devices interrupts +go to PCH-PIC/PCH-LPC and gathered by EIOINTC, and then go to CPUINTC directly:: + + +-----+ +-----------------------+ +-------+ + | IPI | --> | CPUINTC | <-- | Timer | + +-----+ +-----------------------+ +-------+ + ^ ^ ^ + | | | + +---------+ +----------+ +---------+ +-------+ + | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | + +---------+ +----------+ +---------+ +-------+ + ^ ^ + | | + +---------+ +---------+ + | PCH-PIC | | PCH-MSI | + +---------+ +---------+ + ^ ^ ^ + | | | + +---------+ +---------+ +---------+ + | Devices | | PCH-LPC | | Devices | + +---------+ +---------+ +---------+ + ^ + | + +---------+ + | Devices | + +---------+ + ACPI-related definitions ======================== diff --git a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst index f1e9ab18206c..472761938682 100644 --- a/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst +++ b/Documentation/translations/zh_CN/arch/loongarch/irq-chip-model.rst @@ -87,6 +87,38 @@ PCH-LPC/PCH-MSI，然后被EIOINTC统一收集，再直接到达CPUINTC:: | Devices | +---------+ +高级扩展IRQ模型 +=============== + +在这种模型里面，IPI（Inter-Processor Interrupt）和CPU本地时钟中断直接发送到CPUINTC， +CPU串口（UARTs）中断发送到LIOINTC，PCH-MSI中断发送到AVECINTC，而后通过AVECINTC直接 +送达CPUINTC，而其他所有设备的中断则分别发送到所连接的PCH-PIC/PCH-LPC，然后由EIOINTC +统一收集，再直接到达CPUINTC:: + + +-----+ +-----------------------+ +-------+ + | IPI | --> | CPUINTC | <-- | Timer | + +-----+ +-----------------------+ +-------+ + ^ ^ ^ + | | | + +---------+ +----------+ +---------+ +-------+ + | EIOINTC | | AVECINTC | | LIOINTC | <-- | UARTs | + +---------+ +----------+ +---------+ +-------+ + ^ ^ + | | + +---------+ +---------+ + | PCH-PIC | | PCH-MSI | + +---------+ +---------+ + ^ ^ ^ + | | | + +---------+ +---------+ +---------+ + | Devices | | PCH-LPC | | Devices | + +---------+ +---------+ +---------+ + ^ + | + +---------+ + | Devices | + +---------+ + ACPI相关的定义 ============== From 03a9cfc1314bf75cc7a83995f3a029a7ebf49c05 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Mon, 23 Sep 2024 18:06:46 +0900 Subject: [PATCH 066/104] ata: libata-scsi: Fix ata_msense_control_spgt2() ata_msense_control_spgt2() can be called even for devices that do not support CDL when the user requests the ALL_SUB_MPAGES mode sense page, but for such device, this will cause a NULL pointer dereference as dev->cdl is NULL. Similarly, we should not return any data if ata_msense_control_spgt2() is called when the user requested the CDL_T2A_SUB_MPAGE or CDL_T2B_SUB_MPAGE pages for a device that does not support CDL. Avoid this potential NULL pointer dereference by checking if the device support CDL on entry to ata_msense_control_spgt2() and return 0 if it does not support CDL. Reported-by: syzbot+37757dc11ee77ef850bb@syzkaller.appspotmail.com Tested-by: syzbot+37757dc11ee77ef850bb@syzkaller.appspotmail.com Fixes: 602bcf212637 ("ata: libata: Improve CDL resource management") Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke --- drivers/ata/libata-scsi.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 061fe63497bf..97c84b0ec472 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -2245,10 +2245,15 @@ static inline u16 ata_xlat_cdl_limit(u8 *buf) static unsigned int ata_msense_control_spgt2(struct ata_device *dev, u8 *buf, u8 spg) { - u8 *b, *cdl = dev->cdl->desc_log_buf, *desc; + u8 *b, *cdl, *desc; u32 policy; int i; + if (!(dev->flags & ATA_DFLAG_CDL) || !dev->cdl) + return 0; + + cdl = dev->cdl->desc_log_buf; + /* * Fill the subpage. The first four bytes of the T2A/T2B mode pages * are a header. The PAGE LENGTH field is the size of the page From 0e9a2990a93f27daa643b6fa73cfa47b128947a7 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Mon, 23 Sep 2024 18:14:36 +0900 Subject: [PATCH 067/104] ata: libata-scsi: Fix ata_msense_control() CDL page reporting When the user requests the ALL_SUB_MPAGES mode sense page, ata_msense_control() adds the CDL_T2A_SUB_MPAGE twice instead of adding the CDL_T2A_SUB_MPAGE and CDL_T2B_SUB_MPAGE pages information. Correct the second call to ata_msense_control_spgt2() to report the CDL_T2B_SUB_MPAGE page. Fixes: 673b2fe6ff1d ("scsi: ata: libata-scsi: Add support for CDL pages mode sense") Cc: stable@vger.kernel.org Signed-off-by: Damien Le Moal Reviewed-by: Hannes Reinecke --- drivers/ata/libata-scsi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 97c84b0ec472..ea7d365fb7a9 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -2350,7 +2350,7 @@ static unsigned int ata_msense_control(struct ata_device *dev, u8 *buf, case ALL_SUB_MPAGES: n = ata_msense_control_spg0(dev, buf, changeable); n += ata_msense_control_spgt2(dev, buf + n, CDL_T2A_SUB_MPAGE); - n += ata_msense_control_spgt2(dev, buf + n, CDL_T2A_SUB_MPAGE); + n += ata_msense_control_spgt2(dev, buf + n, CDL_T2B_SUB_MPAGE); n += ata_msense_control_ata_feature(dev, buf + n); return n; default: From 9805f39d423a30a7189158905ec3d71774fe98a1 Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Thu, 19 Sep 2024 17:13:59 +0800 Subject: [PATCH 068/104] LoongArch: vDSO: Tune chacha implementation As Christophe pointed out, tuning the chacha implementation by scheduling the instructions like what GCC does can improve the performance. The tuning does not introduce too much complexity (basically it's just reordering some instructions). And the tuning does not hurt readibility too much: actually the tuned code looks even more similar to a textbook-style implementation based on 128-bit vectors. So overall it's a good deal to me. Tested with vdso_test_getchacha and benched with vdso_test_getrandom. On a LA664 the speedup is 5%, and I expect a larger speedup on LA[2-4]64 with a lower issue rate. Suggested-by: Christophe Leroy Link: https://lore.kernel.org/all/77655d9e-fc05-4300-8f0d-7b2ad840d091@csgroup.eu/ Signed-off-by: Xi Ruoyao Reviewed-by: Huacai Chen Signed-off-by: Jason A. Donenfeld --- arch/loongarch/vdso/vgetrandom-chacha.S | 92 +++++++++++++++---------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/arch/loongarch/vdso/vgetrandom-chacha.S b/arch/loongarch/vdso/vgetrandom-chacha.S index 7e86a50f6e85..c2733e6c3a8d 100644 --- a/arch/loongarch/vdso/vgetrandom-chacha.S +++ b/arch/loongarch/vdso/vgetrandom-chacha.S @@ -9,23 +9,11 @@ .text -/* Salsa20 quarter-round */ -.macro QR a b c d - add.w \a, \a, \b - xor \d, \d, \a - rotri.w \d, \d, 16 - - add.w \c, \c, \d - xor \b, \b, \c - rotri.w \b, \b, 20 - - add.w \a, \a, \b - xor \d, \d, \a - rotri.w \d, \d, 24 - - add.w \c, \c, \d - xor \b, \b, \c - rotri.w \b, \b, 25 +.macro OP_4REG op d0 d1 d2 d3 s0 s1 s2 s3 + \op \d0, \d0, \s0 + \op \d1, \d1, \s1 + \op \d2, \d2, \s2 + \op \d3, \d3, \s3 .endm /* @@ -74,6 +62,23 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) /* Reuse i as copy3 */ #define copy3 i +/* Packs to be used with OP_4REG */ +#define line0 state0, state1, state2, state3 +#define line1 state4, state5, state6, state7 +#define line2 state8, state9, state10, state11 +#define line3 state12, state13, state14, state15 + +#define line1_perm state5, state6, state7, state4 +#define line2_perm state10, state11, state8, state9 +#define line3_perm state15, state12, state13, state14 + +#define copy copy0, copy1, copy2, copy3 + +#define _16 16, 16, 16, 16 +#define _20 20, 20, 20, 20 +#define _24 24, 24, 24, 24 +#define _25 25, 25, 25, 25 + /* * The ABI requires s0-s9 saved, and sp aligned to 16-byte. * This does not violate the stack-less requirement: no sensitive data @@ -126,16 +131,38 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) li.w i, 10 .Lpermute: /* odd round */ - QR state0, state4, state8, state12 - QR state1, state5, state9, state13 - QR state2, state6, state10, state14 - QR state3, state7, state11, state15 + OP_4REG add.w line0, line1 + OP_4REG xor line3, line0 + OP_4REG rotri.w line3, _16 + + OP_4REG add.w line2, line3 + OP_4REG xor line1, line2 + OP_4REG rotri.w line1, _20 + + OP_4REG add.w line0, line1 + OP_4REG xor line3, line0 + OP_4REG rotri.w line3, _24 + + OP_4REG add.w line2, line3 + OP_4REG xor line1, line2 + OP_4REG rotri.w line1, _25 /* even round */ - QR state0, state5, state10, state15 - QR state1, state6, state11, state12 - QR state2, state7, state8, state13 - QR state3, state4, state9, state14 + OP_4REG add.w line0, line1_perm + OP_4REG xor line3_perm, line0 + OP_4REG rotri.w line3_perm, _16 + + OP_4REG add.w line2_perm, line3_perm + OP_4REG xor line1_perm, line2_perm + OP_4REG rotri.w line1_perm, _20 + + OP_4REG add.w line0, line1_perm + OP_4REG xor line3_perm, line0 + OP_4REG rotri.w line3_perm, _24 + + OP_4REG add.w line2_perm, line3_perm + OP_4REG xor line1_perm, line2_perm + OP_4REG rotri.w line1_perm, _25 addi.w i, i, -1 bnez i, .Lpermute @@ -147,10 +174,7 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) li.w copy3, 0x6b206574 /* output[0,1,2,3] = copy[0,1,2,3] + state[0,1,2,3] */ - add.w state0, state0, copy0 - add.w state1, state1, copy1 - add.w state2, state2, copy2 - add.w state3, state3, copy3 + OP_4REG add.w line0, copy st.w state0, output, 0 st.w state1, output, 4 st.w state2, output, 8 @@ -165,10 +189,7 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) ld.w state3, key, 12 /* output[4,5,6,7] = state[0,1,2,3] + state[4,5,6,7] */ - add.w state4, state4, state0 - add.w state5, state5, state1 - add.w state6, state6, state2 - add.w state7, state7, state3 + OP_4REG add.w line1, line0 st.w state4, output, 16 st.w state5, output, 20 st.w state6, output, 24 @@ -181,10 +202,7 @@ SYM_FUNC_START(__arch_chacha20_blocks_nostack) ld.w state3, key, 28 /* output[8,9,10,11] = state[0,1,2,3] + state[8,9,10,11] */ - add.w state8, state8, state0 - add.w state9, state9, state1 - add.w state10, state10, state2 - add.w state11, state11, state3 + OP_4REG add.w line2, line0 st.w state8, output, 32 st.w state9, output, 36 st.w state10, output, 40 From 8b985bbfabbe46c8b9200d7d299030232c8ebd05 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Mon, 23 Sep 2024 19:55:50 +0900 Subject: [PATCH 069/104] tomoyo: allow building as a loadable LSM module One of concerns for enabling TOMOYO in prebuilt kernels is that distributor wants to avoid bloating kernel packages. Although boot-time kernel command line options allows selecting built-in LSMs to enable, file size increase of vmlinux and memory footprint increase of vmlinux caused by builtin-but- not-enabled LSMs remains. If it becomes possible to make LSMs dynamically appendable after boot using loadable kernel modules, these problems will go away. Another of concerns for enabling TOMOYO in prebuilt kernels is that who can provide support when distributor cannot provide support. Due to "those who compiled kernel code is expected to provide support for that kernel code" spell, TOMOYO is failing to get enabled in Fedora distribution [1]. The point of loadable kernel module is to share the workload. If it becomes possible to make LSMs dynamically appendable after boot using loadable kernel modules, as with people can use device drivers not supported by distributors but provided by third party device vendors, we can break this spell and can lower the barrier for using TOMOYO. This patch is intended for demonstrating that there is nothing difficult for supporting TOMOYO-like loadable LSM modules. For now we need to live with a mixture of built-in part and loadable part because fully loadable LSM modules are not supported since Linux 2.6.24 [2] and number of LSMs which can reserve static call slots is determined at compile time in Linux 6.12. Major changes in this patch are described below. There are no behavior changes as long as TOMOYO is built into vmlinux. Add CONFIG_SECURITY_TOMOYO_LKM as "bool" instead of changing CONFIG_SECURITY_TOMOYO from "bool" to "tristate", for something went wrong with how Makefile is evaluated if I choose "tristate". Add proxy.c for serving as a bridge between vmlinux and tomoyo.ko . Move callback functions from init.c to proxy.c when building as a loadable LSM module. init.c is built-in part and remains for reserving static call slots. proxy.c contains module's init function and tells init.c location of callback functions, making it possible to use static call for tomoyo.ko . By deferring initialization of "struct tomoyo_task" until tomoyo.ko is loaded, threads created between init.c reserved LSM hooks and proxy.c updates LSM hooks will have NULL "struct tomoyo_task" instances. Assuming that tomoyo.ko is loaded by the moment when the global init process starts, initialize "struct tomoyo_task" instance for current thread as a kernel thread when tomoyo_task(current) is called for the first time. There is a hack for exporting currently not-exported functions. This hack will be removed after all relevant functions are exported. Link: https://bugzilla.redhat.com/show_bug.cgi?id=542986 [1] Link: https://lkml.kernel.org/r/caafb609-8bef-4840-a080-81537356fc60@I-love.SAKURA.ne.jp [2] Signed-off-by: Tetsuo Handa --- security/tomoyo/Kconfig | 15 ++ security/tomoyo/Makefile | 8 +- security/tomoyo/common.c | 14 +- security/tomoyo/common.h | 72 +++++++++ security/tomoyo/init.c | 262 ++++++++++++++++++++++++++++++++ security/tomoyo/load_policy.c | 12 ++ security/tomoyo/proxy.c | 82 ++++++++++ security/tomoyo/securityfs_if.c | 6 +- 8 files changed, 467 insertions(+), 4 deletions(-) create mode 100644 security/tomoyo/proxy.c diff --git a/security/tomoyo/Kconfig b/security/tomoyo/Kconfig index 1e0dd1a6d0b0..90eccc6cd464 100644 --- a/security/tomoyo/Kconfig +++ b/security/tomoyo/Kconfig @@ -13,6 +13,21 @@ config SECURITY_TOMOYO found at . If you are unsure how to answer this question, answer N. +config SECURITY_TOMOYO_LKM + bool "Cut out most of TOMOYO's code to a loadable kernel module" + default n + depends on SECURITY_TOMOYO + depends on MODULES + help + Say Y here if you want to include TOMOYO without bloating + vmlinux file. If you say Y, most of TOMOYO code is cut out to + a loadable kernel module named tomoyo.ko . This option will be + useful for kernels built by Linux distributors where TOMOYO is + included but TOMOYO is not enabled by default. Please be sure + to explicitly load tomoyo.ko if you want to activate TOMOYO + without calling userspace policy loader, for tomoyo.ko is + loaded immediately before calling userspace policy loader. + config SECURITY_TOMOYO_MAX_ACCEPT_ENTRY int "Default maximal count for learning mode" default 2048 diff --git a/security/tomoyo/Makefile b/security/tomoyo/Makefile index e573a5cad732..287a7d16fa15 100644 --- a/security/tomoyo/Makefile +++ b/security/tomoyo/Makefile @@ -1,5 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y = audit.o common.o condition.o domain.o environ.o file.o gc.o group.o init.o load_policy.o memory.o mount.o network.o realpath.o securityfs_if.o util.o +tomoyo-objs := audit.o common.o condition.o domain.o environ.o file.o gc.o group.o memory.o mount.o network.o proxy.o realpath.o securityfs_if.o util.o +obj-y += init.o load_policy.o +ifdef CONFIG_SECURITY_TOMOYO_LKM +obj-m += tomoyo.o +else +obj-y += tomoyo.o +endif targets += builtin-policy.h diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 5c7b059a332a..c0ef014f8009 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -998,8 +998,13 @@ static bool tomoyo_select_domain(struct tomoyo_io_buffer *head, p = find_task_by_pid_ns(pid, &init_pid_ns); else p = find_task_by_vpid(pid); - if (p) + if (p) { domain = tomoyo_task(p)->domain_info; +#ifdef CONFIG_SECURITY_TOMOYO_LKM + if (!domain) + domain = &tomoyo_kernel_domain; +#endif + } rcu_read_unlock(); } else if (!strncmp(data, "domain=", 7)) { if (tomoyo_domain_def(data + 7)) @@ -1710,8 +1715,13 @@ static void tomoyo_read_pid(struct tomoyo_io_buffer *head) p = find_task_by_pid_ns(pid, &init_pid_ns); else p = find_task_by_vpid(pid); - if (p) + if (p) { domain = tomoyo_task(p)->domain_info; +#ifdef CONFIG_SECURITY_TOMOYO_LKM + if (!domain) + domain = &tomoyo_kernel_domain; +#endif + } rcu_read_unlock(); if (!domain) return; diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index 0e8e2e959aef..4f6c52a9f478 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -978,6 +978,7 @@ int tomoyo_get_mode(const struct tomoyo_policy_namespace *ns, const u8 profile, int tomoyo_init_request_info(struct tomoyo_request_info *r, struct tomoyo_domain_info *domain, const u8 index); +int __init tomoyo_interface_init(void); int tomoyo_mkdev_perm(const u8 operation, const struct path *path, const unsigned int mode, unsigned int dev); int tomoyo_mount_permission(const char *dev_name, const struct path *path, @@ -1214,10 +1215,14 @@ static inline void tomoyo_put_group(struct tomoyo_group *group) * * Returns pointer to "struct tomoyo_task" for specified thread. */ +#ifdef CONFIG_SECURITY_TOMOYO_LKM +extern struct tomoyo_task *tomoyo_task(struct task_struct *task); +#else static inline struct tomoyo_task *tomoyo_task(struct task_struct *task) { return task->security + tomoyo_blob_sizes.lbs_task; } +#endif /** * tomoyo_same_name_union - Check for duplicated "struct tomoyo_name_union" entry. @@ -1284,4 +1289,71 @@ static inline struct tomoyo_policy_namespace *tomoyo_current_namespace(void) pos = srcu_dereference((head)->next, &tomoyo_ss); \ for ( ; pos != (head); pos = srcu_dereference(pos->next, &tomoyo_ss)) +#ifdef CONFIG_SECURITY_TOMOYO_LKM + +#define LSM_HOOK(RET, DEFAULT, NAME, ...) typedef RET (NAME##_t)(__VA_ARGS__); +#include +#undef LSM_HOOK + +struct tomoyo_hooks { + cred_prepare_t *cred_prepare; + bprm_committed_creds_t *bprm_committed_creds; + task_alloc_t *task_alloc; + task_free_t *task_free; + bprm_check_security_t *bprm_check_security; + file_fcntl_t *file_fcntl; + file_open_t *file_open; + file_truncate_t *file_truncate; + path_truncate_t *path_truncate; + path_unlink_t *path_unlink; + path_mkdir_t *path_mkdir; + path_rmdir_t *path_rmdir; + path_symlink_t *path_symlink; + path_mknod_t *path_mknod; + path_link_t *path_link; + path_rename_t *path_rename; + inode_getattr_t *inode_getattr; + file_ioctl_t *file_ioctl; + file_ioctl_compat_t *file_ioctl_compat; + path_chmod_t *path_chmod; + path_chown_t *path_chown; + path_chroot_t *path_chroot; + sb_mount_t *sb_mount; + sb_umount_t *sb_umount; + sb_pivotroot_t *sb_pivotroot; + socket_bind_t *socket_bind; + socket_connect_t *socket_connect; + socket_listen_t *socket_listen; + socket_sendmsg_t *socket_sendmsg; +}; + +extern void tomoyo_register_hooks(const struct tomoyo_hooks *tomoyo_hooks); + +struct tomoyo_operations { + void (*check_profile)(void); + int enabled; +}; + +extern struct tomoyo_operations tomoyo_ops; + +/* + * Temporary hack: functions needed by tomoyo.ko . This will be removed + * after all functions are marked as EXPORT_STMBOL_GPL(). + */ +struct tomoyo_tmp_exports { + struct task_struct * (*find_task_by_vpid)(pid_t nr); + struct task_struct * (*find_task_by_pid_ns)(pid_t nr, struct pid_namespace *ns); + void (*put_filesystem)(struct file_system_type *fs); + struct file * (*get_mm_exe_file)(struct mm_struct *mm); + char * (*d_absolute_path)(const struct path *path, char *buf, int buflen); +}; +extern const struct tomoyo_tmp_exports tomoyo_tmp_exports; +#define find_task_by_vpid tomoyo_tmp_exports.find_task_by_vpid +#define find_task_by_pid_ns tomoyo_tmp_exports.find_task_by_pid_ns +#define put_filesystem tomoyo_tmp_exports.put_filesystem +#define get_mm_exe_file tomoyo_tmp_exports.get_mm_exe_file +#define d_absolute_path tomoyo_tmp_exports.d_absolute_path + +#endif /* defined(CONFIG_SECURITY_TOMOYO_LKM) */ + #endif /* !defined(_SECURITY_TOMOYO_COMMON_H) */ diff --git a/security/tomoyo/init.c b/security/tomoyo/init.c index 2b38dbc7a559..034e7db22d4e 100644 --- a/security/tomoyo/init.c +++ b/security/tomoyo/init.c @@ -9,8 +9,248 @@ #include #include "common.h" +#ifndef CONFIG_SECURITY_TOMOYO_LKM + #include "hooks.h" +#else + +#define DEFINE_STATIC_CALL_PROXY(NAME) \ + static NAME##_t tomoyo_##NAME; \ + DEFINE_STATIC_CALL_RET0(tomoyo_##NAME, tomoyo_##NAME); +DEFINE_STATIC_CALL_PROXY(cred_prepare) +DEFINE_STATIC_CALL_PROXY(bprm_committed_creds) +DEFINE_STATIC_CALL_PROXY(bprm_check_security) +DEFINE_STATIC_CALL_PROXY(inode_getattr) +DEFINE_STATIC_CALL_PROXY(path_truncate) +DEFINE_STATIC_CALL_PROXY(file_truncate) +DEFINE_STATIC_CALL_PROXY(path_unlink) +DEFINE_STATIC_CALL_PROXY(path_mkdir) +DEFINE_STATIC_CALL_PROXY(path_rmdir) +DEFINE_STATIC_CALL_PROXY(path_symlink) +DEFINE_STATIC_CALL_PROXY(path_mknod) +DEFINE_STATIC_CALL_PROXY(path_link) +DEFINE_STATIC_CALL_PROXY(path_rename) +DEFINE_STATIC_CALL_PROXY(file_fcntl) +DEFINE_STATIC_CALL_PROXY(file_open) +DEFINE_STATIC_CALL_PROXY(file_ioctl) +DEFINE_STATIC_CALL_PROXY(path_chmod) +DEFINE_STATIC_CALL_PROXY(path_chown) +DEFINE_STATIC_CALL_PROXY(path_chroot) +DEFINE_STATIC_CALL_PROXY(sb_mount) +DEFINE_STATIC_CALL_PROXY(sb_umount) +DEFINE_STATIC_CALL_PROXY(sb_pivotroot) +DEFINE_STATIC_CALL_PROXY(socket_listen) +DEFINE_STATIC_CALL_PROXY(socket_connect) +DEFINE_STATIC_CALL_PROXY(socket_bind) +DEFINE_STATIC_CALL_PROXY(socket_sendmsg) +DEFINE_STATIC_CALL_PROXY(task_alloc) +DEFINE_STATIC_CALL_PROXY(task_free) +#undef DEFINE_STATIC_CALL_PROXY + +static int tomoyo_cred_prepare(struct cred *new, const struct cred *old, gfp_t gfp) +{ + return static_call(tomoyo_cred_prepare)(new, old, gfp); +} + +static void tomoyo_bprm_committed_creds(const struct linux_binprm *bprm) +{ + static_call(tomoyo_bprm_committed_creds)(bprm); +} + +static int tomoyo_bprm_check_security(struct linux_binprm *bprm) +{ + return static_call(tomoyo_bprm_check_security)(bprm); +} + +static int tomoyo_inode_getattr(const struct path *path) +{ + return static_call(tomoyo_inode_getattr)(path); +} + +static int tomoyo_path_truncate(const struct path *path) +{ + return static_call(tomoyo_path_truncate)(path); +} + +static int tomoyo_file_truncate(struct file *file) +{ + return static_call(tomoyo_file_truncate)(file); +} + +static int tomoyo_path_unlink(const struct path *parent, struct dentry *dentry) +{ + return static_call(tomoyo_path_unlink)(parent, dentry); +} + +static int tomoyo_path_mkdir(const struct path *parent, struct dentry *dentry, umode_t mode) +{ + return static_call(tomoyo_path_mkdir)(parent, dentry, mode); +} + +static int tomoyo_path_rmdir(const struct path *parent, struct dentry *dentry) +{ + return static_call(tomoyo_path_rmdir)(parent, dentry); +} + +static int tomoyo_path_symlink(const struct path *parent, struct dentry *dentry, + const char *old_name) +{ + return static_call(tomoyo_path_symlink)(parent, dentry, old_name); +} + +static int tomoyo_path_mknod(const struct path *parent, struct dentry *dentry, + umode_t mode, unsigned int dev) +{ + return static_call(tomoyo_path_mknod)(parent, dentry, mode, dev); +} + +static int tomoyo_path_link(struct dentry *old_dentry, const struct path *new_dir, + struct dentry *new_dentry) +{ + return static_call(tomoyo_path_link)(old_dentry, new_dir, new_dentry); +} + +static int tomoyo_path_rename(const struct path *old_parent, struct dentry *old_dentry, + const struct path *new_parent, struct dentry *new_dentry, + const unsigned int flags) +{ + return static_call(tomoyo_path_rename)(old_parent, old_dentry, new_parent, new_dentry, flags); +} + +static int tomoyo_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg) +{ + return static_call(tomoyo_file_fcntl)(file, cmd, arg); +} + +static int tomoyo_file_open(struct file *f) +{ + return static_call(tomoyo_file_open)(f); +} + +static int tomoyo_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + return static_call(tomoyo_file_ioctl)(file, cmd, arg); +} + +static int tomoyo_path_chmod(const struct path *path, umode_t mode) +{ + return static_call(tomoyo_path_chmod)(path, mode); +} + +static int tomoyo_path_chown(const struct path *path, kuid_t uid, kgid_t gid) +{ + return static_call(tomoyo_path_chown)(path, uid, gid); +} + +static int tomoyo_path_chroot(const struct path *path) +{ + return static_call(tomoyo_path_chroot)(path); +} + +static int tomoyo_sb_mount(const char *dev_name, const struct path *path, + const char *type, unsigned long flags, void *data) +{ + return static_call(tomoyo_sb_mount)(dev_name, path, type, flags, data); +} + +static int tomoyo_sb_umount(struct vfsmount *mnt, int flags) +{ + return static_call(tomoyo_sb_umount)(mnt, flags); +} + +static int tomoyo_sb_pivotroot(const struct path *old_path, const struct path *new_path) +{ + return static_call(tomoyo_sb_pivotroot)(old_path, new_path); +} + +static int tomoyo_socket_listen(struct socket *sock, int backlog) +{ + return static_call(tomoyo_socket_listen)(sock, backlog); +} + +static int tomoyo_socket_connect(struct socket *sock, struct sockaddr *addr, int addr_len) +{ + return static_call(tomoyo_socket_connect)(sock, addr, addr_len); +} + +static int tomoyo_socket_bind(struct socket *sock, struct sockaddr *addr, int addr_len) +{ + return static_call(tomoyo_socket_bind)(sock, addr, addr_len); +} + +static int tomoyo_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size) +{ + return static_call(tomoyo_socket_sendmsg)(sock, msg, size); +} + +static int tomoyo_task_alloc(struct task_struct *task, unsigned long clone_flags) +{ + return static_call(tomoyo_task_alloc)(task, clone_flags); +} + +static void tomoyo_task_free(struct task_struct *task) +{ + static_call(tomoyo_task_free)(task); +} + +void tomoyo_register_hooks(const struct tomoyo_hooks *tomoyo_hooks) +{ + static void *registered; + + if (cmpxchg(®istered, NULL, ®istered)) + panic("%s was called twice!\n", __func__); + static_call_update(tomoyo_task_free, tomoyo_hooks->task_free); + static_call_update(tomoyo_task_alloc, tomoyo_hooks->task_alloc); + static_call_update(tomoyo_cred_prepare, tomoyo_hooks->cred_prepare); + static_call_update(tomoyo_bprm_committed_creds, tomoyo_hooks->bprm_committed_creds); + static_call_update(tomoyo_bprm_check_security, tomoyo_hooks->bprm_check_security); + static_call_update(tomoyo_inode_getattr, tomoyo_hooks->inode_getattr); + static_call_update(tomoyo_path_truncate, tomoyo_hooks->path_truncate); + static_call_update(tomoyo_file_truncate, tomoyo_hooks->file_truncate); + static_call_update(tomoyo_path_unlink, tomoyo_hooks->path_unlink); + static_call_update(tomoyo_path_mkdir, tomoyo_hooks->path_mkdir); + static_call_update(tomoyo_path_rmdir, tomoyo_hooks->path_rmdir); + static_call_update(tomoyo_path_symlink, tomoyo_hooks->path_symlink); + static_call_update(tomoyo_path_mknod, tomoyo_hooks->path_mknod); + static_call_update(tomoyo_path_link, tomoyo_hooks->path_link); + static_call_update(tomoyo_path_rename, tomoyo_hooks->path_rename); + static_call_update(tomoyo_file_fcntl, tomoyo_hooks->file_fcntl); + static_call_update(tomoyo_file_open, tomoyo_hooks->file_open); + static_call_update(tomoyo_file_ioctl, tomoyo_hooks->file_ioctl); + static_call_update(tomoyo_path_chmod, tomoyo_hooks->path_chmod); + static_call_update(tomoyo_path_chown, tomoyo_hooks->path_chown); + static_call_update(tomoyo_path_chroot, tomoyo_hooks->path_chroot); + static_call_update(tomoyo_sb_mount, tomoyo_hooks->sb_mount); + static_call_update(tomoyo_sb_umount, tomoyo_hooks->sb_umount); + static_call_update(tomoyo_sb_pivotroot, tomoyo_hooks->sb_pivotroot); + static_call_update(tomoyo_socket_listen, tomoyo_hooks->socket_listen); + static_call_update(tomoyo_socket_connect, tomoyo_hooks->socket_connect); + static_call_update(tomoyo_socket_bind, tomoyo_hooks->socket_bind); + static_call_update(tomoyo_socket_sendmsg, tomoyo_hooks->socket_sendmsg); +} +EXPORT_SYMBOL_GPL(tomoyo_register_hooks); + +/* + * Temporary hack: functions needed by tomoyo.ko . This hack will be removed + * after all functions are marked as EXPORT_STMBOL_GPL(). + */ +#undef find_task_by_vpid +#undef find_task_by_pid_ns +#undef put_filesystem +#undef get_mm_exe_file +#undef d_absolute_path +const struct tomoyo_tmp_exports tomoyo_tmp_exports = { + .find_task_by_vpid = find_task_by_vpid, + .find_task_by_pid_ns = find_task_by_pid_ns, + .put_filesystem = put_filesystem, + .get_mm_exe_file = get_mm_exe_file, + .d_absolute_path = d_absolute_path, +}; +EXPORT_SYMBOL_GPL(tomoyo_tmp_exports); + +#endif + #ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER static int tomoyo_bprm_creds_for_exec(struct linux_binprm *bprm) { @@ -74,6 +314,27 @@ int tomoyo_enabled __ro_after_init = 1; /* Has /sbin/init started? */ bool tomoyo_policy_loaded; +#ifdef CONFIG_SECURITY_TOMOYO_LKM +EXPORT_SYMBOL_GPL(tomoyo_blob_sizes); +EXPORT_SYMBOL_GPL(tomoyo_policy_loaded); + +struct tomoyo_operations tomoyo_ops; +EXPORT_SYMBOL_GPL(tomoyo_ops); + +/** + * tomoyo_init - Reserve hooks for TOMOYO Linux. + * + * Returns 0. + */ +static int __init tomoyo_init(void) +{ + /* register ourselves with the security framework */ + security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks), &tomoyo_lsmid); + tomoyo_ops.enabled = tomoyo_enabled; + pr_info("Hooks for initializing TOMOYO Linux are ready\n"); + return 0; +} +#else /** * tomoyo_init - Register TOMOYO Linux as a LSM module. * @@ -94,6 +355,7 @@ static int __init tomoyo_init(void) return 0; } +#endif DEFINE_LSM(tomoyo) = { .name = "tomoyo", diff --git a/security/tomoyo/load_policy.c b/security/tomoyo/load_policy.c index 363b65be87ab..6a2a72354a64 100644 --- a/security/tomoyo/load_policy.c +++ b/security/tomoyo/load_policy.c @@ -97,6 +97,14 @@ void tomoyo_load_policy(const char *filename) if (!tomoyo_policy_loader_exists()) return; done = true; +#ifdef CONFIG_SECURITY_TOMOYO_LKM + /* Load tomoyo.ko if not yet loaded. */ + if (!tomoyo_ops.check_profile) + request_module("tomoyo"); + /* Check if tomoyo.ko was successfully loaded. */ + if (!tomoyo_ops.check_profile) + panic("Failed to load tomoyo module."); +#endif pr_info("Calling %s to load policy. Please wait.\n", tomoyo_loader); argv[0] = (char *) tomoyo_loader; argv[1] = NULL; @@ -104,7 +112,11 @@ void tomoyo_load_policy(const char *filename) envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; envp[2] = NULL; call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +#ifdef CONFIG_SECURITY_TOMOYO_LKM + tomoyo_ops.check_profile(); +#else tomoyo_check_profile(); +#endif } #endif diff --git a/security/tomoyo/proxy.c b/security/tomoyo/proxy.c new file mode 100644 index 000000000000..1618cc0f2af8 --- /dev/null +++ b/security/tomoyo/proxy.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * security/tomoyo/proxy.c + * + * Copyright (C) 2005-2011 NTT DATA CORPORATION + */ + +#include +#include "common.h" + +#ifdef CONFIG_SECURITY_TOMOYO_LKM + +struct tomoyo_task *tomoyo_task(struct task_struct *task) +{ + struct tomoyo_task *s = task->security + tomoyo_blob_sizes.lbs_task; + + if (unlikely(!s->domain_info)) { + if (likely(task == current)) { + s->domain_info = &tomoyo_kernel_domain; + atomic_inc(&tomoyo_kernel_domain.users); + } else { + /* Caller handles s->domain_info == NULL case. */ + } + } + return s; +} + +#include "hooks.h" + +/** + * tomoyo_runtime_init - Register TOMOYO Linux as a loadable LSM module. + * + * Returns 0 if TOMOYO is enabled, -EINVAL otherwise. + */ +static int __init tomoyo_runtime_init(void) +{ + const struct tomoyo_hooks tomoyo_hooks = { + .cred_prepare = tomoyo_cred_prepare, + .bprm_committed_creds = tomoyo_bprm_committed_creds, + .task_alloc = tomoyo_task_alloc, + .task_free = tomoyo_task_free, + .bprm_check_security = tomoyo_bprm_check_security, + .file_fcntl = tomoyo_file_fcntl, + .file_open = tomoyo_file_open, + .file_truncate = tomoyo_file_truncate, + .path_truncate = tomoyo_path_truncate, + .path_unlink = tomoyo_path_unlink, + .path_mkdir = tomoyo_path_mkdir, + .path_rmdir = tomoyo_path_rmdir, + .path_symlink = tomoyo_path_symlink, + .path_mknod = tomoyo_path_mknod, + .path_link = tomoyo_path_link, + .path_rename = tomoyo_path_rename, + .inode_getattr = tomoyo_inode_getattr, + .file_ioctl = tomoyo_file_ioctl, + .file_ioctl_compat = tomoyo_file_ioctl, + .path_chmod = tomoyo_path_chmod, + .path_chown = tomoyo_path_chown, + .path_chroot = tomoyo_path_chroot, + .sb_mount = tomoyo_sb_mount, + .sb_umount = tomoyo_sb_umount, + .sb_pivotroot = tomoyo_sb_pivotroot, + .socket_bind = tomoyo_socket_bind, + .socket_connect = tomoyo_socket_connect, + .socket_listen = tomoyo_socket_listen, + .socket_sendmsg = tomoyo_socket_sendmsg, + }; + + if (!tomoyo_ops.enabled) + return -EINVAL; + tomoyo_ops.check_profile = tomoyo_check_profile; + pr_info("TOMOYO Linux initialized\n"); + tomoyo_task(current); + tomoyo_mm_init(); + tomoyo_interface_init(); + tomoyo_register_hooks(&tomoyo_hooks); + return 0; +} +module_init(tomoyo_runtime_init); +MODULE_LICENSE("GPL"); + +#endif diff --git a/security/tomoyo/securityfs_if.c b/security/tomoyo/securityfs_if.c index 7e69747b2f77..a3b821b7f477 100644 --- a/security/tomoyo/securityfs_if.c +++ b/security/tomoyo/securityfs_if.c @@ -233,13 +233,15 @@ static void __init tomoyo_create_entry(const char *name, const umode_t mode, * * Returns 0. */ -static int __init tomoyo_interface_init(void) +int __init tomoyo_interface_init(void) { struct tomoyo_domain_info *domain; struct dentry *tomoyo_dir; +#ifndef CONFIG_SECURITY_TOMOYO_LKM if (!tomoyo_enabled) return 0; +#endif domain = tomoyo_domain(); /* Don't create securityfs entries unless registered. */ if (domain != &tomoyo_kernel_domain) @@ -270,4 +272,6 @@ static int __init tomoyo_interface_init(void) return 0; } +#ifndef CONFIG_SECURITY_TOMOYO_LKM fs_initcall(tomoyo_interface_init); +#endif From 88801d043b1d16caae76a5e2e5991e8b1f55ce7f Mon Sep 17 00:00:00 2001 From: Jiqian Chen Date: Tue, 24 Sep 2024 14:14:35 +0800 Subject: [PATCH 070/104] xen/pci: Add a function to reset device for xen When device on dom0 side has been reset, the vpci on Xen side won't get notification, so that the cached state in vpci is all out of date with the real device state. To solve that problem, add a new function to clear all vpci device state when device is reset on dom0 side. And call that function in pcistub_init_device. Because when using "pci-assignable-add" to assign a passthrough device in Xen, it will reset passthrough device and the vpci state will out of date, and then device will fail to restore bar state. Signed-off-by: Jiqian Chen Signed-off-by: Huang Rui Signed-off-by: Jiqian Chen Reviewed-by: Stefano Stabellini Message-ID: <20240924061437.2636766-2-Jiqian.Chen@amd.com> Signed-off-by: Juergen Gross --- drivers/xen/pci.c | 13 +++++++++++++ drivers/xen/xen-pciback/pci_stub.c | 18 +++++++++++++++--- include/xen/interface/physdev.h | 17 +++++++++++++++++ include/xen/pci.h | 6 ++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/drivers/xen/pci.c b/drivers/xen/pci.c index a2facd8f7e51..416f231809cb 100644 --- a/drivers/xen/pci.c +++ b/drivers/xen/pci.c @@ -173,6 +173,19 @@ static int xen_remove_device(struct device *dev) return r; } +int xen_reset_device(const struct pci_dev *dev) +{ + struct pci_device_reset device = { + .dev.seg = pci_domain_nr(dev->bus), + .dev.bus = dev->bus->number, + .dev.devfn = dev->devfn, + .flags = PCI_DEVICE_RESET_FLR, + }; + + return HYPERVISOR_physdev_op(PHYSDEVOP_pci_device_reset, &device); +} +EXPORT_SYMBOL_GPL(xen_reset_device); + static int xen_pci_notifier(struct notifier_block *nb, unsigned long action, void *data) { diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index 4faebbb84999..3e162c1753e2 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -89,6 +89,16 @@ static struct pcistub_device *pcistub_device_alloc(struct pci_dev *dev) return psdev; } +static int pcistub_reset_device_state(struct pci_dev *dev) +{ + __pci_reset_function_locked(dev); + + if (!xen_pv_domain()) + return xen_reset_device(dev); + else + return 0; +} + /* Don't call this directly as it's called by pcistub_device_put */ static void pcistub_device_release(struct kref *kref) { @@ -107,7 +117,7 @@ static void pcistub_device_release(struct kref *kref) /* Call the reset function which does not take lock as this * is called from "unbind" which takes a device_lock mutex. */ - __pci_reset_function_locked(dev); + pcistub_reset_device_state(dev); if (dev_data && pci_load_and_free_saved_state(dev, &dev_data->pci_saved_state)) dev_info(&dev->dev, "Could not reload PCI state\n"); @@ -284,7 +294,7 @@ void pcistub_put_pci_dev(struct pci_dev *dev) * (so it's ready for the next domain) */ device_lock_assert(&dev->dev); - __pci_reset_function_locked(dev); + pcistub_reset_device_state(dev); dev_data = pci_get_drvdata(dev); ret = pci_load_saved_state(dev, dev_data->pci_saved_state); @@ -420,7 +430,9 @@ static int pcistub_init_device(struct pci_dev *dev) dev_err(&dev->dev, "Could not store PCI conf saved state!\n"); else { dev_dbg(&dev->dev, "resetting (FLR, D3, etc) the device\n"); - __pci_reset_function_locked(dev); + err = pcistub_reset_device_state(dev); + if (err) + goto config_release; pci_restore_state(dev); } /* Now disable the device (this also ensures some private device diff --git a/include/xen/interface/physdev.h b/include/xen/interface/physdev.h index a237af867873..df74e65a884b 100644 --- a/include/xen/interface/physdev.h +++ b/include/xen/interface/physdev.h @@ -256,6 +256,13 @@ struct physdev_pci_device_add { */ #define PHYSDEVOP_prepare_msix 30 #define PHYSDEVOP_release_msix 31 +/* + * Notify the hypervisor that a PCI device has been reset, so that any + * internally cached state is regenerated. Should be called after any + * device reset performed by the hardware domain. + */ +#define PHYSDEVOP_pci_device_reset 32 + struct physdev_pci_device { /* IN */ uint16_t seg; @@ -263,6 +270,16 @@ struct physdev_pci_device { uint8_t devfn; }; +struct pci_device_reset { + struct physdev_pci_device dev; +#define PCI_DEVICE_RESET_COLD 0x0 +#define PCI_DEVICE_RESET_WARM 0x1 +#define PCI_DEVICE_RESET_HOT 0x2 +#define PCI_DEVICE_RESET_FLR 0x3 +#define PCI_DEVICE_RESET_MASK 0x3 + uint32_t flags; +}; + #define PHYSDEVOP_DBGP_RESET_PREPARE 1 #define PHYSDEVOP_DBGP_RESET_DONE 2 diff --git a/include/xen/pci.h b/include/xen/pci.h index b8337cf85fd1..424b8ea89ca8 100644 --- a/include/xen/pci.h +++ b/include/xen/pci.h @@ -4,10 +4,16 @@ #define __XEN_PCI_H__ #if defined(CONFIG_XEN_DOM0) +int xen_reset_device(const struct pci_dev *dev); int xen_find_device_domain_owner(struct pci_dev *dev); int xen_register_device_domain_owner(struct pci_dev *dev, uint16_t domain); int xen_unregister_device_domain_owner(struct pci_dev *dev); #else +static inline int xen_reset_device(const struct pci_dev *dev) +{ + return -1; +} + static inline int xen_find_device_domain_owner(struct pci_dev *dev) { return -1; From b166b8ab4189743a717cb93f50d6fcca3a46770d Mon Sep 17 00:00:00 2001 From: Jiqian Chen Date: Tue, 24 Sep 2024 14:14:36 +0800 Subject: [PATCH 071/104] xen/pvh: Setup gsi for passthrough device In PVH dom0, the gsis don't get registered, but the gsi of a passthrough device must be configured for it to be able to be mapped into a domU. When assigning a device to passthrough, proactively setup the gsi of the device during that process. Signed-off-by: Jiqian Chen Signed-off-by: Huang Rui Signed-off-by: Jiqian Chen Reviewed-by: Stefano Stabellini Message-ID: <20240924061437.2636766-3-Jiqian.Chen@amd.com> Signed-off-by: Juergen Gross --- arch/x86/xen/enlighten_pvh.c | 23 ++++++++++++++ drivers/acpi/pci_irq.c | 2 +- drivers/xen/acpi.c | 50 ++++++++++++++++++++++++++++++ drivers/xen/xen-pciback/pci_stub.c | 20 ++++++++++++ include/linux/acpi.h | 1 + include/xen/acpi.h | 18 +++++++++++ 6 files changed, 113 insertions(+), 1 deletion(-) diff --git a/arch/x86/xen/enlighten_pvh.c b/arch/x86/xen/enlighten_pvh.c index 728a4366ca85..bf68c329fc01 100644 --- a/arch/x86/xen/enlighten_pvh.c +++ b/arch/x86/xen/enlighten_pvh.c @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -28,6 +29,28 @@ bool __ro_after_init xen_pvh; EXPORT_SYMBOL_GPL(xen_pvh); +#ifdef CONFIG_XEN_DOM0 +int xen_pvh_setup_gsi(int gsi, int trigger, int polarity) +{ + int ret; + struct physdev_setup_gsi setup_gsi; + + setup_gsi.gsi = gsi; + setup_gsi.triggering = (trigger == ACPI_EDGE_SENSITIVE ? 0 : 1); + setup_gsi.polarity = (polarity == ACPI_ACTIVE_HIGH ? 0 : 1); + + ret = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi); + if (ret == -EEXIST) { + xen_raw_printk("Already setup the GSI :%d\n", gsi); + ret = 0; + } else if (ret) + xen_raw_printk("Fail to setup GSI (%d)!\n", gsi); + + return ret; +} +EXPORT_SYMBOL_GPL(xen_pvh_setup_gsi); +#endif + /* * Reserve e820 UNUSABLE regions to inflate the memory balloon. * diff --git a/drivers/acpi/pci_irq.c b/drivers/acpi/pci_irq.c index ff30ceca2203..630fe0a34bc6 100644 --- a/drivers/acpi/pci_irq.c +++ b/drivers/acpi/pci_irq.c @@ -288,7 +288,7 @@ static int acpi_reroute_boot_interrupt(struct pci_dev *dev, } #endif /* CONFIG_X86_IO_APIC */ -static struct acpi_prt_entry *acpi_pci_irq_lookup(struct pci_dev *dev, int pin) +struct acpi_prt_entry *acpi_pci_irq_lookup(struct pci_dev *dev, int pin) { struct acpi_prt_entry *entry = NULL; struct pci_dev *bridge; diff --git a/drivers/xen/acpi.c b/drivers/xen/acpi.c index 6893c79fd2a1..9e2096524fbc 100644 --- a/drivers/xen/acpi.c +++ b/drivers/xen/acpi.c @@ -30,6 +30,7 @@ * IN THE SOFTWARE. */ +#include #include #include #include @@ -75,3 +76,52 @@ int xen_acpi_notify_hypervisor_extended_sleep(u8 sleep_state, return xen_acpi_notify_hypervisor_state(sleep_state, val_a, val_b, true); } + +struct acpi_prt_entry { + struct acpi_pci_id id; + u8 pin; + acpi_handle link; + u32 index; +}; + +int xen_acpi_get_gsi_info(struct pci_dev *dev, + int *gsi_out, + int *trigger_out, + int *polarity_out) +{ + int gsi; + u8 pin; + struct acpi_prt_entry *entry; + int trigger = ACPI_LEVEL_SENSITIVE; + int polarity = acpi_irq_model == ACPI_IRQ_MODEL_GIC ? + ACPI_ACTIVE_HIGH : ACPI_ACTIVE_LOW; + + if (!dev || !gsi_out || !trigger_out || !polarity_out) + return -EINVAL; + + pin = dev->pin; + if (!pin) + return -EINVAL; + + entry = acpi_pci_irq_lookup(dev, pin); + if (entry) { + if (entry->link) + gsi = acpi_pci_link_allocate_irq(entry->link, + entry->index, + &trigger, &polarity, + NULL); + else + gsi = entry->index; + } else + gsi = -1; + + if (gsi < 0) + return -EINVAL; + + *gsi_out = gsi; + *trigger_out = trigger; + *polarity_out = polarity; + + return 0; +} +EXPORT_SYMBOL_GPL(xen_acpi_get_gsi_info); diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index 3e162c1753e2..8ce27333f54b 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -21,6 +21,9 @@ #include #include #include +#ifdef CONFIG_XEN_ACPI +#include +#endif #include #include #include "pciback.h" @@ -367,6 +370,9 @@ static int pcistub_match(struct pci_dev *dev) static int pcistub_init_device(struct pci_dev *dev) { struct xen_pcibk_dev_data *dev_data; +#ifdef CONFIG_XEN_ACPI + int gsi, trigger, polarity; +#endif int err = 0; dev_dbg(&dev->dev, "initializing...\n"); @@ -435,6 +441,20 @@ static int pcistub_init_device(struct pci_dev *dev) goto config_release; pci_restore_state(dev); } + +#ifdef CONFIG_XEN_ACPI + if (xen_initial_domain() && xen_pvh_domain()) { + err = xen_acpi_get_gsi_info(dev, &gsi, &trigger, &polarity); + if (err) { + dev_err(&dev->dev, "Fail to get gsi info!\n"); + goto config_release; + } + err = xen_pvh_setup_gsi(gsi, trigger, polarity); + if (err) + goto config_release; + } +#endif + /* Now disable the device (this also ensures some private device * data is setup before we export) */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 0687a442fec7..02ded9f53a6b 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -362,6 +362,7 @@ void acpi_unregister_gsi (u32 gsi); struct pci_dev; +struct acpi_prt_entry *acpi_pci_irq_lookup(struct pci_dev *dev, int pin); int acpi_pci_irq_enable (struct pci_dev *dev); void acpi_penalize_isa_irq(int irq, int active); bool acpi_isa_irq_available(int irq); diff --git a/include/xen/acpi.h b/include/xen/acpi.h index b1e11863144d..3bcfe82d9078 100644 --- a/include/xen/acpi.h +++ b/include/xen/acpi.h @@ -67,10 +67,28 @@ static inline void xen_acpi_sleep_register(void) acpi_suspend_lowlevel = xen_acpi_suspend_lowlevel; } } +int xen_pvh_setup_gsi(int gsi, int trigger, int polarity); +int xen_acpi_get_gsi_info(struct pci_dev *dev, + int *gsi_out, + int *trigger_out, + int *polarity_out); #else static inline void xen_acpi_sleep_register(void) { } + +static inline int xen_pvh_setup_gsi(int gsi, int trigger, int polarity) +{ + return -1; +} + +static inline int xen_acpi_get_gsi_info(struct pci_dev *dev, + int *gsi_out, + int *trigger_out, + int *polarity_out) +{ + return -1; +} #endif #endif /* _XEN_ACPI_H */ From 2fae6bb7be320270801b3c3b040189bd7daa8056 Mon Sep 17 00:00:00 2001 From: Jiqian Chen Date: Tue, 24 Sep 2024 14:14:37 +0800 Subject: [PATCH 072/104] xen/privcmd: Add new syscall to get gsi from dev On PVH dom0, when passthrough a device to domU, QEMU and xl tools want to use gsi number to do pirq mapping, see QEMU code xen_pt_realize->xc_physdev_map_pirq, and xl code pci_add_dm_done->xc_physdev_map_pirq, but in current codes, the gsi number is got from file /sys/bus/pci/devices//irq, that is wrong, because irq is not equal with gsi, they are in different spaces, so pirq mapping fails. And in current linux codes, there is no method to get gsi for userspace. For above purpose, record gsi of pcistub devices when init pcistub and add a new syscall into privcmd to let userspace can get gsi when they have a need. Signed-off-by: Jiqian Chen Signed-off-by: Huang Rui Signed-off-by: Jiqian Chen Reviewed-by: Stefano Stabellini Message-ID: <20240924061437.2636766-4-Jiqian.Chen@amd.com> Signed-off-by: Juergen Gross --- drivers/xen/Kconfig | 1 + drivers/xen/privcmd.c | 32 +++++++++++++++++++++++++ drivers/xen/xen-pciback/pci_stub.c | 38 +++++++++++++++++++++++++++--- include/uapi/xen/privcmd.h | 7 ++++++ include/xen/acpi.h | 9 +++++++ 5 files changed, 84 insertions(+), 3 deletions(-) diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index d5989871dd5d..fd83d51df2f4 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -261,6 +261,7 @@ config XEN_SCSI_BACKEND config XEN_PRIVCMD tristate "Xen hypercall passthrough driver" depends on XEN + imply CONFIG_XEN_PCIDEV_BACKEND default m help The hypercall passthrough driver allows privileged user programs to diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 9563650dfbaf..588f99a2b8df 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -46,6 +46,9 @@ #include #include #include +#ifdef CONFIG_XEN_ACPI +#include +#endif #include "privcmd.h" @@ -844,6 +847,31 @@ out: return rc; } +static long privcmd_ioctl_pcidev_get_gsi(struct file *file, void __user *udata) +{ +#if defined(CONFIG_XEN_ACPI) + int rc = -EINVAL; + struct privcmd_pcidev_get_gsi kdata; + + if (copy_from_user(&kdata, udata, sizeof(kdata))) + return -EFAULT; + + if (IS_REACHABLE(CONFIG_XEN_PCIDEV_BACKEND)) + rc = pcistub_get_gsi_from_sbdf(kdata.sbdf); + + if (rc < 0) + return rc; + + kdata.gsi = rc; + if (copy_to_user(udata, &kdata, sizeof(kdata))) + return -EFAULT; + + return 0; +#else + return -EINVAL; +#endif +} + #ifdef CONFIG_XEN_PRIVCMD_EVENTFD /* Irqfd support */ static struct workqueue_struct *irqfd_cleanup_wq; @@ -1543,6 +1571,10 @@ static long privcmd_ioctl(struct file *file, ret = privcmd_ioctl_ioeventfd(file, udata); break; + case IOCTL_PRIVCMD_PCIDEV_GET_GSI: + ret = privcmd_ioctl_pcidev_get_gsi(file, udata); + break; + default: break; } diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index 8ce27333f54b..d003402ce66b 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -56,6 +56,9 @@ struct pcistub_device { struct pci_dev *dev; struct xen_pcibk_device *pdev;/* non-NULL if struct pci_dev is in use */ +#ifdef CONFIG_XEN_ACPI + int gsi; +#endif }; /* Access to pcistub_devices & seized_devices lists and the initialize_devices @@ -88,6 +91,9 @@ static struct pcistub_device *pcistub_device_alloc(struct pci_dev *dev) kref_init(&psdev->kref); spin_lock_init(&psdev->lock); +#ifdef CONFIG_XEN_ACPI + psdev->gsi = -1; +#endif return psdev; } @@ -220,6 +226,25 @@ static struct pci_dev *pcistub_device_get_pci_dev(struct xen_pcibk_device *pdev, return pci_dev; } +#ifdef CONFIG_XEN_ACPI +int pcistub_get_gsi_from_sbdf(unsigned int sbdf) +{ + struct pcistub_device *psdev; + int domain = (sbdf >> 16) & 0xffff; + int bus = PCI_BUS_NUM(sbdf); + int slot = PCI_SLOT(sbdf); + int func = PCI_FUNC(sbdf); + + psdev = pcistub_device_find(domain, bus, slot, func); + + if (!psdev) + return -ENODEV; + + return psdev->gsi; +} +EXPORT_SYMBOL_GPL(pcistub_get_gsi_from_sbdf); +#endif + struct pci_dev *pcistub_get_pci_dev_by_slot(struct xen_pcibk_device *pdev, int domain, int bus, int slot, int func) @@ -367,14 +392,20 @@ static int pcistub_match(struct pci_dev *dev) return found; } -static int pcistub_init_device(struct pci_dev *dev) +static int pcistub_init_device(struct pcistub_device *psdev) { struct xen_pcibk_dev_data *dev_data; + struct pci_dev *dev; #ifdef CONFIG_XEN_ACPI int gsi, trigger, polarity; #endif int err = 0; + if (!psdev) + return -EINVAL; + + dev = psdev->dev; + dev_dbg(&dev->dev, "initializing...\n"); /* The PCI backend is not intended to be a module (or to work with @@ -452,6 +483,7 @@ static int pcistub_init_device(struct pci_dev *dev) err = xen_pvh_setup_gsi(gsi, trigger, polarity); if (err) goto config_release; + psdev->gsi = gsi; } #endif @@ -494,7 +526,7 @@ static int __init pcistub_init_devices_late(void) spin_unlock_irqrestore(&pcistub_devices_lock, flags); - err = pcistub_init_device(psdev->dev); + err = pcistub_init_device(psdev); if (err) { dev_err(&psdev->dev->dev, "error %d initializing device\n", err); @@ -564,7 +596,7 @@ static int pcistub_seize(struct pci_dev *dev, spin_unlock_irqrestore(&pcistub_devices_lock, flags); /* don't want irqs disabled when calling pcistub_init_device */ - err = pcistub_init_device(psdev->dev); + err = pcistub_init_device(psdev); spin_lock_irqsave(&pcistub_devices_lock, flags); diff --git a/include/uapi/xen/privcmd.h b/include/uapi/xen/privcmd.h index 8b8c5d1420fe..8e2c8fd44764 100644 --- a/include/uapi/xen/privcmd.h +++ b/include/uapi/xen/privcmd.h @@ -126,6 +126,11 @@ struct privcmd_ioeventfd { __u8 pad[2]; }; +struct privcmd_pcidev_get_gsi { + __u32 sbdf; + __u32 gsi; +}; + /* * @cmd: IOCTL_PRIVCMD_HYPERCALL * @arg: &privcmd_hypercall_t @@ -157,5 +162,7 @@ struct privcmd_ioeventfd { _IOW('P', 8, struct privcmd_irqfd) #define IOCTL_PRIVCMD_IOEVENTFD \ _IOW('P', 9, struct privcmd_ioeventfd) +#define IOCTL_PRIVCMD_PCIDEV_GET_GSI \ + _IOC(_IOC_NONE, 'P', 10, sizeof(struct privcmd_pcidev_get_gsi)) #endif /* __LINUX_PUBLIC_PRIVCMD_H__ */ diff --git a/include/xen/acpi.h b/include/xen/acpi.h index 3bcfe82d9078..daa96a22d257 100644 --- a/include/xen/acpi.h +++ b/include/xen/acpi.h @@ -91,4 +91,13 @@ static inline int xen_acpi_get_gsi_info(struct pci_dev *dev, } #endif +#ifdef CONFIG_XEN_PCI_STUB +int pcistub_get_gsi_from_sbdf(unsigned int sbdf); +#else +static inline int pcistub_get_gsi_from_sbdf(unsigned int sbdf) +{ + return -1; +} +#endif + #endif /* _XEN_ACPI_H */ From 8f2f74f2f3ebd9bb7159301cb7560db75d2e801a Mon Sep 17 00:00:00 2001 From: Min-Hua Chen Date: Wed, 18 Sep 2024 07:36:50 +0800 Subject: [PATCH 073/104] xen/pciback: fix cast to restricted pci_ers_result_t and pci_power_t This patch fix the following sparse warning by applying __force cast to pci_ers_result_t and pci_power_t. drivers/xen/xen-pciback/pci_stub.c:760:16: sparse: warning: cast to restricted pci_ers_result_t drivers/xen/xen-pciback/conf_space_capability.c:125:22: sparse: warning: cast to restricted pci_power_t No functional changes intended. Signed-off-by: Min-Hua Chen Reviewed-by: Juergen Gross Message-ID: <20240917233653.61630-1-minhuadotchen@gmail.com> Signed-off-by: Juergen Gross --- drivers/xen/xen-pciback/conf_space_capability.c | 2 +- drivers/xen/xen-pciback/pci_stub.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/xen/xen-pciback/conf_space_capability.c b/drivers/xen/xen-pciback/conf_space_capability.c index 1948a9700c8f..cf568e899ee2 100644 --- a/drivers/xen/xen-pciback/conf_space_capability.c +++ b/drivers/xen/xen-pciback/conf_space_capability.c @@ -122,7 +122,7 @@ static int pm_ctrl_write(struct pci_dev *dev, int offset, u16 new_value, if (err) goto out; - new_state = (pci_power_t)(new_value & PCI_PM_CTRL_STATE_MASK); + new_state = (__force pci_power_t)(new_value & PCI_PM_CTRL_STATE_MASK); new_value &= PM_OK_BITS; if ((old_value & PM_OK_BITS) != new_value) { diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index d003402ce66b..2f3da5ac62cd 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -821,7 +821,7 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev, } clear_bit(_PCIB_op_pending, (unsigned long *)&pdev->flags); - res = (pci_ers_result_t)aer_op->err; + res = (__force pci_ers_result_t)aer_op->err; return res; } From 08377ed24feef66866d0c6aabcae0aec515cf2ca Mon Sep 17 00:00:00 2001 From: Jason Andryuk Date: Fri, 23 Aug 2024 15:36:26 -0400 Subject: [PATCH 074/104] xen: sync elfnote.h from xen tree Sync Xen's elfnote.h header from xen.git to pull in the XEN_ELFNOTE_PHYS32_RELOC define. xen commit dfc9fab00378 ("x86/PVH: Support relocatable dom0 kernels") This is a copy except for the removal of the emacs editor config at the end of the file. Signed-off-by: Jason Andryuk Reviewed-by: Juergen Gross Message-ID: <20240823193630.2583107-2-jason.andryuk@amd.com> Signed-off-by: Juergen Gross --- include/xen/interface/elfnote.h | 93 +++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/include/xen/interface/elfnote.h b/include/xen/interface/elfnote.h index 38deb1214613..918f47d87d7a 100644 --- a/include/xen/interface/elfnote.h +++ b/include/xen/interface/elfnote.h @@ -11,7 +11,9 @@ #define __XEN_PUBLIC_ELFNOTE_H__ /* - * The notes should live in a SHT_NOTE segment and have "Xen" in the + * `incontents 200 elfnotes ELF notes + * + * The notes should live in a PT_NOTE segment and have "Xen" in the * name field. * * Numeric types are either 4 or 8 bytes depending on the content of @@ -22,6 +24,8 @@ * * String values (for non-legacy) are NULL terminated ASCII, also known * as ASCIZ type. + * + * Xen only uses ELF Notes contained in x86 binaries. */ /* @@ -52,7 +56,7 @@ #define XEN_ELFNOTE_VIRT_BASE 3 /* - * The offset of the ELF paddr field from the acutal required + * The offset of the ELF paddr field from the actual required * pseudo-physical address (numeric). * * This is used to maintain backwards compatibility with older kernels @@ -92,7 +96,12 @@ #define XEN_ELFNOTE_LOADER 8 /* - * The kernel supports PAE (x86/32 only, string = "yes" or "no"). + * The kernel supports PAE (x86/32 only, string = "yes", "no" or + * "bimodal"). + * + * For compatibility with Xen 3.0.3 and earlier the "bimodal" setting + * may be given as "yes,bimodal" which will cause older Xen to treat + * this kernel as PAE. * * LEGACY: PAE (n.b. The legacy interface included a provision to * indicate 'extended-cr3' support allowing L3 page tables to be @@ -149,7 +158,9 @@ * The (non-default) location the initial phys-to-machine map should be * placed at by the hypervisor (Dom0) or the tools (DomU). * The kernel must be prepared for this mapping to be established using - * large pages, despite such otherwise not being available to guests. + * large pages, despite such otherwise not being available to guests. Note + * that these large pages may be misaligned in PFN space (they'll obviously + * be aligned in MFN and virtual address spaces). * The kernel must also be able to handle the page table pages used for * this mapping not being accessible through the initial mapping. * (Only x86-64 supports this at present.) @@ -185,9 +196,81 @@ */ #define XEN_ELFNOTE_PHYS32_ENTRY 18 +/* + * Physical loading constraints for PVH kernels + * + * The presence of this note indicates the kernel supports relocating itself. + * + * The note may include up to three 32bit values to place constraints on the + * guest physical loading addresses and alignment for a PVH kernel. Values + * are read in the following order: + * - a required start alignment (default 0x200000) + * - a minimum address for the start of the image (default 0; see below) + * - a maximum address for the last byte of the image (default 0xffffffff) + * + * When this note specifies an alignment value, it is used. Otherwise the + * maximum p_align value from loadable ELF Program Headers is used, if it is + * greater than or equal to 4k (0x1000). Otherwise, the default is used. + */ +#define XEN_ELFNOTE_PHYS32_RELOC 19 + /* * The number of the highest elfnote defined. */ -#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_ENTRY +#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_RELOC + +/* + * System information exported through crash notes. + * + * The kexec / kdump code will create one XEN_ELFNOTE_CRASH_INFO + * note in case of a system crash. This note will contain various + * information about the system, see xen/include/xen/elfcore.h. + */ +#define XEN_ELFNOTE_CRASH_INFO 0x1000001 + +/* + * System registers exported through crash notes. + * + * The kexec / kdump code will create one XEN_ELFNOTE_CRASH_REGS + * note per cpu in case of a system crash. This note is architecture + * specific and will contain registers not saved in the "CORE" note. + * See xen/include/xen/elfcore.h for more information. + */ +#define XEN_ELFNOTE_CRASH_REGS 0x1000002 + + +/* + * xen dump-core none note. + * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_NONE + * in its dump file to indicate that the file is xen dump-core + * file. This note doesn't have any other information. + * See tools/libxc/xc_core.h for more information. + */ +#define XEN_ELFNOTE_DUMPCORE_NONE 0x2000000 + +/* + * xen dump-core header note. + * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_HEADER + * in its dump file. + * See tools/libxc/xc_core.h for more information. + */ +#define XEN_ELFNOTE_DUMPCORE_HEADER 0x2000001 + +/* + * xen dump-core xen version note. + * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_XEN_VERSION + * in its dump file. It contains the xen version obtained via the + * XENVER hypercall. + * See tools/libxc/xc_core.h for more information. + */ +#define XEN_ELFNOTE_DUMPCORE_XEN_VERSION 0x2000002 + +/* + * xen dump-core format version note. + * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_FORMAT_VERSION + * in its dump file. It contains a format version identifier. + * See tools/libxc/xc_core.h for more information. + */ +#define XEN_ELFNOTE_DUMPCORE_FORMAT_VERSION 0x2000003 #endif /* __XEN_PUBLIC_ELFNOTE_H__ */ From 1db29f99edb056d8445876292f53a63459142309 Mon Sep 17 00:00:00 2001 From: Jason Andryuk Date: Fri, 23 Aug 2024 15:36:27 -0400 Subject: [PATCH 075/104] x86/pvh: Make PVH entrypoint PIC for x86-64 The PVH entrypoint is 32bit non-PIC code running the uncompressed vmlinux at its load address CONFIG_PHYSICAL_START - default 0x1000000 (16MB). The kernel is loaded at that physical address inside the VM by the VMM software (Xen/QEMU). When running a Xen PVH Dom0, the host reserved addresses are mapped 1-1 into the PVH container. There exist system firmwares (Coreboot/EDK2) with reserved memory at 16MB. This creates a conflict where the PVH kernel cannot be loaded at that address. Modify the PVH entrypoint to be position-indepedent to allow flexibility in load address. Only the 64bit entry path is converted. A 32bit kernel is not PIC, so calling into other parts of the kernel, like xen_prepare_pvh() and mk_pgtable_32(), don't work properly when relocated. This makes the code PIC, but the page tables need to be updated as well to handle running from the kernel high map. The UNWIND_HINT_END_OF_STACK is to silence: vmlinux.o: warning: objtool: pvh_start_xen+0x7f: unreachable instruction after the lret into 64bit code. Signed-off-by: Jason Andryuk Reviewed-by: Juergen Gross Message-ID: <20240823193630.2583107-3-jason.andryuk@amd.com> Signed-off-by: Juergen Gross --- arch/x86/platform/pvh/head.S | 46 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index f7235ef87bc3..ba4d0eab4436 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -7,6 +7,7 @@ .code32 .text #define _pa(x) ((x) - __START_KERNEL_map) +#define rva(x) ((x) - pvh_start_xen) #include #include @@ -54,7 +55,25 @@ SYM_CODE_START_LOCAL(pvh_start_xen) UNWIND_HINT_END_OF_STACK cld - lgdt (_pa(gdt)) + /* + * See the comment for startup_32 for more details. We need to + * execute a call to get the execution address to be position + * independent, but we don't have a stack. Save and restore the + * magic field of start_info in ebx, and use that as the stack. + */ + mov (%ebx), %eax + leal 4(%ebx), %esp + ANNOTATE_INTRA_FUNCTION_CALL + call 1f +1: popl %ebp + mov %eax, (%ebx) + subl $rva(1b), %ebp + movl $0, %esp + + leal rva(gdt)(%ebp), %eax + leal rva(gdt_start)(%ebp), %ecx + movl %ecx, 2(%eax) + lgdt (%eax) mov $PVH_DS_SEL,%eax mov %eax,%ds @@ -62,14 +81,14 @@ SYM_CODE_START_LOCAL(pvh_start_xen) mov %eax,%ss /* Stash hvm_start_info. */ - mov $_pa(pvh_start_info), %edi + leal rva(pvh_start_info)(%ebp), %edi mov %ebx, %esi - mov _pa(pvh_start_info_sz), %ecx + movl rva(pvh_start_info_sz)(%ebp), %ecx shr $2,%ecx rep movsl - mov $_pa(early_stack_end), %esp + leal rva(early_stack_end)(%ebp), %esp /* Enable PAE mode. */ mov %cr4, %eax @@ -84,30 +103,33 @@ SYM_CODE_START_LOCAL(pvh_start_xen) wrmsr /* Enable pre-constructed page tables. */ - mov $_pa(init_top_pgt), %eax + leal rva(init_top_pgt)(%ebp), %eax mov %eax, %cr3 mov $(X86_CR0_PG | X86_CR0_PE), %eax mov %eax, %cr0 /* Jump to 64-bit mode. */ - ljmp $PVH_CS_SEL, $_pa(1f) + pushl $PVH_CS_SEL + leal rva(1f)(%ebp), %eax + pushl %eax + lretl /* 64-bit entry point. */ .code64 1: + UNWIND_HINT_END_OF_STACK + /* Set base address in stack canary descriptor. */ mov $MSR_GS_BASE,%ecx - mov $_pa(canary), %eax + leal canary(%rip), %eax xor %edx, %edx wrmsr call xen_prepare_pvh /* startup_64 expects boot_params in %rsi. */ - mov $_pa(pvh_bootparams), %rsi - mov $_pa(startup_64), %rax - ANNOTATE_RETPOLINE_SAFE - jmp *%rax + lea pvh_bootparams(%rip), %rsi + jmp startup_64 #else /* CONFIG_X86_64 */ @@ -143,7 +165,7 @@ SYM_CODE_END(pvh_start_xen) .balign 8 SYM_DATA_START_LOCAL(gdt) .word gdt_end - gdt_start - .long _pa(gdt_start) + .long _pa(gdt_start) /* x86-64 will overwrite if relocated. */ .word 0 SYM_DATA_END(gdt) SYM_DATA_START_LOCAL(gdt_start) From b464b461d27d564125db760938643374864c1b1f Mon Sep 17 00:00:00 2001 From: Jason Andryuk Date: Fri, 23 Aug 2024 15:36:28 -0400 Subject: [PATCH 076/104] x86/pvh: Set phys_base when calling xen_prepare_pvh() phys_base needs to be set for __pa() to work in xen_pvh_init() when finding the hypercall page. Set it before calling into xen_prepare_pvh(), which calls xen_pvh_init(). Clear it afterward to avoid __startup_64() adding to it and creating an incorrect value. Signed-off-by: Jason Andryuk Reviewed-by: Juergen Gross Message-ID: <20240823193630.2583107-4-jason.andryuk@amd.com> Signed-off-by: Juergen Gross --- arch/x86/platform/pvh/head.S | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index ba4d0eab4436..14b4345d9bae 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -125,7 +125,20 @@ SYM_CODE_START_LOCAL(pvh_start_xen) xor %edx, %edx wrmsr + /* + * Calculate load offset and store in phys_base. __pa() needs + * phys_base set to calculate the hypercall page in xen_pvh_init(). + */ + movq %rbp, %rbx + subq $_pa(pvh_start_xen), %rbx + movq %rbx, phys_base(%rip) call xen_prepare_pvh + /* + * Clear phys_base. __startup_64 will *add* to its value, + * so reset to 0. + */ + xor %rbx, %rbx + movq %rbx, phys_base(%rip) /* startup_64 expects boot_params in %rsi. */ lea pvh_bootparams(%rip), %rsi From ada1986d07976d60bed5017aa38b7f7cf27883f7 Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Wed, 25 Sep 2024 22:30:59 +0900 Subject: [PATCH 077/104] tomoyo: fallback to realpath if symlink's pathname does not exist Alfred Agrell found that TOMOYO cannot handle execveat(AT_EMPTY_PATH) inside chroot environment where /dev and /proc are not mounted, for commit 51f39a1f0cea ("syscalls: implement execveat() system call") missed that TOMOYO tries to canonicalize argv[0] when the filename fed to the executed program as argv[0] is supplied using potentially nonexistent pathname. Since "/dev/fd/" already lost symlink information used for obtaining that , it is too late to reconstruct symlink's pathname. Although part of "/dev/fd//" might not be canonicalized, TOMOYO cannot use tomoyo_realpath_nofollow() when /dev or /proc is not mounted. Therefore, fallback to tomoyo_realpath_from_path() when tomoyo_realpath_nofollow() failed. Reported-by: Alfred Agrell Closes: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1082001 Fixes: 51f39a1f0cea ("syscalls: implement execveat() system call") Cc: stable@vger.kernel.org # v3.19+ Signed-off-by: Tetsuo Handa --- security/tomoyo/domain.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index 90b53500a236..aed9e3ef2c9e 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -723,10 +723,13 @@ int tomoyo_find_next_domain(struct linux_binprm *bprm) ee->r.obj = &ee->obj; ee->obj.path1 = bprm->file->f_path; /* Get symlink's pathname of program. */ - retval = -ENOENT; exename.name = tomoyo_realpath_nofollow(original_name); - if (!exename.name) - goto out; + if (!exename.name) { + /* Fallback to realpath if symlink's pathname does not exist. */ + exename.name = tomoyo_realpath_from_path(&bprm->file->f_path); + if (!exename.name) + goto out; + } tomoyo_fill_path_info(&exename); retry: /* Check 'aggregator' directive. */ From e3e8cd90f8e2eef67ded38f9c1a5f5520a407a62 Mon Sep 17 00:00:00 2001 From: Jason Andryuk Date: Fri, 23 Aug 2024 15:36:29 -0400 Subject: [PATCH 078/104] x86/kernel: Move page table macros to header The PVH entry point will need an additional set of prebuild page tables. Move the macros and defines to pgtable_64.h, so they can be re-used. Signed-off-by: Jason Andryuk Reviewed-by: Juergen Gross Acked-by: Dave Hansen Message-ID: <20240823193630.2583107-5-jason.andryuk@amd.com> Signed-off-by: Juergen Gross --- arch/x86/include/asm/pgtable_64.h | 23 ++++++++++++++++++++++- arch/x86/kernel/head_64.S | 20 -------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 3c4407271d08..72912b8edfdf 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -271,5 +271,26 @@ static inline bool gup_fast_permitted(unsigned long start, unsigned long end) #include -#endif /* !__ASSEMBLY__ */ +#else /* __ASSEMBLY__ */ + +#define l4_index(x) (((x) >> 39) & 511) +#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) + +L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4) +L4_START_KERNEL = l4_index(__START_KERNEL_map) + +L3_START_KERNEL = pud_index(__START_KERNEL_map) + +#define SYM_DATA_START_PAGE_ALIGNED(name) \ + SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) + +/* Automate the creation of 1 to 1 mapping pmd entries */ +#define PMDS(START, PERM, COUNT) \ + i = 0 ; \ + .rept (COUNT) ; \ + .quad (START) + (i << PMD_SHIFT) + (PERM) ; \ + i = i + 1 ; \ + .endr + +#endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_PGTABLE_64_H */ diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 330922b328bf..16752b8dfa89 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -32,13 +32,6 @@ * We are not able to switch in one step to the final KERNEL ADDRESS SPACE * because we need identity-mapped pages. */ -#define l4_index(x) (((x) >> 39) & 511) -#define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) - -L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4) -L4_START_KERNEL = l4_index(__START_KERNEL_map) - -L3_START_KERNEL = pud_index(__START_KERNEL_map) __HEAD .code64 @@ -577,9 +570,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) SYM_CODE_END(vc_no_ghcb) #endif -#define SYM_DATA_START_PAGE_ALIGNED(name) \ - SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE) - #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION /* * Each PGD needs to be 8k long and 8k aligned. We do not @@ -601,14 +591,6 @@ SYM_CODE_END(vc_no_ghcb) #define PTI_USER_PGD_FILL 0 #endif -/* Automate the creation of 1 to 1 mapping pmd entries */ -#define PMDS(START, PERM, COUNT) \ - i = 0 ; \ - .rept (COUNT) ; \ - .quad (START) + (i << PMD_SHIFT) + (PERM) ; \ - i = i + 1 ; \ - .endr - __INITDATA .balign 4 @@ -708,8 +690,6 @@ SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt) .endr SYM_DATA_END(level1_fixmap_pgt) -#undef PMDS - .data .align 16 From 47ffe0578aee45fed3a06d5dcff76cdebb303163 Mon Sep 17 00:00:00 2001 From: Jason Andryuk Date: Fri, 23 Aug 2024 15:36:30 -0400 Subject: [PATCH 079/104] x86/pvh: Add 64bit relocation page tables The PVH entry point is 32bit. For a 64bit kernel, the entry point must switch to 64bit mode, which requires a set of page tables. In the past, PVH used init_top_pgt. This works fine when the kernel is loaded at LOAD_PHYSICAL_ADDR, as the page tables are prebuilt for this address. If the kernel is loaded at a different address, they need to be adjusted. __startup_64() adjusts the prebuilt page tables for the physical load address, but it is 64bit code. The 32bit PVH entry code can't call it to adjust the page tables, so it can't readily be re-used. 64bit PVH entry needs page tables set up for identity map, the kernel high map and the direct map. pvh_start_xen() enters identity mapped. Inside xen_prepare_pvh(), it jumps through a pv_ops function pointer into the highmap. The direct map is used for __va() on the initramfs and other guest physical addresses. Add a dedicated set of prebuild page tables for PVH entry. They are adjusted in assembly before loading. Add XEN_ELFNOTE_PHYS32_RELOC to indicate support for relocation along with the kernel's loading constraints. The maximum load address, KERNEL_IMAGE_SIZE - 1, is determined by a single pvh_level2_ident_pgt page. It could be larger with more pages. Signed-off-by: Jason Andryuk Reviewed-by: Juergen Gross Message-ID: <20240823193630.2583107-6-jason.andryuk@amd.com> Signed-off-by: Juergen Gross --- arch/x86/platform/pvh/head.S | 104 ++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 14b4345d9bae..64fca49cd88f 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -102,8 +103,47 @@ SYM_CODE_START_LOCAL(pvh_start_xen) btsl $_EFER_LME, %eax wrmsr + mov %ebp, %ebx + subl $_pa(pvh_start_xen), %ebx /* offset */ + jz .Lpagetable_done + + /* Fixup page-tables for relocation. */ + leal rva(pvh_init_top_pgt)(%ebp), %edi + movl $PTRS_PER_PGD, %ecx +2: + testl $_PAGE_PRESENT, 0x00(%edi) + jz 1f + addl %ebx, 0x00(%edi) +1: + addl $8, %edi + decl %ecx + jnz 2b + + /* L3 ident has a single entry. */ + leal rva(pvh_level3_ident_pgt)(%ebp), %edi + addl %ebx, 0x00(%edi) + + leal rva(pvh_level3_kernel_pgt)(%ebp), %edi + addl %ebx, (PAGE_SIZE - 16)(%edi) + addl %ebx, (PAGE_SIZE - 8)(%edi) + + /* pvh_level2_ident_pgt is fine - large pages */ + + /* pvh_level2_kernel_pgt needs adjustment - large pages */ + leal rva(pvh_level2_kernel_pgt)(%ebp), %edi + movl $PTRS_PER_PMD, %ecx +2: + testl $_PAGE_PRESENT, 0x00(%edi) + jz 1f + addl %ebx, 0x00(%edi) +1: + addl $8, %edi + decl %ecx + jnz 2b + +.Lpagetable_done: /* Enable pre-constructed page tables. */ - leal rva(init_top_pgt)(%ebp), %eax + leal rva(pvh_init_top_pgt)(%ebp), %eax mov %eax, %cr3 mov $(X86_CR0_PG | X86_CR0_PE), %eax mov %eax, %cr0 @@ -198,5 +238,67 @@ SYM_DATA_START_LOCAL(early_stack) .fill BOOT_STACK_SIZE, 1, 0 SYM_DATA_END_LABEL(early_stack, SYM_L_LOCAL, early_stack_end) +#ifdef CONFIG_X86_64 +/* + * Xen PVH needs a set of identity mapped and kernel high mapping + * page tables. pvh_start_xen starts running on the identity mapped + * page tables, but xen_prepare_pvh calls into the high mapping. + * These page tables need to be relocatable and are only used until + * startup_64 transitions to init_top_pgt. + */ +SYM_DATA_START_PAGE_ALIGNED(pvh_init_top_pgt) + .quad pvh_level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .org pvh_init_top_pgt + L4_PAGE_OFFSET * 8, 0 + .quad pvh_level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .org pvh_init_top_pgt + L4_START_KERNEL * 8, 0 + /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ + .quad pvh_level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC +SYM_DATA_END(pvh_init_top_pgt) + +SYM_DATA_START_PAGE_ALIGNED(pvh_level3_ident_pgt) + .quad pvh_level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .fill 511, 8, 0 +SYM_DATA_END(pvh_level3_ident_pgt) +SYM_DATA_START_PAGE_ALIGNED(pvh_level2_ident_pgt) + /* + * Since I easily can, map the first 1G. + * Don't set NX because code runs from these pages. + * + * Note: This sets _PAGE_GLOBAL despite whether + * the CPU supports it or it is enabled. But, + * the CPU should ignore the bit. + */ + PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD) +SYM_DATA_END(pvh_level2_ident_pgt) +SYM_DATA_START_PAGE_ALIGNED(pvh_level3_kernel_pgt) + .fill L3_START_KERNEL, 8, 0 + /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */ + .quad pvh_level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC + .quad 0 /* no fixmap */ +SYM_DATA_END(pvh_level3_kernel_pgt) + +SYM_DATA_START_PAGE_ALIGNED(pvh_level2_kernel_pgt) + /* + * Kernel high mapping. + * + * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in + * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled, + * 512 MiB otherwise. + * + * (NOTE: after that starts the module area, see MODULES_VADDR.) + * + * This table is eventually used by the kernel during normal runtime. + * Care must be taken to clear out undesired bits later, like _PAGE_RW + * or _PAGE_GLOBAL in some cases. + */ + PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE / PMD_SIZE) +SYM_DATA_END(pvh_level2_kernel_pgt) + + ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_RELOC, + .long CONFIG_PHYSICAL_ALIGN; + .long LOAD_PHYSICAL_ADDR; + .long KERNEL_IMAGE_SIZE - 1) +#endif + ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, _ASM_PTR (pvh_start_xen - __START_KERNEL_map)) From 977fae6d611764d41de19b9ba01699b1618148f7 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sat, 24 Aug 2024 20:06:09 +0800 Subject: [PATCH 080/104] sh: Remove unused declarations for make_maskreg_irq() and irq_mask_register make_maskreg_irq() and irq_mask_register have been removed since commit 5a4053b23262 ("sh: Kill off dead boards."), so remove the unused declarations. Signed-off-by: Gaosheng Cui Reviewed-by: John Paul Adrian Glaubitz Signed-off-by: John Paul Adrian Glaubitz --- arch/sh/include/asm/irq.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/sh/include/asm/irq.h b/arch/sh/include/asm/irq.h index 0f384b1f45ca..53fc18a3d4c2 100644 --- a/arch/sh/include/asm/irq.h +++ b/arch/sh/include/asm/irq.h @@ -13,12 +13,6 @@ */ #define NO_IRQ_IGNORE ((unsigned int)-1) -/* - * Simple Mask Register Support - */ -extern void make_maskreg_irq(unsigned int irq); -extern unsigned short *irq_mask_register; - /* * PINT IRQs */ From c3e878ca7b6663d2ad77a6e17460fc47a2347f4a Mon Sep 17 00:00:00 2001 From: Hongbo Li Date: Mon, 2 Sep 2024 10:45:34 +0800 Subject: [PATCH 081/104] sh: intc: Replace simple_strtoul() with kstrtoul() The function simple_strtoul() performs no error checking in scenarios where the input value overflows the intended output variable. We can replace the use of simple_strtoul() with the safer alternative kstrtoul(). This also allows us to print an error message in case of failure. Signed-off-by: Hongbo Li Reviewed-by: Geert Uytterhoeven Reviewed-by: John Paul Adrian Glaubitz Signed-off-by: John Paul Adrian Glaubitz --- drivers/sh/intc/userimask.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/sh/intc/userimask.c b/drivers/sh/intc/userimask.c index abe9091827cd..a363f77881d1 100644 --- a/drivers/sh/intc/userimask.c +++ b/drivers/sh/intc/userimask.c @@ -32,8 +32,11 @@ store_intc_userimask(struct device *dev, const char *buf, size_t count) { unsigned long level; + int ret; - level = simple_strtoul(buf, NULL, 10); + ret = kstrtoul(buf, 10, &level); + if (ret != 0) + return ret; /* * Minimal acceptable IRQ levels are in the 2 - 16 range, but From 4feb014bc79a42485b15bc3912dd3b0bca592520 Mon Sep 17 00:00:00 2001 From: Dipendra Khadka Date: Sun, 22 Sep 2024 16:47:01 +0000 Subject: [PATCH 082/104] dm-cache: remove pointless error check Smatch reported following: ''' drivers/md/dm-cache-target.c:3204 parse_cblock_range() warn: sscanf doesn't return error codes drivers/md/dm-cache-target.c:3217 parse_cblock_range() warn: sscanf doesn't return error codes ''' Sscanf doesn't return negative values at all. Signed-off-by: Dipendra Khadka Signed-off-by: Mikulas Patocka --- drivers/md/dm-cache-target.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 17f0fab1e254..621be35c2b00 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -3200,8 +3200,6 @@ static int parse_cblock_range(struct cache *cache, const char *str, * Try and parse form (ii) first. */ r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy); - if (r < 0) - return r; if (r == 2) { result->begin = to_cblock(b); @@ -3213,8 +3211,6 @@ static int parse_cblock_range(struct cache *cache, const char *str, * That didn't work, try form (i). */ r = sscanf(str, "%llu%c", &b, &dummy); - if (r < 0) - return r; if (r == 1) { result->begin = to_cblock(b); From 0a92e5cdeef9fa4cba8bef6cd1d91cff6b5d300b Mon Sep 17 00:00:00 2001 From: Shen Lichuan Date: Tue, 24 Sep 2024 15:21:11 +0200 Subject: [PATCH 083/104] dm: fix spelling errors Fixed some confusing spelling errors that were currently identified, the details are as follows: -in the code comments: dm-cache-target.c: 1371: exclussive ==> exclusive dm-raid.c: 2522: repective ==> respective Signed-off-by: Shen Lichuan Signed-off-by: Mikulas Patocka --- drivers/md/dm-cache-target.c | 2 +- drivers/md/dm-raid.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 621be35c2b00..aaeeabfab09b 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -1368,7 +1368,7 @@ static void mg_copy(struct work_struct *ws) */ bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio); - BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */ + BUG_ON(rb); /* An exclusive lock must _not_ be held for this block */ mg->overwrite_bio = NULL; inc_io_migrations(mg->cache); mg_full_copy(ws); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 0c3323e0adb2..32c8240c1873 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -2519,7 +2519,7 @@ static int super_validate(struct raid_set *rs, struct md_rdev *rdev) rdev->saved_raid_disk = rdev->raid_disk; } - /* Reshape support -> restore repective data offsets */ + /* Reshape support -> restore respective data offsets */ rdev->data_offset = le64_to_cpu(sb->data_offset); rdev->new_data_offset = le64_to_cpu(sb->new_data_offset); From e6a3531dd542cb127c8de32ab1e54a48ae19962b Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Tue, 24 Sep 2024 15:18:29 +0200 Subject: [PATCH 084/104] dm-verity: restart or panic on an I/O error Maxim Suhanov reported that dm-verity doesn't crash if an I/O error happens. In theory, this could be used to subvert security, because an attacker can create sectors that return error with the Write Uncorrectable command. Some programs may misbehave if they have to deal with EIO. This commit fixes dm-verity, so that if "panic_on_corruption" or "restart_on_corruption" was specified and an I/O error happens, the machine will panic or restart. This commit also changes kernel_restart to emergency_restart - kernel_restart calls reboot notifiers and these reboot notifiers may wait for the bio that failed. emergency_restart doesn't call the notifiers. Reported-by: Maxim Suhanov Signed-off-by: Mikulas Patocka Cc: stable@vger.kernel.org --- drivers/md/dm-verity-target.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index cf659c8feb29..a95c1b9cc5b5 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -272,8 +272,10 @@ out: if (v->mode == DM_VERITY_MODE_LOGGING) return 0; - if (v->mode == DM_VERITY_MODE_RESTART) - kernel_restart("dm-verity device corrupted"); + if (v->mode == DM_VERITY_MODE_RESTART) { + pr_emerg("dm-verity device corrupted\n"); + emergency_restart(); + } if (v->mode == DM_VERITY_MODE_PANIC) panic("dm-verity device corrupted"); @@ -596,6 +598,23 @@ static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) verity_fec_finish_io(io); + if (unlikely(status != BLK_STS_OK) && + unlikely(!(bio->bi_opf & REQ_RAHEAD)) && + !verity_is_system_shutting_down()) { + if (v->mode == DM_VERITY_MODE_RESTART || + v->mode == DM_VERITY_MODE_PANIC) + DMERR_LIMIT("%s has error: %s", v->data_dev->name, + blk_status_to_str(status)); + + if (v->mode == DM_VERITY_MODE_RESTART) { + pr_emerg("dm-verity device corrupted\n"); + emergency_restart(); + } + + if (v->mode == DM_VERITY_MODE_PANIC) + panic("dm-verity device corrupted"); + } + bio_endio(bio); } From 579b2ba40ece57f3f9150f59dfe327e60a5445b5 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 22 Sep 2024 18:17:53 +0200 Subject: [PATCH 085/104] dm verity: fallback to platform keyring also if key in trusted keyring is rejected If enabled, we fallback to the platform keyring if the trusted keyring doesn't have the key used to sign the roothash. But if pkcs7_verify() rejects the key for other reasons, such as usage restrictions, we do not fallback. Do so. Follow-up for 6fce1f40e95182ebbfe1ee3096b8fc0b37903269 Suggested-by: Serge Hallyn Signed-off-by: Luca Boccassi Acked-by: Jarkko Sakkinen Signed-off-by: Mikulas Patocka --- drivers/md/dm-verity-verify-sig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c index d351d7d39c60..a9e2c6c0a33c 100644 --- a/drivers/md/dm-verity-verify-sig.c +++ b/drivers/md/dm-verity-verify-sig.c @@ -127,7 +127,7 @@ int verity_verify_root_hash(const void *root_hash, size_t root_hash_len, #endif VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL); #ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG_PLATFORM_KEYRING - if (ret == -ENOKEY) + if (ret == -ENOKEY || ret == -EKEYREJECTED) ret = verify_pkcs7_signature(root_hash, root_hash_len, sig_data, sig_len, VERIFY_USE_PLATFORM_KEYRING, From c234c6534040b1c1f8adcaf44702fc3e584cb1fe Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Tue, 24 Sep 2024 19:07:24 +0100 Subject: [PATCH 086/104] tools: fix shared radix-tree build The shared radix-tree build is not correctly recompiling when lib/maple_tree.c and lib/test_maple_tree.c are modified - fix this by adding these core components to the SHARED_DEPS list. Additionally, add missing header guards to shared header files. Link: https://lkml.kernel.org/r/20240924180724.112169-1-lorenzo.stoakes@oracle.com Fixes: 74579d8dab47 ("tools: separate out shared radix-tree components") Signed-off-by: Lorenzo Stoakes Tested-by: Sidhartha Kumar Cc: "Liam R. Howlett" Cc: Matthew Wilcox Cc: Vlastimil Babka Cc: Signed-off-by: Andrew Morton --- tools/testing/shared/maple-shared.h | 4 ++++ tools/testing/shared/shared.h | 4 ++++ tools/testing/shared/shared.mk | 4 +++- tools/testing/shared/xarray-shared.h | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/testing/shared/maple-shared.h b/tools/testing/shared/maple-shared.h index 3d847edd149d..dc4d30f3860b 100644 --- a/tools/testing/shared/maple-shared.h +++ b/tools/testing/shared/maple-shared.h @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __MAPLE_SHARED_H__ +#define __MAPLE_SHARED_H__ #define CONFIG_DEBUG_MAPLE_TREE #define CONFIG_MAPLE_SEARCH @@ -7,3 +9,5 @@ #include #include #include "linux/init.h" + +#endif /* __MAPLE_SHARED_H__ */ diff --git a/tools/testing/shared/shared.h b/tools/testing/shared/shared.h index f08f683812ad..13fb4d39966b 100644 --- a/tools/testing/shared/shared.h +++ b/tools/testing/shared/shared.h @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SHARED_H__ +#define __SHARED_H__ #include #include @@ -31,3 +33,5 @@ #ifndef dump_stack #define dump_stack() assert(0) #endif + +#endif /* __SHARED_H__ */ diff --git a/tools/testing/shared/shared.mk b/tools/testing/shared/shared.mk index a05f0588513a..a6bc51d0b0bf 100644 --- a/tools/testing/shared/shared.mk +++ b/tools/testing/shared/shared.mk @@ -15,7 +15,9 @@ SHARED_DEPS = Makefile ../shared/shared.mk ../shared/*.h generated/map-shift.h \ ../../../include/linux/maple_tree.h \ ../../../include/linux/radix-tree.h \ ../../../lib/radix-tree.h \ - ../../../include/linux/idr.h + ../../../include/linux/idr.h \ + ../../../lib/maple_tree.c \ + ../../../lib/test_maple_tree.c ifndef SHIFT SHIFT=3 diff --git a/tools/testing/shared/xarray-shared.h b/tools/testing/shared/xarray-shared.h index ac2d16ff53ae..d50de7884803 100644 --- a/tools/testing/shared/xarray-shared.h +++ b/tools/testing/shared/xarray-shared.h @@ -1,4 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __XARRAY_SHARED_H__ +#define __XARRAY_SHARED_H__ #define XA_DEBUG #include "shared.h" + +#endif /* __XARRAY_SHARED_H__ */ From a3344078101ceee46d14a93f7e3a3b91a55d215b Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Tue, 24 Sep 2024 08:42:05 -0700 Subject: [PATCH 087/104] mm: make SPLIT_PTE_PTLOCKS depend on SMP SPLIT_PTE_PTLOCKS depends on "NR_CPUS >= 4". Unfortunately, that evaluates to true if there is no NR_CPUS configuration option. This results in CONFIG_SPLIT_PTE_PTLOCKS=y for mac_defconfig. This in turn causes the m68k "q800" and "virt" machines to crash in qemu if debugging options are enabled. Making CONFIG_SPLIT_PTE_PTLOCKS dependent on the existence of NR_CPUS does not work since a dependency on the existence of a numeric Kconfig entry always evaluates to false. Example: config HAVE_NO_NR_CPUS def_bool y depends on !NR_CPUS After adding this to a Kconfig file, "make defconfig" includes: $ grep NR_CPUS .config CONFIG_NR_CPUS=64 CONFIG_HAVE_NO_NR_CPUS=y Defining NR_CPUS for m68k does not help either since many architectures define NR_CPUS only for SMP configurations. Make SPLIT_PTE_PTLOCKS depend on SMP instead to solve the problem. Link: https://lkml.kernel.org/r/20240924154205.1491376-1-linux@roeck-us.net Fixes: 394290cba966 ("mm: turn USE_SPLIT_PTE_PTLOCKS / USE_SPLIT_PTE_PTLOCKS into Kconfig options") Signed-off-by: Guenter Roeck Acked-by: David Hildenbrand Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Signed-off-by: Andrew Morton --- mm/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/Kconfig b/mm/Kconfig index 09aebca1cae3..4c9f5ea13271 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -595,6 +595,7 @@ config ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE config SPLIT_PTE_PTLOCKS def_bool y depends on MMU + depends on SMP depends on NR_CPUS >= 4 depends on !ARM || CPU_CACHE_VIPT depends on !PARISC || PA20 From c225c4f6056b46a8a5bf2ed35abf17a2d6887691 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 3 Sep 2024 07:25:17 -0700 Subject: [PATCH 088/104] mm/filemap: fix filemap_get_folios_contig THP panic Patch series "memfd-pin huge page fixes". Fix multiple bugs that occur when using memfd_pin_folios with hugetlb pages and THP. The hugetlb bugs only bite when the page is not yet faulted in when memfd_pin_folios is called. The THP bug bites when the starting offset passed to memfd_pin_folios is not huge page aligned. See the commit messages for details. This patch (of 5): memfd_pin_folios on memory backed by THP panics if the requested start offset is not huge page aligned: BUG: kernel NULL pointer dereference, address: 0000000000000036 RIP: 0010:filemap_get_folios_contig+0xdf/0x290 RSP: 0018:ffffc9002092fbe8 EFLAGS: 00010202 RAX: 0000000000000002 RBX: 0000000000000002 RCX: 0000000000000002 The fault occurs here, because xas_load returns a folio with value 2: filemap_get_folios_contig() for (folio = xas_load(&xas); folio && xas.xa_index <= end; folio = xas_next(&xas)) { ... if (!folio_try_get(folio)) <-- BOOM "2" is an xarray sibling entry. We get it because memfd_pin_folios does not round the indices passed to filemap_get_folios_contig to huge page boundaries for THP, so we load from the middle of a huge page range see a sibling. (It does round for hugetlbfs, at the is_file_hugepages test). To fix, if the folio is a sibling, then return the next index as the starting point for the next call to filemap_get_folios_contig. Link: https://lkml.kernel.org/r/1725373521-451395-1-git-send-email-steven.sistare@oracle.com Link: https://lkml.kernel.org/r/1725373521-451395-2-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Vivek Kasireddy Cc: Signed-off-by: Andrew Morton --- mm/filemap.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/filemap.c b/mm/filemap.c index bbaed3dd5049..36d22968be9a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2196,6 +2196,10 @@ unsigned filemap_get_folios_contig(struct address_space *mapping, if (xa_is_value(folio)) goto update_start; + /* If we landed in the middle of a THP, continue at its end. */ + if (xa_is_sibling(folio)) + goto update_start; + if (!folio_try_get(folio)) goto retry; From c56b6f3d801d7ec8965993342bdd9e2972b6cb8e Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 3 Sep 2024 07:25:18 -0700 Subject: [PATCH 089/104] mm/hugetlb: fix memfd_pin_folios free_huge_pages leak memfd_pin_folios followed by unpin_folios fails to restore free_huge_pages if the pages were not already faulted in, because the folio refcount for pages created by memfd_alloc_folio never goes to 0. memfd_pin_folios needs another folio_put to undo the folio_try_get below: memfd_alloc_folio() alloc_hugetlb_folio_nodemask() dequeue_hugetlb_folio_nodemask() dequeue_hugetlb_folio_node_exact() folio_ref_unfreeze(folio, 1); ; adds 1 refcount folio_try_get() ; adds 1 refcount hugetlb_add_to_page_cache() ; adds 512 refcount (on x86) With the fix, after memfd_pin_folios + unpin_folios, the refcount for the (unfaulted) page is 512, which is correct, as the refcount for a faulted unpinned page is 513. Link: https://lkml.kernel.org/r/1725373521-451395-3-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Acked-by: Vivek Kasireddy Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Signed-off-by: Andrew Morton --- mm/gup.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/gup.c b/mm/gup.c index 8232c8c9c372..3e7eac9a4f2b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3615,7 +3615,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, pgoff_t start_idx, end_idx, next_idx; struct folio *folio = NULL; struct folio_batch fbatch; - struct hstate *h; + struct hstate *h = NULL; long ret = -EINVAL; if (start < 0 || start > end || !max_folios) @@ -3659,6 +3659,8 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, &fbatch); if (folio) { folio_put(folio); + if (h) + folio_put(folio); folio = NULL; } From 26a8ea80929c518bdec5e53a5776f95919b7c88e Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 3 Sep 2024 07:25:19 -0700 Subject: [PATCH 090/104] mm/hugetlb: fix memfd_pin_folios resv_huge_pages leak memfd_pin_folios followed by unpin_folios leaves resv_huge_pages elevated if the pages were not already faulted in. During a normal page fault, resv_huge_pages is consumed here: hugetlb_fault() alloc_hugetlb_folio() dequeue_hugetlb_folio_vma() dequeue_hugetlb_folio_nodemask() dequeue_hugetlb_folio_node_exact() free_huge_pages-- resv_huge_pages-- During memfd_pin_folios, the page is created by calling alloc_hugetlb_folio_nodemask instead of alloc_hugetlb_folio, and resv_huge_pages is not modified: memfd_alloc_folio() alloc_hugetlb_folio_nodemask() dequeue_hugetlb_folio_nodemask() dequeue_hugetlb_folio_node_exact() free_huge_pages-- alloc_hugetlb_folio_nodemask has other callers that must not modify resv_huge_pages. Therefore, to fix, define an alternate version of alloc_hugetlb_folio_nodemask for this call site that adjusts resv_huge_pages. Link: https://lkml.kernel.org/r/1725373521-451395-4-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Acked-by: Vivek Kasireddy Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 10 ++++++++++ mm/hugetlb.c | 17 +++++++++++++++++ mm/memfd.c | 9 ++++----- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 98c47c394b89..e4697539b665 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -692,6 +692,9 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback); +struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, + nodemask_t *nmask, gfp_t gfp_mask); + int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping, pgoff_t idx); void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, @@ -1059,6 +1062,13 @@ static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, return NULL; } +static inline struct folio * +alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, + nodemask_t *nmask, gfp_t gfp_mask) +{ + return NULL; +} + static inline struct folio * alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index def84d8bcf2d..190fa05635f4 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2390,6 +2390,23 @@ struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, return folio; } +struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid, + nodemask_t *nmask, gfp_t gfp_mask) +{ + struct folio *folio; + + spin_lock_irq(&hugetlb_lock); + folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid, + nmask); + if (folio) { + VM_BUG_ON(!h->resv_huge_pages); + h->resv_huge_pages--; + } + + spin_unlock_irq(&hugetlb_lock); + return folio; +} + /* folio migration callback function */ struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback) diff --git a/mm/memfd.c b/mm/memfd.c index e7b7c5294d59..bfe0e7189a37 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -82,11 +82,10 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) gfp_mask = htlb_alloc_mask(hstate_file(memfd)); gfp_mask &= ~(__GFP_HIGHMEM | __GFP_MOVABLE); - folio = alloc_hugetlb_folio_nodemask(hstate_file(memfd), - numa_node_id(), - NULL, - gfp_mask, - false); + folio = alloc_hugetlb_folio_reserve(hstate_file(memfd), + numa_node_id(), + NULL, + gfp_mask); if (folio && folio_try_get(folio)) { err = hugetlb_add_to_page_cache(folio, memfd->f_mapping, From 9289f020da47ef04b28865589eeee3d56d4bafea Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 3 Sep 2024 07:25:20 -0700 Subject: [PATCH 091/104] mm/gup: fix memfd_pin_folios hugetlb page allocation When memfd_pin_folios -> memfd_alloc_folio creates a hugetlb page, the index is wrong. The subsequent call to filemap_get_folios_contig thus cannot find it, and fails, and memfd_pin_folios loops forever. To fix, adjust the index for the huge_page_order. memfd_alloc_folio also forgets to unlock the folio, so the next touch of the page calls hugetlb_fault which blocks forever trying to take the lock. Unlock it. Link: https://lkml.kernel.org/r/1725373521-451395-5-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Acked-by: Vivek Kasireddy Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Signed-off-by: Andrew Morton --- mm/memfd.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mm/memfd.c b/mm/memfd.c index bfe0e7189a37..bcb131db829d 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -79,10 +79,13 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) * alloc from. Also, the folio will be pinned for an indefinite * amount of time, so it is not expected to be migrated away. */ - gfp_mask = htlb_alloc_mask(hstate_file(memfd)); - gfp_mask &= ~(__GFP_HIGHMEM | __GFP_MOVABLE); + struct hstate *h = hstate_file(memfd); - folio = alloc_hugetlb_folio_reserve(hstate_file(memfd), + gfp_mask = htlb_alloc_mask(h); + gfp_mask &= ~(__GFP_HIGHMEM | __GFP_MOVABLE); + idx >>= huge_page_order(h); + + folio = alloc_hugetlb_folio_reserve(h, numa_node_id(), NULL, gfp_mask); @@ -95,6 +98,7 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) free_huge_folio(folio); return ERR_PTR(err); } + folio_unlock(folio); return folio; } return ERR_PTR(-ENOMEM); From ce645b9fdc78ec5d28067286e92871ddae6817d5 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Tue, 3 Sep 2024 07:25:21 -0700 Subject: [PATCH 092/104] mm/gup: fix memfd_pin_folios alloc race panic If memfd_pin_folios tries to create a hugetlb page, but someone else already did, then folio gets the value -EEXIST here: folio = memfd_alloc_folio(memfd, start_idx); if (IS_ERR(folio)) { ret = PTR_ERR(folio); if (ret != -EEXIST) goto err; then on the next trip through the "while start_idx" loop we panic here: if (folio) { folio_put(folio); To fix, set the folio to NULL on error. Link: https://lkml.kernel.org/r/1725373521-451395-6-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Acked-by: Vivek Kasireddy Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Signed-off-by: Andrew Morton --- mm/gup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/gup.c b/mm/gup.c index 3e7eac9a4f2b..8d9b560a0fec 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3702,6 +3702,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, ret = PTR_ERR(folio); if (ret != -EEXIST) goto err; + folio = NULL; } } } From dc677b5f3765cfd0944c8873d1ea57f1a3439676 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Wed, 4 Sep 2024 12:41:08 -0700 Subject: [PATCH 093/104] mm/hugetlb: simplify refs in memfd_alloc_folio The folio_try_get in memfd_alloc_folio is not necessary. Delete it, and delete the matching folio_put in memfd_pin_folios. This also avoids leaking a ref if the memfd_alloc_folio call to hugetlb_add_to_page_cache fails. That error path is also broken in a second way -- when its folio_put causes the ref to become 0, it will implicitly call free_huge_folio, but then the path *explicitly* calls free_huge_folio. Delete the latter. This is a continuation of the fix "mm/hugetlb: fix memfd_pin_folios free_huge_pages leak" [steven.sistare@oracle.com: remove explicit call to free_huge_folio(), per Matthew] Link: https://lkml.kernel.org/r/Zti-7nPVMcGgpcbi@casper.infradead.org Link: https://lkml.kernel.org/r/1725481920-82506-1-git-send-email-steven.sistare@oracle.com Link: https://lkml.kernel.org/r/1725478868-61732-1-git-send-email-steven.sistare@oracle.com Fixes: 89c1905d9c14 ("mm/gup: introduce memfd_pin_folios() for pinning memfd folios") Signed-off-by: Steve Sistare Suggested-by: Vivek Kasireddy Cc: David Hildenbrand Cc: Jason Gunthorpe Cc: Matthew Wilcox Cc: Muchun Song Cc: Peter Xu Cc: Signed-off-by: Andrew Morton --- mm/gup.c | 4 +--- mm/memfd.c | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 8d9b560a0fec..a82890b46a36 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3615,7 +3615,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, pgoff_t start_idx, end_idx, next_idx; struct folio *folio = NULL; struct folio_batch fbatch; - struct hstate *h = NULL; + struct hstate *h; long ret = -EINVAL; if (start < 0 || start > end || !max_folios) @@ -3659,8 +3659,6 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, &fbatch); if (folio) { folio_put(folio); - if (h) - folio_put(folio); folio = NULL; } diff --git a/mm/memfd.c b/mm/memfd.c index bcb131db829d..c17c3ea701a1 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -89,13 +89,12 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) numa_node_id(), NULL, gfp_mask); - if (folio && folio_try_get(folio)) { + if (folio) { err = hugetlb_add_to_page_cache(folio, memfd->f_mapping, idx); if (err) { folio_put(folio); - free_huge_folio(folio); return ERR_PTR(err); } folio_unlock(folio); From 8001070cfbec5cd4ea00b8b48ea51df91122f265 Mon Sep 17 00:00:00 2001 From: Jeongjun Park Date: Tue, 24 Sep 2024 22:00:53 +0900 Subject: [PATCH 094/104] mm: migrate: annotate data-race in migrate_folio_unmap() I found a report from syzbot [1] This report shows that the value can be changed, but in reality, the value of __folio_set_movable() cannot be changed because it holds the folio refcount. Therefore, it is appropriate to add an annotate to make KCSAN ignore that data-race. [1] ================================================================== BUG: KCSAN: data-race in __filemap_remove_folio / migrate_pages_batch write to 0xffffea0004b81dd8 of 8 bytes by task 6348 on cpu 0: page_cache_delete mm/filemap.c:153 [inline] __filemap_remove_folio+0x1ac/0x2c0 mm/filemap.c:233 filemap_remove_folio+0x6b/0x1f0 mm/filemap.c:265 truncate_inode_folio+0x42/0x50 mm/truncate.c:178 shmem_undo_range+0x25b/0xa70 mm/shmem.c:1028 shmem_truncate_range mm/shmem.c:1144 [inline] shmem_evict_inode+0x14d/0x530 mm/shmem.c:1272 evict+0x2f0/0x580 fs/inode.c:731 iput_final fs/inode.c:1883 [inline] iput+0x42a/0x5b0 fs/inode.c:1909 dentry_unlink_inode+0x24f/0x260 fs/dcache.c:412 __dentry_kill+0x18b/0x4c0 fs/dcache.c:615 dput+0x5c/0xd0 fs/dcache.c:857 __fput+0x3fb/0x6d0 fs/file_table.c:439 ____fput+0x1c/0x30 fs/file_table.c:459 task_work_run+0x13a/0x1a0 kernel/task_work.c:228 resume_user_mode_work include/linux/resume_user_mode.h:50 [inline] exit_to_user_mode_loop kernel/entry/common.c:114 [inline] exit_to_user_mode_prepare include/linux/entry-common.h:328 [inline] __syscall_exit_to_user_mode_work kernel/entry/common.c:207 [inline] syscall_exit_to_user_mode+0xbe/0x130 kernel/entry/common.c:218 do_syscall_64+0xd6/0x1c0 arch/x86/entry/common.c:89 entry_SYSCALL_64_after_hwframe+0x77/0x7f read to 0xffffea0004b81dd8 of 8 bytes by task 6342 on cpu 1: __folio_test_movable include/linux/page-flags.h:699 [inline] migrate_folio_unmap mm/migrate.c:1199 [inline] migrate_pages_batch+0x24c/0x1940 mm/migrate.c:1797 migrate_pages_sync mm/migrate.c:1963 [inline] migrate_pages+0xff1/0x1820 mm/migrate.c:2072 do_mbind mm/mempolicy.c:1390 [inline] kernel_mbind mm/mempolicy.c:1533 [inline] __do_sys_mbind mm/mempolicy.c:1607 [inline] __se_sys_mbind+0xf76/0x1160 mm/mempolicy.c:1603 __x64_sys_mbind+0x78/0x90 mm/mempolicy.c:1603 x64_sys_call+0x2b4d/0x2d60 arch/x86/include/generated/asm/syscalls_64.h:238 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xc9/0x1c0 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f value changed: 0xffff888127601078 -> 0x0000000000000000 Link: https://lkml.kernel.org/r/20240924130053.107490-1-aha310510@gmail.com Fixes: 7e2a5e5ab217 ("mm: migrate: use __folio_test_movable()") Signed-off-by: Jeongjun Park Reported-by: syzbot Acked-by: David Hildenbrand Cc: Kefeng Wang Cc: Matthew Wilcox Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/migrate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/migrate.c b/mm/migrate.c index dfdb3a136bf8..df91248755e4 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1196,7 +1196,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, int rc = -EAGAIN; int old_page_state = 0; struct anon_vma *anon_vma = NULL; - bool is_lru = !__folio_test_movable(src); + bool is_lru = data_race(!__folio_test_movable(src)); bool locked = false; bool dst_locked = false; From 5ca60b86f57a4d9648f68418a725b3a7de2816b0 Mon Sep 17 00:00:00 2001 From: Gautham Ananthakrishna Date: Wed, 18 Sep 2024 06:38:44 +0000 Subject: [PATCH 095/104] ocfs2: reserve space for inline xattr before attaching reflink tree One of our customers reported a crash and a corrupted ocfs2 filesystem. The crash was due to the detection of corruption. Upon troubleshooting, the fsck -fn output showed the below corruption [EXTENT_LIST_FREE] Extent list in owner 33080590 claims 230 as the next free chain record, but fsck believes the largest valid value is 227. Clamp the next record value? n The stat output from the debugfs.ocfs2 showed the following corruption where the "Next Free Rec:" had overshot the "Count:" in the root metadata block. Inode: 33080590 Mode: 0640 Generation: 2619713622 (0x9c25a856) FS Generation: 904309833 (0x35e6ac49) CRC32: 00000000 ECC: 0000 Type: Regular Attr: 0x0 Flags: Valid Dynamic Features: (0x16) HasXattr InlineXattr Refcounted Extended Attributes Block: 0 Extended Attributes Inline Size: 256 User: 0 (root) Group: 0 (root) Size: 281320357888 Links: 1 Clusters: 141738 ctime: 0x66911b56 0x316edcb8 -- Fri Jul 12 06:02:30.829349048 2024 atime: 0x66911d6b 0x7f7a28d -- Fri Jul 12 06:11:23.133669517 2024 mtime: 0x66911b56 0x12ed75d7 -- Fri Jul 12 06:02:30.317552087 2024 dtime: 0x0 -- Wed Dec 31 17:00:00 1969 Refcount Block: 2777346 Last Extblk: 2886943 Orphan Slot: 0 Sub Alloc Slot: 0 Sub Alloc Bit: 14 Tree Depth: 1 Count: 227 Next Free Rec: 230 ## Offset Clusters Block# 0 0 2310 2776351 1 2310 2139 2777375 2 4449 1221 2778399 3 5670 731 2779423 4 6401 566 2780447 ....... .... ....... ....... .... ....... The issue was in the reflink workfow while reserving space for inline xattr. The problematic function is ocfs2_reflink_xattr_inline(). By the time this function is called the reflink tree is already recreated at the destination inode from the source inode. At this point, this function reserves space for inline xattrs at the destination inode without even checking if there is space at the root metadata block. It simply reduces the l_count from 243 to 227 thereby making space of 256 bytes for inline xattr whereas the inode already has extents beyond this index (in this case up to 230), thereby causing corruption. The fix for this is to reserve space for inline metadata at the destination inode before the reflink tree gets recreated. The customer has verified the fix. Link: https://lkml.kernel.org/r/20240918063844.1830332-1-gautham.ananthakrishna@oracle.com Fixes: ef962df057aa ("ocfs2: xattr: fix inlined xattr reflink") Signed-off-by: Gautham Ananthakrishna Reviewed-by: Joseph Qi Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/refcounttree.c | 26 ++++++++++++++++++++++++-- fs/ocfs2/xattr.c | 11 +---------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 4f85508538fc..004393b13c0a 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -25,6 +25,7 @@ #include "namei.h" #include "ocfs2_trace.h" #include "file.h" +#include "symlink.h" #include #include @@ -4148,8 +4149,9 @@ static int __ocfs2_reflink(struct dentry *old_dentry, int ret; struct inode *inode = d_inode(old_dentry); struct buffer_head *new_bh = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); - if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SYSTEM_FILE) { + if (oi->ip_flags & OCFS2_INODE_SYSTEM_FILE) { ret = -EINVAL; mlog_errno(ret); goto out; @@ -4175,6 +4177,26 @@ static int __ocfs2_reflink(struct dentry *old_dentry, goto out_unlock; } + if ((oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) && + (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) { + /* + * Adjust extent record count to reserve space for extended attribute. + * Inline data count had been adjusted in ocfs2_duplicate_inline_data(). + */ + struct ocfs2_inode_info *new_oi = OCFS2_I(new_inode); + + if (!(new_oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) && + !(ocfs2_inode_is_fast_symlink(new_inode))) { + struct ocfs2_dinode *new_di = (struct ocfs2_dinode *)new_bh->b_data; + struct ocfs2_dinode *old_di = (struct ocfs2_dinode *)old_bh->b_data; + struct ocfs2_extent_list *el = &new_di->id2.i_list; + int inline_size = le16_to_cpu(old_di->i_xattr_inline_size); + + le16_add_cpu(&el->l_count, -(inline_size / + sizeof(struct ocfs2_extent_rec))); + } + } + ret = ocfs2_create_reflink_node(inode, old_bh, new_inode, new_bh, preserve); if (ret) { @@ -4182,7 +4204,7 @@ static int __ocfs2_reflink(struct dentry *old_dentry, goto inode_unlock; } - if (OCFS2_I(inode)->ip_dyn_features & OCFS2_HAS_XATTR_FL) { + if (oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) { ret = ocfs2_reflink_xattrs(inode, old_bh, new_inode, new_bh, preserve); diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 0e58a5ce539e..dd0a05365e79 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -6511,16 +6511,7 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) } new_oi = OCFS2_I(args->new_inode); - /* - * Adjust extent record count to reserve space for extended attribute. - * Inline data count had been adjusted in ocfs2_duplicate_inline_data(). - */ - if (!(new_oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) && - !(ocfs2_inode_is_fast_symlink(args->new_inode))) { - struct ocfs2_extent_list *el = &new_di->id2.i_list; - le16_add_cpu(&el->l_count, -(inline_size / - sizeof(struct ocfs2_extent_rec))); - } + spin_lock(&new_oi->ip_lock); new_oi->ip_dyn_features |= OCFS2_HAS_XATTR_FL | OCFS2_INLINE_XATTR_FL; new_di->i_dyn_features = cpu_to_le16(new_oi->ip_dyn_features); From 7bf1823e010e8db2fb649c790bd1b449a75f52d8 Mon Sep 17 00:00:00 2001 From: Mohammed Anees Date: Tue, 24 Sep 2024 09:32:57 +0000 Subject: [PATCH 096/104] ocfs2: fix deadlock in ocfs2_get_system_file_inode syzbot has found a possible deadlock in ocfs2_get_system_file_inode [1]. The scenario is depicted here, CPU0 CPU1 lock(&ocfs2_file_ip_alloc_sem_key); lock(&osb->system_file_mutex); lock(&ocfs2_file_ip_alloc_sem_key); lock(&osb->system_file_mutex); The function calls which could lead to this are: CPU0 ocfs2_mknod - lock(&ocfs2_file_ip_alloc_sem_key); . . . ocfs2_get_system_file_inode - lock(&osb->system_file_mutex); CPU1 - ocfs2_fill_super - lock(&osb->system_file_mutex); . . . ocfs2_read_virt_blocks - lock(&ocfs2_file_ip_alloc_sem_key); This issue can be resolved by making the down_read -> down_read_try in the ocfs2_read_virt_blocks. [1] https://syzkaller.appspot.com/bug?extid=e0055ea09f1f5e6fabdd Link: https://lkml.kernel.org/r/20240924093257.7181-1-pvmohammedanees2003@gmail.com Signed-off-by: Mohammed Anees Reviewed-by: Joseph Qi Reported-by: Closes: https://syzkaller.appspot.com/bug?extid=e0055ea09f1f5e6fabdd Tested-by: syzbot+e0055ea09f1f5e6fabdd@syzkaller.appspotmail.com Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/extent_map.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/ocfs2/extent_map.c b/fs/ocfs2/extent_map.c index 70a768b623cf..f7672472fa82 100644 --- a/fs/ocfs2/extent_map.c +++ b/fs/ocfs2/extent_map.c @@ -973,7 +973,13 @@ int ocfs2_read_virt_blocks(struct inode *inode, u64 v_block, int nr, } while (done < nr) { - down_read(&OCFS2_I(inode)->ip_alloc_sem); + if (!down_read_trylock(&OCFS2_I(inode)->ip_alloc_sem)) { + rc = -EAGAIN; + mlog(ML_ERROR, + "Inode #%llu ip_alloc_sem is temporarily unavailable\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + break; + } rc = ocfs2_extent_map_get_blocks(inode, v_block + done, &p_block, &p_count, NULL); up_read(&OCFS2_I(inode)->ip_alloc_sem); From ff7f5ad7bce4fd14f8ed057f1f593ade2840e84d Mon Sep 17 00:00:00 2001 From: "qiwu.chen" Date: Tue, 24 Sep 2024 16:50:04 +0800 Subject: [PATCH 097/104] mm: kfence: fix elapsed time for allocated/freed track Fix elapsed time for the allocated/freed track introduced by commit 62e73fd85d7bf. Link: https://lkml.kernel.org/r/20240924085004.75401-1-qiwu.chen@transsion.com Fixes: 62e73fd85d7b ("mm: kfence: print the elapsed time for allocated/freed track") Signed-off-by: qiwu.chen Reviewed-by: Marco Elver Cc: Alexander Potapenko Cc: Dmitry Vyukov Signed-off-by: Andrew Morton --- mm/kfence/report.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/kfence/report.c b/mm/kfence/report.c index 451991a3a8f2..6370c5207d1a 100644 --- a/mm/kfence/report.c +++ b/mm/kfence/report.c @@ -109,7 +109,7 @@ static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadat const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; u64 ts_sec = track->ts_nsec; unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC); - u64 interval_nsec = local_clock() - meta->alloc_track.ts_nsec; + u64 interval_nsec = local_clock() - track->ts_nsec; unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC); /* Timestamp matches printk timestamp format. */ From 6901cf55de224b2ca51a5675b86c8ef241ae640c Mon Sep 17 00:00:00 2001 From: Diederik de Haas Date: Tue, 24 Sep 2024 10:21:46 +0200 Subject: [PATCH 098/104] mm/damon/Kconfig: update DAMON doc URL The old URL doesn't really work anymore and as the documentation has been integrated in the main kernel documentation site, change the URL to point to that. Link: https://lkml.kernel.org/r/20240924082331.11499-1-didi.debian@cknow.org Signed-off-by: Diederik de Haas Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton --- mm/damon/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index fecb8172410c..35b72f88983a 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -9,7 +9,7 @@ config DAMON access frequency of each memory region. The information can be useful for performance-centric DRAM level memory management. - See https://damonitor.github.io/doc/html/latest-damon/index.html for + See https://www.kernel.org/doc/html/latest/mm/damon/index.html for more information. config DAMON_KUNIT_TEST From c5b1184decc819756ae549ba54c63b6790c4ddfd Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 24 Sep 2024 14:27:10 +0800 Subject: [PATCH 099/104] compiler.h: specify correct attribute for .rodata..c_jump_table Currently, there is an assembler message when generating kernel/bpf/core.o under CONFIG_OBJTOOL with LoongArch compiler toolchain: Warning: setting incorrect section attributes for .rodata..c_jump_table This is because the section ".rodata..c_jump_table" should be readonly, but there is a "W" (writable) part of the flags: $ readelf -S kernel/bpf/core.o | grep -A 1 "rodata..c" [34] .rodata..c_j[...] PROGBITS 0000000000000000 0000d2e0 0000000000000800 0000000000000000 WA 0 0 8 There is no above issue on x86 due to the generated section flag is only "A" (allocatable). In order to silence the warning on LoongArch, specify the attribute like ".rodata..c_jump_table,\"a\",@progbits #" explicitly, then the section attribute of ".rodata..c_jump_table" must be readonly in the kernel/bpf/core.o file. Before: $ objdump -h kernel/bpf/core.o | grep -A 1 "rodata..c" 21 .rodata..c_jump_table 00000800 0000000000000000 0000000000000000 0000d2e0 2**3 CONTENTS, ALLOC, LOAD, RELOC, DATA After: $ objdump -h kernel/bpf/core.o | grep -A 1 "rodata..c" 21 .rodata..c_jump_table 00000800 0000000000000000 0000000000000000 0000d2e0 2**3 CONTENTS, ALLOC, LOAD, RELOC, READONLY, DATA By the way, AFAICT, maybe the root cause is related with the different compiler behavior of various archs, so to some extent this change is a workaround for LoongArch, and also there is no effect for x86 which is the only port supported by objtool before LoongArch with this patch. Link: https://lkml.kernel.org/r/20240924062710.1243-1-yangtiezhu@loongson.cn Signed-off-by: Tiezhu Yang Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: [6.9+] Signed-off-by: Andrew Morton --- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index ec55bcce4146..4d4e23b6e3e7 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -133,7 +133,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #define annotate_unreachable() __annotate_unreachable(__COUNTER__) /* Annotate a C jump table to allow objtool to follow the code flow */ -#define __annotate_jump_table __section(".rodata..c_jump_table") +#define __annotate_jump_table __section(".rodata..c_jump_table,\"a\",@progbits #") #else /* !CONFIG_OBJTOOL */ #define annotate_reachable() From f30beffd977e98c33550bbeb6f278d157ff54844 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Anjum Date: Mon, 23 Sep 2024 10:38:36 +0500 Subject: [PATCH 100/104] kselftests: mm: fix wrong __NR_userfaultfd value grep -rnIF "#define __NR_userfaultfd" tools/include/uapi/asm-generic/unistd.h:681:#define __NR_userfaultfd 282 arch/x86/include/generated/uapi/asm/unistd_32.h:374:#define __NR_userfaultfd 374 arch/x86/include/generated/uapi/asm/unistd_64.h:327:#define __NR_userfaultfd 323 arch/x86/include/generated/uapi/asm/unistd_x32.h:282:#define __NR_userfaultfd (__X32_SYSCALL_BIT + 323) arch/arm/include/generated/uapi/asm/unistd-eabi.h:347:#define __NR_userfaultfd (__NR_SYSCALL_BASE + 388) arch/arm/include/generated/uapi/asm/unistd-oabi.h:359:#define __NR_userfaultfd (__NR_SYSCALL_BASE + 388) include/uapi/asm-generic/unistd.h:681:#define __NR_userfaultfd 282 The number is dependent on the architecture. The above data shows that: x86 374 x86_64 323 The value of __NR_userfaultfd was changed to 282 when asm-generic/unistd.h was included. It makes the test to fail every time as the correct number of this syscall on x86_64 is 323. Fix the header to asm/unistd.h. Link: https://lkml.kernel.org/r/20240923053836.3270393-1-usama.anjum@collabora.com Fixes: a5c6bc590094 ("selftests/mm: remove local __NR_* definitions") Signed-off-by: Muhammad Usama Anjum Reviewed-by: Shuah Khan Reviewed-by: David Hildenbrand Cc: John Hubbard Cc: Signed-off-by: Andrew Morton --- tools/testing/selftests/mm/pagemap_ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index fc90af2a97b8..bcc73b4e805c 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include From c509f67df398f985717601563db577e91e67787f Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Sun, 22 Sep 2024 08:05:07 -0700 Subject: [PATCH 101/104] Revert "list: test: fix tests for list_cut_position()" This reverts commit e620799c414a035dea1208bcb51c869744931dbb. The commit introduces unit test failures. Expected cur == &entries[i], but cur == 0000037fffadfd80 &entries[i] == 0000037fffadfd60 # list_test_list_cut_position: pass:0 fail:1 skip:0 total:1 not ok 21 list_test_list_cut_position # list_test_list_cut_before: EXPECTATION FAILED at lib/list-test.c:444 Expected cur == &entries[i], but cur == 0000037fffa9fd70 &entries[i] == 0000037fffa9fd60 # list_test_list_cut_before: EXPECTATION FAILED at lib/list-test.c:444 Expected cur == &entries[i], but cur == 0000037fffa9fd80 &entries[i] == 0000037fffa9fd70 Revert it. Link: https://lkml.kernel.org/r/20240922150507.553814-1-linux@roeck-us.net Fixes: e620799c414a ("list: test: fix tests for list_cut_position()") Signed-off-by: Guenter Roeck Cc: I Hsin Cheng Cc: David Gow Signed-off-by: Andrew Morton --- lib/list-test.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/list-test.c b/lib/list-test.c index 4f3dc75baec1..e207c4c98d70 100644 --- a/lib/list-test.c +++ b/lib/list-test.c @@ -408,13 +408,10 @@ static void list_test_list_cut_position(struct kunit *test) KUNIT_EXPECT_EQ(test, i, 2); - i = 0; list_for_each(cur, &list1) { KUNIT_EXPECT_PTR_EQ(test, cur, &entries[i]); i++; } - - KUNIT_EXPECT_EQ(test, i, 1); } static void list_test_list_cut_before(struct kunit *test) @@ -439,13 +436,10 @@ static void list_test_list_cut_before(struct kunit *test) KUNIT_EXPECT_EQ(test, i, 1); - i = 0; list_for_each(cur, &list1) { KUNIT_EXPECT_PTR_EQ(test, cur, &entries[i]); i++; } - - KUNIT_EXPECT_EQ(test, i, 2); } static void list_test_list_splice(struct kunit *test) From a530bbc53826c607f64e8ee466c3351efaf6aea5 Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Fri, 20 Sep 2024 09:47:40 +0800 Subject: [PATCH 102/104] memory tiers: use default_dram_perf_ref_source in log message Commit 3718c02dbd4c ("acpi, hmat: calculate abstract distance with HMAT") added a default_dram_perf_ref_source variable that was initialized but never used. This causes kmemleak to report the following memory leak: unreferenced object 0xff11000225a47b60 (size 16): comm "swapper/0", pid 1, jiffies 4294761654 hex dump (first 16 bytes): 41 43 50 49 20 48 4d 41 54 00 c1 4b 7d b7 75 7c ACPI HMAT..K}.u| backtrace (crc e6d0e7b2): [] __kmalloc_node_track_caller_noprof+0x36b/0x440 [] kstrdup+0x36/0x60 [] mt_set_default_dram_perf+0x23a/0x2c0 [] hmat_init+0x2b3/0x660 [] do_one_initcall+0x11c/0x5c0 [] do_initcalls+0x1b4/0x1f0 [] kernel_init_freeable+0x4ae/0x520 [] kernel_init+0x1c/0x150 [] ret_from_fork+0x31/0x70 [] ret_from_fork_asm+0x1a/0x30 This reminds us that we forget to use the performance data source information. So, use the variable in the error log message to help identify the root cause of inconsistent performance number. Link: https://lkml.kernel.org/r/87y13mvo0n.fsf@yhuang6-desk2.ccr.corp.intel.com Fixes: 3718c02dbd4c ("acpi, hmat: calculate abstract distance with HMAT") Signed-off-by: "Huang, Ying" Reported-by: Waiman Long Acked-by: Waiman Long Cc: Alistair Popple Cc: Dave Jiang Signed-off-by: Andrew Morton --- mm/memory-tiers.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index 9842acebd05e..fc14fe53e9b7 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -768,10 +768,10 @@ int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, pr_info( "memory-tiers: the performance of DRAM node %d mismatches that of the reference\n" "DRAM node %d.\n", nid, default_dram_perf_ref_nid); - pr_info(" performance of reference DRAM node %d:\n", - default_dram_perf_ref_nid); + pr_info(" performance of reference DRAM node %d from %s:\n", + default_dram_perf_ref_nid, default_dram_perf_ref_source); dump_hmem_attrs(&default_dram_perf, " "); - pr_info(" performance of DRAM node %d:\n", nid); + pr_info(" performance of DRAM node %d from %s:\n", nid, source); dump_hmem_attrs(perf, " "); pr_info( " disable default DRAM node performance based abstract distance algorithm.\n"); From 486fd58af7ac1098b68370b1d4d9f94a2a1c7124 Mon Sep 17 00:00:00 2001 From: Andrey Skvortsov Date: Mon, 23 Sep 2024 19:48:43 +0300 Subject: [PATCH 103/104] zram: don't free statically defined names When CONFIG_ZRAM_MULTI_COMP isn't set ZRAM_SECONDARY_COMP can hold default_compressor, because it's the same offset as ZRAM_PRIMARY_COMP, so we need to make sure that we don't attempt to kfree() the statically defined compressor name. This is detected by KASAN. ================================================================== Call trace: kfree+0x60/0x3a0 zram_destroy_comps+0x98/0x198 [zram] zram_reset_device+0x22c/0x4a8 [zram] reset_store+0x1bc/0x2d8 [zram] dev_attr_store+0x44/0x80 sysfs_kf_write+0xfc/0x188 kernfs_fop_write_iter+0x28c/0x428 vfs_write+0x4dc/0x9b8 ksys_write+0x100/0x1f8 __arm64_sys_write+0x74/0xb8 invoke_syscall+0xd8/0x260 el0_svc_common.constprop.0+0xb4/0x240 do_el0_svc+0x48/0x68 el0_svc+0x40/0xc8 el0t_64_sync_handler+0x120/0x130 el0t_64_sync+0x190/0x198 ================================================================== Link: https://lkml.kernel.org/r/20240923164843.1117010-1-andrej.skvortzov@gmail.com Fixes: 684826f8271a ("zram: free secondary algorithms names") Signed-off-by: Andrey Skvortsov Reviewed-by: Sergey Senozhatsky Reported-by: Venkat Rao Bagalkote Closes: https://lore.kernel.org/lkml/57130e48-dbb6-4047-a8c7-ebf5aaea93f4@linux.vnet.ibm.com/ Tested-by: Venkat Rao Bagalkote Cc: Christophe JAILLET Cc: Jens Axboe Cc: Minchan Kim Cc: Sergey Senozhatsky Cc: Venkat Rao Bagalkote Cc: Chris Li Signed-off-by: Andrew Morton --- drivers/block/zram/zram_drv.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index c3d245617083..ad9c9bc3ccfc 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2115,8 +2115,10 @@ static void zram_destroy_comps(struct zram *zram) zram->num_active_comps--; } - for (prio = ZRAM_SECONDARY_COMP; prio < ZRAM_MAX_COMPS; prio++) { - kfree(zram->comp_algs[prio]); + for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) { + /* Do not free statically defined compression algorithms */ + if (zram->comp_algs[prio] != default_compressor) + kfree(zram->comp_algs[prio]); zram->comp_algs[prio] = NULL; } From 2af148ef8549a12f8025286b8825c2833ee6bcb8 Mon Sep 17 00:00:00 2001 From: Joseph Qi Date: Wed, 25 Sep 2024 17:06:00 +0800 Subject: [PATCH 104/104] ocfs2: fix uninit-value in ocfs2_get_block() syzbot reported an uninit-value BUG: BUG: KMSAN: uninit-value in ocfs2_get_block+0xed2/0x2710 fs/ocfs2/aops.c:159 ocfs2_get_block+0xed2/0x2710 fs/ocfs2/aops.c:159 do_mpage_readpage+0xc45/0x2780 fs/mpage.c:225 mpage_readahead+0x43f/0x840 fs/mpage.c:374 ocfs2_readahead+0x269/0x320 fs/ocfs2/aops.c:381 read_pages+0x193/0x1110 mm/readahead.c:160 page_cache_ra_unbounded+0x901/0x9f0 mm/readahead.c:273 do_page_cache_ra mm/readahead.c:303 [inline] force_page_cache_ra+0x3b1/0x4b0 mm/readahead.c:332 force_page_cache_readahead mm/internal.h:347 [inline] generic_fadvise+0x6b0/0xa90 mm/fadvise.c:106 vfs_fadvise mm/fadvise.c:185 [inline] ksys_fadvise64_64 mm/fadvise.c:199 [inline] __do_sys_fadvise64 mm/fadvise.c:214 [inline] __se_sys_fadvise64 mm/fadvise.c:212 [inline] __x64_sys_fadvise64+0x1fb/0x3a0 mm/fadvise.c:212 x64_sys_call+0xe11/0x3ba0 arch/x86/include/generated/asm/syscalls_64.h:222 do_syscall_x64 arch/x86/entry/common.c:52 [inline] do_syscall_64+0xcd/0x1e0 arch/x86/entry/common.c:83 entry_SYSCALL_64_after_hwframe+0x77/0x7f This is because when ocfs2_extent_map_get_blocks() fails, p_blkno is uninitialized. So the error log will trigger the above uninit-value access. The error log is out-of-date since get_blocks() was removed long time ago. And the error code will be logged in ocfs2_extent_map_get_blocks() once ocfs2_get_cluster() fails, so fix this by only logging inode and block. Link: https://syzkaller.appspot.com/bug?extid=9709e73bae885b05314b Link: https://lkml.kernel.org/r/20240925090600.3643376-1-joseph.qi@linux.alibaba.com Fixes: ccd979bdbce9 ("[PATCH] OCFS2: The Second Oracle Cluster Filesystem") Signed-off-by: Joseph Qi Reported-by: syzbot+9709e73bae885b05314b@syzkaller.appspotmail.com Tested-by: syzbot+9709e73bae885b05314b@syzkaller.appspotmail.com Cc: Heming Zhao Cc: Mark Fasheh Cc: Joel Becker Cc: Junxiao Bi Cc: Changwei Ge Cc: Gang He Cc: Jun Piao Cc: Signed-off-by: Andrew Morton --- fs/ocfs2/aops.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 1fea43c33b6b..db72b3e924b3 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -156,9 +156,8 @@ int ocfs2_get_block(struct inode *inode, sector_t iblock, err = ocfs2_extent_map_get_blocks(inode, iblock, &p_blkno, &count, &ext_flags); if (err) { - mlog(ML_ERROR, "Error %d from get_blocks(0x%p, %llu, 1, " - "%llu, NULL)\n", err, inode, (unsigned long long)iblock, - (unsigned long long)p_blkno); + mlog(ML_ERROR, "get_blocks() failed, inode: 0x%p, " + "block: %llu\n", inode, (unsigned long long)iblock); goto bail; }