From 455ca134a7b2121dd739d425d3be313fb52f0651 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 30 Apr 2024 11:17:20 +0200 Subject: [PATCH 001/280] x86/percpu: Introduce the pcpu_binary_op() macro Introduce the pcpu_binary_op() macro, a copy of the percpu_to_op() macro. Update percpu binary operators to use the new macro, since percpu_to_op() will be re-purposed as a raw percpu write accessor in a follow-up patch. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240430091833.196482-1-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 47 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 3bedee1801e2..cc40d8d9c272 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -163,6 +163,19 @@ do { \ : [var] "+m" (__my_cpu_var(_var))); \ }) +#define percpu_binary_op(size, qual, op, _var, _val) \ +do { \ + __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ + if (0) { \ + typeof(_var) pto_tmp__; \ + pto_tmp__ = (_val); \ + (void)pto_tmp__; \ + } \ + asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + : [var] "+m" (__my_cpu_var(_var)) \ + : [val] __pcpu_reg_imm_##size(pto_val__)); \ +} while (0) + /* * Generate a percpu add to memory instruction and optimize code * if one is added or subtracted. @@ -182,7 +195,7 @@ do { \ else if (pao_ID__ == -1) \ percpu_unary_op(size, qual, "dec", var); \ else \ - percpu_to_op(size, qual, "add", var, val); \ + percpu_binary_op(size, qual, "add", var, val); \ } while (0) #define percpu_from_op(size, qual, op, _var) \ @@ -492,12 +505,12 @@ do { \ #define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) #define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) #define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val) -#define raw_cpu_and_1(pcp, val) percpu_to_op(1, , "and", (pcp), val) -#define raw_cpu_and_2(pcp, val) percpu_to_op(2, , "and", (pcp), val) -#define raw_cpu_and_4(pcp, val) percpu_to_op(4, , "and", (pcp), val) -#define raw_cpu_or_1(pcp, val) percpu_to_op(1, , "or", (pcp), val) -#define raw_cpu_or_2(pcp, val) percpu_to_op(2, , "or", (pcp), val) -#define raw_cpu_or_4(pcp, val) percpu_to_op(4, , "or", (pcp), val) +#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val) +#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val) +#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val) +#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val) +#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val) +#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val) #define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val) #define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val) #define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val) @@ -505,12 +518,12 @@ do { \ #define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val) #define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val) #define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val) -#define this_cpu_and_1(pcp, val) percpu_to_op(1, volatile, "and", (pcp), val) -#define this_cpu_and_2(pcp, val) percpu_to_op(2, volatile, "and", (pcp), val) -#define this_cpu_and_4(pcp, val) percpu_to_op(4, volatile, "and", (pcp), val) -#define this_cpu_or_1(pcp, val) percpu_to_op(1, volatile, "or", (pcp), val) -#define this_cpu_or_2(pcp, val) percpu_to_op(2, volatile, "or", (pcp), val) -#define this_cpu_or_4(pcp, val) percpu_to_op(4, volatile, "or", (pcp), val) +#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) +#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) +#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) +#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) +#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) +#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val) #define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval) #define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval) #define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval) @@ -543,16 +556,16 @@ do { \ #define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp) #define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) -#define raw_cpu_and_8(pcp, val) percpu_to_op(8, , "and", (pcp), val) -#define raw_cpu_or_8(pcp, val) percpu_to_op(8, , "or", (pcp), val) +#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val) +#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val) #define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val) #define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval) #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval) #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval) #define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val) -#define this_cpu_and_8(pcp, val) percpu_to_op(8, volatile, "and", (pcp), val) -#define this_cpu_or_8(pcp, val) percpu_to_op(8, volatile, "or", (pcp), val) +#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) +#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val) #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val) #define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval) #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval) From 08d564ad699ef32ceaf99d238b3d9c1f4ce5c998 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 30 Apr 2024 11:17:21 +0200 Subject: [PATCH 002/280] x86/percpu: Move some percpu macros around for readability Move some percpu macros around to make a follow-up patch more readable. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240430091833.196482-2-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 63 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index cc40d8d9c272..08113a2e5377 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -144,6 +144,29 @@ #define __pcpu_reg_imm_4(x) "ri" (x) #define __pcpu_reg_imm_8(x) "re" (x) +#ifdef CONFIG_USE_X86_SEG_SUPPORT + +#define __raw_cpu_read(qual, pcp) \ +({ \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ +}) + +#define __raw_cpu_write(qual, pcp, val) \ +do { \ + *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ +} while (0) + +#else /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_from_op(size, qual, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "m" (__my_cpu_var(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) + #define percpu_to_op(size, qual, op, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ @@ -157,6 +180,17 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +#endif /* CONFIG_USE_X86_SEG_SUPPORT */ + +#define percpu_stable_op(size, op, _var) \ +({ \ + __pcpu_type_##size pfo_val__; \ + asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ + : [val] __pcpu_reg_##size("=", pfo_val__) \ + : [var] "i" (&(_var))); \ + (typeof(_var))(unsigned long) pfo_val__; \ +}) + #define percpu_unary_op(size, qual, op, _var) \ ({ \ asm qual (__pcpu_op1_##size(op, __percpu_arg([var])) \ @@ -198,24 +232,6 @@ do { \ percpu_binary_op(size, qual, "add", var, val); \ } while (0) -#define percpu_from_op(size, qual, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "m" (__my_cpu_var(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) - -#define percpu_stable_op(size, op, _var) \ -({ \ - __pcpu_type_##size pfo_val__; \ - asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ - : [val] __pcpu_reg_##size("=", pfo_val__) \ - : [var] "i" (&(_var))); \ - (typeof(_var))(unsigned long) pfo_val__; \ -}) - /* * Add return operation */ @@ -433,17 +449,6 @@ do { \ #define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) #ifdef CONFIG_USE_X86_SEG_SUPPORT - -#define __raw_cpu_read(qual, pcp) \ -({ \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ -}) - -#define __raw_cpu_write(qual, pcp, val) \ -do { \ - *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ -} while (0) - #define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp) #define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp) #define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp) From a50ea641296699af1947336c2e75f6234f53548a Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 30 Apr 2024 11:17:22 +0200 Subject: [PATCH 003/280] x86/percpu: Unify percpu read-write accessors Redefine percpu_from_op() and percpu_to_op() as __raw_cpu_read() and __raw_cpu_write(). Unify __raw_cpu_{read,write}() macros between configs w/ and w/o USE_X86_SEG_SUPPORT in order to unify {raw,this}_cpu{read_write}_N() accessors between configs. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240430091833.196482-3-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 80 +++++++++++++---------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 08113a2e5377..f360ac5fccde 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -146,28 +146,28 @@ #ifdef CONFIG_USE_X86_SEG_SUPPORT -#define __raw_cpu_read(qual, pcp) \ +#define __raw_cpu_read(size, qual, pcp) \ ({ \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ }) -#define __raw_cpu_write(qual, pcp, val) \ +#define __raw_cpu_write(size, qual, pcp, val) \ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0) #else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define percpu_from_op(size, qual, op, _var) \ +#define __raw_cpu_read(size, qual, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ - asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]") \ + asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "m" (__my_cpu_var(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ }) -#define percpu_to_op(size, qual, op, _var, _val) \ +#define __raw_cpu_write(size, qual, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ if (0) { \ @@ -175,7 +175,7 @@ do { \ pto_tmp__ = (_val); \ (void)pto_tmp__; \ } \ - asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var])) \ + asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ : [var] "+m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) @@ -448,54 +448,32 @@ do { \ */ #define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) +#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) +#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) +#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp) +#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val) +#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val) +#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val) + +#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp) +#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp) +#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp) +#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val) +#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val) +#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val) + +#ifdef CONFIG_X86_64 +#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) +#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) + +#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) +#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) +#endif + #ifdef CONFIG_USE_X86_SEG_SUPPORT -#define raw_cpu_read_1(pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_2(pcp) __raw_cpu_read(, pcp) -#define raw_cpu_read_4(pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_1(pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_2(pcp, val) __raw_cpu_write(, pcp, val) -#define raw_cpu_write_4(pcp, val) __raw_cpu_write(, pcp, val) - -#define this_cpu_read_1(pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_2(pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_read_4(pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_1(pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_2(pcp, val) __raw_cpu_write(volatile, pcp, val) -#define this_cpu_write_4(pcp, val) __raw_cpu_write(volatile, pcp, val) - -#ifdef CONFIG_X86_64 -#define raw_cpu_read_8(pcp) __raw_cpu_read(, pcp) -#define raw_cpu_write_8(pcp, val) __raw_cpu_write(, pcp, val) - -#define this_cpu_read_8(pcp) __raw_cpu_read(volatile, pcp) -#define this_cpu_write_8(pcp, val) __raw_cpu_write(volatile, pcp, val) -#endif - -#define this_cpu_read_const(pcp) __raw_cpu_read(, pcp) +#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp) #else /* CONFIG_USE_X86_SEG_SUPPORT */ -#define raw_cpu_read_1(pcp) percpu_from_op(1, , "mov", pcp) -#define raw_cpu_read_2(pcp) percpu_from_op(2, , "mov", pcp) -#define raw_cpu_read_4(pcp) percpu_from_op(4, , "mov", pcp) -#define raw_cpu_write_1(pcp, val) percpu_to_op(1, , "mov", (pcp), val) -#define raw_cpu_write_2(pcp, val) percpu_to_op(2, , "mov", (pcp), val) -#define raw_cpu_write_4(pcp, val) percpu_to_op(4, , "mov", (pcp), val) - -#define this_cpu_read_1(pcp) percpu_from_op(1, volatile, "mov", pcp) -#define this_cpu_read_2(pcp) percpu_from_op(2, volatile, "mov", pcp) -#define this_cpu_read_4(pcp) percpu_from_op(4, volatile, "mov", pcp) -#define this_cpu_write_1(pcp, val) percpu_to_op(1, volatile, "mov", (pcp), val) -#define this_cpu_write_2(pcp, val) percpu_to_op(2, volatile, "mov", (pcp), val) -#define this_cpu_write_4(pcp, val) percpu_to_op(4, volatile, "mov", (pcp), val) - -#ifdef CONFIG_X86_64 -#define raw_cpu_read_8(pcp) percpu_from_op(8, , "mov", pcp) -#define raw_cpu_write_8(pcp, val) percpu_to_op(8, , "mov", (pcp), val) - -#define this_cpu_read_8(pcp) percpu_from_op(8, volatile, "mov", pcp) -#define this_cpu_write_8(pcp, val) percpu_to_op(8, volatile, "mov", (pcp), val) -#endif - /* * The generic per-cpu infrastrucutre is not suitable for * reading const-qualified variables. From 539615de7004a46778020183622856f4ca14e4ac Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 30 Apr 2024 11:17:23 +0200 Subject: [PATCH 004/280] x86/percpu: Introduce the __raw_cpu_read_const() macro Introduce the __raw_cpu_read_const() macro to further reduce ifdeffery and differences between configs w/ and w/o USE_X86_SEG_SUPPORT. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240430091833.196482-4-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index f360ac5fccde..d20255138426 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -156,6 +156,8 @@ do { \ *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ } while (0) +#define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp) + #else /* CONFIG_USE_X86_SEG_SUPPORT */ #define __raw_cpu_read(size, qual, _var) \ @@ -180,6 +182,12 @@ do { \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) +/* + * The generic per-cpu infrastrucutre is not suitable for + * reading const-qualified variables. + */ +#define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) + #endif /* CONFIG_USE_X86_SEG_SUPPORT */ #define percpu_stable_op(size, op, _var) \ @@ -470,16 +478,7 @@ do { \ #define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) #endif -#ifdef CONFIG_USE_X86_SEG_SUPPORT -#define this_cpu_read_const(pcp) __raw_cpu_read(, , pcp) -#else /* CONFIG_USE_X86_SEG_SUPPORT */ - -/* - * The generic per-cpu infrastrucutre is not suitable for - * reading const-qualified variables. - */ -#define this_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) -#endif /* CONFIG_USE_X86_SEG_SUPPORT */ +#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) #define this_cpu_read_stable_1(pcp) percpu_stable_op(1, "mov", pcp) #define this_cpu_read_stable_2(pcp) percpu_stable_op(2, "mov", pcp) From 1fe67aee8ab3fdab4357afc983a9e9ff3892d694 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 30 Apr 2024 11:17:24 +0200 Subject: [PATCH 005/280] x86/percpu: Fix operand constraint modifier in __raw_cpu_write() __raw_cpu_write() with !USE_X86_SEG_SUPPORT config uses read/write operand constraint modifier "+" for its memory location. This signals the compiler that the location is both read and written by the asm. This is not true, because MOV insn only writes to the output. Correct the modifier to "=" to inform the compiler that the memory location is only written to. This also prevents the compiler from value tracking the undefined value from the uninitialized memory. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240430091833.196482-5-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index d20255138426..c77393cd0273 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -178,7 +178,7 @@ do { \ (void)pto_tmp__; \ } \ asm qual(__pcpu_op2_##size("mov", "%[val]", __percpu_arg([var])) \ - : [var] "+m" (__my_cpu_var(_var)) \ + : [var] "=m" (__my_cpu_var(_var)) \ : [val] __pcpu_reg_imm_##size(pto_val__)); \ } while (0) From 48908919c9062bf9472def7389dd7cd9c6a45b70 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 20 May 2024 10:09:24 +0200 Subject: [PATCH 006/280] x86/percpu: Rename percpu_stable_op() to __raw_cpu_read_stable() Rename percpu_stable_op() to __raw_cpu_read_stable() to be in line with other read/write percpu accessors. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Uros Bizjak Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240520080951.121049-1-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index c77393cd0273..39762fcfe328 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -190,10 +190,10 @@ do { \ #endif /* CONFIG_USE_X86_SEG_SUPPORT */ -#define percpu_stable_op(size, op, _var) \ +#define __raw_cpu_read_stable(size, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ - asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]") \ + asm(__pcpu_op2_##size("mov", __force_percpu_arg(a[var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "i" (&(_var))); \ (typeof(_var))(unsigned long) pfo_val__; \ @@ -480,9 +480,9 @@ do { \ #define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) -#define this_cpu_read_stable_1(pcp) percpu_stable_op(1, "mov", pcp) -#define this_cpu_read_stable_2(pcp) percpu_stable_op(2, "mov", pcp) -#define this_cpu_read_stable_4(pcp) percpu_stable_op(4, "mov", pcp) +#define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp) +#define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp) +#define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp) #define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) #define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) @@ -535,7 +535,7 @@ do { \ * 32 bit must fall back to generic operations. */ #ifdef CONFIG_X86_64 -#define this_cpu_read_stable_8(pcp) percpu_stable_op(8, "mov", pcp) +#define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp) #define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) #define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val) From 47c9dbd2fb5f98453840e18ebced9138ec8b4cc5 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 20 May 2024 10:09:25 +0200 Subject: [PATCH 007/280] x86/percpu: Move some percpu accessors around to reduce ifdeffery Move some percpu accessors around, mainly to reduce ifdeffery and improve readabilty by following dependencies between accessors. No functional change intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Link: https://lore.kernel.org/r/20240520080951.121049-2-ubizjak@gmail.com --- arch/x86/include/asm/percpu.h | 40 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 39762fcfe328..0f0d8973f8df 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -445,17 +445,6 @@ do { \ #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval) #endif -/* - * this_cpu_read() makes gcc load the percpu variable every time it is - * accessed while this_cpu_read_stable() allows the value to be cached. - * this_cpu_read_stable() is more efficient and can be used if its value - * is guaranteed to be valid across cpus. The current users include - * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are - * actually per-thread variables implemented as per-CPU variables and - * thus stable for the duration of the respective task. - */ -#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) - #define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) #define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) #define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp) @@ -470,16 +459,6 @@ do { \ #define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val) #define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val) -#ifdef CONFIG_X86_64 -#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) -#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) - -#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) -#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) -#endif - -#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) - #define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp) #define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp) #define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp) @@ -535,6 +514,12 @@ do { \ * 32 bit must fall back to generic operations. */ #ifdef CONFIG_X86_64 +#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) +#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) + +#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) +#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) + #define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp) #define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) @@ -561,6 +546,19 @@ do { \ #define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp) #endif +#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) + +/* + * this_cpu_read() makes gcc load the percpu variable every time it is + * accessed while this_cpu_read_stable() allows the value to be cached. + * this_cpu_read_stable() is more efficient and can be used if its value + * is guaranteed to be valid across cpus. The current users include + * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are + * actually per-thread variables implemented as per-CPU variables and + * thus stable for the duration of the respective task. + */ +#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) + #define x86_this_cpu_constant_test_bit(_nr, _var) \ ({ \ unsigned long __percpu *addr__ = \ From 61d73e4f7d538f3907d954a531169e8164aef56b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 20 May 2024 10:22:39 +0200 Subject: [PATCH 008/280] x86/percpu: Clean up a bit - Fix misc typos - There's 4 variants of the same spelling right now: 'per-CPU', 'per CPU', 'percpu' and 'per-cpu' Standardize on 'per-CPU' only. - s/makes gcc load /makes the compiler load - Instead of: #ifdef CONFIG_XXXX #define YYYY FOO #else #define YYYY BAR #endif Use the slightly more readable form of: #ifdef CONFIG_XXXX # define YYYY FOO #else # define YYYY BAR #endif - Standardize & expand '#else' and '#endif' comments - Fix comment style - Capitalize x86 instruction names in comments No change in code. Signed-off-by: Ingo Molnar Cc: Uros Bizjak Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/percpu.h | 91 +++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 0f0d8973f8df..b424cb1008f7 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -3,30 +3,30 @@ #define _ASM_X86_PERCPU_H #ifdef CONFIG_X86_64 -#define __percpu_seg gs -#define __percpu_rel (%rip) +# define __percpu_seg gs +# define __percpu_rel (%rip) #else -#define __percpu_seg fs -#define __percpu_rel +# define __percpu_seg fs +# define __percpu_rel #endif #ifdef __ASSEMBLY__ #ifdef CONFIG_SMP -#define __percpu %__percpu_seg: +# define __percpu %__percpu_seg: #else -#define __percpu +# define __percpu #endif #define PER_CPU_VAR(var) __percpu(var)__percpu_rel #ifdef CONFIG_X86_64_SMP -#define INIT_PER_CPU_VAR(var) init_per_cpu__##var +# define INIT_PER_CPU_VAR(var) init_per_cpu__##var #else -#define INIT_PER_CPU_VAR(var) var +# define INIT_PER_CPU_VAR(var) var #endif -#else /* ...!ASSEMBLY */ +#else /* !__ASSEMBLY__: */ #include #include @@ -37,19 +37,19 @@ #ifdef CONFIG_CC_HAS_NAMED_AS #ifdef __CHECKER__ -#define __seg_gs __attribute__((address_space(__seg_gs))) -#define __seg_fs __attribute__((address_space(__seg_fs))) +# define __seg_gs __attribute__((address_space(__seg_gs))) +# define __seg_fs __attribute__((address_space(__seg_fs))) #endif #ifdef CONFIG_X86_64 -#define __percpu_seg_override __seg_gs +# define __percpu_seg_override __seg_gs #else -#define __percpu_seg_override __seg_fs +# define __percpu_seg_override __seg_fs #endif #define __percpu_prefix "" -#else /* CONFIG_CC_HAS_NAMED_AS */ +#else /* !CONFIG_CC_HAS_NAMED_AS: */ #define __percpu_seg_override #define __percpu_prefix "%%"__stringify(__percpu_seg)":" @@ -80,7 +80,8 @@ #define PER_CPU_VAR(var) %__percpu_seg:(var)__percpu_rel -#else /* CONFIG_SMP */ +#else /* !CONFIG_SMP: */ + #define __percpu_seg_override #define __percpu_prefix "" #define __force_percpu_prefix "" @@ -96,7 +97,7 @@ #define __force_percpu_arg(x) __force_percpu_prefix "%" #x /* - * Initialized pointers to per-cpu variables needed for the boot + * Initialized pointers to per-CPU variables needed for the boot * processor need to use these macros to get the proper address * offset from __per_cpu_load on SMP. * @@ -106,13 +107,15 @@ extern typeof(var) init_per_cpu_var(var) #ifdef CONFIG_X86_64_SMP -#define init_per_cpu_var(var) init_per_cpu__##var +# define init_per_cpu_var(var) init_per_cpu__##var #else -#define init_per_cpu_var(var) var +# define init_per_cpu_var(var) var #endif -/* For arch-specific code, we can use direct single-insn ops (they - * don't give an lvalue though). */ +/* + * For arch-specific code, we can use direct single-insn ops (they + * don't give an lvalue though). + */ #define __pcpu_type_1 u8 #define __pcpu_type_2 u16 @@ -158,7 +161,7 @@ do { \ #define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp) -#else /* CONFIG_USE_X86_SEG_SUPPORT */ +#else /* !CONFIG_USE_X86_SEG_SUPPORT: */ #define __raw_cpu_read(size, qual, _var) \ ({ \ @@ -183,7 +186,7 @@ do { \ } while (0) /* - * The generic per-cpu infrastrucutre is not suitable for + * The generic per-CPU infrastrucutre is not suitable for * reading const-qualified variables. */ #define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) @@ -219,7 +222,7 @@ do { \ } while (0) /* - * Generate a percpu add to memory instruction and optimize code + * Generate a per-CPU add to memory instruction and optimize code * if one is added or subtracted. */ #define percpu_add_op(size, qual, var, val) \ @@ -266,9 +269,9 @@ do { \ }) /* - * this_cpu_xchg() is implemented using cmpxchg without a lock prefix. - * xchg is expensive due to the implied lock prefix. The processor - * cannot prefetch cachelines if xchg is used. + * this_cpu_xchg() is implemented using CMPXCHG without a LOCK prefix. + * XCHG is expensive due to the implied LOCK prefix. The processor + * cannot prefetch cachelines if XCHG is used. */ #define this_percpu_xchg_op(_var, _nval) \ ({ \ @@ -278,8 +281,8 @@ do { \ }) /* - * cmpxchg has no such implied lock semantics as a result it is much - * more efficient for cpu local operations. + * CMPXCHG has no such implied lock semantics as a result it is much + * more efficient for CPU-local operations. */ #define percpu_cmpxchg_op(size, qual, _var, _oval, _nval) \ ({ \ @@ -314,6 +317,7 @@ do { \ }) #if defined(CONFIG_X86_32) && !defined(CONFIG_UML) + #define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \ ({ \ union { \ @@ -374,7 +378,8 @@ do { \ #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, , pcp, ovalp, nval) #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, volatile, pcp, ovalp, nval) -#endif + +#endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */ #ifdef CONFIG_X86_64 #define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval); @@ -443,7 +448,8 @@ do { \ #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, , pcp, ovalp, nval) #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval) -#endif + +#endif /* CONFIG_X86_64 */ #define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) #define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) @@ -510,8 +516,8 @@ do { \ #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval) /* - * Per cpu atomic 64 bit operations are only available under 64 bit. - * 32 bit must fall back to generic operations. + * Per-CPU atomic 64-bit operations are only available under 64-bit kernels. + * 32-bit kernels must fall back to generic operations. */ #ifdef CONFIG_X86_64 #define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) @@ -539,20 +545,23 @@ do { \ #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval) #define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp) -#else -/* There is no generic 64 bit read stable operation for 32 bit targets. */ + +#else /* !CONFIG_X86_64: */ + +/* There is no generic 64-bit read stable operation for 32-bit targets. */ #define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) #define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp) -#endif + +#endif /* CONFIG_X86_64 */ #define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) /* - * this_cpu_read() makes gcc load the percpu variable every time it is - * accessed while this_cpu_read_stable() allows the value to be cached. + * this_cpu_read() makes the compiler load the per-CPU variable every time + * it is accessed while this_cpu_read_stable() allows the value to be cached. * this_cpu_read_stable() is more efficient and can be used if its value - * is guaranteed to be valid across cpus. The current users include + * is guaranteed to be valid across CPUs. The current users include * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are * actually per-thread variables implemented as per-CPU variables and * thus stable for the duration of the respective task. @@ -626,12 +635,12 @@ DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off); #define early_per_cpu_ptr(_name) (_name##_early_ptr) #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) -#define early_per_cpu(_name, _cpu) \ +#define early_per_cpu(_name, _cpu) \ *(early_per_cpu_ptr(_name) ? \ &early_per_cpu_ptr(_name)[_cpu] : \ &per_cpu(_name, _cpu)) -#else /* !CONFIG_SMP */ +#else /* !CONFIG_SMP: */ #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ DEFINE_PER_CPU(_type, _name) = _initvalue @@ -651,6 +660,6 @@ DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off); #define early_per_cpu_ptr(_name) NULL /* no early_per_cpu_map() */ -#endif /* !CONFIG_SMP */ +#endif /* CONFIG_SMP */ #endif /* _ASM_X86_PERCPU_H */ From 9130ea06163fc229665b9ec4666de9f4ef68284d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 20 May 2024 10:45:06 +0200 Subject: [PATCH 009/280] x86/percpu: Clean up vertical alignment details - Fix/unify misc vertical alignment inconsistencies - Make CPP macros look a bit more like C code by adding an empty line after local variable declaration blocks, and before final rvalue statements. No change in code. Signed-off-by: Ingo Molnar Cc: Uros Bizjak Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Cc: linux-kernel@vger.kernel.org --- arch/x86/include/asm/percpu.h | 301 ++++++++++++++++++---------------- 1 file changed, 161 insertions(+), 140 deletions(-) diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index b424cb1008f7..c55a79d5feae 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -68,11 +68,12 @@ * sizeof(this_cpu_off) becames 4. */ #ifndef BUILD_VDSO32_64 -#define arch_raw_cpu_ptr(_ptr) \ -({ \ - unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \ - tcp_ptr__ += (__force unsigned long)(_ptr); \ - (typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \ +#define arch_raw_cpu_ptr(_ptr) \ +({ \ + unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \ + \ + tcp_ptr__ += (__force unsigned long)(_ptr); \ + (typeof(*(_ptr)) __kernel __force *)tcp_ptr__; \ }) #else #define arch_raw_cpu_ptr(_ptr) ({ BUILD_BUG(); (typeof(_ptr))0; }) @@ -117,35 +118,35 @@ * don't give an lvalue though). */ -#define __pcpu_type_1 u8 -#define __pcpu_type_2 u16 -#define __pcpu_type_4 u32 -#define __pcpu_type_8 u64 +#define __pcpu_type_1 u8 +#define __pcpu_type_2 u16 +#define __pcpu_type_4 u32 +#define __pcpu_type_8 u64 -#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff)) -#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff)) -#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff)) -#define __pcpu_cast_8(val) ((u64)(val)) +#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff)) +#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff)) +#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff)) +#define __pcpu_cast_8(val) ((u64)(val)) -#define __pcpu_op1_1(op, dst) op "b " dst -#define __pcpu_op1_2(op, dst) op "w " dst -#define __pcpu_op1_4(op, dst) op "l " dst -#define __pcpu_op1_8(op, dst) op "q " dst +#define __pcpu_op1_1(op, dst) op "b " dst +#define __pcpu_op1_2(op, dst) op "w " dst +#define __pcpu_op1_4(op, dst) op "l " dst +#define __pcpu_op1_8(op, dst) op "q " dst #define __pcpu_op2_1(op, src, dst) op "b " src ", " dst #define __pcpu_op2_2(op, src, dst) op "w " src ", " dst #define __pcpu_op2_4(op, src, dst) op "l " src ", " dst #define __pcpu_op2_8(op, src, dst) op "q " src ", " dst -#define __pcpu_reg_1(mod, x) mod "q" (x) -#define __pcpu_reg_2(mod, x) mod "r" (x) -#define __pcpu_reg_4(mod, x) mod "r" (x) -#define __pcpu_reg_8(mod, x) mod "r" (x) +#define __pcpu_reg_1(mod, x) mod "q" (x) +#define __pcpu_reg_2(mod, x) mod "r" (x) +#define __pcpu_reg_4(mod, x) mod "r" (x) +#define __pcpu_reg_8(mod, x) mod "r" (x) -#define __pcpu_reg_imm_1(x) "qi" (x) -#define __pcpu_reg_imm_2(x) "ri" (x) -#define __pcpu_reg_imm_4(x) "ri" (x) -#define __pcpu_reg_imm_8(x) "re" (x) +#define __pcpu_reg_imm_1(x) "qi" (x) +#define __pcpu_reg_imm_2(x) "ri" (x) +#define __pcpu_reg_imm_4(x) "ri" (x) +#define __pcpu_reg_imm_8(x) "re" (x) #ifdef CONFIG_USE_X86_SEG_SUPPORT @@ -166,15 +167,18 @@ do { \ #define __raw_cpu_read(size, qual, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ + \ asm qual (__pcpu_op2_##size("mov", __percpu_arg([var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "m" (__my_cpu_var(_var))); \ + \ (typeof(_var))(unsigned long) pfo_val__; \ }) #define __raw_cpu_write(size, qual, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ + \ if (0) { \ typeof(_var) pto_tmp__; \ pto_tmp__ = (_val); \ @@ -196,9 +200,11 @@ do { \ #define __raw_cpu_read_stable(size, _var) \ ({ \ __pcpu_type_##size pfo_val__; \ + \ asm(__pcpu_op2_##size("mov", __force_percpu_arg(a[var]), "%[val]") \ : [val] __pcpu_reg_##size("=", pfo_val__) \ : [var] "i" (&(_var))); \ + \ (typeof(_var))(unsigned long) pfo_val__; \ }) @@ -211,6 +217,7 @@ do { \ #define percpu_binary_op(size, qual, op, _var, _val) \ do { \ __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ + \ if (0) { \ typeof(_var) pto_tmp__; \ pto_tmp__ = (_val); \ @@ -230,6 +237,7 @@ do { \ const int pao_ID__ = (__builtin_constant_p(val) && \ ((val) == 1 || (val) == -1)) ? \ (int)(val) : 0; \ + \ if (0) { \ typeof(var) pao_tmp__; \ pao_tmp__ = (val); \ @@ -249,6 +257,7 @@ do { \ #define percpu_add_return_op(size, qual, _var, _val) \ ({ \ __pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val); \ + \ asm qual (__pcpu_op2_##size("xadd", "%[tmp]", \ __percpu_arg([var])) \ : [tmp] __pcpu_reg_##size("+", paro_tmp__), \ @@ -264,7 +273,9 @@ do { \ #define raw_percpu_xchg_op(_var, _nval) \ ({ \ typeof(_var) pxo_old__ = raw_cpu_read(_var); \ + \ raw_cpu_write(_var, _nval); \ + \ pxo_old__; \ }) @@ -276,7 +287,9 @@ do { \ #define this_percpu_xchg_op(_var, _nval) \ ({ \ typeof(_var) pxo_old__ = this_cpu_read(_var); \ + \ do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval)); \ + \ pxo_old__; \ }) @@ -288,12 +301,14 @@ do { \ ({ \ __pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval); \ __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \ + \ asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \ __percpu_arg([var])) \ : [oval] "+a" (pco_old__), \ [var] "+m" (__my_cpu_var(_var)) \ : [nval] __pcpu_reg_##size(, pco_new__) \ : "memory"); \ + \ (typeof(_var))(unsigned long) pco_old__; \ }) @@ -303,6 +318,7 @@ do { \ __pcpu_type_##size *pco_oval__ = (__pcpu_type_##size *)(_ovalp); \ __pcpu_type_##size pco_old__ = *pco_oval__; \ __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \ + \ asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]", \ __percpu_arg([var])) \ CC_SET(z) \ @@ -313,6 +329,7 @@ do { \ : "memory"); \ if (unlikely(!success)) \ *pco_oval__ = pco_old__; \ + \ likely(success); \ }) @@ -343,8 +360,8 @@ do { \ old__.var; \ }) -#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval) -#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval) +#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval) +#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval) #define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval) \ ({ \ @@ -373,6 +390,7 @@ do { \ : "memory"); \ if (unlikely(!success)) \ *_oval = old__.var; \ + \ likely(success); \ }) @@ -382,8 +400,8 @@ do { \ #endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */ #ifdef CONFIG_X86_64 -#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval); -#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval); +#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval); +#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval); #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval); #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval); @@ -413,8 +431,8 @@ do { \ old__.var; \ }) -#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval) -#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval) +#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval) +#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval) #define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval) \ ({ \ @@ -451,66 +469,66 @@ do { \ #endif /* CONFIG_X86_64 */ -#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) -#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) -#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp) -#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val) -#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val) -#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val) +#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) +#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) +#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp) +#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val) +#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val) +#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val) -#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp) -#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp) -#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp) -#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val) -#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val) -#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val) +#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp) +#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp) +#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp) +#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val) +#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val) +#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val) -#define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp) -#define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp) -#define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp) +#define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp) +#define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp) +#define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp) -#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) -#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) -#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val) -#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val) -#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val) -#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val) -#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val) -#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val) -#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val) -#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val) -#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val) -#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val) +#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) +#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) +#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val) +#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val) +#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val) +#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val) +#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val) +#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val) +#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val) +#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val) +#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val) -#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val) -#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val) -#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val) -#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) -#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) -#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) -#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) -#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) -#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val) -#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval) -#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval) -#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval) +#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val) +#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val) +#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val) +#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) +#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) +#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) +#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) +#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) +#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val) +#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval) +#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval) +#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval) -#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val) -#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val) -#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val) -#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval) -#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval) -#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval) -#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval) -#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval) -#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval) +#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val) +#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val) +#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val) +#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval) +#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval) +#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval) +#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval) +#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval) +#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval) -#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val) -#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val) -#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val) -#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval) -#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval) -#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval) +#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val) +#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val) +#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val) +#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval) +#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval) +#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval) #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, volatile, pcp, ovalp, nval) #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, volatile, pcp, ovalp, nval) #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval) @@ -520,42 +538,43 @@ do { \ * 32-bit kernels must fall back to generic operations. */ #ifdef CONFIG_X86_64 -#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) -#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) -#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) -#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) +#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) +#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) -#define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp) +#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) +#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) -#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) -#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val) -#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val) -#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val) -#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval) -#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval) -#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval) +#define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp) -#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val) -#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) -#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val) -#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val) -#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval) -#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval) +#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) +#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val) +#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val) +#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val) +#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval) +#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval) +#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval) + +#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val) +#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) +#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val) +#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val) +#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval) +#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval) #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval) -#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp) +#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp) #else /* !CONFIG_X86_64: */ /* There is no generic 64-bit read stable operation for 32-bit targets. */ -#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) +#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) -#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp) +#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp) #endif /* CONFIG_X86_64 */ -#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) +#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) /* * this_cpu_read() makes the compiler load the per-CPU variable every time @@ -566,30 +585,31 @@ do { \ * actually per-thread variables implemented as per-CPU variables and * thus stable for the duration of the respective task. */ -#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) +#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) #define x86_this_cpu_constant_test_bit(_nr, _var) \ ({ \ unsigned long __percpu *addr__ = \ (unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \ + \ !!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__)); \ }) -#define x86_this_cpu_variable_test_bit(_nr, _var) \ -({ \ - bool oldbit; \ - \ - asm volatile("btl %[nr], " __percpu_arg([var]) \ - CC_SET(c) \ - : CC_OUT(c) (oldbit) \ - : [var] "m" (__my_cpu_var(_var)), \ - [nr] "rI" (_nr)); \ - oldbit; \ +#define x86_this_cpu_variable_test_bit(_nr, _var) \ +({ \ + bool oldbit; \ + \ + asm volatile("btl %[nr], " __percpu_arg([var]) \ + CC_SET(c) \ + : CC_OUT(c) (oldbit) \ + : [var] "m" (__my_cpu_var(_var)), \ + [nr] "rI" (_nr)); \ + oldbit; \ }) -#define x86_this_cpu_test_bit(_nr, _var) \ - (__builtin_constant_p(_nr) \ - ? x86_this_cpu_constant_test_bit(_nr, _var) \ +#define x86_this_cpu_test_bit(_nr, _var) \ + (__builtin_constant_p(_nr) \ + ? x86_this_cpu_constant_test_bit(_nr, _var) \ : x86_this_cpu_variable_test_bit(_nr, _var)) @@ -620,46 +640,47 @@ DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off); { [0 ... NR_CPUS-1] = _initvalue }; \ __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map -#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ EXPORT_PER_CPU_SYMBOL(_name) -#define DECLARE_EARLY_PER_CPU(_type, _name) \ - DECLARE_PER_CPU(_type, _name); \ - extern __typeof__(_type) *_name##_early_ptr; \ +#define DECLARE_EARLY_PER_CPU(_type, _name) \ + DECLARE_PER_CPU(_type, _name); \ + extern __typeof__(_type) *_name##_early_ptr; \ extern __typeof__(_type) _name##_early_map[] -#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ - DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ - extern __typeof__(_type) *_name##_early_ptr; \ +#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ + DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ + extern __typeof__(_type) *_name##_early_ptr; \ extern __typeof__(_type) _name##_early_map[] -#define early_per_cpu_ptr(_name) (_name##_early_ptr) -#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) -#define early_per_cpu(_name, _cpu) \ - *(early_per_cpu_ptr(_name) ? \ - &early_per_cpu_ptr(_name)[_cpu] : \ +#define early_per_cpu_ptr(_name) (_name##_early_ptr) +#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) + +#define early_per_cpu(_name, _cpu) \ + *(early_per_cpu_ptr(_name) ? \ + &early_per_cpu_ptr(_name)[_cpu] : \ &per_cpu(_name, _cpu)) #else /* !CONFIG_SMP: */ -#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ +#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ DEFINE_PER_CPU(_type, _name) = _initvalue #define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue -#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ EXPORT_PER_CPU_SYMBOL(_name) -#define DECLARE_EARLY_PER_CPU(_type, _name) \ +#define DECLARE_EARLY_PER_CPU(_type, _name) \ DECLARE_PER_CPU(_type, _name) -#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ +#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ DECLARE_PER_CPU_READ_MOSTLY(_type, _name) -#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) -#define early_per_cpu_ptr(_name) NULL +#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) +#define early_per_cpu_ptr(_name) NULL /* no early_per_cpu_map() */ -#endif /* CONFIG_SMP */ +#endif /* !CONFIG_SMP */ #endif /* _ASM_X86_PERCPU_H */ From 47ff30cc1be7bf426c03ecc84371452109b416e4 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 20 May 2024 10:21:14 +0200 Subject: [PATCH 010/280] x86/percpu: Enable named address spaces for all capable GCC versions Enable named address spaces also for GCC 6, GCC 7 and GCC 8 releases. These compilers all produce kernel images that boot without problems. Use compile-time test to detect compiler support for named address spaces. The test passes with GCC 6 as the earliest compiler version where the support for named address spaces was introduced. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Josh Poimboeuf Cc: Linus Torvalds Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/20240520082134.121320-1-ubizjak@gmail.com --- arch/x86/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9d16fee6bdb8..c9e0a54f469e 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2430,7 +2430,8 @@ source "kernel/livepatch/Kconfig" endmenu config CC_HAS_NAMED_AS - def_bool CC_IS_GCC && GCC_VERSION >= 90100 + def_bool $(success,echo 'int __seg_fs fs; int __seg_gs gs;' | $(CC) -x c - -S -o /dev/null) + depends on CC_IS_GCC config CC_HAS_NAMED_AS_FIXED_SANITIZERS def_bool CC_IS_GCC && GCC_VERSION >= 130300 From a0a44d9175b349df2462089140fb7f292100bd7c Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 27 May 2024 11:01:28 +0200 Subject: [PATCH 011/280] mm, slab: don't wrap internal functions with alloc_hooks() The functions __kmalloc_noprof(), kmalloc_large_noprof(), kmalloc_trace_noprof() and their _node variants are all internal to the implementations of kmalloc_noprof() and kmalloc_node_noprof() and are only declared in the "public" slab.h and exported so that those implementations can be static inline and distinguish the build-time constant size variants. The only other users for some of the internal functions are slub_kunit and fortify_kunit tests which make very short-lived allocations. Therefore we can stop wrapping them with the alloc_hooks() macro. Instead add a __ prefix to all of them and a comment documenting these as internal. Also rename __kmalloc_trace() to __kmalloc_cache() which is more descriptive - it is a variant of __kmalloc() where the exact kmalloc cache has been already determined. The usage in fortify_kunit can be removed completely, as the internal functions should be tested already through kmalloc() tests in the test variant that passes non-constant allocation size. Reported-by: Kent Overstreet Cc: Suren Baghdasaryan Cc: Kees Cook Reviewed-by: Kent Overstreet Acked-by: David Rientjes Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 48 ++++++++++++++++++++++---------------------- lib/fortify_kunit.c | 5 ----- lib/slub_kunit.c | 2 +- mm/slub.c | 26 ++++++++++++------------ 4 files changed, 38 insertions(+), 43 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index 7247e217e21b..ed6bee5ec2b6 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -528,9 +528,6 @@ static_assert(PAGE_SHIFT <= 20); #include -void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); -#define __kmalloc(...) alloc_hooks(__kmalloc_noprof(__VA_ARGS__)) - /** * kmem_cache_alloc - Allocate an object * @cachep: The cache to allocate from. @@ -568,31 +565,34 @@ static __always_inline void kfree_bulk(size_t size, void **p) kmem_cache_free_bulk(NULL, size, p); } -void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment - __alloc_size(1); -#define __kmalloc_node(...) alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__)) - void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment __malloc; #define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__)) -void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t flags, size_t size) - __assume_kmalloc_alignment __alloc_size(3); +/* + * The following functions are not to be used directly and are intended only + * for internal use from kmalloc() and kmalloc_node() + * with the exception of kunit tests + */ -void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) __assume_kmalloc_alignment - __alloc_size(4); -#define kmalloc_trace(...) alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__)) +void *__kmalloc_noprof(size_t size, gfp_t flags) + __assume_kmalloc_alignment __alloc_size(1); -#define kmalloc_node_trace(...) alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__)) +void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) + __assume_kmalloc_alignment __alloc_size(1); -void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment - __alloc_size(1); -#define kmalloc_large(...) alloc_hooks(kmalloc_large_noprof(__VA_ARGS__)) +void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size) + __assume_kmalloc_alignment __alloc_size(3); -void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment - __alloc_size(1); -#define kmalloc_large_node(...) alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__)) +void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) + __assume_kmalloc_alignment __alloc_size(4); + +void *__kmalloc_large_noprof(size_t size, gfp_t flags) + __assume_page_alignment __alloc_size(1); + +void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) + __assume_page_alignment __alloc_size(1); /** * kmalloc - allocate kernel memory @@ -654,10 +654,10 @@ static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t f unsigned int index; if (size > KMALLOC_MAX_CACHE_SIZE) - return kmalloc_large_noprof(size, flags); + return __kmalloc_large_noprof(size, flags); index = kmalloc_index(size); - return kmalloc_trace_noprof( + return __kmalloc_cache_noprof( kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], flags, size); } @@ -671,10 +671,10 @@ static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gf unsigned int index; if (size > KMALLOC_MAX_CACHE_SIZE) - return kmalloc_large_node_noprof(size, flags, node); + return __kmalloc_large_node_noprof(size, flags, node); index = kmalloc_index(size); - return kmalloc_node_trace_noprof( + return __kmalloc_cache_node_noprof( kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], flags, node, size); } diff --git a/lib/fortify_kunit.c b/lib/fortify_kunit.c index 39da5b3bc649..044f409ef856 100644 --- a/lib/fortify_kunit.c +++ b/lib/fortify_kunit.c @@ -233,11 +233,6 @@ static void fortify_test_alloc_size_##allocator##_dynamic(struct kunit *test) \ kfree(p)); \ checker(expected_size, \ kmalloc_array_node(alloc_size, 1, gfp, NUMA_NO_NODE), \ - kfree(p)); \ - checker(expected_size, __kmalloc(alloc_size, gfp), \ - kfree(p)); \ - checker(expected_size, \ - __kmalloc_node(alloc_size, gfp, NUMA_NO_NODE), \ kfree(p)); \ \ orig = kmalloc(alloc_size, gfp); \ diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c index 4ce960438806..e6667a28c014 100644 --- a/lib/slub_kunit.c +++ b/lib/slub_kunit.c @@ -140,7 +140,7 @@ static void test_kmalloc_redzone_access(struct kunit *test) { struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_kmalloc", 32, SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE); - u8 *p = kmalloc_trace(s, GFP_KERNEL, 18); + u8 *p = __kmalloc_cache_noprof(s, GFP_KERNEL, 18); kasan_disable_current(); diff --git a/mm/slub.c b/mm/slub.c index 0809760cf789..95e0a3332c44 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4053,7 +4053,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); * directly to the page allocator. We use __GFP_COMP, because we will need to * know the allocation order to free the pages properly in kfree. */ -static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) +static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) { struct folio *folio; void *ptr = NULL; @@ -4078,25 +4078,25 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) return ptr; } -void *kmalloc_large_noprof(size_t size, gfp_t flags) +void *__kmalloc_large_noprof(size_t size, gfp_t flags) { - void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE); + void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE); trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), flags, NUMA_NO_NODE); return ret; } -EXPORT_SYMBOL(kmalloc_large_noprof); +EXPORT_SYMBOL(__kmalloc_large_noprof); -void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) +void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) { - void *ret = __kmalloc_large_node(size, flags, node); + void *ret = ___kmalloc_large_node(size, flags, node); trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), flags, node); return ret; } -EXPORT_SYMBOL(kmalloc_large_node_noprof); +EXPORT_SYMBOL(__kmalloc_large_node_noprof); static __always_inline void *__do_kmalloc_node(size_t size, gfp_t flags, int node, @@ -4106,7 +4106,7 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node, void *ret; if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { - ret = __kmalloc_large_node(size, flags, node); + ret = __kmalloc_large_node_noprof(size, flags, node); trace_kmalloc(caller, ret, size, PAGE_SIZE << get_order(size), flags, node); return ret; @@ -4142,7 +4142,7 @@ void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, } EXPORT_SYMBOL(kmalloc_node_track_caller_noprof); -void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) +void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, size); @@ -4152,10 +4152,10 @@ void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) ret = kasan_kmalloc(s, ret, size, gfpflags); return ret; } -EXPORT_SYMBOL(kmalloc_trace_noprof); +EXPORT_SYMBOL(__kmalloc_cache_noprof); -void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) +void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) { void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); @@ -4164,7 +4164,7 @@ void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, ret = kasan_kmalloc(s, ret, size, gfpflags); return ret; } -EXPORT_SYMBOL(kmalloc_node_trace_noprof); +EXPORT_SYMBOL(__kmalloc_cache_node_noprof); static noinline void free_to_partial_list( struct kmem_cache *s, struct slab *slab, From 3241d46f5f5450ddff255a136f2ebf3282065435 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 14 May 2024 23:01:13 -0700 Subject: [PATCH 012/280] perf pmus: Sort/merge/aggregate PMUs like mrvl_ddr_pmu The mrvl_ddr_pmu is uncore and has a hexadecimal address suffix while the previous PMU sorting/merging code assumes uncore PMU names start with uncore_ and have a decimal suffix. Because of the previous assumption it isn't possible to wildcard the mrvl_ddr_pmu. Modify pmu_name_len_no_suffix but also remove the suffix number out argument, this is because we don't know if a suffix number of say 100 is in hexadecimal or decimal. As the only use of the suffix number is in comparisons, it is safe there to compare the values as hexadecimal. Modify perf_pmu__match_ignoring_suffix so that hexadecimal suffixes are ignored. Only allow hexadecimal suffixes to be greater than length 2 (ie 3 or more) so that S390's cpum_cf PMU doesn't lose its suffix. Change the return type of pmu_name_len_no_suffix to size_t to workaround GCC incorrectly determining the result could be negative. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: James Clark Cc: Robin Murphy Cc: Stephane Eranian Cc: Will Deacon Cc: Thomas Richter Cc: Bharat Bhushan Cc: Bhaskara Budiredla Cc: Tuan Phan Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240515060114.3268149-2-irogers@google.com --- tools/perf/util/pmu.c | 33 +++++++++++++-------- tools/perf/util/pmus.c | 67 ++++++++++++++++++++++++------------------ tools/perf/util/pmus.h | 7 ++++- 3 files changed, 65 insertions(+), 42 deletions(-) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 888ce9912275..c94a91645b21 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -856,26 +856,34 @@ __weak const struct pmu_metrics_table *pmu_metrics_table__find(void) */ static bool perf_pmu__match_ignoring_suffix(const char *pmu_name, const char *tok) { - const char *p; + const char *p, *suffix; + bool has_hex = false; if (strncmp(pmu_name, tok, strlen(tok))) return false; - p = pmu_name + strlen(tok); + suffix = p = pmu_name + strlen(tok); if (*p == 0) return true; - if (*p == '_') + if (*p == '_') { ++p; + ++suffix; + } /* Ensure we end in a number */ while (1) { - if (!isdigit(*p)) + if (!isxdigit(*p)) return false; + if (!has_hex) + has_hex = !isdigit(*p); if (*(++p) == 0) break; } + if (has_hex) + return (p - suffix) > 2; + return true; } @@ -1788,10 +1796,10 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, const struct perf_pmu_alias *alias, bool skip_duplicate_pmus) { struct parse_events_term *term; - int pmu_name_len = skip_duplicate_pmus - ? pmu_name_len_no_suffix(pmu->name, /*num=*/NULL) - : (int)strlen(pmu->name); - int used = snprintf(buf, len, "%.*s/%s", pmu_name_len, pmu->name, alias->name); + size_t pmu_name_len = skip_duplicate_pmus + ? pmu_name_len_no_suffix(pmu->name) + : strlen(pmu->name); + int used = snprintf(buf, len, "%.*s/%s", (int)pmu_name_len, pmu->name, alias->name); list_for_each_entry(term, &alias->terms.terms, list) { if (term->type_val == PARSE_EVENTS__TERM_TYPE_STR) @@ -1828,13 +1836,12 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, pmu_aliases_parse(pmu); pmu_add_cpu_aliases(pmu); list_for_each_entry(event, &pmu->aliases, list) { - size_t buf_used; - int pmu_name_len; + size_t buf_used, pmu_name_len; info.pmu_name = event->pmu_name ?: pmu->name; pmu_name_len = skip_duplicate_pmus - ? pmu_name_len_no_suffix(info.pmu_name, /*num=*/NULL) - : (int)strlen(info.pmu_name); + ? pmu_name_len_no_suffix(info.pmu_name) + : strlen(info.pmu_name); info.alias = NULL; if (event->desc) { info.name = event->name; @@ -1859,7 +1866,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, info.encoding_desc = buf + buf_used; parse_events_terms__to_strbuf(&event->terms, &sb); buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, - "%.*s/%s/", pmu_name_len, info.pmu_name, sb.buf) + 1; + "%.*s/%s/", (int)pmu_name_len, info.pmu_name, sb.buf) + 1; info.topic = event->topic; info.str = sb.buf; info.deprecated = event->deprecated; diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index b9b4c5eb5002..63b9cf9ccfa7 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -40,31 +40,52 @@ static bool read_sysfs_all_pmus; static void pmu_read_sysfs(bool core_only); -int pmu_name_len_no_suffix(const char *str, unsigned long *num) +size_t pmu_name_len_no_suffix(const char *str) { int orig_len, len; + bool has_hex_digits = false; orig_len = len = strlen(str); - /* Non-uncore PMUs have their full length, for example, i915. */ - if (!strstarts(str, "uncore_")) - return len; - - /* - * Count trailing digits and '_', if '_{num}' suffix isn't present use - * the full length. - */ - while (len > 0 && isdigit(str[len - 1])) + /* Count trailing digits. */ + while (len > 0 && isxdigit(str[len - 1])) { + if (!isdigit(str[len - 1])) + has_hex_digits = true; len--; + } if (len > 0 && len != orig_len && str[len - 1] == '_') { - if (num) - *num = strtoul(&str[len], NULL, 10); - return len - 1; + /* + * There is a '_{num}' suffix. For decimal suffixes any length + * will do, for hexadecimal ensure more than 2 hex digits so + * that S390's cpum_cf PMU doesn't match. + */ + if (!has_hex_digits || (orig_len - len) > 2) + return len - 1; } + /* Use the full length. */ return orig_len; } +int pmu_name_cmp(const char *lhs_pmu_name, const char *rhs_pmu_name) +{ + unsigned long lhs_num = 0, rhs_num = 0; + size_t lhs_pmu_name_len = pmu_name_len_no_suffix(lhs_pmu_name); + size_t rhs_pmu_name_len = pmu_name_len_no_suffix(rhs_pmu_name); + int ret = strncmp(lhs_pmu_name, rhs_pmu_name, + lhs_pmu_name_len < rhs_pmu_name_len ? lhs_pmu_name_len : rhs_pmu_name_len); + + if (lhs_pmu_name_len != rhs_pmu_name_len || ret != 0 || lhs_pmu_name_len == 0) + return ret; + + if (lhs_pmu_name_len + 1 < strlen(lhs_pmu_name)) + lhs_num = strtoul(&lhs_pmu_name[lhs_pmu_name_len + 1], NULL, 16); + if (rhs_pmu_name_len + 1 < strlen(rhs_pmu_name)) + rhs_num = strtoul(&rhs_pmu_name[rhs_pmu_name_len + 1], NULL, 16); + + return lhs_num < rhs_num ? -1 : (lhs_num > rhs_num ? 1 : 0); +} + void perf_pmus__destroy(void) { struct perf_pmu *pmu, *tmp; @@ -167,20 +188,10 @@ static struct perf_pmu *perf_pmu__find2(int dirfd, const char *name) static int pmus_cmp(void *priv __maybe_unused, const struct list_head *lhs, const struct list_head *rhs) { - unsigned long lhs_num = 0, rhs_num = 0; struct perf_pmu *lhs_pmu = container_of(lhs, struct perf_pmu, list); struct perf_pmu *rhs_pmu = container_of(rhs, struct perf_pmu, list); - const char *lhs_pmu_name = lhs_pmu->name ?: ""; - const char *rhs_pmu_name = rhs_pmu->name ?: ""; - int lhs_pmu_name_len = pmu_name_len_no_suffix(lhs_pmu_name, &lhs_num); - int rhs_pmu_name_len = pmu_name_len_no_suffix(rhs_pmu_name, &rhs_num); - int ret = strncmp(lhs_pmu_name, rhs_pmu_name, - lhs_pmu_name_len < rhs_pmu_name_len ? lhs_pmu_name_len : rhs_pmu_name_len); - if (lhs_pmu_name_len != rhs_pmu_name_len || ret != 0 || lhs_pmu_name_len == 0) - return ret; - - return lhs_num < rhs_num ? -1 : (lhs_num > rhs_num ? 1 : 0); + return pmu_name_cmp(lhs_pmu->name ?: "", rhs_pmu->name ?: ""); } /* Add all pmus in sysfs to pmu list: */ @@ -300,11 +311,11 @@ static struct perf_pmu *perf_pmus__scan_skip_duplicates(struct perf_pmu *pmu) pmu_read_sysfs(/*core_only=*/false); pmu = list_prepare_entry(pmu, &core_pmus, list); } else - last_pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: "", NULL); + last_pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: ""); if (use_core_pmus) { list_for_each_entry_continue(pmu, &core_pmus, list) { - int pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: "", /*num=*/NULL); + int pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: ""); if (last_pmu_name_len == pmu_name_len && !strncmp(last_pmu_name, pmu->name ?: "", pmu_name_len)) @@ -316,7 +327,7 @@ static struct perf_pmu *perf_pmus__scan_skip_duplicates(struct perf_pmu *pmu) pmu = list_prepare_entry(pmu, &other_pmus, list); } list_for_each_entry_continue(pmu, &other_pmus, list) { - int pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: "", /*num=*/NULL); + int pmu_name_len = pmu_name_len_no_suffix(pmu->name ?: ""); if (last_pmu_name_len == pmu_name_len && !strncmp(last_pmu_name, pmu->name ?: "", pmu_name_len)) @@ -566,7 +577,7 @@ void perf_pmus__print_raw_pmu_events(const struct print_callbacks *print_cb, voi .long_string = STRBUF_INIT, .num_formats = 0, }; - int len = pmu_name_len_no_suffix(pmu->name, /*num=*/NULL); + int len = pmu_name_len_no_suffix(pmu->name); const char *desc = "(see 'man perf-list' or 'man perf-record' on how to encode it)"; if (!pmu->is_core) diff --git a/tools/perf/util/pmus.h b/tools/perf/util/pmus.h index 9d4ded80b8e9..bdbff02324bb 100644 --- a/tools/perf/util/pmus.h +++ b/tools/perf/util/pmus.h @@ -2,10 +2,15 @@ #ifndef __PMUS_H #define __PMUS_H +#include +#include + struct perf_pmu; struct print_callbacks; -int pmu_name_len_no_suffix(const char *str, unsigned long *num); +size_t pmu_name_len_no_suffix(const char *str); +/* Exposed for testing only. */ +int pmu_name_cmp(const char *lhs_pmu_name, const char *rhs_pmu_name); void perf_pmus__destroy(void); From 678be1ca30cc939e0180c85b4cc9150b3d5ef0c8 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 14 May 2024 23:01:14 -0700 Subject: [PATCH 013/280] perf tests: Add some pmu core functionality tests Test behavior of PMU names and comparisons wrt suffixes using Intel uncore_cha, marvell mrvl_ddr_pmu and S390's cpum_cf as examples. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: James Clark Cc: Robin Murphy Cc: Stephane Eranian Cc: Will Deacon Cc: Thomas Richter Cc: Bharat Bhushan Cc: Bhaskara Budiredla Cc: Tuan Phan Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240515060114.3268149-3-irogers@google.com --- tools/perf/tests/pmu.c | 99 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index 06cc0e46cb28..cc88b5920c3e 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -3,6 +3,7 @@ #include "evsel.h" #include "parse-events.h" #include "pmu.h" +#include "pmus.h" #include "tests.h" #include "debug.h" #include "fncache.h" @@ -340,10 +341,108 @@ static int test__pmu_event_names(struct test_suite *test __maybe_unused, return ret; } +static const char * const uncore_chas[] = { + "uncore_cha_0", + "uncore_cha_1", + "uncore_cha_2", + "uncore_cha_3", + "uncore_cha_4", + "uncore_cha_5", + "uncore_cha_6", + "uncore_cha_7", + "uncore_cha_8", + "uncore_cha_9", + "uncore_cha_10", + "uncore_cha_11", + "uncore_cha_12", + "uncore_cha_13", + "uncore_cha_14", + "uncore_cha_15", + "uncore_cha_16", + "uncore_cha_17", + "uncore_cha_18", + "uncore_cha_19", + "uncore_cha_20", + "uncore_cha_21", + "uncore_cha_22", + "uncore_cha_23", + "uncore_cha_24", + "uncore_cha_25", + "uncore_cha_26", + "uncore_cha_27", + "uncore_cha_28", + "uncore_cha_29", + "uncore_cha_30", + "uncore_cha_31", +}; + +static const char * const mrvl_ddrs[] = { + "mrvl_ddr_pmu_87e1b0000000", + "mrvl_ddr_pmu_87e1b1000000", + "mrvl_ddr_pmu_87e1b2000000", + "mrvl_ddr_pmu_87e1b3000000", + "mrvl_ddr_pmu_87e1b4000000", + "mrvl_ddr_pmu_87e1b5000000", + "mrvl_ddr_pmu_87e1b6000000", + "mrvl_ddr_pmu_87e1b7000000", + "mrvl_ddr_pmu_87e1b8000000", + "mrvl_ddr_pmu_87e1b9000000", + "mrvl_ddr_pmu_87e1ba000000", + "mrvl_ddr_pmu_87e1bb000000", + "mrvl_ddr_pmu_87e1bc000000", + "mrvl_ddr_pmu_87e1bd000000", + "mrvl_ddr_pmu_87e1be000000", + "mrvl_ddr_pmu_87e1bf000000", +}; + +static int test__name_len(struct test_suite *test __maybe_unused, int subtest __maybe_unused) +{ + TEST_ASSERT_VAL("cpu", pmu_name_len_no_suffix("cpu") == strlen("cpu")); + TEST_ASSERT_VAL("i915", pmu_name_len_no_suffix("i915") == strlen("i915")); + TEST_ASSERT_VAL("cpum_cf", pmu_name_len_no_suffix("cpum_cf") == strlen("cpum_cf")); + for (size_t i = 0; i < ARRAY_SIZE(uncore_chas); i++) { + TEST_ASSERT_VAL("Strips uncore_cha suffix", + pmu_name_len_no_suffix(uncore_chas[i]) == + strlen("uncore_cha")); + } + for (size_t i = 0; i < ARRAY_SIZE(mrvl_ddrs); i++) { + TEST_ASSERT_VAL("Strips mrvl_ddr_pmu suffix", + pmu_name_len_no_suffix(mrvl_ddrs[i]) == + strlen("mrvl_ddr_pmu")); + } + return TEST_OK; +} + +static int test__name_cmp(struct test_suite *test __maybe_unused, int subtest __maybe_unused) +{ + TEST_ASSERT_EQUAL("cpu", pmu_name_cmp("cpu", "cpu"), 0); + TEST_ASSERT_EQUAL("i915", pmu_name_cmp("i915", "i915"), 0); + TEST_ASSERT_EQUAL("cpum_cf", pmu_name_cmp("cpum_cf", "cpum_cf"), 0); + TEST_ASSERT_VAL("i915", pmu_name_cmp("cpu", "i915") < 0); + TEST_ASSERT_VAL("i915", pmu_name_cmp("i915", "cpu") > 0); + TEST_ASSERT_VAL("cpum_cf", pmu_name_cmp("cpum_cf", "cpum_ce") > 0); + TEST_ASSERT_VAL("cpum_cf", pmu_name_cmp("cpum_cf", "cpum_d0") < 0); + for (size_t i = 1; i < ARRAY_SIZE(uncore_chas); i++) { + TEST_ASSERT_VAL("uncore_cha suffixes ordered lt", + pmu_name_cmp(uncore_chas[i-1], uncore_chas[i]) < 0); + TEST_ASSERT_VAL("uncore_cha suffixes ordered gt", + pmu_name_cmp(uncore_chas[i], uncore_chas[i-1]) > 0); + } + for (size_t i = 1; i < ARRAY_SIZE(mrvl_ddrs); i++) { + TEST_ASSERT_VAL("mrvl_ddr_pmu suffixes ordered lt", + pmu_name_cmp(mrvl_ddrs[i-1], mrvl_ddrs[i]) < 0); + TEST_ASSERT_VAL("mrvl_ddr_pmu suffixes ordered gt", + pmu_name_cmp(mrvl_ddrs[i], mrvl_ddrs[i-1]) > 0); + } + return TEST_OK; +} + static struct test_case tests__pmu[] = { TEST_CASE("Parsing with PMU format directory", pmu_format), TEST_CASE("Parsing with PMU event", pmu_events), TEST_CASE("PMU event names", pmu_event_names), + TEST_CASE("PMU name combining", name_len), + TEST_CASE("PMU name comparison", name_cmp), { .name = NULL, } }; From cbd446b4db7efce27311f3186f81c2a2d906dd60 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 13 May 2024 22:24:02 -0700 Subject: [PATCH 014/280] perf arm-spe: Unaligned pointer work around Use get_unaligned_leXX instead of leXX_to_cpu to handle unaligned pointers. Such pointers occur with libFuzzer testing. A similar change for intel-pt was done in: https://lore.kernel.org/r/20231005190451.175568-6-adrian.hunter@intel.com Signed-off-by: Ian Rogers Reviewed-by: James Clark Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240514052402.3031871-1-irogers@google.com --- .../arm-spe-decoder/arm-spe-pkt-decoder.c | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c index a454c6737563..7bf607d0f6d8 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -10,24 +10,11 @@ #include #include #include +#include +#include #include "arm-spe-pkt-decoder.h" -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define le16_to_cpu bswap_16 -#define le32_to_cpu bswap_32 -#define le64_to_cpu bswap_64 -#define memcpy_le64(d, s, n) do { \ - memcpy((d), (s), (n)); \ - *(d) = le64_to_cpu(*(d)); \ -} while (0) -#else -#define le16_to_cpu -#define le32_to_cpu -#define le64_to_cpu -#define memcpy_le64 memcpy -#endif - static const char * const arm_spe_packet_name[] = { [ARM_SPE_PAD] = "PAD", [ARM_SPE_END] = "END", @@ -70,9 +57,9 @@ static int arm_spe_get_payload(const unsigned char *buf, size_t len, switch (payload_len) { case 1: packet->payload = *(uint8_t *)buf; break; - case 2: packet->payload = le16_to_cpu(*(uint16_t *)buf); break; - case 4: packet->payload = le32_to_cpu(*(uint32_t *)buf); break; - case 8: packet->payload = le64_to_cpu(*(uint64_t *)buf); break; + case 2: packet->payload = get_unaligned_le16(buf); break; + case 4: packet->payload = get_unaligned_le32(buf); break; + case 8: packet->payload = get_unaligned_le64(buf); break; default: return ARM_SPE_BAD_PACKET; } From 265b71153e1ac270546f1d0d2a59a565947f2ed3 Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 17 May 2024 07:14:26 -0700 Subject: [PATCH 015/280] perf list: Fix the --no-desc option Currently, the --no-desc option in perf list isn't functioning as intended. This issue arises from the overwriting of struct option->desc with the opposite value of struct option->long_desc. Consequently, whatever parse_options() returns at struct option->desc gets overridden later, rendering the --desc or --no-desc arguments ineffective. To resolve this, set ->desc as true by default and allow parse_options() to adjust it accordingly. This adjustment will fix the --no-desc option while preserving the functionality of the other parameters. Signed-off-by: Breno Leitao Reviewed-by: Ian Rogers Cc: leit@meta.com Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240517141427.1905691-1-leitao@debian.org --- tools/perf/builtin-list.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tools/perf/builtin-list.c b/tools/perf/builtin-list.c index 5cab31231551..82cb4b1010aa 100644 --- a/tools/perf/builtin-list.c +++ b/tools/perf/builtin-list.c @@ -162,7 +162,11 @@ static void default_print_event(void *ps, const char *pmu_name, const char *topi } else fputc('\n', fp); - if (desc && print_state->desc) { + if (long_desc && print_state->long_desc) { + fprintf(fp, "%*s", 8, "["); + wordwrap(fp, long_desc, 8, pager_get_columns(), 0); + fprintf(fp, "]\n"); + } else if (desc && print_state->desc) { char *desc_with_unit = NULL; int desc_len = -1; @@ -178,12 +182,6 @@ static void default_print_event(void *ps, const char *pmu_name, const char *topi fprintf(fp, "]\n"); free(desc_with_unit); } - long_desc = long_desc ?: desc; - if (long_desc && print_state->long_desc) { - fprintf(fp, "%*s", 8, "["); - wordwrap(fp, long_desc, 8, pager_get_columns(), 0); - fprintf(fp, "]\n"); - } if (print_state->detailed && encoding_desc) { fprintf(fp, "%*s", 8, ""); @@ -256,15 +254,14 @@ static void default_print_metric(void *ps, } fprintf(fp, " %s\n", name); - if (desc && print_state->desc) { - fprintf(fp, "%*s", 8, "["); - wordwrap(fp, desc, 8, pager_get_columns(), 0); - fprintf(fp, "]\n"); - } if (long_desc && print_state->long_desc) { fprintf(fp, "%*s", 8, "["); wordwrap(fp, long_desc, 8, pager_get_columns(), 0); fprintf(fp, "]\n"); + } else if (desc && print_state->desc) { + fprintf(fp, "%*s", 8, "["); + wordwrap(fp, desc, 8, pager_get_columns(), 0); + fprintf(fp, "]\n"); } if (expr && print_state->detailed) { fprintf(fp, "%*s", 8, "["); @@ -507,6 +504,7 @@ int cmd_list(int argc, const char **argv) int i, ret = 0; struct print_state default_ps = { .fp = stdout, + .desc = true, }; struct print_state json_ps = { .fp = stdout, @@ -579,7 +577,6 @@ int cmd_list(int argc, const char **argv) }; ps = &json_ps; } else { - default_ps.desc = !default_ps.long_desc; default_ps.last_topic = strdup(""); assert(default_ps.last_topic); default_ps.visited_metrics = strlist__new(NULL, NULL); From a93c83eca48a4ffb8e57cb0c7cc2e3935744d2c6 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 21 May 2024 15:35:55 -0700 Subject: [PATCH 016/280] perf docs: Fix typos Assorted typo fixes. Signed-off-by: Ian Rogers Reviewed-by: James Clark Cc: Changbin Du Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240521223555.858859-1-irogers@google.com --- tools/perf/Documentation/perf-kwork.txt | 4 ++-- tools/perf/Documentation/perf-mem.txt | 2 +- tools/perf/Documentation/perf-record.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 109ace1d5e90..21e607669d78 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -1,4 +1,4 @@ -perf-kowrk(1) +perf-kwork(1) ============= NAME @@ -35,7 +35,7 @@ There are several variants of 'perf kwork': perf kwork top perf kwork top -b - By default it shows the individual work events such as irq, workqeueu, + By default it shows the individual work events such as irq, workqueue, including the run time and delay (time between raise and actually entry): Runtime start Runtime end Cpu Kwork name Runtime Delaytime diff --git a/tools/perf/Documentation/perf-mem.txt b/tools/perf/Documentation/perf-mem.txt index 19862572e3f2..47456b212e99 100644 --- a/tools/perf/Documentation/perf-mem.txt +++ b/tools/perf/Documentation/perf-mem.txt @@ -21,7 +21,7 @@ and stores are sampled. Use the -t option to limit to loads or stores. Note that on Intel systems the memory latency reported is the use-latency, not the pure load (or store latency). Use latency includes any pipeline -queueing delays in addition to the memory subsystem latency. +queuing delays in addition to the memory subsystem latency. On Arm64 this uses SPE to sample load and store operations, therefore hardware and kernel support is required. See linkperf:perf-arm-spe[1] for a setup guide. diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index 6015fdd08fb6..77578c0a142a 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -311,7 +311,7 @@ OPTIONS User can change the size by passing the size after comma like "--call-graph dwarf,4096". - When "fp" recording is used, perf tries to save stack enties + When "fp" recording is used, perf tries to save stack entries up to the number specified in sysctl.kernel.perf_event_max_stack by default. User can change the number by passing it after comma like "--call-graph fp,32". From 92968dcc037fed045dab5c8e52b51255d77f5432 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Wed, 22 May 2024 11:35:41 +0800 Subject: [PATCH 017/280] perf trace beauty: Always show param if show_zero is set For some parameters, it is best to also display them when they are 0, e.g. flags. Here we only check the show_zero property and let arg printer handle special cases. Signed-off-by: Changbin Du Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240522033542.1359421-2-changbin.du@huawei.com --- tools/perf/builtin-trace.c | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 51eca671c797..a36e98c7a1c5 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -2091,17 +2091,11 @@ static size_t syscall__scnprintf_args(struct syscall *sc, char *bf, size_t size, val = syscall_arg_fmt__mask_val(&sc->arg_fmt[arg.idx], &arg, val); /* - * Suppress this argument if its value is zero and - * and we don't have a string associated in an - * strarray for it. - */ - if (val == 0 && - !trace->show_zeros && - !(sc->arg_fmt && - (sc->arg_fmt[arg.idx].show_zero || - sc->arg_fmt[arg.idx].scnprintf == SCA_STRARRAY || - sc->arg_fmt[arg.idx].scnprintf == SCA_STRARRAYS) && - sc->arg_fmt[arg.idx].parm)) + * Suppress this argument if its value is zero and show_zero + * property isn't set. + */ + if (val == 0 && !trace->show_zeros && + !(sc->arg_fmt && sc->arg_fmt[arg.idx].show_zero)) continue; printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); @@ -2796,17 +2790,8 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, */ val = syscall_arg_fmt__mask_val(arg, &syscall_arg, val); - /* - * Suppress this argument if its value is zero and - * we don't have a string associated in an - * strarray for it. - */ - if (val == 0 && - !trace->show_zeros && - !((arg->show_zero || - arg->scnprintf == SCA_STRARRAY || - arg->scnprintf == SCA_STRARRAYS) && - arg->parm)) + /* Suppress this argument if its value is zero and show_zero property isn't set. */ + if (val == 0 && !trace->show_zeros && !arg->show_zero) continue; printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); From f975c13d2a34a335fc559aeff76dcaba456cced0 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Wed, 22 May 2024 11:35:42 +0800 Subject: [PATCH 018/280] perf trace beauty: Always show mmap prot even though PROT_NONE PROT_NONE is also useful information, so do not omit the mmap prot even though it is 0. syscall_arg__scnprintf_mmap_prot() could print PROT_NONE for prot 0. Before: PROT_NONE is not shown. $ sudo perf trace -e syscalls:sys_enter_mmap --filter prot==0 -- ls 0.000 ls/2979231 syscalls:sys_enter_mmap(len: 4220888, flags: PRIVATE|ANONYMOUS) After: PROT_NONE is displayed. $ sudo perf trace -e syscalls:sys_enter_mmap --filter prot==0 -- ls 0.000 ls/2975708 syscalls:sys_enter_mmap(len: 4220888, prot: NONE, flags: PRIVATE|ANONYMOUS) Signed-off-by: Changbin Du Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240522033542.1359421-3-changbin.du@huawei.com --- tools/perf/builtin-trace.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index a36e98c7a1c5..c42bc608954e 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -1033,7 +1033,7 @@ static const struct syscall_fmt syscall_fmts[] = { #if defined(__s390x__) .alias = "old_mmap", #endif - .arg = { [2] = { .scnprintf = SCA_MMAP_PROT, /* prot */ }, + .arg = { [2] = { .scnprintf = SCA_MMAP_PROT, .show_zero = true, /* prot */ }, [3] = { .scnprintf = SCA_MMAP_FLAGS, /* flags */ .strtoul = STUL_STRARRAY_FLAGS, .parm = &strarray__mmap_flags, }, @@ -1050,7 +1050,7 @@ static const struct syscall_fmt syscall_fmts[] = { [4] = { .scnprintf = SCA_MOVE_MOUNT_FLAGS, /* flags */ }, }, }, { .name = "mprotect", .arg = { [0] = { .scnprintf = SCA_HEX, /* start */ }, - [2] = { .scnprintf = SCA_MMAP_PROT, /* prot */ }, }, }, + [2] = { .scnprintf = SCA_MMAP_PROT, .show_zero = true, /* prot */ }, }, }, { .name = "mq_unlink", .arg = { [0] = { .scnprintf = SCA_FILENAME, /* u_name */ }, }, }, { .name = "mremap", .hexret = true, @@ -1084,7 +1084,7 @@ static const struct syscall_fmt syscall_fmts[] = { .arg = { [0] = { .scnprintf = SCA_INT, /* key */ }, }, }, { .name = "pkey_mprotect", .arg = { [0] = { .scnprintf = SCA_HEX, /* start */ }, - [2] = { .scnprintf = SCA_MMAP_PROT, /* prot */ }, + [2] = { .scnprintf = SCA_MMAP_PROT, .show_zero = true, /* prot */ }, [3] = { .scnprintf = SCA_INT, /* pkey */ }, }, }, { .name = "poll", .timeout = true, }, { .name = "ppoll", .timeout = true, }, From d163d60258c755845cbc9cfe0e45fca71e649488 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Sun, 19 May 2024 11:17:16 -0700 Subject: [PATCH 019/280] tools api io: Move filling the io buffer to its own function In general a read fills 4kb so filling the buffer is a 1 in 4096 operation, move it out of the io__get_char function to avoid some checking overhead and to better hint the function is good to inline. For perf's IO intensive internal (non-rigorous) benchmarks there's a small improvement to kallsyms-parsing with a default build. Before: ``` $ perf bench internals all Computing performance of single threaded perf event synthesis by synthesizing events on the perf process itself: Average synthesis took: 146.322 usec (+- 0.305 usec) Average num. events: 61.000 (+- 0.000) Average time per event 2.399 usec Average data synthesis took: 145.056 usec (+- 0.155 usec) Average num. events: 329.000 (+- 0.000) Average time per event 0.441 usec Average kallsyms__parse took: 162.313 ms (+- 0.599 ms) ... Computing performance of sysfs PMU event scan for 100 times Average core PMU scanning took: 53.720 usec (+- 7.823 usec) Average PMU scanning took: 375.145 usec (+- 23.974 usec) ``` After: ``` $ perf bench internals all Computing performance of single threaded perf event synthesis by synthesizing events on the perf process itself: Average synthesis took: 127.829 usec (+- 0.079 usec) Average num. events: 61.000 (+- 0.000) Average time per event 2.096 usec Average data synthesis took: 133.652 usec (+- 0.101 usec) Average num. events: 327.000 (+- 0.000) Average time per event 0.409 usec Average kallsyms__parse took: 150.415 ms (+- 0.313 ms) ... Computing performance of sysfs PMU event scan for 100 times Average core PMU scanning took: 47.790 usec (+- 1.178 usec) Average PMU scanning took: 376.945 usec (+- 23.683 usec) ``` Signed-off-by: Ian Rogers Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240519181716.4088459-1-irogers@google.com --- tools/lib/api/io.h | 67 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/tools/lib/api/io.h b/tools/lib/api/io.h index 84adf8102018..d3eb04d1bc89 100644 --- a/tools/lib/api/io.h +++ b/tools/lib/api/io.h @@ -43,48 +43,55 @@ static inline void io__init(struct io *io, int fd, io->eof = false; } -/* Reads one character from the "io" file with similar semantics to fgetc. */ -static inline int io__get_char(struct io *io) +/* Read from fd filling the buffer. Called when io->data == io->end. */ +static inline int io__fill_buffer(struct io *io) { - char *ptr = io->data; + ssize_t n; if (io->eof) return -1; - if (ptr == io->end) { - ssize_t n; + if (io->timeout_ms != 0) { + struct pollfd pfds[] = { + { + .fd = io->fd, + .events = POLLIN, + }, + }; - if (io->timeout_ms != 0) { - struct pollfd pfds[] = { - { - .fd = io->fd, - .events = POLLIN, - }, - }; - - n = poll(pfds, 1, io->timeout_ms); - if (n == 0) - errno = ETIMEDOUT; - if (n > 0 && !(pfds[0].revents & POLLIN)) { - errno = EIO; - n = -1; - } - if (n <= 0) { - io->eof = true; - return -1; - } + n = poll(pfds, 1, io->timeout_ms); + if (n == 0) + errno = ETIMEDOUT; + if (n > 0 && !(pfds[0].revents & POLLIN)) { + errno = EIO; + n = -1; } - n = read(io->fd, io->buf, io->buf_len); - if (n <= 0) { io->eof = true; return -1; } - ptr = &io->buf[0]; - io->end = &io->buf[n]; } - io->data = ptr + 1; - return *ptr; + n = read(io->fd, io->buf, io->buf_len); + + if (n <= 0) { + io->eof = true; + return -1; + } + io->data = &io->buf[0]; + io->end = &io->buf[n]; + return 0; +} + +/* Reads one character from the "io" file with similar semantics to fgetc. */ +static inline int io__get_char(struct io *io) +{ + if (io->data == io->end) { + int ret = io__fill_buffer(io); + + if (ret) + return ret; + } + return *io->data++; } /* Read a hexadecimal value with no 0x prefix into the out argument hex. If the From 63b9cbd7941aa9ec5cb61567042176c4ce04b020 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 24 May 2024 13:52:25 -0700 Subject: [PATCH 020/280] perf bpf filter: Give terms their own enum Give the term types their own enum so that additional terms can be added that don't correspond to a PERF_SAMPLE_xx flag. The term values are numerically ascending rather than bit field positions, this means they need translating to a PERF_SAMPLE_xx bit field in certain places using a shift. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: John Fastabend Cc: Changbin Du Cc: Yang Jihong Cc: Andrii Nakryiko Cc: bpf@vger.kernel.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240524205227.244375-2-irogers@google.com --- tools/perf/util/bpf-filter.c | 28 ++++---- tools/perf/util/bpf-filter.h | 5 +- tools/perf/util/bpf-filter.l | 64 +++++++++--------- tools/perf/util/bpf-filter.y | 7 +- tools/perf/util/bpf_skel/sample-filter.h | 37 ++++++++++- tools/perf/util/bpf_skel/sample_filter.bpf.c | 69 +++++++++++++++----- 6 files changed, 141 insertions(+), 69 deletions(-) diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index b51544996046..f10148623a8e 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -17,11 +17,11 @@ #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) -#define __PERF_SAMPLE_TYPE(st, opt) { st, #st, opt } -#define PERF_SAMPLE_TYPE(_st, opt) __PERF_SAMPLE_TYPE(PERF_SAMPLE_##_st, opt) +#define __PERF_SAMPLE_TYPE(tt, st, opt) { tt, #st, opt } +#define PERF_SAMPLE_TYPE(_st, opt) __PERF_SAMPLE_TYPE(PBF_TERM_##_st, PERF_SAMPLE_##_st, opt) static const struct perf_sample_info { - u64 type; + enum perf_bpf_filter_term type; const char *name; const char *option; } sample_table[] = { @@ -44,12 +44,12 @@ static const struct perf_sample_info { PERF_SAMPLE_TYPE(DATA_PAGE_SIZE, "--data-page-size"), }; -static const struct perf_sample_info *get_sample_info(u64 flags) +static const struct perf_sample_info *get_sample_info(enum perf_bpf_filter_term type) { size_t i; for (i = 0; i < ARRAY_SIZE(sample_table); i++) { - if (sample_table[i].type == flags) + if (sample_table[i].type == type) return &sample_table[i]; } return NULL; @@ -59,7 +59,8 @@ static int check_sample_flags(struct evsel *evsel, struct perf_bpf_filter_expr * { const struct perf_sample_info *info; - if (evsel->core.attr.sample_type & expr->sample_flags) + if (expr->term >= PBF_TERM_SAMPLE_START && expr->term <= PBF_TERM_SAMPLE_END && + (evsel->core.attr.sample_type & (1 << (expr->term - PBF_TERM_SAMPLE_START)))) return 0; if (expr->op == PBF_OP_GROUP_BEGIN) { @@ -72,10 +73,10 @@ static int check_sample_flags(struct evsel *evsel, struct perf_bpf_filter_expr * return 0; } - info = get_sample_info(expr->sample_flags); + info = get_sample_info(expr->term); if (info == NULL) { - pr_err("Error: %s event does not have sample flags %lx\n", - evsel__name(evsel), expr->sample_flags); + pr_err("Error: %s event does not have sample flags %d\n", + evsel__name(evsel), expr->term); return -1; } @@ -105,7 +106,7 @@ int perf_bpf_filter__prepare(struct evsel *evsel) struct perf_bpf_filter_entry entry = { .op = expr->op, .part = expr->part, - .flags = expr->sample_flags, + .term = expr->term, .value = expr->val, }; @@ -122,7 +123,7 @@ int perf_bpf_filter__prepare(struct evsel *evsel) struct perf_bpf_filter_entry group_entry = { .op = group->op, .part = group->part, - .flags = group->sample_flags, + .term = group->term, .value = group->val, }; bpf_map_update_elem(fd, &i, &group_entry, BPF_ANY); @@ -173,7 +174,8 @@ u64 perf_bpf_filter__lost_count(struct evsel *evsel) return skel ? skel->bss->dropped : 0; } -struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flags, int part, +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, enum perf_bpf_filter_op op, unsigned long val) { @@ -181,7 +183,7 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flag expr = malloc(sizeof(*expr)); if (expr != NULL) { - expr->sample_flags = sample_flags; + expr->term = term; expr->part = part; expr->op = op; expr->val = val; diff --git a/tools/perf/util/bpf-filter.h b/tools/perf/util/bpf-filter.h index 7afd159411b8..cd6764442c16 100644 --- a/tools/perf/util/bpf-filter.h +++ b/tools/perf/util/bpf-filter.h @@ -11,14 +11,15 @@ struct perf_bpf_filter_expr { struct list_head groups; enum perf_bpf_filter_op op; int part; - unsigned long sample_flags; + enum perf_bpf_filter_term term; unsigned long val; }; struct evsel; #ifdef HAVE_BPF_SKEL -struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flags, int part, +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, enum perf_bpf_filter_op op, unsigned long val); int perf_bpf_filter__parse(struct list_head *expr_head, const char *str); diff --git a/tools/perf/util/bpf-filter.l b/tools/perf/util/bpf-filter.l index d4ff0f1345cd..62c959813466 100644 --- a/tools/perf/util/bpf-filter.l +++ b/tools/perf/util/bpf-filter.l @@ -9,16 +9,16 @@ #include "bpf-filter.h" #include "bpf-filter-bison.h" -static int sample(unsigned long sample_flag) +static int sample(enum perf_bpf_filter_term term) { - perf_bpf_filter_lval.sample.type = sample_flag; + perf_bpf_filter_lval.sample.term = term; perf_bpf_filter_lval.sample.part = 0; return BFT_SAMPLE; } -static int sample_part(unsigned long sample_flag, int part) +static int sample_part(enum perf_bpf_filter_term term, int part) { - perf_bpf_filter_lval.sample.type = sample_flag; + perf_bpf_filter_lval.sample.term = term; perf_bpf_filter_lval.sample.part = part; return BFT_SAMPLE; } @@ -67,34 +67,34 @@ ident [_a-zA-Z][_a-zA-Z0-9]+ {num_hex} { return value(16); } {space} { } -ip { return sample(PERF_SAMPLE_IP); } -id { return sample(PERF_SAMPLE_ID); } -tid { return sample(PERF_SAMPLE_TID); } -pid { return sample_part(PERF_SAMPLE_TID, 1); } -cpu { return sample(PERF_SAMPLE_CPU); } -time { return sample(PERF_SAMPLE_TIME); } -addr { return sample(PERF_SAMPLE_ADDR); } -period { return sample(PERF_SAMPLE_PERIOD); } -txn { return sample(PERF_SAMPLE_TRANSACTION); } -weight { return sample(PERF_SAMPLE_WEIGHT); } -weight1 { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 1); } -weight2 { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 2); } -weight3 { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 3); } -ins_lat { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 2); } /* alias for weight2 */ -p_stage_cyc { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 3); } /* alias for weight3 */ -retire_lat { return sample_part(PERF_SAMPLE_WEIGHT_STRUCT, 3); } /* alias for weight3 */ -phys_addr { return sample(PERF_SAMPLE_PHYS_ADDR); } -code_pgsz { return sample(PERF_SAMPLE_CODE_PAGE_SIZE); } -data_pgsz { return sample(PERF_SAMPLE_DATA_PAGE_SIZE); } -mem_op { return sample_part(PERF_SAMPLE_DATA_SRC, 1); } -mem_lvlnum { return sample_part(PERF_SAMPLE_DATA_SRC, 2); } -mem_lvl { return sample_part(PERF_SAMPLE_DATA_SRC, 2); } /* alias for mem_lvlnum */ -mem_snoop { return sample_part(PERF_SAMPLE_DATA_SRC, 3); } /* include snoopx */ -mem_remote { return sample_part(PERF_SAMPLE_DATA_SRC, 4); } -mem_lock { return sample_part(PERF_SAMPLE_DATA_SRC, 5); } -mem_dtlb { return sample_part(PERF_SAMPLE_DATA_SRC, 6); } -mem_blk { return sample_part(PERF_SAMPLE_DATA_SRC, 7); } -mem_hops { return sample_part(PERF_SAMPLE_DATA_SRC, 8); } +ip { return sample(PBF_TERM_IP); } +id { return sample(PBF_TERM_ID); } +tid { return sample(PBF_TERM_TID); } +pid { return sample_part(PBF_TERM_TID, 1); } +cpu { return sample(PBF_TERM_CPU); } +time { return sample(PBF_TERM_TIME); } +addr { return sample(PBF_TERM_ADDR); } +period { return sample(PBF_TERM_PERIOD); } +txn { return sample(PBF_TERM_TRANSACTION); } +weight { return sample(PBF_TERM_WEIGHT); } +weight1 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 1); } +weight2 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } +weight3 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } +ins_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } /* alias for weight2 */ +p_stage_cyc { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +retire_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +phys_addr { return sample(PBF_TERM_PHYS_ADDR); } +code_pgsz { return sample(PBF_TERM_CODE_PAGE_SIZE); } +data_pgsz { return sample(PBF_TERM_DATA_PAGE_SIZE); } +mem_op { return sample_part(PBF_TERM_DATA_SRC, 1); } +mem_lvlnum { return sample_part(PBF_TERM_DATA_SRC, 2); } +mem_lvl { return sample_part(PBF_TERM_DATA_SRC, 2); } /* alias for mem_lvlnum */ +mem_snoop { return sample_part(PBF_TERM_DATA_SRC, 3); } /* include snoopx */ +mem_remote { return sample_part(PBF_TERM_DATA_SRC, 4); } +mem_lock { return sample_part(PBF_TERM_DATA_SRC, 5); } +mem_dtlb { return sample_part(PBF_TERM_DATA_SRC, 6); } +mem_blk { return sample_part(PBF_TERM_DATA_SRC, 7); } +mem_hops { return sample_part(PBF_TERM_DATA_SRC, 8); } "==" { return operator(PBF_OP_EQ); } "!=" { return operator(PBF_OP_NEQ); } diff --git a/tools/perf/util/bpf-filter.y b/tools/perf/util/bpf-filter.y index 0e4d6de3c2ad..0c56fccb8874 100644 --- a/tools/perf/util/bpf-filter.y +++ b/tools/perf/util/bpf-filter.y @@ -27,7 +27,7 @@ static void perf_bpf_filter_error(struct list_head *expr __maybe_unused, { unsigned long num; struct { - unsigned long type; + enum perf_bpf_filter_term term; int part; } sample; enum perf_bpf_filter_op op; @@ -62,7 +62,8 @@ filter_term BFT_LOGICAL_OR filter_expr if ($1->op == PBF_OP_GROUP_BEGIN) { expr = $1; } else { - expr = perf_bpf_filter_expr__new(0, 0, PBF_OP_GROUP_BEGIN, 1); + expr = perf_bpf_filter_expr__new(PBF_TERM_NONE, /*part=*/0, + PBF_OP_GROUP_BEGIN, /*val=*/1); list_add_tail(&$1->list, &expr->groups); } expr->val++; @@ -78,7 +79,7 @@ filter_expr filter_expr: BFT_SAMPLE BFT_OP BFT_NUM { - $$ = perf_bpf_filter_expr__new($1.type, $1.part, $2, $3); + $$ = perf_bpf_filter_expr__new($1.term, $1.part, $2, $3); } %% diff --git a/tools/perf/util/bpf_skel/sample-filter.h b/tools/perf/util/bpf_skel/sample-filter.h index 2e96e1ab084a..25f780022951 100644 --- a/tools/perf/util/bpf_skel/sample-filter.h +++ b/tools/perf/util/bpf_skel/sample-filter.h @@ -16,12 +16,45 @@ enum perf_bpf_filter_op { PBF_OP_GROUP_END, }; +enum perf_bpf_filter_term { + /* No term is in use. */ + PBF_TERM_NONE = 0, + /* Terms that correspond to PERF_SAMPLE_xx values. */ + PBF_TERM_SAMPLE_START = PBF_TERM_NONE + 1, + PBF_TERM_IP = PBF_TERM_SAMPLE_START + 0, /* SAMPLE_IP = 1U << 0 */ + PBF_TERM_TID = PBF_TERM_SAMPLE_START + 1, /* SAMPLE_TID = 1U << 1 */ + PBF_TERM_TIME = PBF_TERM_SAMPLE_START + 2, /* SAMPLE_TIME = 1U << 2 */ + PBF_TERM_ADDR = PBF_TERM_SAMPLE_START + 3, /* SAMPLE_ADDR = 1U << 3 */ + __PBF_UNUSED_TERM4 = PBF_TERM_SAMPLE_START + 4, /* SAMPLE_READ = 1U << 4 */ + __PBF_UNUSED_TERM5 = PBF_TERM_SAMPLE_START + 5, /* SAMPLE_CALLCHAIN = 1U << 5 */ + PBF_TERM_ID = PBF_TERM_SAMPLE_START + 6, /* SAMPLE_ID = 1U << 6 */ + PBF_TERM_CPU = PBF_TERM_SAMPLE_START + 7, /* SAMPLE_CPU = 1U << 7 */ + PBF_TERM_PERIOD = PBF_TERM_SAMPLE_START + 8, /* SAMPLE_PERIOD = 1U << 8 */ + __PBF_UNUSED_TERM9 = PBF_TERM_SAMPLE_START + 9, /* SAMPLE_STREAM_ID = 1U << 9 */ + __PBF_UNUSED_TERM10 = PBF_TERM_SAMPLE_START + 10, /* SAMPLE_RAW = 1U << 10 */ + __PBF_UNUSED_TERM11 = PBF_TERM_SAMPLE_START + 11, /* SAMPLE_BRANCH_STACK = 1U << 11 */ + __PBF_UNUSED_TERM12 = PBF_TERM_SAMPLE_START + 12, /* SAMPLE_REGS_USER = 1U << 12 */ + __PBF_UNUSED_TERM13 = PBF_TERM_SAMPLE_START + 13, /* SAMPLE_STACK_USER = 1U << 13 */ + PBF_TERM_WEIGHT = PBF_TERM_SAMPLE_START + 14, /* SAMPLE_WEIGHT = 1U << 14 */ + PBF_TERM_DATA_SRC = PBF_TERM_SAMPLE_START + 15, /* SAMPLE_DATA_SRC = 1U << 15 */ + __PBF_UNUSED_TERM16 = PBF_TERM_SAMPLE_START + 16, /* SAMPLE_IDENTIFIER = 1U << 16 */ + PBF_TERM_TRANSACTION = PBF_TERM_SAMPLE_START + 17, /* SAMPLE_TRANSACTION = 1U << 17 */ + __PBF_UNUSED_TERM18 = PBF_TERM_SAMPLE_START + 18, /* SAMPLE_REGS_INTR = 1U << 18 */ + PBF_TERM_PHYS_ADDR = PBF_TERM_SAMPLE_START + 19, /* SAMPLE_PHYS_ADDR = 1U << 19 */ + __PBF_UNUSED_TERM20 = PBF_TERM_SAMPLE_START + 20, /* SAMPLE_AUX = 1U << 20 */ + __PBF_UNUSED_TERM21 = PBF_TERM_SAMPLE_START + 21, /* SAMPLE_CGROUP = 1U << 21 */ + PBF_TERM_DATA_PAGE_SIZE = PBF_TERM_SAMPLE_START + 22, /* SAMPLE_DATA_PAGE_SIZE = 1U << 22 */ + PBF_TERM_CODE_PAGE_SIZE = PBF_TERM_SAMPLE_START + 23, /* SAMPLE_CODE_PAGE_SIZE = 1U << 23 */ + PBF_TERM_WEIGHT_STRUCT = PBF_TERM_SAMPLE_START + 24, /* SAMPLE_WEIGHT_STRUCT = 1U << 24 */ + PBF_TERM_SAMPLE_END = PBF_TERM_WEIGHT_STRUCT, +}; + /* BPF map entry for filtering */ struct perf_bpf_filter_entry { enum perf_bpf_filter_op op; __u32 part; /* sub-sample type info when it has multiple values */ - __u64 flags; /* perf sample type flags */ + enum perf_bpf_filter_term term; __u64 value; }; -#endif /* PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H */ \ No newline at end of file +#endif /* PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H */ diff --git a/tools/perf/util/bpf_skel/sample_filter.bpf.c b/tools/perf/util/bpf_skel/sample_filter.bpf.c index fb94f5280626..5ac1778ff66e 100644 --- a/tools/perf/util/bpf_skel/sample_filter.bpf.c +++ b/tools/perf/util/bpf_skel/sample_filter.bpf.c @@ -48,31 +48,54 @@ static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, { struct perf_sample_data___new *data = (void *)kctx->data; - if (!bpf_core_field_exists(data->sample_flags) || - (data->sample_flags & entry->flags) == 0) + if (!bpf_core_field_exists(data->sample_flags)) return 0; - switch (entry->flags) { - case PERF_SAMPLE_IP: +#define BUILD_CHECK_SAMPLE(x) \ + _Static_assert((1 << (PBF_TERM_##x - PBF_TERM_SAMPLE_START)) == PERF_SAMPLE_##x, \ + "Mismatched PBF term to sample bit " #x) + BUILD_CHECK_SAMPLE(IP); + BUILD_CHECK_SAMPLE(TID); + BUILD_CHECK_SAMPLE(TIME); + BUILD_CHECK_SAMPLE(ADDR); + BUILD_CHECK_SAMPLE(ID); + BUILD_CHECK_SAMPLE(CPU); + BUILD_CHECK_SAMPLE(PERIOD); + BUILD_CHECK_SAMPLE(WEIGHT); + BUILD_CHECK_SAMPLE(DATA_SRC); + BUILD_CHECK_SAMPLE(TRANSACTION); + BUILD_CHECK_SAMPLE(PHYS_ADDR); + BUILD_CHECK_SAMPLE(DATA_PAGE_SIZE); + BUILD_CHECK_SAMPLE(CODE_PAGE_SIZE); + BUILD_CHECK_SAMPLE(WEIGHT_STRUCT); +#undef BUILD_CHECK_SAMPLE + + /* For sample terms check the sample bit is set. */ + if (entry->term >= PBF_TERM_SAMPLE_START && entry->term <= PBF_TERM_SAMPLE_END && + (data->sample_flags & (1 << (entry->term - PBF_TERM_SAMPLE_START))) == 0) + return 0; + + switch (entry->term) { + case PBF_TERM_IP: return kctx->data->ip; - case PERF_SAMPLE_ID: + case PBF_TERM_ID: return kctx->data->id; - case PERF_SAMPLE_TID: + case PBF_TERM_TID: if (entry->part) return kctx->data->tid_entry.pid; else return kctx->data->tid_entry.tid; - case PERF_SAMPLE_CPU: + case PBF_TERM_CPU: return kctx->data->cpu_entry.cpu; - case PERF_SAMPLE_TIME: + case PBF_TERM_TIME: return kctx->data->time; - case PERF_SAMPLE_ADDR: + case PBF_TERM_ADDR: return kctx->data->addr; - case PERF_SAMPLE_PERIOD: + case PBF_TERM_PERIOD: return kctx->data->period; - case PERF_SAMPLE_TRANSACTION: + case PBF_TERM_TRANSACTION: return kctx->data->txn; - case PERF_SAMPLE_WEIGHT_STRUCT: + case PBF_TERM_WEIGHT_STRUCT: if (entry->part == 1) return kctx->data->weight.var1_dw; if (entry->part == 2) @@ -80,15 +103,15 @@ static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, if (entry->part == 3) return kctx->data->weight.var3_w; /* fall through */ - case PERF_SAMPLE_WEIGHT: + case PBF_TERM_WEIGHT: return kctx->data->weight.full; - case PERF_SAMPLE_PHYS_ADDR: + case PBF_TERM_PHYS_ADDR: return kctx->data->phys_addr; - case PERF_SAMPLE_CODE_PAGE_SIZE: + case PBF_TERM_CODE_PAGE_SIZE: return kctx->data->code_page_size; - case PERF_SAMPLE_DATA_PAGE_SIZE: + case PBF_TERM_DATA_PAGE_SIZE: return kctx->data->data_page_size; - case PERF_SAMPLE_DATA_SRC: + case PBF_TERM_DATA_SRC: if (entry->part == 1) return kctx->data->data_src.mem_op; if (entry->part == 2) @@ -117,6 +140,18 @@ static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, } /* return the whole word */ return kctx->data->data_src.val; + case PBF_TERM_NONE: + case __PBF_UNUSED_TERM4: + case __PBF_UNUSED_TERM5: + case __PBF_UNUSED_TERM9: + case __PBF_UNUSED_TERM10: + case __PBF_UNUSED_TERM11: + case __PBF_UNUSED_TERM12: + case __PBF_UNUSED_TERM13: + case __PBF_UNUSED_TERM16: + case __PBF_UNUSED_TERM18: + case __PBF_UNUSED_TERM20: + case __PBF_UNUSED_TERM21: default: break; } From d92aa899fe0a66350303a1986d6dc7ec4b3a1ea7 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 24 May 2024 13:52:26 -0700 Subject: [PATCH 021/280] perf bpf filter: Add uid and gid terms Allow the BPF filter to use the uid and gid terms determined by the bpf_get_current_uid_gid BPF helper. For example, the following will record the cpu-clock event system wide discarding samples that don't belong to the current user. $ perf record -e cpu-clock --filter "uid == $(id -u)" -a sleep 0.1 Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: John Fastabend Cc: Changbin Du Cc: Yang Jihong Cc: Andrii Nakryiko Cc: bpf@vger.kernel.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240524205227.244375-3-irogers@google.com --- tools/perf/Documentation/perf-record.txt | 2 +- tools/perf/util/bpf-filter.c | 5 +++++ tools/perf/util/bpf-filter.l | 2 ++ tools/perf/util/bpf_skel/sample-filter.h | 3 +++ tools/perf/util/bpf_skel/sample_filter.bpf.c | 4 ++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index 77578c0a142a..d6532ed97c02 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -200,7 +200,7 @@ OPTIONS ip, id, tid, pid, cpu, time, addr, period, txn, weight, phys_addr, code_pgsz, data_pgsz, weight1, weight2, weight3, ins_lat, retire_lat, p_stage_cyc, mem_op, mem_lvl, mem_snoop, mem_remote, mem_lock, - mem_dtlb, mem_blk, mem_hops + mem_dtlb, mem_blk, mem_hops, uid, gid The can be one of: ==, !=, >, >=, <, <=, & diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index f10148623a8e..04f98b6bb291 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -63,6 +63,11 @@ static int check_sample_flags(struct evsel *evsel, struct perf_bpf_filter_expr * (evsel->core.attr.sample_type & (1 << (expr->term - PBF_TERM_SAMPLE_START)))) return 0; + if (expr->term == PBF_TERM_UID || expr->term == PBF_TERM_GID) { + /* Not dependent on the sample_type as computed from a BPF helper. */ + return 0; + } + if (expr->op == PBF_OP_GROUP_BEGIN) { struct perf_bpf_filter_expr *group; diff --git a/tools/perf/util/bpf-filter.l b/tools/perf/util/bpf-filter.l index 62c959813466..2a7c839f3fae 100644 --- a/tools/perf/util/bpf-filter.l +++ b/tools/perf/util/bpf-filter.l @@ -95,6 +95,8 @@ mem_lock { return sample_part(PBF_TERM_DATA_SRC, 5); } mem_dtlb { return sample_part(PBF_TERM_DATA_SRC, 6); } mem_blk { return sample_part(PBF_TERM_DATA_SRC, 7); } mem_hops { return sample_part(PBF_TERM_DATA_SRC, 8); } +uid { return sample(PBF_TERM_UID); } +gid { return sample(PBF_TERM_GID); } "==" { return operator(PBF_OP_EQ); } "!=" { return operator(PBF_OP_NEQ); } diff --git a/tools/perf/util/bpf_skel/sample-filter.h b/tools/perf/util/bpf_skel/sample-filter.h index 25f780022951..350efa121026 100644 --- a/tools/perf/util/bpf_skel/sample-filter.h +++ b/tools/perf/util/bpf_skel/sample-filter.h @@ -47,6 +47,9 @@ enum perf_bpf_filter_term { PBF_TERM_CODE_PAGE_SIZE = PBF_TERM_SAMPLE_START + 23, /* SAMPLE_CODE_PAGE_SIZE = 1U << 23 */ PBF_TERM_WEIGHT_STRUCT = PBF_TERM_SAMPLE_START + 24, /* SAMPLE_WEIGHT_STRUCT = 1U << 24 */ PBF_TERM_SAMPLE_END = PBF_TERM_WEIGHT_STRUCT, + /* Terms computed from BPF helpers. */ + PBF_TERM_UID, + PBF_TERM_GID, }; /* BPF map entry for filtering */ diff --git a/tools/perf/util/bpf_skel/sample_filter.bpf.c b/tools/perf/util/bpf_skel/sample_filter.bpf.c index 5ac1778ff66e..f59985101973 100644 --- a/tools/perf/util/bpf_skel/sample_filter.bpf.c +++ b/tools/perf/util/bpf_skel/sample_filter.bpf.c @@ -140,6 +140,10 @@ static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, } /* return the whole word */ return kctx->data->data_src.val; + case PBF_TERM_UID: + return bpf_get_current_uid_gid() & 0xFFFFFFFF; + case PBF_TERM_GID: + return bpf_get_current_uid_gid() >> 32; case PBF_TERM_NONE: case __PBF_UNUSED_TERM4: case __PBF_UNUSED_TERM5: From af752016340021d433a962063067e819dba889b1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 24 May 2024 13:52:27 -0700 Subject: [PATCH 022/280] perf top: Allow filters on events Allow filters to be added to perf top events. One use is to workaround issues with: ``` $ perf top --uid="$(id -u)" ``` which tries to scan /proc find processes belonging to the uid and can fail in such a pid terminates between the scan and the perf_event_open reporting: ``` Error: The sys_perf_event_open() syscall returned with 3 (No such process) for event (cycles:P). /bin/dmesg | grep -i perf may provide additional information. ``` A similar filter: ``` $ perf top -e cycles:P --filter "uid == $(id -u)" ``` doesn't fail this way. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: John Fastabend Cc: Changbin Du Cc: Yang Jihong Cc: Andrii Nakryiko Cc: bpf@vger.kernel.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240524205227.244375-4-irogers@google.com --- tools/perf/Documentation/perf-top.txt | 4 ++++ tools/perf/builtin-top.c | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/tools/perf/Documentation/perf-top.txt b/tools/perf/Documentation/perf-top.txt index a754875fa5bb..667e5102075e 100644 --- a/tools/perf/Documentation/perf-top.txt +++ b/tools/perf/Documentation/perf-top.txt @@ -43,6 +43,10 @@ Default is to monitor all CPUS. encoding with the layout of the event control registers as described by entries in /sys/bus/event_source/devices/cpu/format/*. +--filter=:: + Event filter. This option should follow an event selector (-e). For + syntax see linkperf:perf-record[1]. + -E :: --entries=:: Display this many functions. diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index 1d6aef51c122..e8cbbf10d361 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -1055,6 +1055,13 @@ try_again: } } + if (evlist__apply_filters(evlist, &counter)) { + pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n", + counter->filter ?: "BPF", evsel__name(counter), errno, + str_error_r(errno, msg, sizeof(msg))); + goto out_err; + } + if (evlist__mmap(evlist, opts->mmap_pages) < 0) { ui__error("Failed to mmap with %d (%s)\n", errno, str_error_r(errno, msg, sizeof(msg))); @@ -1462,6 +1469,8 @@ int cmd_top(int argc, const char **argv) OPT_CALLBACK('e', "event", &parse_events_option_args, "event", "event selector. use 'perf list' to list available events", parse_events_option), + OPT_CALLBACK(0, "filter", &top.evlist, "filter", + "event filter", parse_filter), OPT_U64('c', "count", &opts->user_interval, "event period to sample"), OPT_STRING('p', "pid", &target->pid, "pid", "profile events on existing process id"), From 4d2bcefa965b06a1f2be6912456bcfa86a34f184 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 31 May 2024 13:29:02 +0100 Subject: [PATCH 023/280] mm: Reduce the number of slab->folio casts Mark a few more folio functions as taking a const folio pointer, which allows us to remove a few places in slab which cast away the const. Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Vlastimil Babka --- include/linux/mm.h | 6 +++--- mm/slab.h | 4 ++-- mm/slub.c | 6 ++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 9849dfda44d4..a983d371fafa 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1105,7 +1105,7 @@ static inline unsigned int compound_order(struct page *page) * * Return: The order of the folio. */ -static inline unsigned int folio_order(struct folio *folio) +static inline unsigned int folio_order(const struct folio *folio) { if (!folio_test_large(folio)) return 0; @@ -2145,7 +2145,7 @@ static inline struct folio *folio_next(struct folio *folio) * it from being split. It is not necessary for the folio to be locked. * Return: The base-2 logarithm of the size of this folio. */ -static inline unsigned int folio_shift(struct folio *folio) +static inline unsigned int folio_shift(const struct folio *folio) { return PAGE_SHIFT + folio_order(folio); } @@ -2158,7 +2158,7 @@ static inline unsigned int folio_shift(struct folio *folio) * it from being split. It is not necessary for the folio to be locked. * Return: The number of bytes in this folio. */ -static inline size_t folio_size(struct folio *folio) +static inline size_t folio_size(const struct folio *folio) { return PAGE_SIZE << folio_order(folio); } diff --git a/mm/slab.h b/mm/slab.h index 5f8f47c5bee0..b16e63191578 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -166,7 +166,7 @@ static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t) */ static inline bool slab_test_pfmemalloc(const struct slab *slab) { - return folio_test_active((struct folio *)slab_folio(slab)); + return folio_test_active(slab_folio(slab)); } static inline void slab_set_pfmemalloc(struct slab *slab) @@ -211,7 +211,7 @@ static inline struct slab *virt_to_slab(const void *addr) static inline int slab_order(const struct slab *slab) { - return folio_order((struct folio *)slab_folio(slab)); + return folio_order(slab_folio(slab)); } static inline size_t slab_size(const struct slab *slab) diff --git a/mm/slub.c b/mm/slub.c index 95e0a3332c44..b8ba068ca079 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -962,11 +962,9 @@ void print_tracking(struct kmem_cache *s, void *object) static void print_slab_info(const struct slab *slab) { - struct folio *folio = (struct folio *)slab_folio(slab); - pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n", slab, slab->objects, slab->inuse, slab->freelist, - folio_flags(folio, 0)); + &slab->__page_flags); } /* @@ -2532,7 +2530,7 @@ static void discard_slab(struct kmem_cache *s, struct slab *slab) */ static inline bool slab_test_node_partial(const struct slab *slab) { - return folio_test_workingset((struct folio *)slab_folio(slab)); + return folio_test_workingset(slab_folio(slab)); } static inline void slab_set_node_partial(struct slab *slab) From f7d4485fcedcb4978148bad5fcde570f63790323 Mon Sep 17 00:00:00 2001 From: Nick Forrington Date: Mon, 13 May 2024 09:14:12 +0000 Subject: [PATCH 024/280] perf lock info: Display both map and thread by default Change "perf lock info" argument handling to: Display both map and thread info (rather than an error) when neither are specified. Display both map and thread info (rather than just thread info) when both are requested. Signed-off-by: Nick Forrington Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240513091413.738537-2-nick.forrington@arm.com --- tools/perf/Documentation/perf-lock.txt | 4 ++-- tools/perf/builtin-lock.c | 29 ++++++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index f5938d616d75..57a940399de0 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -111,11 +111,11 @@ INFO OPTIONS -t:: --threads:: - dump thread list in perf.data + dump only the thread list in perf.data -m:: --map:: - dump map of lock instances (address:name table) + dump only the map of lock instances (address:name table) CONTENTION OPTIONS diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 7007d26fe654..0253184b3b58 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -1477,20 +1477,16 @@ static void dump_map(void) fprintf(lock_output, " %#llx: %s\n", (unsigned long long)st->addr, st->name); } -static int dump_info(void) +static void dump_info(void) { - int rc = 0; - if (info_threads) dump_threads(); - else if (info_map) - dump_map(); - else { - rc = -1; - pr_err("Unknown type of information\n"); - } - return rc; + if (info_map) { + if (info_threads) + fputc('\n', lock_output); + dump_map(); + } } static const struct evsel_str_handler lock_tracepoints[] = { @@ -1992,7 +1988,7 @@ static int __cmd_report(bool display_info) setup_pager(); if (display_info) /* used for info subcommand */ - err = dump_info(); + dump_info(); else { combine_result(); sort_result(); @@ -2568,9 +2564,9 @@ int cmd_lock(int argc, const char **argv) const struct option info_options[] = { OPT_BOOLEAN('t', "threads", &info_threads, - "dump thread list in perf.data"), + "dump the thread list in perf.data"), OPT_BOOLEAN('m', "map", &info_map, - "map of lock instances (address:name table)"), + "dump the map of lock instances (address:name table)"), OPT_PARENT(lock_options) }; @@ -2684,6 +2680,13 @@ int cmd_lock(int argc, const char **argv) if (argc) usage_with_options(info_usage, info_options); } + + /* If neither threads nor map requested, display both */ + if (!info_threads && !info_map) { + info_threads = true; + info_map = true; + } + /* recycling report_lock_ops */ trace_handler = &report_lock_ops; rc = __cmd_report(true); From f7abc0cfa8be0aa872842569583a69e6bdec0e76 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sun, 2 Jun 2024 01:05:05 +0100 Subject: [PATCH 025/280] perf genelf: remove unused struct 'options' 'options' has been unused since commit fa7f7e735495 ("perf jit: Move test functionality in to a test"). Remove it. Signed-off-by: Dr. David Alan Gilbert Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240602000505.213032-1-linux@treblig.org --- tools/perf/util/genelf.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index ac17a3cb59dc..c8f6bee1fa61 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -54,11 +54,6 @@ typedef struct { char name[0]; /* Start of the name+desc data */ } Elf_Note; -struct options { - char *output; - int fd; -}; - static char shd_string_table[] = { 0, '.', 't', 'e', 'x', 't', 0, /* 1 */ From 42675b723b4842bca7bfb0f209aa9a493a10324a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:05 -0400 Subject: [PATCH 026/280] function_graph: Convert ret_stack to a series of longs In order to make it possible to have multiple callbacks registered with the function_graph tracer, the retstack needs to be converted from an array of ftrace_ret_stack structures to an array of longs. This will allow to store the list of callbacks on the stack for the return side of the functions. Link: https://lore.kernel.org/linux-trace-kernel/171509092742.162236.4427737821399314856.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.073111754@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/sched.h | 2 +- kernel/trace/fgraph.c | 136 +++++++++++++++++++++++++----------------- 2 files changed, 83 insertions(+), 55 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 61591ac6eab6..352939dab3a5 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1402,7 +1402,7 @@ struct task_struct { int curr_ret_depth; /* Stack of return addresses for return function tracing: */ - struct ftrace_ret_stack *ret_stack; + unsigned long *ret_stack; /* Timestamp for last schedule: */ unsigned long long ftrace_timestamp; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index a130b2d898f7..c62e6db718a0 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -25,6 +25,30 @@ #define ASSIGN_OPS_HASH(opsname, val) #endif +/* + * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack + * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame + * SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack + * SHADOW_STACK_OFFSET: The size in long words of the shadow stack + * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added + */ +#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack) +#define FGRAPH_FRAME_OFFSET (ALIGN(FGRAPH_FRAME_SIZE, sizeof(long)) / sizeof(long)) +#define SHADOW_STACK_SIZE (PAGE_SIZE) +#define SHADOW_STACK_OFFSET \ + (ALIGN(SHADOW_STACK_SIZE, sizeof(long)) / sizeof(long)) +/* Leave on a buffer at the end */ +#define SHADOW_STACK_MAX_INDEX (SHADOW_STACK_OFFSET - FGRAPH_FRAME_OFFSET) + +/* + * RET_STACK(): Return the frame from a given @offset from task @t + * RET_STACK_INC(): Reserve one frame size on the stack. + * RET_STACK_DEC(): Remove one frame size from the stack. + */ +#define RET_STACK(t, index) ((struct ftrace_ret_stack *)(&(t)->ret_stack[index])) +#define RET_STACK_INC(c) ({ c += FGRAPH_FRAME_OFFSET; }) +#define RET_STACK_DEC(c) ({ c -= FGRAPH_FRAME_OFFSET; }) + DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; @@ -69,6 +93,7 @@ static int ftrace_push_return_trace(unsigned long ret, unsigned long func, unsigned long frame_pointer, unsigned long *retp) { + struct ftrace_ret_stack *ret_stack; unsigned long long calltime; int index; @@ -85,23 +110,25 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, smp_rmb(); /* The return trace stack is full */ - if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) { + if (current->curr_ret_stack >= SHADOW_STACK_MAX_INDEX) { atomic_inc(¤t->trace_overrun); return -EBUSY; } calltime = trace_clock_local(); - index = ++current->curr_ret_stack; + index = current->curr_ret_stack; + RET_STACK_INC(current->curr_ret_stack); + ret_stack = RET_STACK(current, index); barrier(); - current->ret_stack[index].ret = ret; - current->ret_stack[index].func = func; - current->ret_stack[index].calltime = calltime; + ret_stack->ret = ret; + ret_stack->func = func; + ret_stack->calltime = calltime; #ifdef HAVE_FUNCTION_GRAPH_FP_TEST - current->ret_stack[index].fp = frame_pointer; + ret_stack->fp = frame_pointer; #endif #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR - current->ret_stack[index].retp = retp; + ret_stack->retp = retp; #endif return 0; } @@ -137,7 +164,7 @@ int function_graph_enter(unsigned long ret, unsigned long func, return 0; out_ret: - current->curr_ret_stack--; + RET_STACK_DEC(current->curr_ret_stack); out: current->curr_ret_depth--; return -EBUSY; @@ -148,11 +175,13 @@ static void ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, unsigned long frame_pointer) { + struct ftrace_ret_stack *ret_stack; int index; index = current->curr_ret_stack; + RET_STACK_DEC(index); - if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) { + if (unlikely(index < 0 || index > SHADOW_STACK_MAX_INDEX)) { ftrace_graph_stop(); WARN_ON(1); /* Might as well panic, otherwise we have no where to go */ @@ -160,6 +189,7 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, return; } + ret_stack = RET_STACK(current, index); #ifdef HAVE_FUNCTION_GRAPH_FP_TEST /* * The arch may choose to record the frame pointer used @@ -175,22 +205,22 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, * Note, -mfentry does not use frame pointers, and this test * is not needed if CC_USING_FENTRY is set. */ - if (unlikely(current->ret_stack[index].fp != frame_pointer)) { + if (unlikely(ret_stack->fp != frame_pointer)) { ftrace_graph_stop(); WARN(1, "Bad frame pointer: expected %lx, received %lx\n" " from func %ps return to %lx\n", current->ret_stack[index].fp, frame_pointer, - (void *)current->ret_stack[index].func, - current->ret_stack[index].ret); + (void *)ret_stack->func, + ret_stack->ret); *ret = (unsigned long)panic; return; } #endif - *ret = current->ret_stack[index].ret; - trace->func = current->ret_stack[index].func; - trace->calltime = current->ret_stack[index].calltime; + *ret = ret_stack->ret; + trace->func = ret_stack->func; + trace->calltime = ret_stack->calltime; trace->overrun = atomic_read(¤t->trace_overrun); trace->depth = current->curr_ret_depth--; /* @@ -251,7 +281,7 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs * curr_ret_stack is after that. */ barrier(); - current->curr_ret_stack--; + RET_STACK_DEC(current->curr_ret_stack); if (unlikely(!ret)) { ftrace_graph_stop(); @@ -294,12 +324,13 @@ unsigned long ftrace_return_to_handler(unsigned long frame_pointer) struct ftrace_ret_stack * ftrace_graph_get_ret_stack(struct task_struct *task, int idx) { - idx = task->curr_ret_stack - idx; + int index = task->curr_ret_stack; - if (idx >= 0 && idx <= task->curr_ret_stack) - return &task->ret_stack[idx]; + index -= FGRAPH_FRAME_OFFSET * (idx + 1); + if (index < 0) + return NULL; - return NULL; + return RET_STACK(task, index); } /** @@ -321,18 +352,20 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int idx) unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { + struct ftrace_ret_stack *ret_stack; int index = task->curr_ret_stack; int i; if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler)) return ret; - if (index < 0) - return ret; + RET_STACK_DEC(index); - for (i = 0; i <= index; i++) - if (task->ret_stack[i].retp == retp) - return task->ret_stack[i].ret; + for (i = index; i >= 0; RET_STACK_DEC(i)) { + ret_stack = RET_STACK(task, i); + if (ret_stack->retp == retp) + return ret_stack->ret; + } return ret; } @@ -346,14 +379,15 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, return ret; task_idx = task->curr_ret_stack; + RET_STACK_DEC(task_idx); if (!task->ret_stack || task_idx < *idx) return ret; task_idx -= *idx; - (*idx)++; + RET_STACK_INC(*idx); - return task->ret_stack[task_idx].ret; + return RET_STACK(task, task_idx); } #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ @@ -391,7 +425,7 @@ trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ -static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) +static int alloc_retstack_tasklist(unsigned long **ret_stack_list) { int i; int ret = 0; @@ -399,10 +433,7 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) struct task_struct *g, *t; for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { - ret_stack_list[i] = - kmalloc_array(FTRACE_RETFUNC_DEPTH, - sizeof(struct ftrace_ret_stack), - GFP_KERNEL); + ret_stack_list[i] = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL); if (!ret_stack_list[i]) { start = 0; end = i; @@ -420,9 +451,9 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) if (t->ret_stack == NULL) { atomic_set(&t->trace_overrun, 0); - t->curr_ret_stack = -1; + t->curr_ret_stack = 0; t->curr_ret_depth = -1; - /* Make sure the tasks see the -1 first: */ + /* Make sure the tasks see the 0 first: */ smp_wmb(); t->ret_stack = ret_stack_list[start++]; } @@ -442,6 +473,7 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt, struct task_struct *next, unsigned int prev_state) { + struct ftrace_ret_stack *ret_stack; unsigned long long timestamp; int index; @@ -466,8 +498,11 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt, */ timestamp -= next->ftrace_timestamp; - for (index = next->curr_ret_stack; index >= 0; index--) - next->ret_stack[index].calltime += timestamp; + for (index = next->curr_ret_stack - FGRAPH_FRAME_OFFSET; index >= 0; ) { + ret_stack = RET_STACK(next, index); + ret_stack->calltime += timestamp; + index -= FGRAPH_FRAME_OFFSET; + } } static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) @@ -510,10 +545,10 @@ void update_function_graph_func(void) ftrace_graph_entry = __ftrace_graph_entry; } -static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); +static DEFINE_PER_CPU(unsigned long *, idle_ret_stack); static void -graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) +graph_init_task(struct task_struct *t, unsigned long *ret_stack) { atomic_set(&t->trace_overrun, 0); t->ftrace_timestamp = 0; @@ -528,7 +563,7 @@ graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) */ void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { - t->curr_ret_stack = -1; + t->curr_ret_stack = 0; t->curr_ret_depth = -1; /* * The idle task has no parent, it either has its own @@ -538,14 +573,11 @@ void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); if (ftrace_graph_active) { - struct ftrace_ret_stack *ret_stack; + unsigned long *ret_stack; ret_stack = per_cpu(idle_ret_stack, cpu); if (!ret_stack) { - ret_stack = - kmalloc_array(FTRACE_RETFUNC_DEPTH, - sizeof(struct ftrace_ret_stack), - GFP_KERNEL); + ret_stack = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL); if (!ret_stack) return; per_cpu(idle_ret_stack, cpu) = ret_stack; @@ -559,15 +591,13 @@ void ftrace_graph_init_task(struct task_struct *t) { /* Make sure we do not use the parent ret_stack */ t->ret_stack = NULL; - t->curr_ret_stack = -1; + t->curr_ret_stack = 0; t->curr_ret_depth = -1; if (ftrace_graph_active) { - struct ftrace_ret_stack *ret_stack; + unsigned long *ret_stack; - ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH, - sizeof(struct ftrace_ret_stack), - GFP_KERNEL); + ret_stack = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL); if (!ret_stack) return; graph_init_task(t, ret_stack); @@ -576,7 +606,7 @@ void ftrace_graph_init_task(struct task_struct *t) void ftrace_graph_exit_task(struct task_struct *t) { - struct ftrace_ret_stack *ret_stack = t->ret_stack; + unsigned long *ret_stack = t->ret_stack; t->ret_stack = NULL; /* NULL must become visible to IRQs before we free it: */ @@ -588,12 +618,10 @@ void ftrace_graph_exit_task(struct task_struct *t) /* Allocate a return stack for each task */ static int start_graph_tracing(void) { - struct ftrace_ret_stack **ret_stack_list; + unsigned long **ret_stack_list; int ret, cpu; - ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE, - sizeof(struct ftrace_ret_stack *), - GFP_KERNEL); + ret_stack_list = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL); if (!ret_stack_list) return -ENOMEM; From 59e5f04e4184181227889663618e01dce676e671 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:06 -0400 Subject: [PATCH 027/280] fgraph: Use BUILD_BUG_ON() to make sure we have structures divisible by long Instead of using "ALIGN()", use BUILD_BUG_ON() as the structures should always be divisible by sizeof(long). Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509093949.162236.14518699447151894536.stgit@devnote2 Link: http://lkml.kernel.org/r/20190524111144.GI2589@hirez.programming.kicks-ass.net Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.232168933@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Suggested-by: Peter Zijlstra Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index c62e6db718a0..fdb206aeffe3 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -33,7 +33,7 @@ * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added */ #define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack) -#define FGRAPH_FRAME_OFFSET (ALIGN(FGRAPH_FRAME_SIZE, sizeof(long)) / sizeof(long)) +#define FGRAPH_FRAME_OFFSET DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long)) #define SHADOW_STACK_SIZE (PAGE_SIZE) #define SHADOW_STACK_OFFSET \ (ALIGN(SHADOW_STACK_SIZE, sizeof(long)) / sizeof(long)) @@ -103,6 +103,8 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, if (!current->ret_stack) return -EBUSY; + BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long)); + /* * We must make sure the ret_stack is tested before we read * anything else. @@ -326,6 +328,8 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int idx) { int index = task->curr_ret_stack; + BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long)); + index -= FGRAPH_FRAME_OFFSET * (idx + 1); if (index < 0) return NULL; From 518d6804a865772735588c5b2203d6c0c0bf98a9 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:07 -0400 Subject: [PATCH 028/280] function_graph: Add an array structure that will allow multiple callbacks Add an array structure that will eventually allow the function graph tracer to have up to 16 simultaneous callbacks attached. It's an array of 16 fgraph_ops pointers, that is assigned when one is registered. On entry of a function the entry of the first item in the array is called, and if it returns zero, then the callback returns non zero if it wants the return callback to be called on exit of the function. The array will simplify the process of having more than one callback attached to the same function, as its index into the array can be stored on the shadow stack. We need to only save the index, because this will allow the fgraph_ops to be freed before the function returns (which may happen if the function call schedule for a long time). Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509095075.162236.8272148192748284581.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.392113213@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 118 +++++++++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index fdb206aeffe3..d2ce5d651cf0 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -52,6 +52,11 @@ DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; +static int fgraph_array_cnt; +#define FGRAPH_ARRAY_SIZE 16 + +static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE]; + /* Both enabled by default (can be cleared by function_graph tracer flags */ static bool fgraph_sleep_time = true; @@ -75,6 +80,20 @@ int __weak ftrace_disable_ftrace_graph_caller(void) } #endif +int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) +{ + return 0; +} + +static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace) +{ +} + +static struct fgraph_ops fgraph_stub = { + .entryfunc = ftrace_graph_entry_stub, + .retfunc = ftrace_graph_ret_stub, +}; + /** * ftrace_graph_stop - set to permanently disable function graph tracing * @@ -161,7 +180,7 @@ int function_graph_enter(unsigned long ret, unsigned long func, goto out; /* Only trace if the calling function expects to */ - if (!ftrace_graph_entry(&trace)) + if (!fgraph_array[0]->entryfunc(&trace)) goto out_ret; return 0; @@ -276,7 +295,7 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs trace.retval = fgraph_ret_regs_return_value(ret_regs); #endif trace.rettime = trace_clock_local(); - ftrace_graph_return(&trace); + fgraph_array[0]->retfunc(&trace); /* * The ftrace_graph_return() may still access the current * ret_stack structure, we need to make sure the update of @@ -412,11 +431,6 @@ void ftrace_graph_sleep_time_control(bool enable) fgraph_sleep_time = enable; } -int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) -{ - return 0; -} - /* * Simply points to ftrace_stub, but with the proper protocol. * Defined by the linker script in linux/vmlinux.lds.h @@ -654,37 +668,54 @@ static int start_graph_tracing(void) int register_ftrace_graph(struct fgraph_ops *gops) { int ret = 0; + int i; mutex_lock(&ftrace_lock); - /* we currently allow only one tracer registered at a time */ - if (ftrace_graph_active) { + if (!fgraph_array[0]) { + /* The array must always have real data on it */ + for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) + fgraph_array[i] = &fgraph_stub; + } + + /* Look for an available spot */ + for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { + if (fgraph_array[i] == &fgraph_stub) + break; + } + if (i >= FGRAPH_ARRAY_SIZE) { ret = -EBUSY; goto out; } - register_pm_notifier(&ftrace_suspend_notifier); + fgraph_array[i] = gops; + if (i + 1 > fgraph_array_cnt) + fgraph_array_cnt = i + 1; ftrace_graph_active++; - ret = start_graph_tracing(); - if (ret) { - ftrace_graph_active--; - goto out; + + if (ftrace_graph_active == 1) { + register_pm_notifier(&ftrace_suspend_notifier); + ret = start_graph_tracing(); + if (ret) { + ftrace_graph_active--; + goto out; + } + + ftrace_graph_return = gops->retfunc; + + /* + * Update the indirect function to the entryfunc, and the + * function that gets called to the entry_test first. Then + * call the update fgraph entry function to determine if + * the entryfunc should be called directly or not. + */ + __ftrace_graph_entry = gops->entryfunc; + ftrace_graph_entry = ftrace_graph_entry_test; + update_function_graph_func(); + + ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); } - - ftrace_graph_return = gops->retfunc; - - /* - * Update the indirect function to the entryfunc, and the - * function that gets called to the entry_test first. Then - * call the update fgraph entry function to determine if - * the entryfunc should be called directly or not. - */ - __ftrace_graph_entry = gops->entryfunc; - ftrace_graph_entry = ftrace_graph_entry_test; - update_function_graph_func(); - - ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); out: mutex_unlock(&ftrace_lock); return ret; @@ -692,19 +723,36 @@ out: void unregister_ftrace_graph(struct fgraph_ops *gops) { + int i; + mutex_lock(&ftrace_lock); if (unlikely(!ftrace_graph_active)) goto out; - ftrace_graph_active--; - ftrace_graph_return = ftrace_stub_graph; - ftrace_graph_entry = ftrace_graph_entry_stub; - __ftrace_graph_entry = ftrace_graph_entry_stub; - ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); - unregister_pm_notifier(&ftrace_suspend_notifier); - unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); + for (i = 0; i < fgraph_array_cnt; i++) + if (gops == fgraph_array[i]) + break; + if (i >= fgraph_array_cnt) + goto out; + fgraph_array[i] = &fgraph_stub; + if (i + 1 == fgraph_array_cnt) { + for (; i >= 0; i--) + if (fgraph_array[i] != &fgraph_stub) + break; + fgraph_array_cnt = i + 1; + } + + ftrace_graph_active--; + if (!ftrace_graph_active) { + ftrace_graph_return = ftrace_stub_graph; + ftrace_graph_entry = ftrace_graph_entry_stub; + __ftrace_graph_entry = ftrace_graph_entry_stub; + ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); + unregister_pm_notifier(&ftrace_suspend_notifier); + unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); + } out: mutex_unlock(&ftrace_lock); } From 7aa1eaef9f4282c9acd39588b1fdc9dda7e73f34 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:08 -0400 Subject: [PATCH 029/280] function_graph: Allow multiple users to attach to function graph Allow for multiple users to attach to function graph tracer at the same time. Only 16 simultaneous users can attach to the tracer. This is because there's an array that stores the pointers to the attached fgraph_ops. When a function being traced is entered, each of the ftrace_ops entryfunc is called and if it returns non zero, its index into the array will be added to the shadow stack. On exit of the function being traced, the shadow stack will contain the indexes of the ftrace_ops on the array that want their retfunc to be called. Because a function may sleep for a long time (if a task sleeps itself), the return of the function may be literally days later. If the ftrace_ops is removed, its place on the array is replaced with a ftrace_ops that contains the stub functions and that will be called when the function finally returns. If another ftrace_ops is added that happens to get the same index into the array, its return function may be called. But that's actually the way things current work with the old function graph tracer. If one tracer is removed and another is added, the new one will get the return calls of the function traced by the previous one, thus this is not a regression. This can be fixed by adding a counter to each time the array item is updated and save that on the shadow stack as well, such that it won't be called if the index saved does not match the index on the array. Note, being able to filter functions when both are called is not completely handled yet, but that shouldn't be too hard to manage. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509096221.162236.8806372072523195752.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.555493396@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 3 +- kernel/trace/fgraph.c | 379 ++++++++++++++++++++++++++++++++--------- 2 files changed, 304 insertions(+), 78 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 800995c425e0..8f00a7415bd5 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1038,6 +1038,7 @@ extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace); struct fgraph_ops { trace_func_graph_ent_t entryfunc; trace_func_graph_ret_t retfunc; + int idx; }; /* @@ -1072,7 +1073,7 @@ function_graph_enter(unsigned long ret, unsigned long func, unsigned long frame_pointer, unsigned long *retp); struct ftrace_ret_stack * -ftrace_graph_get_ret_stack(struct task_struct *task, int idx); +ftrace_graph_get_ret_stack(struct task_struct *task, int skip); unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp); diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index d2ce5d651cf0..aae51f746828 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -7,6 +7,7 @@ * * Highly modified by Steven Rostedt (VMware). */ +#include #include #include #include @@ -28,35 +29,177 @@ /* * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame + */ +#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack) +#define FGRAPH_FRAME_OFFSET DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long)) + +/* + * On entry to a function (via function_graph_enter()), a new fgraph frame + * (ftrace_ret_stack) is pushed onto the stack as well as a word that + * holds a bitmask and a type (called "bitmap"). The bitmap is defined as: + * + * bits: 0 - 9 offset in words from the previous ftrace_ret_stack + * Currently, this will always be set to FGRAPH_FRAME_OFFSET + * to get to the fgraph frame. + * + * bits: 10 - 11 Type of storage + * 0 - reserved + * 1 - bitmap of fgraph_array index + * + * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP): + * bits: 12 - 27 The bitmap of fgraph_ops fgraph_array index + * That is, it's a bitmask of 0-15 (16 bits) + * where if a corresponding ops in the fgraph_array[] + * expects a callback from the return of the function + * it's corresponding bit will be set. + * + * + * The top of the ret_stack (when not empty) will always have a reference + * word that points to the last fgraph frame that was saved. + * + * That is, at the end of function_graph_enter, if the first and forth + * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called + * on the return of the function being traced, this is what will be on the + * task's shadow ret_stack: (the stack grows upward) + * + * ret_stack[SHADOW_STACK_MAX_OFFSET] + * ... + * | | <- task->curr_ret_stack + * +--------------------------------------------+ + * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET| + * | *or put another way* | + * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \ | + * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ | + * | (offset:FGRAPH_FRAME_OFFSET) | <- the offset is from here + * +--------------------------------------------+ + * | struct ftrace_ret_stack | + * | (stores the saved ret pointer) | <- the offset points here + * +--------------------------------------------+ + * | (X) | (N) | ( N words away from + * | | previous ret_stack) + * ... + * ret_stack[0] + * + * If a backtrace is required, and the real return pointer needs to be + * fetched, then it looks at the task's curr_ret_stack offset, if it + * is greater than zero (reserved, or right before popped), it would mask + * the value by FGRAPH_FRAME_OFFSET_MASK to get the offset of the + * ftrace_ret_stack structure stored on the shadow stack. + */ + +/* + * The following is for the top word on the stack: + * + * FGRAPH_FRAME_OFFSET (0-9) holds the offset delta to the fgraph frame + * FGRAPH_TYPE (10-11) holds the type of word this is. + * (RESERVED or BITMAP) + */ +#define FGRAPH_FRAME_OFFSET_BITS 10 +#define FGRAPH_FRAME_OFFSET_MASK GENMASK(FGRAPH_FRAME_OFFSET_BITS - 1, 0) + +#define FGRAPH_TYPE_BITS 2 +#define FGRAPH_TYPE_MASK GENMASK(FGRAPH_TYPE_BITS - 1, 0) +#define FGRAPH_TYPE_SHIFT FGRAPH_FRAME_OFFSET_BITS + +enum { + FGRAPH_TYPE_RESERVED = 0, + FGRAPH_TYPE_BITMAP = 1, +}; + +/* + * For BITMAP type: + * FGRAPH_INDEX (12-27) bits holding the gops index wanting return callback called + */ +#define FGRAPH_INDEX_BITS 16 +#define FGRAPH_INDEX_MASK GENMASK(FGRAPH_INDEX_BITS - 1, 0) +#define FGRAPH_INDEX_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS) + +#define FGRAPH_ARRAY_SIZE FGRAPH_INDEX_BITS + +/* * SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack * SHADOW_STACK_OFFSET: The size in long words of the shadow stack * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added */ -#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack) -#define FGRAPH_FRAME_OFFSET DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long)) -#define SHADOW_STACK_SIZE (PAGE_SIZE) -#define SHADOW_STACK_OFFSET \ - (ALIGN(SHADOW_STACK_SIZE, sizeof(long)) / sizeof(long)) +#define SHADOW_STACK_SIZE (PAGE_SIZE) +#define SHADOW_STACK_OFFSET (SHADOW_STACK_SIZE / sizeof(long)) /* Leave on a buffer at the end */ -#define SHADOW_STACK_MAX_INDEX (SHADOW_STACK_OFFSET - FGRAPH_FRAME_OFFSET) +#define SHADOW_STACK_MAX_OFFSET (SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1)) -/* - * RET_STACK(): Return the frame from a given @offset from task @t - * RET_STACK_INC(): Reserve one frame size on the stack. - * RET_STACK_DEC(): Remove one frame size from the stack. - */ -#define RET_STACK(t, index) ((struct ftrace_ret_stack *)(&(t)->ret_stack[index])) -#define RET_STACK_INC(c) ({ c += FGRAPH_FRAME_OFFSET; }) -#define RET_STACK_DEC(c) ({ c -= FGRAPH_FRAME_OFFSET; }) +/* RET_STACK(): Return the frame from a given @offset from task @t */ +#define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset])) DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; static int fgraph_array_cnt; -#define FGRAPH_ARRAY_SIZE 16 static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE]; +/* Get the FRAME_OFFSET from the word from the @offset on ret_stack */ +static inline int get_frame_offset(struct task_struct *t, int offset) +{ + return t->ret_stack[offset] & FGRAPH_FRAME_OFFSET_MASK; +} + +/* Get FGRAPH_TYPE from the word from the @offset at ret_stack */ +static inline int get_fgraph_type(struct task_struct *t, int offset) +{ + return (t->ret_stack[offset] >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK; +} + +/* For BITMAP type: get the bitmask from the @offset at ret_stack */ +static inline unsigned long +get_bitmap_bits(struct task_struct *t, int offset) +{ + return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK; +} + +/* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */ +static inline void +set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) +{ + t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) | + (FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET; +} + +/* + * @offset: The offset into @t->ret_stack to find the ret_stack entry + * @frame_offset: Where to place the offset into @t->ret_stack of that entry + * + * Returns a pointer to the previous ret_stack below @offset or NULL + * when it reaches the bottom of the stack. + * + * Calling this with: + * + * offset = task->curr_ret_stack; + * do { + * ret_stack = get_ret_stack(task, offset, &offset); + * } while (ret_stack); + * + * Will iterate through all the ret_stack entries from curr_ret_stack + * down to the first one. + */ +static inline struct ftrace_ret_stack * +get_ret_stack(struct task_struct *t, int offset, int *frame_offset) +{ + int offs; + + BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long)); + + if (unlikely(offset <= 0)) + return NULL; + + offs = get_frame_offset(t, --offset); + if (WARN_ON_ONCE(offs <= 0 || offs > offset)) + return NULL; + + offset -= offs; + + *frame_offset = offset; + return RET_STACK(t, offset); +} + /* Both enabled by default (can be cleared by function_graph tracer flags */ static bool fgraph_sleep_time = true; @@ -110,11 +253,13 @@ void ftrace_graph_stop(void) /* Add a function return address to the trace stack on thread info.*/ static int ftrace_push_return_trace(unsigned long ret, unsigned long func, - unsigned long frame_pointer, unsigned long *retp) + unsigned long frame_pointer, unsigned long *retp, + int fgraph_idx) { struct ftrace_ret_stack *ret_stack; unsigned long long calltime; - int index; + unsigned long val; + int offset; if (unlikely(ftrace_graph_is_dead())) return -EBUSY; @@ -124,24 +269,57 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, BUILD_BUG_ON(SHADOW_STACK_SIZE % sizeof(long)); + /* Set val to "reserved" with the delta to the new fgraph frame */ + val = (FGRAPH_TYPE_RESERVED << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET; + /* * We must make sure the ret_stack is tested before we read * anything else. */ smp_rmb(); - /* The return trace stack is full */ - if (current->curr_ret_stack >= SHADOW_STACK_MAX_INDEX) { + /* + * Check if there's room on the shadow stack to fit a fraph frame + * and a bitmap word. + */ + if (current->curr_ret_stack + FGRAPH_FRAME_OFFSET + 1 >= SHADOW_STACK_MAX_OFFSET) { atomic_inc(¤t->trace_overrun); return -EBUSY; } calltime = trace_clock_local(); - index = current->curr_ret_stack; - RET_STACK_INC(current->curr_ret_stack); - ret_stack = RET_STACK(current, index); + offset = READ_ONCE(current->curr_ret_stack); + ret_stack = RET_STACK(current, offset); + offset += FGRAPH_FRAME_OFFSET; + + /* ret offset = FGRAPH_FRAME_OFFSET ; type = reserved */ + current->ret_stack[offset] = val; + ret_stack->ret = ret; + /* + * The unwinders expect curr_ret_stack to point to either zero + * or an offset where to find the next ret_stack. Even though the + * ret stack might be bogus, we want to write the ret and the + * offset to find the ret_stack before we increment the stack point. + * If an interrupt comes in now before we increment the curr_ret_stack + * it may blow away what we wrote. But that's fine, because the + * offset will still be correct (even though the 'ret' won't be). + * What we worry about is the offset being correct after we increment + * the curr_ret_stack and before we update that offset, as if an + * interrupt comes in and does an unwind stack dump, it will need + * at least a correct offset! + */ barrier(); + WRITE_ONCE(current->curr_ret_stack, offset + 1); + /* + * This next barrier is to ensure that an interrupt coming in + * will not corrupt what we are about to write. + */ + barrier(); + + /* Still keep it reserved even if an interrupt came in */ + current->ret_stack[offset] = val; + ret_stack->ret = ret; ret_stack->func = func; ret_stack->calltime = calltime; @@ -151,7 +329,7 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR ret_stack->retp = retp; #endif - return 0; + return offset; } /* @@ -168,49 +346,67 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, # define MCOUNT_INSN_SIZE 0 #endif +/* If the caller does not use ftrace, call this function. */ int function_graph_enter(unsigned long ret, unsigned long func, unsigned long frame_pointer, unsigned long *retp) { struct ftrace_graph_ent trace; + unsigned long bitmap = 0; + int offset; + int i; trace.func = func; trace.depth = ++current->curr_ret_depth; - if (ftrace_push_return_trace(ret, func, frame_pointer, retp)) + offset = ftrace_push_return_trace(ret, func, frame_pointer, retp, 0); + if (offset < 0) goto out; - /* Only trace if the calling function expects to */ - if (!fgraph_array[0]->entryfunc(&trace)) + for (i = 0; i < fgraph_array_cnt; i++) { + struct fgraph_ops *gops = fgraph_array[i]; + + if (gops == &fgraph_stub) + continue; + + if (gops->entryfunc(&trace)) + bitmap |= BIT(i); + } + + if (!bitmap) goto out_ret; + /* + * Since this function uses fgraph_idx = 0 as a tail-call checking + * flag, set that bit always. + */ + set_bitmap(current, offset, bitmap | BIT(0)); + return 0; out_ret: - RET_STACK_DEC(current->curr_ret_stack); + current->curr_ret_stack -= FGRAPH_FRAME_OFFSET + 1; out: current->curr_ret_depth--; return -EBUSY; } /* Retrieve a function return address to the trace stack on thread info.*/ -static void +static struct ftrace_ret_stack * ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, - unsigned long frame_pointer) + unsigned long frame_pointer, int *offset) { struct ftrace_ret_stack *ret_stack; - int index; - index = current->curr_ret_stack; - RET_STACK_DEC(index); + ret_stack = get_ret_stack(current, current->curr_ret_stack, offset); - if (unlikely(index < 0 || index > SHADOW_STACK_MAX_INDEX)) { + if (unlikely(!ret_stack)) { ftrace_graph_stop(); - WARN_ON(1); + WARN(1, "Bad function graph ret_stack pointer: %d", + current->curr_ret_stack); /* Might as well panic, otherwise we have no where to go */ *ret = (unsigned long)panic; - return; + return NULL; } - ret_stack = RET_STACK(current, index); #ifdef HAVE_FUNCTION_GRAPH_FP_TEST /* * The arch may choose to record the frame pointer used @@ -230,26 +426,29 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, ftrace_graph_stop(); WARN(1, "Bad frame pointer: expected %lx, received %lx\n" " from func %ps return to %lx\n", - current->ret_stack[index].fp, + ret_stack->fp, frame_pointer, (void *)ret_stack->func, ret_stack->ret); *ret = (unsigned long)panic; - return; + return NULL; } #endif + *offset += FGRAPH_FRAME_OFFSET; *ret = ret_stack->ret; trace->func = ret_stack->func; trace->calltime = ret_stack->calltime; trace->overrun = atomic_read(¤t->trace_overrun); - trace->depth = current->curr_ret_depth--; + trace->depth = current->curr_ret_depth; /* * We still want to trace interrupts coming in if * max_depth is set to 1. Make sure the decrement is * seen before ftrace_graph_return. */ barrier(); + + return ret_stack; } /* @@ -287,30 +486,47 @@ struct fgraph_ret_regs; static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs, unsigned long frame_pointer) { + struct ftrace_ret_stack *ret_stack; struct ftrace_graph_ret trace; + unsigned long bitmap; unsigned long ret; + int offset; + int i; - ftrace_pop_return_trace(&trace, &ret, frame_pointer); + ret_stack = ftrace_pop_return_trace(&trace, &ret, frame_pointer, &offset); + + if (unlikely(!ret_stack)) { + ftrace_graph_stop(); + WARN_ON(1); + /* Might as well panic. What else to do? */ + return (unsigned long)panic; + } + + trace.rettime = trace_clock_local(); #ifdef CONFIG_FUNCTION_GRAPH_RETVAL trace.retval = fgraph_ret_regs_return_value(ret_regs); #endif - trace.rettime = trace_clock_local(); - fgraph_array[0]->retfunc(&trace); + + bitmap = get_bitmap_bits(current, offset); + for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { + struct fgraph_ops *gops = fgraph_array[i]; + + if (!(bitmap & BIT(i))) + continue; + if (gops == &fgraph_stub) + continue; + + gops->retfunc(&trace); + } + /* * The ftrace_graph_return() may still access the current * ret_stack structure, we need to make sure the update of * curr_ret_stack is after that. */ barrier(); - RET_STACK_DEC(current->curr_ret_stack); - - if (unlikely(!ret)) { - ftrace_graph_stop(); - WARN_ON(1); - /* Might as well panic. What else to do? */ - ret = (unsigned long)panic; - } - + current->curr_ret_stack -= FGRAPH_FRAME_OFFSET + 1; + current->curr_ret_depth--; return ret; } @@ -333,7 +549,7 @@ unsigned long ftrace_return_to_handler(unsigned long frame_pointer) /** * ftrace_graph_get_ret_stack - return the entry of the shadow stack - * @task: The task to read the shadow stack from + * @task: The task to read the shadow stack from. * @idx: Index down the shadow stack * * Return the ret_struct on the shadow stack of the @task at the @@ -345,15 +561,17 @@ unsigned long ftrace_return_to_handler(unsigned long frame_pointer) struct ftrace_ret_stack * ftrace_graph_get_ret_stack(struct task_struct *task, int idx) { - int index = task->curr_ret_stack; + struct ftrace_ret_stack *ret_stack = NULL; + int offset = task->curr_ret_stack; - BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long)); - - index -= FGRAPH_FRAME_OFFSET * (idx + 1); - if (index < 0) + if (offset < 0) return NULL; - return RET_STACK(task, index); + do { + ret_stack = get_ret_stack(task, offset, &offset); + } while (ret_stack && --idx >= 0); + + return ret_stack; } /** @@ -376,16 +594,15 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { struct ftrace_ret_stack *ret_stack; - int index = task->curr_ret_stack; - int i; + int i = task->curr_ret_stack; if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler)) return ret; - RET_STACK_DEC(index); - - for (i = index; i >= 0; RET_STACK_DEC(i)) { - ret_stack = RET_STACK(task, i); + while (i > 0) { + ret_stack = get_ret_stack(current, i, &i); + if (!ret_stack) + break; if (ret_stack->retp == retp) return ret_stack->ret; } @@ -396,21 +613,26 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { - int task_idx; + struct ftrace_ret_stack *ret_stack; + int offset = task->curr_ret_stack; + int i; if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler)) return ret; - task_idx = task->curr_ret_stack; - RET_STACK_DEC(task_idx); - - if (!task->ret_stack || task_idx < *idx) + if (!idx) return ret; - task_idx -= *idx; - RET_STACK_INC(*idx); + i = *idx; + do { + ret_stack = get_ret_stack(task, offset, &offset); + i--; + } while (i >= 0 && ret_stack); - return RET_STACK(task, task_idx); + if (ret_stack) + return ret_stack->ret; + + return ret; } #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ @@ -493,7 +715,7 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt, { struct ftrace_ret_stack *ret_stack; unsigned long long timestamp; - int index; + int offset; /* * Does the user want to count the time a function was asleep. @@ -516,10 +738,10 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt, */ timestamp -= next->ftrace_timestamp; - for (index = next->curr_ret_stack - FGRAPH_FRAME_OFFSET; index >= 0; ) { - ret_stack = RET_STACK(next, index); - ret_stack->calltime += timestamp; - index -= FGRAPH_FRAME_OFFSET; + for (offset = next->curr_ret_stack; offset > 0; ) { + ret_stack = get_ret_stack(next, offset, &offset); + if (ret_stack) + ret_stack->calltime += timestamp; } } @@ -570,6 +792,8 @@ graph_init_task(struct task_struct *t, unsigned long *ret_stack) { atomic_set(&t->trace_overrun, 0); t->ftrace_timestamp = 0; + t->curr_ret_stack = 0; + t->curr_ret_depth = -1; /* make curr_ret_stack visible before we add the ret_stack */ smp_wmb(); t->ret_stack = ret_stack; @@ -691,6 +915,7 @@ int register_ftrace_graph(struct fgraph_ops *gops) fgraph_array[i] = gops; if (i + 1 > fgraph_array_cnt) fgraph_array_cnt = i + 1; + gops->idx = i; ftrace_graph_active++; From 375bb57292f49fa0956cc2739c81749b88e69510 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 3 Jun 2024 15:07:09 -0400 Subject: [PATCH 030/280] function_graph: Handle tail calls for stack unwinding For the tail-call, there would be 2 or more ftrace_ret_stacks on the ret_stack, which records "return_to_handler" as the return address except for the last one. But on the real stack, there should be 1 entry because tail-call reuses the return address on the stack and jump to the next function. In ftrace_graph_ret_addr() that is used for stack unwinding, skip tail calls as a real stack unwinder would do. Link: https://lore.kernel.org/linux-trace-kernel/171509096221.162236.8806372072523195752.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.717065217@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index aae51f746828..8de2a2662281 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -594,16 +594,26 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { struct ftrace_ret_stack *ret_stack; + unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler); int i = task->curr_ret_stack; - if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler)) + if (ret != return_handler) return ret; while (i > 0) { ret_stack = get_ret_stack(current, i, &i); if (!ret_stack) break; - if (ret_stack->retp == retp) + /* + * For the tail-call, there would be 2 or more ftrace_ret_stacks on + * the ret_stack, which records "return_to_handler" as the return + * address except for the last one. + * But on the real stack, there should be 1 entry because tail-call + * reuses the return address on the stack and jump to the next function. + * Thus we will continue to find real return address. + */ + if (ret_stack->retp == retp && + ret_stack->ret != return_handler) return ret_stack->ret; } @@ -614,10 +624,11 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { struct ftrace_ret_stack *ret_stack; + unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler); int offset = task->curr_ret_stack; int i; - if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler)) + if (ret != return_handler) return ret; if (!idx) @@ -626,6 +637,8 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, i = *idx; do { ret_stack = get_ret_stack(task, offset, &offset); + if (ret_stack && ret_stack->ret == return_handler) + continue; i--; } while (i >= 0 && ret_stack); From 2fbb549983763b2cc32a1ab840fe59cc1822e06d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:10 -0400 Subject: [PATCH 031/280] function_graph: Remove logic around ftrace_graph_entry and return The function pointers ftrace_graph_entry and ftrace_graph_return are no longer called via the function_graph tracer. Instead, an array structure is now used that will allow for multiple users of the function_graph infrastructure. The variables are still used by the architecture code for non dynamic ftrace configs, where a test is made against them to see if they point to the default stub function or not. This is how the static function tracing knows to call into the function graph tracer infrastructure or not. Two new stub functions are made. entry_run() and return_run(). The ftrace_graph_entry and ftrace_graph_return are set to them respectively when the function graph tracer is enabled, and this will trigger the architecture specific function graph code to be executed. This also requires checking the global_ops hash for all calls into the function_graph tracer. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509097408.162236.17387844142114638932.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.872127216@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 67 ++++++++-------------------------- kernel/trace/ftrace.c | 2 - kernel/trace/ftrace_internal.h | 2 - 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 8de2a2662281..2b52afa03ab4 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -163,6 +163,17 @@ set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) (FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET; } +/* ftrace_graph_entry set to this to tell some archs to run function graph */ +static int entry_run(struct ftrace_graph_ent *trace) +{ + return 0; +} + +/* ftrace_graph_return set to this to tell some archs to run function graph */ +static void return_run(struct ftrace_graph_ret *trace) +{ +} + /* * @offset: The offset into @t->ret_stack to find the ret_stack entry * @frame_offset: Where to place the offset into @t->ret_stack of that entry @@ -675,7 +686,6 @@ extern void ftrace_stub_graph(struct ftrace_graph_ret *); /* The callbacks that hook a function */ trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph; trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; -static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ static int alloc_retstack_tasklist(unsigned long **ret_stack_list) @@ -758,46 +768,6 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt, } } -static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) -{ - if (!ftrace_ops_test(&global_ops, trace->func, NULL)) - return 0; - return __ftrace_graph_entry(trace); -} - -/* - * The function graph tracer should only trace the functions defined - * by set_ftrace_filter and set_ftrace_notrace. If another function - * tracer ops is registered, the graph tracer requires testing the - * function against the global ops, and not just trace any function - * that any ftrace_ops registered. - */ -void update_function_graph_func(void) -{ - struct ftrace_ops *op; - bool do_test = false; - - /* - * The graph and global ops share the same set of functions - * to test. If any other ops is on the list, then - * the graph tracing needs to test if its the function - * it should call. - */ - do_for_each_ftrace_op(op, ftrace_ops_list) { - if (op != &global_ops && op != &graph_ops && - op != &ftrace_list_end) { - do_test = true; - /* in double loop, break out with goto */ - goto out; - } - } while_for_each_ftrace_op(op); - out: - if (do_test) - ftrace_graph_entry = ftrace_graph_entry_test; - else - ftrace_graph_entry = __ftrace_graph_entry; -} - static DEFINE_PER_CPU(unsigned long *, idle_ret_stack); static void @@ -939,18 +909,12 @@ int register_ftrace_graph(struct fgraph_ops *gops) ftrace_graph_active--; goto out; } - - ftrace_graph_return = gops->retfunc; - /* - * Update the indirect function to the entryfunc, and the - * function that gets called to the entry_test first. Then - * call the update fgraph entry function to determine if - * the entryfunc should be called directly or not. + * Some archs just test to see if these are not + * the default function */ - __ftrace_graph_entry = gops->entryfunc; - ftrace_graph_entry = ftrace_graph_entry_test; - update_function_graph_func(); + ftrace_graph_return = return_run; + ftrace_graph_entry = entry_run; ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); } @@ -986,7 +950,6 @@ void unregister_ftrace_graph(struct fgraph_ops *gops) if (!ftrace_graph_active) { ftrace_graph_return = ftrace_stub_graph; ftrace_graph_entry = ftrace_graph_entry_stub; - __ftrace_graph_entry = ftrace_graph_entry_stub; ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); unregister_pm_notifier(&ftrace_suspend_notifier); unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 65208d3b5ed9..789950a4f977 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -235,8 +235,6 @@ static void update_ftrace_function(void) func = ftrace_ops_list_func; } - update_function_graph_func(); - /* If there's no change, then do nothing more here */ if (ftrace_trace_function == func) return; diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h index 5012c04f92c0..19eddcb91584 100644 --- a/kernel/trace/ftrace_internal.h +++ b/kernel/trace/ftrace_internal.h @@ -42,10 +42,8 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) #ifdef CONFIG_FUNCTION_GRAPH_TRACER extern int ftrace_graph_active; -void update_function_graph_func(void); #else /* !CONFIG_FUNCTION_GRAPH_TRACER */ # define ftrace_graph_active 0 -static inline void update_function_graph_func(void) { } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #else /* !CONFIG_FUNCTION_TRACER */ From 37238abe3cb47b8daaa8706c9949f67b2a705cf1 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:11 -0400 Subject: [PATCH 032/280] ftrace/function_graph: Pass fgraph_ops to function graph callbacks Pass the fgraph_ops structure to the function graph callbacks. This will allow callbacks to add a descriptor to a fgraph_ops private field that wil be added in the future and use it for the callbacks. This will be useful when more than one callback can be registered to the function graph tracer. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509098588.162236.4787930115997357578.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.035147698@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 12 ++++++++---- kernel/trace/fgraph.c | 16 +++++++++------- kernel/trace/ftrace.c | 6 ++++-- kernel/trace/trace.h | 4 ++-- kernel/trace/trace_functions_graph.c | 11 +++++++---- kernel/trace/trace_irqsoff.c | 6 ++++-- kernel/trace/trace_sched_wakeup.c | 6 ++++-- kernel/trace/trace_selftest.c | 5 +++-- 8 files changed, 41 insertions(+), 25 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 8f00a7415bd5..4d817b85ac0d 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1027,11 +1027,15 @@ struct ftrace_graph_ret { unsigned long long rettime; } __packed; -/* Type of the callback handlers for tracing function graph*/ -typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *); /* return */ -typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */ +struct fgraph_ops; -extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace); +/* Type of the callback handlers for tracing function graph*/ +typedef void (*trace_func_graph_ret_t)(struct ftrace_graph_ret *, + struct fgraph_ops *); /* return */ +typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *, + struct fgraph_ops *); /* entry */ + +extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, struct fgraph_ops *gops); #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 2b52afa03ab4..54ed2ed2036b 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -164,13 +164,13 @@ set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) } /* ftrace_graph_entry set to this to tell some archs to run function graph */ -static int entry_run(struct ftrace_graph_ent *trace) +static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops) { return 0; } /* ftrace_graph_return set to this to tell some archs to run function graph */ -static void return_run(struct ftrace_graph_ret *trace) +static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops) { } @@ -234,12 +234,14 @@ int __weak ftrace_disable_ftrace_graph_caller(void) } #endif -int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) +int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { return 0; } -static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace) +static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { } @@ -379,7 +381,7 @@ int function_graph_enter(unsigned long ret, unsigned long func, if (gops == &fgraph_stub) continue; - if (gops->entryfunc(&trace)) + if (gops->entryfunc(&trace, gops)) bitmap |= BIT(i); } @@ -527,7 +529,7 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs if (gops == &fgraph_stub) continue; - gops->retfunc(&trace); + gops->retfunc(&trace, gops); } /* @@ -681,7 +683,7 @@ void ftrace_graph_sleep_time_control(bool enable) * Simply points to ftrace_stub, but with the proper protocol. * Defined by the linker script in linux/vmlinux.lds.h */ -extern void ftrace_stub_graph(struct ftrace_graph_ret *); +void ftrace_stub_graph(struct ftrace_graph_ret *trace, struct fgraph_ops *gops); /* The callbacks that hook a function */ trace_func_graph_ret_t ftrace_graph_return = ftrace_stub_graph; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 789950a4f977..d18387c0642d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -815,7 +815,8 @@ void ftrace_graph_graph_time_control(bool enable) fgraph_graph_time = enable; } -static int profile_graph_entry(struct ftrace_graph_ent *trace) +static int profile_graph_entry(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { struct ftrace_ret_stack *ret_stack; @@ -832,7 +833,8 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace) return 1; } -static void profile_graph_return(struct ftrace_graph_ret *trace) +static void profile_graph_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { struct ftrace_ret_stack *ret_stack; struct ftrace_profile_stat *stat; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 749a182dab48..2575ec243350 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -679,8 +679,8 @@ void trace_latency_header(struct seq_file *m); void trace_default_header(struct seq_file *m); void print_trace_header(struct seq_file *m, struct trace_iterator *iter); -void trace_graph_return(struct ftrace_graph_ret *trace); -int trace_graph_entry(struct ftrace_graph_ent *trace); +void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops); +int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops); void set_graph_array(struct trace_array *tr); void tracing_start_cmdline_record(void); diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index c35fbaab2a47..b7b142b65299 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -129,7 +129,8 @@ static inline int ftrace_graph_ignore_irqs(void) return in_hardirq(); } -int trace_graph_entry(struct ftrace_graph_ent *trace) +int trace_graph_entry(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { struct trace_array *tr = graph_array; struct trace_array_cpu *data; @@ -238,7 +239,8 @@ void __trace_graph_return(struct trace_array *tr, trace_buffer_unlock_commit_nostack(buffer, event); } -void trace_graph_return(struct ftrace_graph_ret *trace) +void trace_graph_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { struct trace_array *tr = graph_array; struct trace_array_cpu *data; @@ -275,7 +277,8 @@ void set_graph_array(struct trace_array *tr) smp_mb(); } -static void trace_graph_thresh_return(struct ftrace_graph_ret *trace) +static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { ftrace_graph_addr_finish(trace); @@ -288,7 +291,7 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace) (trace->rettime - trace->calltime < tracing_thresh)) return; else - trace_graph_return(trace); + trace_graph_return(trace, gops); } static struct fgraph_ops funcgraph_thresh_ops = { diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index ba37f768e2f2..5478f4c4f708 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c @@ -175,7 +175,8 @@ static int irqsoff_display_graph(struct trace_array *tr, int set) return start_irqsoff_tracer(irqsoff_trace, set); } -static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) +static int irqsoff_graph_entry(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; @@ -205,7 +206,8 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) return ret; } -static void irqsoff_graph_return(struct ftrace_graph_ret *trace) +static void irqsoff_graph_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 0469a04a355f..49bcc812652c 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -112,7 +112,8 @@ static int wakeup_display_graph(struct trace_array *tr, int set) return start_func_tracer(tr, set); } -static int wakeup_graph_entry(struct ftrace_graph_ent *trace) +static int wakeup_graph_entry(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { struct trace_array *tr = wakeup_trace; struct trace_array_cpu *data; @@ -141,7 +142,8 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace) return ret; } -static void wakeup_graph_return(struct ftrace_graph_ret *trace) +static void wakeup_graph_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) { struct trace_array *tr = wakeup_trace; struct trace_array_cpu *data; diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index e9c5058a8efd..56f269c0560a 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -762,7 +762,8 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr) static unsigned int graph_hang_thresh; /* Wrap the real function entry probe to avoid possible hanging */ -static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace) +static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) { /* This is harmlessly racy, we want to approximately detect a hang */ if (unlikely(++graph_hang_thresh > GRAPH_MAX_FUNC_TEST)) { @@ -776,7 +777,7 @@ static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace) return 0; } - return trace_graph_entry(trace); + return trace_graph_entry(trace, gops); } static struct fgraph_ops fgraph_ops __initdata = { From 26dda5631d1bb2f254f4c94aa87ee6c92a89cfdb Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:12 -0400 Subject: [PATCH 033/280] ftrace: Allow function_graph tracer to be enabled in instances Now that function graph tracing can handle more than one user, allow it to be enabled in the ftrace instances. Note, the filtering of the functions is still joined by the top level set_ftrace_filter and friends, as well as the graph and nograph files. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509099743.162236.1699959255446248163.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.190630762@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 1 + kernel/trace/ftrace.c | 1 + kernel/trace/trace.h | 13 +++++- kernel/trace/trace_functions.c | 8 ++++ kernel/trace/trace_functions_graph.c | 65 +++++++++++++++++----------- kernel/trace/trace_selftest.c | 4 +- 6 files changed, 64 insertions(+), 28 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 4d817b85ac0d..fd656e6d6b7c 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1042,6 +1042,7 @@ extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, struct fgraph struct fgraph_ops { trace_func_graph_ent_t entryfunc; trace_func_graph_ret_t retfunc; + void *private; int idx; }; diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index d18387c0642d..b85f00b0ffe7 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7327,6 +7327,7 @@ __init void ftrace_init_global_array_ops(struct trace_array *tr) tr->ops = &global_ops; tr->ops->private = tr; ftrace_init_trace_array(tr); + init_array_fgraph_ops(tr); } void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 2575ec243350..a5070f9b977b 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -397,6 +397,9 @@ struct trace_array { struct ftrace_ops *ops; struct trace_pid_list __rcu *function_pids; struct trace_pid_list __rcu *function_no_pids; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + struct fgraph_ops *gops; +#endif #ifdef CONFIG_DYNAMIC_FTRACE /* All of these are protected by the ftrace_lock */ struct list_head func_probes; @@ -681,7 +684,6 @@ void print_trace_header(struct seq_file *m, struct trace_iterator *iter); void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops); int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops); -void set_graph_array(struct trace_array *tr); void tracing_start_cmdline_record(void); void tracing_stop_cmdline_record(void); @@ -892,6 +894,9 @@ extern int __trace_graph_entry(struct trace_array *tr, extern void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, unsigned int trace_ctx); +extern void init_array_fgraph_ops(struct trace_array *tr); +extern int allocate_fgraph_ops(struct trace_array *tr); +extern void free_fgraph_ops(struct trace_array *tr); #ifdef CONFIG_DYNAMIC_FTRACE extern struct ftrace_hash __rcu *ftrace_graph_hash; @@ -1004,6 +1009,12 @@ print_graph_function_flags(struct trace_iterator *iter, u32 flags) { return TRACE_TYPE_UNHANDLED; } +static inline void init_array_fgraph_ops(struct trace_array *tr) { } +static inline int allocate_fgraph_ops(struct trace_array *tr) +{ + return 0; +} +static inline void free_fgraph_ops(struct trace_array *tr) { } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ extern struct list_head ftrace_pids; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 9f1bfbe105e8..8e8da0d0ee52 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -80,6 +80,7 @@ void ftrace_free_ftrace_ops(struct trace_array *tr) int ftrace_create_function_files(struct trace_array *tr, struct dentry *parent) { + int ret; /* * The top level array uses the "global_ops", and the files are * created on boot up. @@ -90,6 +91,12 @@ int ftrace_create_function_files(struct trace_array *tr, if (!tr->ops) return -EINVAL; + ret = allocate_fgraph_ops(tr); + if (ret) { + kfree(tr->ops); + return ret; + } + ftrace_create_filter_files(tr->ops, parent); return 0; @@ -99,6 +106,7 @@ void ftrace_destroy_function_files(struct trace_array *tr) { ftrace_destroy_filter_files(tr->ops); ftrace_free_ftrace_ops(tr); + free_fgraph_ops(tr); } static ftrace_func_t select_trace_function(u32 flags_val) diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index b7b142b65299..9ccc904a7703 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -83,8 +83,6 @@ static struct tracer_flags tracer_flags = { .opts = trace_opts }; -static struct trace_array *graph_array; - /* * DURATION column is being also used to display IRQ signs, * following values are used by print_graph_irq and others @@ -132,7 +130,7 @@ static inline int ftrace_graph_ignore_irqs(void) int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { - struct trace_array *tr = graph_array; + struct trace_array *tr = gops->private; struct trace_array_cpu *data; unsigned long flags; unsigned int trace_ctx; @@ -242,7 +240,7 @@ void __trace_graph_return(struct trace_array *tr, void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { - struct trace_array *tr = graph_array; + struct trace_array *tr = gops->private; struct trace_array_cpu *data; unsigned long flags; unsigned int trace_ctx; @@ -268,15 +266,6 @@ void trace_graph_return(struct ftrace_graph_ret *trace, local_irq_restore(flags); } -void set_graph_array(struct trace_array *tr) -{ - graph_array = tr; - - /* Make graph_array visible before we start tracing */ - - smp_mb(); -} - static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { @@ -294,25 +283,53 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, trace_graph_return(trace, gops); } -static struct fgraph_ops funcgraph_thresh_ops = { - .entryfunc = &trace_graph_entry, - .retfunc = &trace_graph_thresh_return, -}; - static struct fgraph_ops funcgraph_ops = { .entryfunc = &trace_graph_entry, .retfunc = &trace_graph_return, }; +int allocate_fgraph_ops(struct trace_array *tr) +{ + struct fgraph_ops *gops; + + gops = kzalloc(sizeof(*gops), GFP_KERNEL); + if (!gops) + return -ENOMEM; + + gops->entryfunc = &trace_graph_entry; + gops->retfunc = &trace_graph_return; + + tr->gops = gops; + gops->private = tr; + return 0; +} + +void free_fgraph_ops(struct trace_array *tr) +{ + kfree(tr->gops); +} + +__init void init_array_fgraph_ops(struct trace_array *tr) +{ + tr->gops = &funcgraph_ops; + funcgraph_ops.private = tr; +} + static int graph_trace_init(struct trace_array *tr) { int ret; - set_graph_array(tr); + tr->gops->entryfunc = trace_graph_entry; + if (tracing_thresh) - ret = register_ftrace_graph(&funcgraph_thresh_ops); + tr->gops->retfunc = trace_graph_thresh_return; else - ret = register_ftrace_graph(&funcgraph_ops); + tr->gops->retfunc = trace_graph_return; + + /* Make gops functions are visible before we start tracing */ + smp_mb(); + + ret = register_ftrace_graph(tr->gops); if (ret) return ret; tracing_start_cmdline_record(); @@ -323,10 +340,7 @@ static int graph_trace_init(struct trace_array *tr) static void graph_trace_reset(struct trace_array *tr) { tracing_stop_cmdline_record(); - if (tracing_thresh) - unregister_ftrace_graph(&funcgraph_thresh_ops); - else - unregister_ftrace_graph(&funcgraph_ops); + unregister_ftrace_graph(tr->gops); } static int graph_trace_update_thresh(struct trace_array *tr) @@ -1365,6 +1379,7 @@ static struct tracer graph_trace __tracer_data = { .print_header = print_graph_headers, .flags = &tracer_flags, .set_flag = func_graph_set_flag, + .allow_instances = true, #ifdef CONFIG_FTRACE_SELFTEST .selftest = trace_selftest_startup_function_graph, #endif diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 56f269c0560a..f8f55fd79e53 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -813,7 +813,7 @@ trace_selftest_startup_function_graph(struct tracer *trace, * to detect and recover from possible hangs */ tracing_reset_online_cpus(&tr->array_buffer); - set_graph_array(tr); + fgraph_ops.private = tr; ret = register_ftrace_graph(&fgraph_ops); if (ret) { warn_failed_init_tracer(trace, ret); @@ -856,7 +856,7 @@ trace_selftest_startup_function_graph(struct tracer *trace, cond_resched(); tracing_reset_online_cpus(&tr->array_buffer); - set_graph_array(tr); + fgraph_ops.private = tr; /* * Some archs *cough*PowerPC*cough* add characters to the From ab6b84630382914ffcbab59f4913c9a60971d034 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:13 -0400 Subject: [PATCH 034/280] ftrace: Allow ftrace startup flags to exist without dynamic ftrace Some of the flags for ftrace_startup() may be exposed even when CONFIG_DYNAMIC_FTRACE is not configured in. This is fine as the difference between dynamic ftrace and static ftrace is done within the internals of ftrace itself. No need to have use cases fail to compile because dynamic ftrace is disabled. This change is needed to move some of the logic of what is passed to ftrace_startup() out of the parameters of ftrace_startup(). Link: https://lore.kernel.org/linux-trace-kernel/171509100890.162236.4362350342549122222.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.350654104@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index fd656e6d6b7c..586018744785 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -509,6 +509,15 @@ static inline void stack_tracer_disable(void) { } static inline void stack_tracer_enable(void) { } #endif +enum { + FTRACE_UPDATE_CALLS = (1 << 0), + FTRACE_DISABLE_CALLS = (1 << 1), + FTRACE_UPDATE_TRACE_FUNC = (1 << 2), + FTRACE_START_FUNC_RET = (1 << 3), + FTRACE_STOP_FUNC_RET = (1 << 4), + FTRACE_MAY_SLEEP = (1 << 5), +}; + #ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void); @@ -603,15 +612,6 @@ void ftrace_set_global_notrace(unsigned char *buf, int len, int reset); void ftrace_free_filter(struct ftrace_ops *ops); void ftrace_ops_set_global_filter(struct ftrace_ops *ops); -enum { - FTRACE_UPDATE_CALLS = (1 << 0), - FTRACE_DISABLE_CALLS = (1 << 1), - FTRACE_UPDATE_TRACE_FUNC = (1 << 2), - FTRACE_START_FUNC_RET = (1 << 3), - FTRACE_STOP_FUNC_RET = (1 << 4), - FTRACE_MAY_SLEEP = (1 << 5), -}; - /* * The FTRACE_UPDATE_* enum is used to pass information back * from the ftrace_update_record() and ftrace_test_record() From 5fccc7552ccbc521bad61653ee739b1196b1bc53 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:14 -0400 Subject: [PATCH 035/280] ftrace: Add subops logic to allow one ops to manage many There are cases where a single system will use a single function callback to handle multiple users. For example, to allow function_graph tracer to have multiple users where each can trace their own set of functions, it is useful to only have one ftrace_ops registered to ftrace that will call a function by the function_graph tracer to handle the multiplexing with the different registered function_graph tracers. Add a "subop_list" to the ftrace_ops that will hold a list of other ftrace_ops that the top ftrace_ops will manage. The function ftrace_startup_subops() that takes the manager ftrace_ops and a subop ftrace_ops it will manage. If there are no subops with the ftrace_ops yet, it will copy the ftrace_ops subop filters to the manager ftrace_ops and register that with ftrace_startup(), and adds the subop to its subop_list. If the manager ops already has something registered, it will then merge the new subop filters with what it has and enable the new functions that covers all the subops it has. To remove a subop, ftrace_shutdown_subops() is called which will use the subop_list of the manager ops to rebuild all the functions it needs to trace, and update the ftrace records to only call the functions it now has registered. If there are no more functions registered, it will then call ftrace_shutdown() to disable itself completely. Note, it is up to the manager ops callback to always make sure that the subops callbacks are called if its filter matches, as there are times in the update where the callback could be calling more functions than those that are currently registered. This could be updated to handle other systems other than function_graph, for example, fprobes could use this (but will need an interface to call ftrace_startup_subops()). Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.508431129@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 1 + kernel/trace/fgraph.c | 3 +- kernel/trace/ftrace.c | 401 ++++++++++++++++++++++++++++++++- kernel/trace/ftrace_internal.h | 1 + kernel/trace/trace.h | 1 + 5 files changed, 405 insertions(+), 2 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 586018744785..978a1d3b270a 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -334,6 +334,7 @@ struct ftrace_ops { unsigned long trampoline; unsigned long trampoline_size; struct list_head list; + struct list_head subop_list; ftrace_ops_func_t ops_func; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS unsigned long direct_call; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 54ed2ed2036b..e39042c40937 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -21,7 +21,8 @@ #ifdef CONFIG_DYNAMIC_FTRACE #define ASSIGN_OPS_HASH(opsname, val) \ .func_hash = val, \ - .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), + .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ + .subop_list = LIST_HEAD_INIT(opsname.subop_list), #else #define ASSIGN_OPS_HASH(opsname, val) #endif diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b85f00b0ffe7..63b8d1fe1dd8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -74,7 +74,8 @@ #ifdef CONFIG_DYNAMIC_FTRACE #define INIT_OPS_HASH(opsname) \ .func_hash = &opsname.local_hash, \ - .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), + .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ + .subop_list = LIST_HEAD_INIT(opsname.subop_list), #else #define INIT_OPS_HASH(opsname) #endif @@ -161,6 +162,7 @@ static inline void ftrace_ops_init(struct ftrace_ops *ops) #ifdef CONFIG_DYNAMIC_FTRACE if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { mutex_init(&ops->local_hash.regex_lock); + INIT_LIST_HEAD(&ops->subop_list); ops->func_hash = &ops->local_hash; ops->flags |= FTRACE_OPS_FL_INITIALIZED; } @@ -3164,6 +3166,403 @@ out: return 0; } +/* Simply make a copy of @src and return it */ +static struct ftrace_hash *copy_hash(struct ftrace_hash *src) +{ + if (ftrace_hash_empty(src)) + return EMPTY_HASH; + + return alloc_and_copy_ftrace_hash(src->size_bits, src); +} + +/* + * Append @new_hash entries to @hash: + * + * If @hash is the EMPTY_HASH then it traces all functions and nothing + * needs to be done. + * + * If @new_hash is the EMPTY_HASH, then make *hash the EMPTY_HASH so + * that it traces everything. + * + * Otherwise, go through all of @new_hash and add anything that @hash + * doesn't already have, to @hash. + * + * The filter_hash updates uses just the append_hash() function + * and the notrace_hash does not. + */ +static int append_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash) +{ + struct ftrace_func_entry *entry; + int size; + int i; + + /* An empty hash does everything */ + if (ftrace_hash_empty(*hash)) + return 0; + + /* If new_hash has everything make hash have everything */ + if (ftrace_hash_empty(new_hash)) { + free_ftrace_hash(*hash); + *hash = EMPTY_HASH; + return 0; + } + + size = 1 << new_hash->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &new_hash->buckets[i], hlist) { + /* Only add if not already in hash */ + if (!__ftrace_lookup_ip(*hash, entry->ip) && + add_hash_entry(*hash, entry->ip) == NULL) + return -ENOMEM; + } + } + return 0; +} + +/* + * Add to @hash only those that are in both @new_hash1 and @new_hash2 + * + * The notrace_hash updates uses just the intersect_hash() function + * and the filter_hash does not. + */ +static int intersect_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash1, + struct ftrace_hash *new_hash2) +{ + struct ftrace_func_entry *entry; + int size; + int i; + + /* + * If new_hash1 or new_hash2 is the EMPTY_HASH then make the hash + * empty as well as empty for notrace means none are notraced. + */ + if (ftrace_hash_empty(new_hash1) || ftrace_hash_empty(new_hash2)) { + free_ftrace_hash(*hash); + *hash = EMPTY_HASH; + return 0; + } + + size = 1 << new_hash1->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &new_hash1->buckets[i], hlist) { + /* Only add if in both @new_hash1 and @new_hash2 */ + if (__ftrace_lookup_ip(new_hash2, entry->ip) && + add_hash_entry(*hash, entry->ip) == NULL) + return -ENOMEM; + } + } + /* If nothing intersects, make it the empty set */ + if (ftrace_hash_empty(*hash)) { + free_ftrace_hash(*hash); + *hash = EMPTY_HASH; + } + return 0; +} + +/* Return a new hash that has a union of all @ops->filter_hash entries */ +static struct ftrace_hash *append_hashes(struct ftrace_ops *ops) +{ + struct ftrace_hash *new_hash; + struct ftrace_ops *subops; + int ret; + + new_hash = alloc_ftrace_hash(ops->func_hash->filter_hash->size_bits); + if (!new_hash) + return NULL; + + list_for_each_entry(subops, &ops->subop_list, list) { + ret = append_hash(&new_hash, subops->func_hash->filter_hash); + if (ret < 0) { + free_ftrace_hash(new_hash); + return NULL; + } + /* Nothing more to do if new_hash is empty */ + if (ftrace_hash_empty(new_hash)) + break; + } + return new_hash; +} + +/* Make @ops trace evenything except what all its subops do not trace */ +static struct ftrace_hash *intersect_hashes(struct ftrace_ops *ops) +{ + struct ftrace_hash *new_hash = NULL; + struct ftrace_ops *subops; + int size_bits; + int ret; + + list_for_each_entry(subops, &ops->subop_list, list) { + struct ftrace_hash *next_hash; + + if (!new_hash) { + size_bits = subops->func_hash->notrace_hash->size_bits; + new_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->notrace_hash); + if (!new_hash) + return NULL; + continue; + } + size_bits = new_hash->size_bits; + next_hash = new_hash; + new_hash = alloc_ftrace_hash(size_bits); + ret = intersect_hash(&new_hash, next_hash, subops->func_hash->notrace_hash); + free_ftrace_hash(next_hash); + if (ret < 0) { + free_ftrace_hash(new_hash); + return NULL; + } + /* Nothing more to do if new_hash is empty */ + if (ftrace_hash_empty(new_hash)) + break; + } + return new_hash; +} + +static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B) +{ + struct ftrace_func_entry *entry; + int size; + int i; + + if (ftrace_hash_empty(A)) + return ftrace_hash_empty(B); + + if (ftrace_hash_empty(B)) + return ftrace_hash_empty(A); + + if (A->count != B->count) + return false; + + size = 1 << A->size_bits; + for (i = 0; i < size; i++) { + hlist_for_each_entry(entry, &A->buckets[i], hlist) { + if (!__ftrace_lookup_ip(B, entry->ip)) + return false; + } + } + + return true; +} + +static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, + struct ftrace_hash **orig_hash, + struct ftrace_hash *hash, + int enable); + +static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_hash, + struct ftrace_hash *notrace_hash) +{ + int ret; + + if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) { + ret = ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, + filter_hash, 1); + if (ret < 0) + return ret; + } + + if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) { + ret = ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, + notrace_hash, 0); + if (ret < 0) + return ret; + } + + return 0; +} + +/** + * ftrace_startup_subops - enable tracing for subops of an ops + * @ops: Manager ops (used to pick all the functions of its subops) + * @subops: A new ops to add to @ops + * @command: Extra commands to use to enable tracing + * + * The @ops is a manager @ops that has the filter that includes all the functions + * that its list of subops are tracing. Adding a new @subops will add the + * functions of @subops to @ops. + */ +int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) +{ + struct ftrace_hash *filter_hash; + struct ftrace_hash *notrace_hash; + struct ftrace_hash *save_filter_hash; + struct ftrace_hash *save_notrace_hash; + int size_bits; + int ret; + + if (unlikely(ftrace_disabled)) + return -ENODEV; + + ftrace_ops_init(ops); + ftrace_ops_init(subops); + + if (WARN_ON_ONCE(subops->flags & FTRACE_OPS_FL_ENABLED)) + return -EBUSY; + + /* Make everything canonical (Just in case!) */ + if (!ops->func_hash->filter_hash) + ops->func_hash->filter_hash = EMPTY_HASH; + if (!ops->func_hash->notrace_hash) + ops->func_hash->notrace_hash = EMPTY_HASH; + if (!subops->func_hash->filter_hash) + subops->func_hash->filter_hash = EMPTY_HASH; + if (!subops->func_hash->notrace_hash) + subops->func_hash->notrace_hash = EMPTY_HASH; + + /* For the first subops to ops just enable it normally */ + if (list_empty(&ops->subop_list)) { + /* Just use the subops hashes */ + filter_hash = copy_hash(subops->func_hash->filter_hash); + notrace_hash = copy_hash(subops->func_hash->notrace_hash); + if (!filter_hash || !notrace_hash) { + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + return -ENOMEM; + } + + save_filter_hash = ops->func_hash->filter_hash; + save_notrace_hash = ops->func_hash->notrace_hash; + + ops->func_hash->filter_hash = filter_hash; + ops->func_hash->notrace_hash = notrace_hash; + list_add(&subops->list, &ops->subop_list); + ret = ftrace_startup(ops, command); + if (ret < 0) { + list_del(&subops->list); + ops->func_hash->filter_hash = save_filter_hash; + ops->func_hash->notrace_hash = save_notrace_hash; + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + } else { + free_ftrace_hash(save_filter_hash); + free_ftrace_hash(save_notrace_hash); + subops->flags |= FTRACE_OPS_FL_ENABLED; + } + return ret; + } + + /* + * Here there's already something attached. Here are the rules: + * o If either filter_hash is empty then the final stays empty + * o Otherwise, the final is a superset of both hashes + * o If either notrace_hash is empty then the final stays empty + * o Otherwise, the final is an intersection between the hashes + */ + if (ftrace_hash_empty(ops->func_hash->filter_hash) || + ftrace_hash_empty(subops->func_hash->filter_hash)) { + filter_hash = EMPTY_HASH; + } else { + size_bits = max(ops->func_hash->filter_hash->size_bits, + subops->func_hash->filter_hash->size_bits); + filter_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->filter_hash); + if (!filter_hash) + return -ENOMEM; + ret = append_hash(&filter_hash, subops->func_hash->filter_hash); + if (ret < 0) { + free_ftrace_hash(filter_hash); + return ret; + } + } + + if (ftrace_hash_empty(ops->func_hash->notrace_hash) || + ftrace_hash_empty(subops->func_hash->notrace_hash)) { + notrace_hash = EMPTY_HASH; + } else { + size_bits = max(ops->func_hash->filter_hash->size_bits, + subops->func_hash->filter_hash->size_bits); + notrace_hash = alloc_ftrace_hash(size_bits); + if (!notrace_hash) { + free_ftrace_hash(filter_hash); + return -ENOMEM; + } + + ret = intersect_hash(¬race_hash, ops->func_hash->filter_hash, + subops->func_hash->filter_hash); + if (ret < 0) { + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + return ret; + } + } + + list_add(&subops->list, &ops->subop_list); + + ret = ftrace_update_ops(ops, filter_hash, notrace_hash); + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + if (ret < 0) + list_del(&subops->list); + else + subops->flags |= FTRACE_OPS_FL_ENABLED; + + return ret; +} + +/** + * ftrace_shutdown_subops - Remove a subops from a manager ops + * @ops: A manager ops to remove @subops from + * @subops: The subops to remove from @ops + * @command: Any extra command flags to add to modifying the text + * + * Removes the functions being traced by the @subops from @ops. Note, it + * will not affect functions that are being traced by other subops that + * still exist in @ops. + * + * If the last subops is removed from @ops, then @ops is shutdown normally. + */ +int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) +{ + struct ftrace_hash *filter_hash; + struct ftrace_hash *notrace_hash; + int ret; + + if (unlikely(ftrace_disabled)) + return -ENODEV; + + if (WARN_ON_ONCE(!(subops->flags & FTRACE_OPS_FL_ENABLED))) + return -EINVAL; + + list_del(&subops->list); + + if (list_empty(&ops->subop_list)) { + /* Last one, just disable the current ops */ + + ret = ftrace_shutdown(ops, command); + if (ret < 0) { + list_add(&subops->list, &ops->subop_list); + return ret; + } + + subops->flags &= ~FTRACE_OPS_FL_ENABLED; + + free_ftrace_hash(ops->func_hash->filter_hash); + free_ftrace_hash(ops->func_hash->notrace_hash); + ops->func_hash->filter_hash = EMPTY_HASH; + ops->func_hash->notrace_hash = EMPTY_HASH; + + return 0; + } + + /* Rebuild the hashes without subops */ + filter_hash = append_hashes(ops); + notrace_hash = intersect_hashes(ops); + if (!filter_hash || !notrace_hash) { + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + list_add(&subops->list, &ops->subop_list); + return -ENOMEM; + } + + ret = ftrace_update_ops(ops, filter_hash, notrace_hash); + if (ret < 0) + list_add(&subops->list, &ops->subop_list); + else + subops->flags &= ~FTRACE_OPS_FL_ENABLED; + + free_ftrace_hash(filter_hash); + free_ftrace_hash(notrace_hash); + return ret; +} + static u64 ftrace_update_time; unsigned long ftrace_update_tot_cnt; unsigned long ftrace_number_of_pages; diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h index 19eddcb91584..cdfd12c44ab4 100644 --- a/kernel/trace/ftrace_internal.h +++ b/kernel/trace/ftrace_internal.h @@ -15,6 +15,7 @@ extern struct ftrace_ops global_ops; int ftrace_startup(struct ftrace_ops *ops, int command); int ftrace_shutdown(struct ftrace_ops *ops, int command); int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs); +int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command); #else /* !CONFIG_DYNAMIC_FTRACE */ diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index a5070f9b977b..9a70beb2cc46 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1136,6 +1136,7 @@ extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, int len, int reset); extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, int len, int reset); +extern int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command); #else struct ftrace_func_command; From d9bbfbd14f58d2955cc7a3efa8ae6d4e09ee5995 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:15 -0400 Subject: [PATCH 036/280] ftrace: Allow subops filtering to be modified The subops filters use a "manager" ops to enable and disable its filters. The manager ops can handle more than one subops, and its filter is what controls what functions get set. Add a ftrace_hash_move_and_update_subops() function that will update the manager ops when the subops filters change. Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.673932251@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 3 + kernel/trace/ftrace.c | 141 +++++++++++++++++++++++++++++++++-------- 2 files changed, 116 insertions(+), 28 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 978a1d3b270a..63238a9a9270 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -227,6 +227,7 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops); * ftrace_enabled. * DIRECT - Used by the direct ftrace_ops helper for direct functions * (internal ftrace only, should not be used by others) + * SUBOP - Is controlled by another op in field managed. */ enum { FTRACE_OPS_FL_ENABLED = BIT(0), @@ -247,6 +248,7 @@ enum { FTRACE_OPS_FL_TRACE_ARRAY = BIT(15), FTRACE_OPS_FL_PERMANENT = BIT(16), FTRACE_OPS_FL_DIRECT = BIT(17), + FTRACE_OPS_FL_SUBOP = BIT(18), }; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS @@ -336,6 +338,7 @@ struct ftrace_ops { struct list_head list; struct list_head subop_list; ftrace_ops_func_t ops_func; + struct ftrace_ops *managed; #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS unsigned long direct_call; #endif diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 63b8d1fe1dd8..cbb91b0afcc8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3343,10 +3343,28 @@ static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B) return true; } -static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, - struct ftrace_hash **orig_hash, - struct ftrace_hash *hash, - int enable); +static void ftrace_ops_update_code(struct ftrace_ops *ops, + struct ftrace_ops_hash *old_hash); + +static int __ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, + struct ftrace_hash **orig_hash, + struct ftrace_hash *hash, + int enable) +{ + struct ftrace_ops_hash old_hash_ops; + struct ftrace_hash *old_hash; + int ret; + + old_hash = *orig_hash; + old_hash_ops.filter_hash = ops->func_hash->filter_hash; + old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; + ret = ftrace_hash_move(ops, enable, orig_hash, hash); + if (!ret) { + ftrace_ops_update_code(ops, &old_hash_ops); + free_ftrace_hash_rcu(old_hash); + } + return ret; +} static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_hash, struct ftrace_hash *notrace_hash) @@ -3354,15 +3372,15 @@ static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_ int ret; if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) { - ret = ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, - filter_hash, 1); + ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, + filter_hash, 1); if (ret < 0) return ret; } if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) { - ret = ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, - notrace_hash, 0); + ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, + notrace_hash, 0); if (ret < 0) return ret; } @@ -3435,7 +3453,8 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int } else { free_ftrace_hash(save_filter_hash); free_ftrace_hash(save_notrace_hash); - subops->flags |= FTRACE_OPS_FL_ENABLED; + subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; + subops->managed = ops; } return ret; } @@ -3489,11 +3508,12 @@ int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int ret = ftrace_update_ops(ops, filter_hash, notrace_hash); free_ftrace_hash(filter_hash); free_ftrace_hash(notrace_hash); - if (ret < 0) + if (ret < 0) { list_del(&subops->list); - else - subops->flags |= FTRACE_OPS_FL_ENABLED; - + } else { + subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; + subops->managed = ops; + } return ret; } @@ -3538,6 +3558,8 @@ int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, in free_ftrace_hash(ops->func_hash->notrace_hash); ops->func_hash->filter_hash = EMPTY_HASH; ops->func_hash->notrace_hash = EMPTY_HASH; + subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); + subops->managed = NULL; return 0; } @@ -3553,16 +3575,65 @@ int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, in } ret = ftrace_update_ops(ops, filter_hash, notrace_hash); - if (ret < 0) + if (ret < 0) { list_add(&subops->list, &ops->subop_list); - else - subops->flags &= ~FTRACE_OPS_FL_ENABLED; - + } else { + subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); + subops->managed = NULL; + } free_ftrace_hash(filter_hash); free_ftrace_hash(notrace_hash); return ret; } +static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops, + struct ftrace_hash **orig_subhash, + struct ftrace_hash *hash, + int enable) +{ + struct ftrace_ops *ops = subops->managed; + struct ftrace_hash **orig_hash; + struct ftrace_hash *save_hash; + struct ftrace_hash *new_hash; + int ret; + + /* Manager ops can not be subops (yet) */ + if (WARN_ON_ONCE(!ops || ops->flags & FTRACE_OPS_FL_SUBOP)) + return -EINVAL; + + /* Move the new hash over to the subops hash */ + save_hash = *orig_subhash; + *orig_subhash = __ftrace_hash_move(hash); + if (!*orig_subhash) { + *orig_subhash = save_hash; + return -ENOMEM; + } + + /* Create a new_hash to hold the ops new functions */ + if (enable) { + orig_hash = &ops->func_hash->filter_hash; + new_hash = append_hashes(ops); + } else { + orig_hash = &ops->func_hash->notrace_hash; + new_hash = intersect_hashes(ops); + } + + /* Move the hash over to the new hash */ + ret = __ftrace_hash_move_and_update_ops(ops, orig_hash, new_hash, enable); + + free_ftrace_hash(new_hash); + + if (ret) { + /* Put back the original hash */ + free_ftrace_hash_rcu(*orig_subhash); + *orig_subhash = save_hash; + } else { + free_ftrace_hash_rcu(save_hash); + } + return ret; +} + + static u64 ftrace_update_time; unsigned long ftrace_update_tot_cnt; unsigned long ftrace_number_of_pages; @@ -4779,19 +4850,33 @@ static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, struct ftrace_hash *hash, int enable) { - struct ftrace_ops_hash old_hash_ops; - struct ftrace_hash *old_hash; - int ret; + if (ops->flags & FTRACE_OPS_FL_SUBOP) + return ftrace_hash_move_and_update_subops(ops, orig_hash, hash, enable); - old_hash = *orig_hash; - old_hash_ops.filter_hash = ops->func_hash->filter_hash; - old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; - ret = ftrace_hash_move(ops, enable, orig_hash, hash); - if (!ret) { - ftrace_ops_update_code(ops, &old_hash_ops); - free_ftrace_hash_rcu(old_hash); + /* + * If this ops is not enabled, it could be sharing its filters + * with a subop. If that's the case, update the subop instead of + * this ops. Shared filters are only allowed to have one ops set + * at a time, and if we update the ops that is not enabled, + * it will not affect subops that share it. + */ + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) { + struct ftrace_ops *op; + + /* Check if any other manager subops maps to this hash */ + do_for_each_ftrace_op(op, ftrace_ops_list) { + struct ftrace_ops *subops; + + list_for_each_entry(subops, &op->subop_list, list) { + if ((subops->flags & FTRACE_OPS_FL_ENABLED) && + subops->func_hash == ops->func_hash) { + return ftrace_hash_move_and_update_subops(subops, orig_hash, hash, enable); + } + } + } while_for_each_ftrace_op(op); } - return ret; + + return __ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); } static bool module_exists(const char *module) From c132be2c4fcc1150ad0791c2a85dd4c9ad0bd0c8 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:16 -0400 Subject: [PATCH 037/280] function_graph: Have the instances use their own ftrace_ops for filtering Allow for instances to have their own ftrace_ops part of the fgraph_ops that makes the funtion_graph tracer filter on the set_ftrace_filter file of the instance and not the top instance. This uses the new ftrace_startup_subops(), by using graph_ops as the "manager ops" that defines the callback function and adds the functions defined by the filters of the ops for each trace instance. The callback defined by the manager ops will call the registered fgraph ops that were added to the fgraph_array. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509102088.162236.15758883237657317789.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.832946261@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 1 + kernel/trace/fgraph.c | 81 +++++++++++++++++----------- kernel/trace/ftrace.c | 2 +- kernel/trace/trace.h | 15 +++--- kernel/trace/trace_functions.c | 2 +- kernel/trace/trace_functions_graph.c | 8 ++- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 63238a9a9270..8f865689e868 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1046,6 +1046,7 @@ extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, struct fgraph struct fgraph_ops { trace_func_graph_ent_t entryfunc; trace_func_graph_ret_t retfunc; + struct ftrace_ops ops; /* for the hash lists */ void *private; int idx; }; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index e39042c40937..3ef6db53c0bf 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -18,15 +18,6 @@ #include "ftrace_internal.h" #include "trace.h" -#ifdef CONFIG_DYNAMIC_FTRACE -#define ASSIGN_OPS_HASH(opsname, val) \ - .func_hash = val, \ - .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ - .subop_list = LIST_HEAD_INIT(opsname.subop_list), -#else -#define ASSIGN_OPS_HASH(opsname, val) -#endif - /* * FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack * FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame @@ -156,6 +147,13 @@ get_bitmap_bits(struct task_struct *t, int offset) return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK; } +/* For BITMAP type: set the bits in the bitmap bitmask at @offset on ret_stack */ +static inline void +set_bitmap_bits(struct task_struct *t, int offset, unsigned long bitmap) +{ + t->ret_stack[offset] |= (bitmap << FGRAPH_INDEX_SHIFT); +} + /* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */ static inline void set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) @@ -382,7 +380,8 @@ int function_graph_enter(unsigned long ret, unsigned long func, if (gops == &fgraph_stub) continue; - if (gops->entryfunc(&trace, gops)) + if (ftrace_ops_test(&gops->ops, func, NULL) && + gops->entryfunc(&trace, gops)) bitmap |= BIT(i); } @@ -665,16 +664,28 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, static struct ftrace_ops graph_ops = { .func = ftrace_graph_func, - .flags = FTRACE_OPS_FL_INITIALIZED | - FTRACE_OPS_FL_PID | - FTRACE_OPS_GRAPH_STUB, + .flags = FTRACE_OPS_GRAPH_STUB, #ifdef FTRACE_GRAPH_TRAMP_ADDR .trampoline = FTRACE_GRAPH_TRAMP_ADDR, /* trampoline_size is only needed for dynamically allocated tramps */ #endif - ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) }; +void fgraph_init_ops(struct ftrace_ops *dst_ops, + struct ftrace_ops *src_ops) +{ + dst_ops->flags = FTRACE_OPS_FL_PID | FTRACE_OPS_GRAPH_STUB; + +#ifdef CONFIG_DYNAMIC_FTRACE + if (src_ops) { + dst_ops->func_hash = &src_ops->local_hash; + mutex_init(&dst_ops->local_hash.regex_lock); + INIT_LIST_HEAD(&dst_ops->subop_list); + dst_ops->flags |= FTRACE_OPS_FL_INITIALIZED; + } +#endif +} + void ftrace_graph_sleep_time_control(bool enable) { fgraph_sleep_time = enable; @@ -877,6 +888,7 @@ static int start_graph_tracing(void) int register_ftrace_graph(struct fgraph_ops *gops) { + int command = 0; int ret = 0; int i; @@ -894,7 +906,7 @@ int register_ftrace_graph(struct fgraph_ops *gops) break; } if (i >= FGRAPH_ARRAY_SIZE) { - ret = -EBUSY; + ret = -ENOSPC; goto out; } @@ -908,18 +920,22 @@ int register_ftrace_graph(struct fgraph_ops *gops) if (ftrace_graph_active == 1) { register_pm_notifier(&ftrace_suspend_notifier); ret = start_graph_tracing(); - if (ret) { - ftrace_graph_active--; - goto out; - } + if (ret) + goto error; /* * Some archs just test to see if these are not * the default function */ ftrace_graph_return = return_run; ftrace_graph_entry = entry_run; + command = FTRACE_START_FUNC_RET; + } - ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); + ret = ftrace_startup_subops(&graph_ops, &gops->ops, command); +error: + if (ret) { + fgraph_array[i] = &fgraph_stub; + ftrace_graph_active--; } out: mutex_unlock(&ftrace_lock); @@ -928,6 +944,7 @@ out: void unregister_ftrace_graph(struct fgraph_ops *gops) { + int command = 0; int i; mutex_lock(&ftrace_lock); @@ -935,25 +952,29 @@ void unregister_ftrace_graph(struct fgraph_ops *gops) if (unlikely(!ftrace_graph_active)) goto out; - for (i = 0; i < fgraph_array_cnt; i++) - if (gops == fgraph_array[i]) - break; - if (i >= fgraph_array_cnt) + if (unlikely(gops->idx < 0 || gops->idx >= fgraph_array_cnt)) goto out; - fgraph_array[i] = &fgraph_stub; - if (i + 1 == fgraph_array_cnt) { - for (; i >= 0; i--) - if (fgraph_array[i] != &fgraph_stub) - break; + WARN_ON_ONCE(fgraph_array[gops->idx] != gops); + + fgraph_array[gops->idx] = &fgraph_stub; + if (gops->idx + 1 == fgraph_array_cnt) { + i = gops->idx; + while (i >= 0 && fgraph_array[i] == &fgraph_stub) + i--; fgraph_array_cnt = i + 1; } ftrace_graph_active--; + + if (!ftrace_graph_active) + command = FTRACE_STOP_FUNC_RET; + + ftrace_shutdown_subops(&graph_ops, &gops->ops, command); + if (!ftrace_graph_active) { ftrace_graph_return = ftrace_stub_graph; ftrace_graph_entry = ftrace_graph_entry_stub; - ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); unregister_pm_notifier(&ftrace_suspend_notifier); unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index cbb91b0afcc8..58e0f4bc0241 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7811,7 +7811,7 @@ __init void ftrace_init_global_array_ops(struct trace_array *tr) tr->ops = &global_ops; tr->ops->private = tr; ftrace_init_trace_array(tr); - init_array_fgraph_ops(tr); + init_array_fgraph_ops(tr, tr->ops); } void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 9a70beb2cc46..f06b5ddd3580 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -894,8 +894,8 @@ extern int __trace_graph_entry(struct trace_array *tr, extern void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, unsigned int trace_ctx); -extern void init_array_fgraph_ops(struct trace_array *tr); -extern int allocate_fgraph_ops(struct trace_array *tr); +extern void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops); +extern int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops); extern void free_fgraph_ops(struct trace_array *tr); #ifdef CONFIG_DYNAMIC_FTRACE @@ -1003,18 +1003,19 @@ static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace) (fgraph_max_depth && trace->depth >= fgraph_max_depth); } +void fgraph_init_ops(struct ftrace_ops *dst_ops, + struct ftrace_ops *src_ops); + #else /* CONFIG_FUNCTION_GRAPH_TRACER */ static inline enum print_line_t print_graph_function_flags(struct trace_iterator *iter, u32 flags) { return TRACE_TYPE_UNHANDLED; } -static inline void init_array_fgraph_ops(struct trace_array *tr) { } -static inline int allocate_fgraph_ops(struct trace_array *tr) -{ - return 0; -} static inline void free_fgraph_ops(struct trace_array *tr) { } +/* ftrace_ops may not be defined */ +#define init_array_fgraph_ops(tr, ops) do { } while (0) +#define allocate_fgraph_ops(tr, ops) ({ 0; }) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ extern struct list_head ftrace_pids; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 8e8da0d0ee52..13bf2415245d 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -91,7 +91,7 @@ int ftrace_create_function_files(struct trace_array *tr, if (!tr->ops) return -EINVAL; - ret = allocate_fgraph_ops(tr); + ret = allocate_fgraph_ops(tr, tr->ops); if (ret) { kfree(tr->ops); return ret; diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 9ccc904a7703..7f30652f0e97 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -288,7 +288,7 @@ static struct fgraph_ops funcgraph_ops = { .retfunc = &trace_graph_return, }; -int allocate_fgraph_ops(struct trace_array *tr) +int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) { struct fgraph_ops *gops; @@ -301,6 +301,9 @@ int allocate_fgraph_ops(struct trace_array *tr) tr->gops = gops; gops->private = tr; + + fgraph_init_ops(&gops->ops, ops); + return 0; } @@ -309,10 +312,11 @@ void free_fgraph_ops(struct trace_array *tr) kfree(tr->gops); } -__init void init_array_fgraph_ops(struct trace_array *tr) +__init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) { tr->gops = &funcgraph_ops; funcgraph_ops.private = tr; + fgraph_init_ops(&tr->gops->ops, ops); } static int graph_trace_init(struct trace_array *tr) From df3ec5da6a1e7f6e142680d7c5266d3af187170b Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:17 -0400 Subject: [PATCH 038/280] function_graph: Add pid tracing back to function graph tracer Now that the function_graph has a main callback that handles the function graph subops tracing, it no longer honors the pid filtering of ftrace. Add back this logic in the function_graph code to update the gops callback for the entry function to test if it should trace the current task or not. Link: https://lore.kernel.org/linux-trace-kernel/20240603190822.991720703@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 2 ++ kernel/trace/fgraph.c | 40 ++++++++++++++++++++++++++++++++++ kernel/trace/ftrace.c | 5 +++-- kernel/trace/ftrace_internal.h | 2 ++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 8f865689e868..e31ec8516de1 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1040,6 +1040,7 @@ typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *, struct fgraph_ops *); /* entry */ extern int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace, struct fgraph_ops *gops); +bool ftrace_pids_enabled(struct ftrace_ops *ops); #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -1048,6 +1049,7 @@ struct fgraph_ops { trace_func_graph_ret_t retfunc; struct ftrace_ops ops; /* for the hash lists */ void *private; + trace_func_graph_ent_t saved_func; int idx; }; diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 3ef6db53c0bf..30bed20c655f 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -854,6 +854,41 @@ void ftrace_graph_exit_task(struct task_struct *t) kfree(ret_stack); } +static int fgraph_pid_func(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) +{ + struct trace_array *tr = gops->ops.private; + int pid; + + if (tr) { + pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); + if (pid == FTRACE_PID_IGNORE) + return 0; + if (pid != FTRACE_PID_TRACE && + pid != current->pid) + return 0; + } + + return gops->saved_func(trace, gops); +} + +void fgraph_update_pid_func(void) +{ + struct fgraph_ops *gops; + struct ftrace_ops *op; + + if (!(graph_ops.flags & FTRACE_OPS_FL_INITIALIZED)) + return; + + list_for_each_entry(op, &graph_ops.subop_list, list) { + if (op->flags & FTRACE_OPS_FL_PID) { + gops = container_of(op, struct fgraph_ops, ops); + gops->entryfunc = ftrace_pids_enabled(op) ? + fgraph_pid_func : gops->saved_func; + } + } +} + /* Allocate a return stack for each task */ static int start_graph_tracing(void) { @@ -931,11 +966,15 @@ int register_ftrace_graph(struct fgraph_ops *gops) command = FTRACE_START_FUNC_RET; } + /* Always save the function, and reset at unregistering */ + gops->saved_func = gops->entryfunc; + ret = ftrace_startup_subops(&graph_ops, &gops->ops, command); error: if (ret) { fgraph_array[i] = &fgraph_stub; ftrace_graph_active--; + gops->saved_func = NULL; } out: mutex_unlock(&ftrace_lock); @@ -979,5 +1018,6 @@ void unregister_ftrace_graph(struct fgraph_ops *gops) unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); } out: + gops->saved_func = NULL; mutex_unlock(&ftrace_lock); } diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 58e0f4bc0241..da7e6abf48b4 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -100,7 +100,7 @@ struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; /* What to set function_trace_op to */ static struct ftrace_ops *set_function_trace_op; -static bool ftrace_pids_enabled(struct ftrace_ops *ops) +bool ftrace_pids_enabled(struct ftrace_ops *ops) { struct trace_array *tr; @@ -402,10 +402,11 @@ static void ftrace_update_pid_func(void) if (op->flags & FTRACE_OPS_FL_PID) { op->func = ftrace_pids_enabled(op) ? ftrace_pid_func : op->saved_func; - ftrace_update_trampoline(op); } } while_for_each_ftrace_op(op); + fgraph_update_pid_func(); + update_ftrace_function(); } diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h index cdfd12c44ab4..bfba10c2fcf1 100644 --- a/kernel/trace/ftrace_internal.h +++ b/kernel/trace/ftrace_internal.h @@ -43,8 +43,10 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) #ifdef CONFIG_FUNCTION_GRAPH_TRACER extern int ftrace_graph_active; +extern void fgraph_update_pid_func(void); #else /* !CONFIG_FUNCTION_GRAPH_TRACER */ # define ftrace_graph_active 0 +static inline void fgraph_update_pid_func(void) {} #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #else /* !CONFIG_FUNCTION_TRACER */ From 6d4786592ac88aa31f45fde6bfaad3162e3a92a4 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 3 Jun 2024 15:07:18 -0400 Subject: [PATCH 039/280] function_graph: Use a simple LRU for fgraph_array index number Since the fgraph_array index is used for the bitmap on the shadow stack, it may leave some entries after a function_graph instance is removed. Thus if another instance reuses the fgraph_array index soon after releasing it, the fgraph may confuse to call the newer callback for the entries which are pushed by the older instance. To avoid reusing the fgraph_array index soon after releasing, introduce a simple LRU table for managing the index number. This will reduce the possibility of this confusion. Link: https://lore.kernel.org/linux-trace-kernel/171509103267.162236.6885097397289135378.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.147421545@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 71 ++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 30bed20c655f..7fd9b03bd170 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -124,10 +124,48 @@ enum { DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; -static int fgraph_array_cnt; - static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE]; +/* LRU index table for fgraph_array */ +static int fgraph_lru_table[FGRAPH_ARRAY_SIZE]; +static int fgraph_lru_next; +static int fgraph_lru_last; + +/* Initialize fgraph_lru_table with unused index */ +static void fgraph_lru_init(void) +{ + int i; + + for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) + fgraph_lru_table[i] = i; +} + +/* Release the used index to the LRU table */ +static int fgraph_lru_release_index(int idx) +{ + if (idx < 0 || idx >= FGRAPH_ARRAY_SIZE || + WARN_ON_ONCE(fgraph_lru_table[fgraph_lru_last] != -1)) + return -1; + + fgraph_lru_table[fgraph_lru_last] = idx; + fgraph_lru_last = (fgraph_lru_last + 1) % FGRAPH_ARRAY_SIZE; + return 0; +} + +/* Allocate a new index from LRU table */ +static int fgraph_lru_alloc_index(void) +{ + int idx = fgraph_lru_table[fgraph_lru_next]; + + /* No id is available */ + if (idx == -1) + return -1; + + fgraph_lru_table[fgraph_lru_next] = -1; + fgraph_lru_next = (fgraph_lru_next + 1) % FGRAPH_ARRAY_SIZE; + return idx; +} + /* Get the FRAME_OFFSET from the word from the @offset on ret_stack */ static inline int get_frame_offset(struct task_struct *t, int offset) { @@ -374,7 +412,7 @@ int function_graph_enter(unsigned long ret, unsigned long func, if (offset < 0) goto out; - for (i = 0; i < fgraph_array_cnt; i++) { + for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { struct fgraph_ops *gops = fgraph_array[i]; if (gops == &fgraph_stub) @@ -925,7 +963,7 @@ int register_ftrace_graph(struct fgraph_ops *gops) { int command = 0; int ret = 0; - int i; + int i = -1; mutex_lock(&ftrace_lock); @@ -933,21 +971,16 @@ int register_ftrace_graph(struct fgraph_ops *gops) /* The array must always have real data on it */ for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) fgraph_array[i] = &fgraph_stub; + fgraph_lru_init(); } - /* Look for an available spot */ - for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { - if (fgraph_array[i] == &fgraph_stub) - break; - } - if (i >= FGRAPH_ARRAY_SIZE) { + i = fgraph_lru_alloc_index(); + if (i < 0 || WARN_ON_ONCE(fgraph_array[i] != &fgraph_stub)) { ret = -ENOSPC; goto out; } fgraph_array[i] = gops; - if (i + 1 > fgraph_array_cnt) - fgraph_array_cnt = i + 1; gops->idx = i; ftrace_graph_active++; @@ -975,6 +1008,7 @@ error: fgraph_array[i] = &fgraph_stub; ftrace_graph_active--; gops->saved_func = NULL; + fgraph_lru_release_index(i); } out: mutex_unlock(&ftrace_lock); @@ -984,25 +1018,20 @@ out: void unregister_ftrace_graph(struct fgraph_ops *gops) { int command = 0; - int i; mutex_lock(&ftrace_lock); if (unlikely(!ftrace_graph_active)) goto out; - if (unlikely(gops->idx < 0 || gops->idx >= fgraph_array_cnt)) + if (unlikely(gops->idx < 0 || gops->idx >= FGRAPH_ARRAY_SIZE || + fgraph_array[gops->idx] != gops)) goto out; - WARN_ON_ONCE(fgraph_array[gops->idx] != gops); + if (fgraph_lru_release_index(gops->idx) < 0) + goto out; fgraph_array[gops->idx] = &fgraph_stub; - if (gops->idx + 1 == fgraph_array_cnt) { - i = gops->idx; - while (i >= 0 && fgraph_array[i] == &fgraph_stub) - i--; - fgraph_array_cnt = i + 1; - } ftrace_graph_active--; From 4497412a1f7b5d9e0849f125652f2cc58cdba562 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:19 -0400 Subject: [PATCH 040/280] function_graph: Add "task variables" per task for fgraph_ops Add a "task variables" array on the tasks shadow ret_stack that is the size of longs for each possible registered fgraph_ops. That's a total of 16, taking up 8 * 16 = 128 bytes (out of a page size 4k). This will allow for fgraph_ops to do specific features on a per task basis having a way to maintain state for each task. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509104383.162236.12239656156685718550.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.308806126@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 1 + kernel/trace/fgraph.c | 74 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index e31ec8516de1..82cfc8e09cfd 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1089,6 +1089,7 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int skip); unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp); +unsigned long *fgraph_get_task_var(struct fgraph_ops *gops); /* * Sometimes we don't want to trace a function with the function diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 7fd9b03bd170..baa08249ffd5 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -54,6 +54,10 @@ * on the return of the function being traced, this is what will be on the * task's shadow ret_stack: (the stack grows upward) * + * ret_stack[SHADOW_STACK_OFFSET] + * | SHADOW_STACK_TASK_VARS(ret_stack)[15] | + * ... + * | SHADOW_STACK_TASK_VARS(ret_stack)[0] | * ret_stack[SHADOW_STACK_MAX_OFFSET] * ... * | | <- task->curr_ret_stack @@ -116,11 +120,19 @@ enum { #define SHADOW_STACK_SIZE (PAGE_SIZE) #define SHADOW_STACK_OFFSET (SHADOW_STACK_SIZE / sizeof(long)) /* Leave on a buffer at the end */ -#define SHADOW_STACK_MAX_OFFSET (SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1)) +#define SHADOW_STACK_MAX_OFFSET \ + (SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1 + FGRAPH_ARRAY_SIZE)) /* RET_STACK(): Return the frame from a given @offset from task @t */ #define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset])) +/* + * Each fgraph_ops has a reservered unsigned long at the end (top) of the + * ret_stack to store task specific state. + */ +#define SHADOW_STACK_TASK_VARS(ret_stack) \ + ((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE])) + DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; @@ -211,6 +223,44 @@ static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops) { } +static void ret_stack_set_task_var(struct task_struct *t, int idx, long val) +{ + unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack); + + gvals[idx] = val; +} + +static unsigned long * +ret_stack_get_task_var(struct task_struct *t, int idx) +{ + unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack); + + return &gvals[idx]; +} + +static void ret_stack_init_task_vars(unsigned long *ret_stack) +{ + unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack); + + memset(gvals, 0, sizeof(*gvals) * FGRAPH_ARRAY_SIZE); +} + +/** + * fgraph_get_task_var - retrieve a task specific state variable + * @gops: The ftrace_ops that owns the task specific variable + * + * Every registered fgraph_ops has a task state variable + * reserved on the task's ret_stack. This function returns the + * address to that variable. + * + * Returns the address to the fgraph_ops @gops tasks specific + * unsigned long variable. + */ +unsigned long *fgraph_get_task_var(struct fgraph_ops *gops) +{ + return ret_stack_get_task_var(current, gops->idx); +} + /* * @offset: The offset into @t->ret_stack to find the ret_stack entry * @frame_offset: Where to place the offset into @t->ret_stack of that entry @@ -766,6 +816,7 @@ static int alloc_retstack_tasklist(unsigned long **ret_stack_list) if (t->ret_stack == NULL) { atomic_set(&t->trace_overrun, 0); + ret_stack_init_task_vars(ret_stack_list[start]); t->curr_ret_stack = 0; t->curr_ret_depth = -1; /* Make sure the tasks see the 0 first: */ @@ -826,6 +877,7 @@ static void graph_init_task(struct task_struct *t, unsigned long *ret_stack) { atomic_set(&t->trace_overrun, 0); + ret_stack_init_task_vars(ret_stack); t->ftrace_timestamp = 0; t->curr_ret_stack = 0; t->curr_ret_depth = -1; @@ -959,6 +1011,24 @@ static int start_graph_tracing(void) return ret; } +static void init_task_vars(int idx) +{ + struct task_struct *g, *t; + int cpu; + + for_each_online_cpu(cpu) { + if (idle_task(cpu)->ret_stack) + ret_stack_set_task_var(idle_task(cpu), idx, 0); + } + + read_lock(&tasklist_lock); + for_each_process_thread(g, t) { + if (t->ret_stack) + ret_stack_set_task_var(t, idx, 0); + } + read_unlock(&tasklist_lock); +} + int register_ftrace_graph(struct fgraph_ops *gops) { int command = 0; @@ -997,6 +1067,8 @@ int register_ftrace_graph(struct fgraph_ops *gops) ftrace_graph_return = return_run; ftrace_graph_entry = entry_run; command = FTRACE_START_FUNC_RET; + } else { + init_task_vars(gops->idx); } /* Always save the function, and reset at unregistering */ From 12117f3307b63f287756d7ec8cc4f11b94e1206a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:20 -0400 Subject: [PATCH 041/280] function_graph: Move set_graph_function tests to shadow stack global var The use of the task->trace_recursion for the logic used for the set_graph_function was a bit of an abuse of that variable. Now that there exists global vars that are per stack for registered graph traces, use that instead. Link: https://lore.kernel.org/linux-trace-kernel/171509105520.162236.10339831553995971290.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.472955399@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_recursion.h | 5 +---- kernel/trace/trace.h | 32 ++++++++++++++++++---------- kernel/trace/trace_functions_graph.c | 6 +++--- kernel/trace/trace_irqsoff.c | 4 ++-- kernel/trace/trace_sched_wakeup.c | 4 ++-- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index 24ea8ac049b4..02e6afc6d7fe 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -44,9 +44,6 @@ enum { */ TRACE_IRQ_BIT, - /* Set if the function is in the set_graph_function file */ - TRACE_GRAPH_BIT, - /* * In the very unlikely case that an interrupt came in * at a start of graph tracing, and we want to trace @@ -60,7 +57,7 @@ enum { * that preempted a softirq start of a function that * preempted normal context!!!! Luckily, it can't be * greater than 3, so the next two bits are a mask - * of what the depth is when we set TRACE_GRAPH_BIT + * of what the depth is when we set TRACE_GRAPH_FL */ TRACE_GRAPH_DEPTH_START_BIT, diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index f06b5ddd3580..73919129e57c 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -898,11 +898,16 @@ extern void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops extern int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops); extern void free_fgraph_ops(struct trace_array *tr); +enum { + TRACE_GRAPH_FL = 1, +}; + #ifdef CONFIG_DYNAMIC_FTRACE extern struct ftrace_hash __rcu *ftrace_graph_hash; extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash; -static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace) +static inline int +ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace) { unsigned long addr = trace->func; int ret = 0; @@ -924,12 +929,11 @@ static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace) } if (ftrace_lookup_ip(hash, addr)) { - /* * This needs to be cleared on the return functions * when the depth is zero. */ - trace_recursion_set(TRACE_GRAPH_BIT); + *task_var |= TRACE_GRAPH_FL; trace_recursion_set_depth(trace->depth); /* @@ -949,11 +953,14 @@ out: return ret; } -static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace) +static inline void +ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace) { - if (trace_recursion_test(TRACE_GRAPH_BIT) && + unsigned long *task_var = fgraph_get_task_var(gops); + + if ((*task_var & TRACE_GRAPH_FL) && trace->depth == trace_recursion_depth()) - trace_recursion_clear(TRACE_GRAPH_BIT); + *task_var &= ~TRACE_GRAPH_FL; } static inline int ftrace_graph_notrace_addr(unsigned long addr) @@ -979,7 +986,7 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr) return ret; } #else -static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace) +static inline int ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace) { return 1; } @@ -988,17 +995,20 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr) { return 0; } -static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace) +static inline void ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace) { } #endif /* CONFIG_DYNAMIC_FTRACE */ extern unsigned int fgraph_max_depth; -static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace) +static inline bool +ftrace_graph_ignore_func(struct fgraph_ops *gops, struct ftrace_graph_ent *trace) { + unsigned long *task_var = fgraph_get_task_var(gops); + /* trace it when it is-nested-in or is a function enabled. */ - return !(trace_recursion_test(TRACE_GRAPH_BIT) || - ftrace_graph_addr(trace)) || + return !((*task_var & TRACE_GRAPH_FL) || + ftrace_graph_addr(task_var, trace)) || (trace->depth < 0) || (fgraph_max_depth && trace->depth >= fgraph_max_depth); } diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 7f30652f0e97..66cce73e94f8 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -160,7 +160,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace, if (!ftrace_trace_task(tr)) return 0; - if (ftrace_graph_ignore_func(trace)) + if (ftrace_graph_ignore_func(gops, trace)) return 0; if (ftrace_graph_ignore_irqs()) @@ -247,7 +247,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace, long disabled; int cpu; - ftrace_graph_addr_finish(trace); + ftrace_graph_addr_finish(gops, trace); if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); @@ -269,7 +269,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace, static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { - ftrace_graph_addr_finish(trace); + ftrace_graph_addr_finish(gops, trace); if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index 5478f4c4f708..fce064e20570 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c @@ -184,7 +184,7 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace, unsigned int trace_ctx; int ret; - if (ftrace_graph_ignore_func(trace)) + if (ftrace_graph_ignore_func(gops, trace)) return 0; /* * Do not trace a function if it's filtered by set_graph_notrace. @@ -214,7 +214,7 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace, unsigned long flags; unsigned int trace_ctx; - ftrace_graph_addr_finish(trace); + ftrace_graph_addr_finish(gops, trace); if (!func_prolog_dec(tr, &data, &flags)) return; diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 49bcc812652c..130ca7e7787e 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -120,7 +120,7 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace, unsigned int trace_ctx; int ret = 0; - if (ftrace_graph_ignore_func(trace)) + if (ftrace_graph_ignore_func(gops, trace)) return 0; /* * Do not trace a function if it's filtered by set_graph_notrace. @@ -149,7 +149,7 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace, struct trace_array_cpu *data; unsigned int trace_ctx; - ftrace_graph_addr_finish(trace); + ftrace_graph_addr_finish(gops, trace); if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) return; From 068da098eb504469dc195137ae35eeacfe0c8de9 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:21 -0400 Subject: [PATCH 042/280] function_graph: Move graph depth stored data to shadow stack global var The use of the task->trace_recursion for the logic used for the function graph depth was a bit of an abuse of that variable. Now that there exists global vars that are per stack for registered graph traces, use that instead. Link: https://lore.kernel.org/linux-trace-kernel/171509106728.162236.2398372644430125344.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.634870264@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_recursion.h | 29 ---------------------------- kernel/trace/trace.h | 34 +++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index 02e6afc6d7fe..fdfb6f66718a 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -44,25 +44,6 @@ enum { */ TRACE_IRQ_BIT, - /* - * In the very unlikely case that an interrupt came in - * at a start of graph tracing, and we want to trace - * the function in that interrupt, the depth can be greater - * than zero, because of the preempted start of a previous - * trace. In an even more unlikely case, depth could be 2 - * if a softirq interrupted the start of graph tracing, - * followed by an interrupt preempting a start of graph - * tracing in the softirq, and depth can even be 3 - * if an NMI came in at the start of an interrupt function - * that preempted a softirq start of a function that - * preempted normal context!!!! Luckily, it can't be - * greater than 3, so the next two bits are a mask - * of what the depth is when we set TRACE_GRAPH_FL - */ - - TRACE_GRAPH_DEPTH_START_BIT, - TRACE_GRAPH_DEPTH_END_BIT, - /* * To implement set_graph_notrace, if this bit is set, we ignore * function graph tracing of called functions, until the return @@ -78,16 +59,6 @@ enum { #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0) #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit))) -#define trace_recursion_depth() \ - (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3) -#define trace_recursion_set_depth(depth) \ - do { \ - current->trace_recursion &= \ - ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \ - current->trace_recursion |= \ - ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \ - } while (0) - #define TRACE_CONTEXT_BITS 4 #define TRACE_FTRACE_START TRACE_FTRACE_BIT diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 73919129e57c..82d879dc63ff 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -900,8 +900,38 @@ extern void free_fgraph_ops(struct trace_array *tr); enum { TRACE_GRAPH_FL = 1, + + /* + * In the very unlikely case that an interrupt came in + * at a start of graph tracing, and we want to trace + * the function in that interrupt, the depth can be greater + * than zero, because of the preempted start of a previous + * trace. In an even more unlikely case, depth could be 2 + * if a softirq interrupted the start of graph tracing, + * followed by an interrupt preempting a start of graph + * tracing in the softirq, and depth can even be 3 + * if an NMI came in at the start of an interrupt function + * that preempted a softirq start of a function that + * preempted normal context!!!! Luckily, it can't be + * greater than 3, so the next two bits are a mask + * of what the depth is when we set TRACE_GRAPH_FL + */ + + TRACE_GRAPH_DEPTH_START_BIT, + TRACE_GRAPH_DEPTH_END_BIT, }; +static inline unsigned long ftrace_graph_depth(unsigned long *task_var) +{ + return (*task_var >> TRACE_GRAPH_DEPTH_START_BIT) & 3; +} + +static inline void ftrace_graph_set_depth(unsigned long *task_var, int depth) +{ + *task_var &= ~(3 << TRACE_GRAPH_DEPTH_START_BIT); + *task_var |= (depth & 3) << TRACE_GRAPH_DEPTH_START_BIT; +} + #ifdef CONFIG_DYNAMIC_FTRACE extern struct ftrace_hash __rcu *ftrace_graph_hash; extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash; @@ -934,7 +964,7 @@ ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace) * when the depth is zero. */ *task_var |= TRACE_GRAPH_FL; - trace_recursion_set_depth(trace->depth); + ftrace_graph_set_depth(task_var, trace->depth); /* * If no irqs are to be traced, but a set_graph_function @@ -959,7 +989,7 @@ ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace unsigned long *task_var = fgraph_get_task_var(gops); if ((*task_var & TRACE_GRAPH_FL) && - trace->depth == trace_recursion_depth()) + trace->depth == ftrace_graph_depth(task_var)) *task_var &= ~TRACE_GRAPH_FL; } From b84214890a9bc56f0fe4ec4fc72f2307ed05096d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:22 -0400 Subject: [PATCH 043/280] function_graph: Move graph notrace bit to shadow stack global var The use of the task->trace_recursion for the logic used for the function graph no-trace was a bit of an abuse of that variable. Now that there exists global vars that are per stack for registered graph traces, use that instead. Link: https://lore.kernel.org/linux-trace-kernel/171509107907.162236.6564679266777519065.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.796709456@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_recursion.h | 7 ------- kernel/trace/trace.h | 9 +++++++++ kernel/trace/trace_functions_graph.c | 10 ++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/linux/trace_recursion.h b/include/linux/trace_recursion.h index fdfb6f66718a..ae04054a1be3 100644 --- a/include/linux/trace_recursion.h +++ b/include/linux/trace_recursion.h @@ -44,13 +44,6 @@ enum { */ TRACE_IRQ_BIT, - /* - * To implement set_graph_notrace, if this bit is set, we ignore - * function graph tracing of called functions, until the return - * function is called to clear it. - */ - TRACE_GRAPH_NOTRACE_BIT, - /* Used to prevent recursion recording from recursing. */ TRACE_RECORD_RECURSION_BIT, }; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 82d879dc63ff..b37402e3f0c9 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -919,8 +919,17 @@ enum { TRACE_GRAPH_DEPTH_START_BIT, TRACE_GRAPH_DEPTH_END_BIT, + + /* + * To implement set_graph_notrace, if this bit is set, we ignore + * function graph tracing of called functions, until the return + * function is called to clear it. + */ + TRACE_GRAPH_NOTRACE_BIT, }; +#define TRACE_GRAPH_NOTRACE (1 << TRACE_GRAPH_NOTRACE_BIT) + static inline unsigned long ftrace_graph_depth(unsigned long *task_var) { return (*task_var >> TRACE_GRAPH_DEPTH_START_BIT) & 3; diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 66cce73e94f8..13d0387ac6a6 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -130,6 +130,7 @@ static inline int ftrace_graph_ignore_irqs(void) int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { + unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; unsigned long flags; @@ -138,7 +139,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace, int ret; int cpu; - if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) + if (*task_var & TRACE_GRAPH_NOTRACE) return 0; /* @@ -149,7 +150,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace, * returning from the function. */ if (ftrace_graph_notrace_addr(trace->func)) { - trace_recursion_set(TRACE_GRAPH_NOTRACE_BIT); + *task_var |= TRACE_GRAPH_NOTRACE_BIT; /* * Need to return 1 to have the return called * that will clear the NOTRACE bit. @@ -240,6 +241,7 @@ void __trace_graph_return(struct trace_array *tr, void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { + unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; unsigned long flags; @@ -249,8 +251,8 @@ void trace_graph_return(struct ftrace_graph_ret *trace, ftrace_graph_addr_finish(gops, trace); - if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { - trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); + if (*task_var & TRACE_GRAPH_NOTRACE) { + *task_var &= ~TRACE_GRAPH_NOTRACE; return; } From 91c46b0aa917546432b5b219494859cda0edc39e Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:23 -0400 Subject: [PATCH 044/280] function_graph: Implement fgraph_reserve_data() and fgraph_retrieve_data() Added functions that can be called by a fgraph_ops entryfunc and retfunc to store state between the entry of the function being traced to the exit of the same function. The fgraph_ops entryfunc() may call fgraph_reserve_data() to store up to 32 words onto the task's shadow ret_stack and this then can be retrieved by fgraph_retrieve_data() called by the corresponding retfunc(). Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509109089.162236.11372474169781184034.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190823.959703050@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 3 + kernel/trace/fgraph.c | 191 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 186 insertions(+), 8 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 82cfc8e09cfd..9f61556a9491 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1053,6 +1053,9 @@ struct fgraph_ops { int idx; }; +void *fgraph_reserve_data(int idx, int size_bytes); +void *fgraph_retrieve_data(int idx, int *size_bytes); + /* * Stack of return addresses for functions * of a thread. diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index baa08249ffd5..f207b7ae5f46 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -31,12 +31,11 @@ * holds a bitmask and a type (called "bitmap"). The bitmap is defined as: * * bits: 0 - 9 offset in words from the previous ftrace_ret_stack - * Currently, this will always be set to FGRAPH_FRAME_OFFSET - * to get to the fgraph frame. * * bits: 10 - 11 Type of storage * 0 - reserved * 1 - bitmap of fgraph_array index + * 2 - reserved data * * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP): * bits: 12 - 27 The bitmap of fgraph_ops fgraph_array index @@ -49,10 +48,15 @@ * The top of the ret_stack (when not empty) will always have a reference * word that points to the last fgraph frame that was saved. * + * For reserved data: + * bits: 12 - 17 The size in words that is stored + * bits: 18 - 23 The index of fgraph_array, which shows who is stored + * * That is, at the end of function_graph_enter, if the first and forth * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called - * on the return of the function being traced, this is what will be on the - * task's shadow ret_stack: (the stack grows upward) + * on the return of the function being traced, and the forth fgraph_ops + * stored two words of data, this is what will be on the task's shadow + * ret_stack: (the stack grows upward) * * ret_stack[SHADOW_STACK_OFFSET] * | SHADOW_STACK_TASK_VARS(ret_stack)[15] | @@ -62,11 +66,21 @@ * ... * | | <- task->curr_ret_stack * +--------------------------------------------+ + * | (3 << 12) | (3 << 10) | FGRAPH_FRAME_OFFSET| + * | *or put another way* | + * | (3 << FGRAPH_DATA_INDEX_SHIFT)| \ | This is for fgraph_ops[3]. + * | ((2 - 1) << FGRAPH_DATA_SHIFT)| \ | The data size is 2 words. + * | (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT)| \ | + * | (offset2:FGRAPH_FRAME_OFFSET+3) | <- the offset2 is from here + * +--------------------------------------------+ ( It is 4 words from the ret_stack) + * | STORED DATA WORD 2 | + * | STORED DATA WORD 1 | + * +--------------------------------------------+ * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET| * | *or put another way* | * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \ | * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ | - * | (offset:FGRAPH_FRAME_OFFSET) | <- the offset is from here + * | (offset1:FGRAPH_FRAME_OFFSET) | <- the offset1 is from here * +--------------------------------------------+ * | struct ftrace_ret_stack | * | (stores the saved ret pointer) | <- the offset points here @@ -100,6 +114,7 @@ enum { FGRAPH_TYPE_RESERVED = 0, FGRAPH_TYPE_BITMAP = 1, + FGRAPH_TYPE_DATA = 2, }; /* @@ -110,6 +125,26 @@ enum { #define FGRAPH_INDEX_MASK GENMASK(FGRAPH_INDEX_BITS - 1, 0) #define FGRAPH_INDEX_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS) +/* + * For DATA type: + * FGRAPH_DATA (12-17) bits hold the size of data (in words) + * FGRAPH_INDEX (18-23) bits hold the index for which gops->idx the data is for + * + * Note: + * data_size == 0 means 1 word, and 31 (=2^5 - 1) means 32 words. + */ +#define FGRAPH_DATA_BITS 5 +#define FGRAPH_DATA_MASK GENMASK(FGRAPH_DATA_BITS - 1, 0) +#define FGRAPH_DATA_SHIFT (FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS) +#define FGRAPH_MAX_DATA_SIZE (sizeof(long) * (1 << FGRAPH_DATA_BITS)) + +#define FGRAPH_DATA_INDEX_BITS 4 +#define FGRAPH_DATA_INDEX_MASK GENMASK(FGRAPH_DATA_INDEX_BITS - 1, 0) +#define FGRAPH_DATA_INDEX_SHIFT (FGRAPH_DATA_SHIFT + FGRAPH_DATA_BITS) + +#define FGRAPH_MAX_INDEX \ + ((FGRAPH_INDEX_SIZE << FGRAPH_DATA_BITS) + FGRAPH_RET_INDEX) + #define FGRAPH_ARRAY_SIZE FGRAPH_INDEX_BITS /* @@ -178,16 +213,46 @@ static int fgraph_lru_alloc_index(void) return idx; } +/* Get the offset to the fgraph frame from a ret_stack value */ +static inline int __get_offset(unsigned long val) +{ + return val & FGRAPH_FRAME_OFFSET_MASK; +} + +/* Get the type of word from a ret_stack value */ +static inline int __get_type(unsigned long val) +{ + return (val >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK; +} + +/* Get the data_index for a DATA type ret_stack word */ +static inline int __get_data_index(unsigned long val) +{ + return (val >> FGRAPH_DATA_INDEX_SHIFT) & FGRAPH_DATA_INDEX_MASK; +} + +/* Get the data_size for a DATA type ret_stack word */ +static inline int __get_data_size(unsigned long val) +{ + return ((val >> FGRAPH_DATA_SHIFT) & FGRAPH_DATA_MASK) + 1; +} + +/* Get the word from the ret_stack at @offset */ +static inline unsigned long get_fgraph_entry(struct task_struct *t, int offset) +{ + return t->ret_stack[offset]; +} + /* Get the FRAME_OFFSET from the word from the @offset on ret_stack */ static inline int get_frame_offset(struct task_struct *t, int offset) { - return t->ret_stack[offset] & FGRAPH_FRAME_OFFSET_MASK; + return __get_offset(t->ret_stack[offset]); } /* Get FGRAPH_TYPE from the word from the @offset at ret_stack */ static inline int get_fgraph_type(struct task_struct *t, int offset) { - return (t->ret_stack[offset] >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK; + return __get_type(t->ret_stack[offset]); } /* For BITMAP type: get the bitmask from the @offset at ret_stack */ @@ -212,6 +277,25 @@ set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) (FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET; } +/* For DATA type: get the data saved under the ret_stack word at @offset */ +static inline void *get_data_type_data(struct task_struct *t, int offset) +{ + unsigned long val = t->ret_stack[offset]; + + if (__get_type(val) != FGRAPH_TYPE_DATA) + return NULL; + offset -= __get_data_size(val); + return (void *)&t->ret_stack[offset]; +} + +/* Create the ret_stack word for a DATA type */ +static inline unsigned long make_data_type_val(int idx, int size, int offset) +{ + return (idx << FGRAPH_DATA_INDEX_SHIFT) | + ((size - 1) << FGRAPH_DATA_SHIFT) | + (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT) | offset; +} + /* ftrace_graph_entry set to this to tell some archs to run function graph */ static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops) { @@ -245,6 +329,91 @@ static void ret_stack_init_task_vars(unsigned long *ret_stack) memset(gvals, 0, sizeof(*gvals) * FGRAPH_ARRAY_SIZE); } +/** + * fgraph_reserve_data - Reserve storage on the task's ret_stack + * @idx: The index of fgraph_array + * @size_bytes: The size in bytes to reserve + * + * Reserves space of up to FGRAPH_MAX_DATA_SIZE bytes on the + * task's ret_stack shadow stack, for a given fgraph_ops during + * the entryfunc() call. If entryfunc() returns zero, the storage + * is discarded. An entryfunc() can only call this once per iteration. + * The fgraph_ops retfunc() can retrieve this stored data with + * fgraph_retrieve_data(). + * + * Returns: On success, a pointer to the data on the stack. + * Otherwise, NULL if there's not enough space left on the + * ret_stack for the data, or if fgraph_reserve_data() was called + * more than once for a single entryfunc() call. + */ +void *fgraph_reserve_data(int idx, int size_bytes) +{ + unsigned long val; + void *data; + int curr_ret_stack = current->curr_ret_stack; + int data_size; + + if (size_bytes > FGRAPH_MAX_DATA_SIZE) + return NULL; + + /* Convert the data size to number of longs. */ + data_size = (size_bytes + sizeof(long) - 1) >> (sizeof(long) == 4 ? 2 : 3); + + val = get_fgraph_entry(current, curr_ret_stack - 1); + data = ¤t->ret_stack[curr_ret_stack]; + + curr_ret_stack += data_size + 1; + if (unlikely(curr_ret_stack >= SHADOW_STACK_MAX_OFFSET)) + return NULL; + + val = make_data_type_val(idx, data_size, __get_offset(val) + data_size + 1); + + /* Set the last word to be reserved */ + current->ret_stack[curr_ret_stack - 1] = val; + + /* Make sure interrupts see this */ + barrier(); + current->curr_ret_stack = curr_ret_stack; + /* Again sync with interrupts, and reset reserve */ + current->ret_stack[curr_ret_stack - 1] = val; + + return data; +} + +/** + * fgraph_retrieve_data - Retrieve stored data from fgraph_reserve_data() + * @idx: the index of fgraph_array (fgraph_ops::idx) + * @size_bytes: pointer to retrieved data size. + * + * This is to be called by a fgraph_ops retfunc(), to retrieve data that + * was stored by the fgraph_ops entryfunc() on the function entry. + * That is, this will retrieve the data that was reserved on the + * entry of the function that corresponds to the exit of the function + * that the fgraph_ops retfunc() is called on. + * + * Returns: The stored data from fgraph_reserve_data() called by the + * matching entryfunc() for the retfunc() this is called from. + * Or NULL if there was nothing stored. + */ +void *fgraph_retrieve_data(int idx, int *size_bytes) +{ + int offset = current->curr_ret_stack - 1; + unsigned long val; + + val = get_fgraph_entry(current, offset); + while (__get_type(val) == FGRAPH_TYPE_DATA) { + if (__get_data_index(val) == idx) + goto found; + offset -= __get_data_size(val) + 1; + val = get_fgraph_entry(current, offset); + } + return NULL; +found: + if (size_bytes) + *size_bytes = __get_data_size(val) * sizeof(long); + return get_data_type_data(current, offset); +} + /** * fgraph_get_task_var - retrieve a task specific state variable * @gops: The ftrace_ops that owns the task specific variable @@ -464,13 +633,18 @@ int function_graph_enter(unsigned long ret, unsigned long func, for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { struct fgraph_ops *gops = fgraph_array[i]; + int save_curr_ret_stack; if (gops == &fgraph_stub) continue; + save_curr_ret_stack = current->curr_ret_stack; if (ftrace_ops_test(&gops->ops, func, NULL) && gops->entryfunc(&trace, gops)) bitmap |= BIT(i); + else + /* Clear out any saved storage */ + current->curr_ret_stack = save_curr_ret_stack; } if (!bitmap) @@ -626,7 +800,8 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs * curr_ret_stack is after that. */ barrier(); - current->curr_ret_stack -= FGRAPH_FRAME_OFFSET + 1; + current->curr_ret_stack = offset - FGRAPH_FRAME_OFFSET; + current->curr_ret_depth--; return ret; } From 47c3c70aa36971c90e32e91f9254110195d67a02 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Jun 2024 15:07:24 -0400 Subject: [PATCH 045/280] function_graph: Add selftest for passing local variables Add boot up selftest that passes variables from a function entry to a function exit, and make sure that they do get passed around. Co-developed with Masami Hiramatsu: Link: https://lore.kernel.org/linux-trace-kernel/171509110271.162236.11047551496319744627.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.122952310@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_selftest.c | 169 ++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index f8f55fd79e53..fcdc744c245e 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -756,6 +756,173 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr) #ifdef CONFIG_FUNCTION_GRAPH_TRACER +#ifdef CONFIG_DYNAMIC_FTRACE + +#define BYTE_NUMBER 123 +#define SHORT_NUMBER 12345 +#define WORD_NUMBER 1234567890 +#define LONG_NUMBER 1234567890123456789LL + +static int fgraph_store_size __initdata; +static const char *fgraph_store_type_name __initdata; +static char *fgraph_error_str __initdata; +static char fgraph_error_str_buf[128] __initdata; + +static __init int store_entry(struct ftrace_graph_ent *trace, + struct fgraph_ops *gops) +{ + const char *type = fgraph_store_type_name; + int size = fgraph_store_size; + void *p; + + p = fgraph_reserve_data(gops->idx, size); + if (!p) { + snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + "Failed to reserve %s\n", type); + fgraph_error_str = fgraph_error_str_buf; + return 0; + } + + switch (fgraph_store_size) { + case 1: + *(char *)p = BYTE_NUMBER; + break; + case 2: + *(short *)p = SHORT_NUMBER; + break; + case 4: + *(int *)p = WORD_NUMBER; + break; + case 8: + *(long long *)p = LONG_NUMBER; + break; + } + + return 1; +} + +static __init void store_return(struct ftrace_graph_ret *trace, + struct fgraph_ops *gops) +{ + const char *type = fgraph_store_type_name; + long long expect = 0; + long long found = -1; + int size; + char *p; + + p = fgraph_retrieve_data(gops->idx, &size); + if (!p) { + snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + "Failed to retrieve %s\n", type); + fgraph_error_str = fgraph_error_str_buf; + return; + } + if (fgraph_store_size > size) { + snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + "Retrieved size %d is smaller than expected %d\n", + size, (int)fgraph_store_size); + fgraph_error_str = fgraph_error_str_buf; + return; + } + + switch (fgraph_store_size) { + case 1: + expect = BYTE_NUMBER; + found = *(char *)p; + break; + case 2: + expect = SHORT_NUMBER; + found = *(short *)p; + break; + case 4: + expect = WORD_NUMBER; + found = *(int *)p; + break; + case 8: + expect = LONG_NUMBER; + found = *(long long *)p; + break; + } + + if (found != expect) { + snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + "%s returned not %lld but %lld\n", type, expect, found); + fgraph_error_str = fgraph_error_str_buf; + return; + } + fgraph_error_str = NULL; +} + +static struct fgraph_ops store_bytes __initdata = { + .entryfunc = store_entry, + .retfunc = store_return, +}; + +static int __init test_graph_storage_type(const char *name, int size) +{ + char *func_name; + int len; + int ret; + + fgraph_store_type_name = name; + fgraph_store_size = size; + + snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + "Failed to execute storage %s\n", name); + fgraph_error_str = fgraph_error_str_buf; + + pr_cont("PASSED\n"); + pr_info("Testing fgraph storage of %d byte%s: ", size, size > 1 ? "s" : ""); + + func_name = "*" __stringify(DYN_FTRACE_TEST_NAME); + len = strlen(func_name); + + ret = ftrace_set_filter(&store_bytes.ops, func_name, len, 1); + if (ret && ret != -ENODEV) { + pr_cont("*Could not set filter* "); + return -1; + } + + ret = register_ftrace_graph(&store_bytes); + if (ret) { + pr_warn("Failed to init store_bytes fgraph tracing\n"); + return -1; + } + + DYN_FTRACE_TEST_NAME(); + + unregister_ftrace_graph(&store_bytes); + + if (fgraph_error_str) { + pr_cont("*** %s ***", fgraph_error_str); + return -1; + } + + return 0; +} +/* Test the storage passed across function_graph entry and return */ +static __init int test_graph_storage(void) +{ + int ret; + + ret = test_graph_storage_type("byte", 1); + if (ret) + return ret; + ret = test_graph_storage_type("short", 2); + if (ret) + return ret; + ret = test_graph_storage_type("word", 4); + if (ret) + return ret; + ret = test_graph_storage_type("long long", 8); + if (ret) + return ret; + return 0; +} +#else +static inline int test_graph_storage(void) { return 0; } +#endif /* CONFIG_DYNAMIC_FTRACE */ + /* Maximum number of functions to trace before diagnosing a hang */ #define GRAPH_MAX_FUNC_TEST 100000000 @@ -913,6 +1080,8 @@ trace_selftest_startup_function_graph(struct tracer *trace, ftrace_set_global_filter(NULL, 0, 1); #endif + ret = test_graph_storage(); + /* Don't test dynamic tracing, the function tracer already did */ out: /* Stop it if we failed */ From dd120af2d5f8f3d2d742a64cefc4a529d382ab06 Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Mon, 3 Jun 2024 15:07:25 -0400 Subject: [PATCH 046/280] ftrace: Add multiple fgraph storage selftest Add a selftest for multiple function graph tracer with storage on a same function. In this case, the shadow stack entry will be shared among those fgraph with different data storage. So this will ensure the fgraph will not mixed those storage data. Link: https://lore.kernel.org/linux-trace-kernel/171509111465.162236.3795819216426570800.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.284049716@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) Suggested-by: Steven Rostedt (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_selftest.c | 171 +++++++++++++++++++++++++--------- 1 file changed, 126 insertions(+), 45 deletions(-) diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index fcdc744c245e..369efc569238 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -762,28 +762,32 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr) #define SHORT_NUMBER 12345 #define WORD_NUMBER 1234567890 #define LONG_NUMBER 1234567890123456789LL +#define ERRSTR_BUFLEN 128 -static int fgraph_store_size __initdata; -static const char *fgraph_store_type_name __initdata; -static char *fgraph_error_str __initdata; -static char fgraph_error_str_buf[128] __initdata; +struct fgraph_fixture { + struct fgraph_ops gops; + int store_size; + const char *store_type_name; + char error_str_buf[ERRSTR_BUFLEN]; + char *error_str; +}; static __init int store_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { - const char *type = fgraph_store_type_name; - int size = fgraph_store_size; + struct fgraph_fixture *fixture = container_of(gops, struct fgraph_fixture, gops); + const char *type = fixture->store_type_name; + int size = fixture->store_size; void *p; p = fgraph_reserve_data(gops->idx, size); if (!p) { - snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + snprintf(fixture->error_str_buf, ERRSTR_BUFLEN, "Failed to reserve %s\n", type); - fgraph_error_str = fgraph_error_str_buf; return 0; } - switch (fgraph_store_size) { + switch (size) { case 1: *(char *)p = BYTE_NUMBER; break; @@ -804,7 +808,8 @@ static __init int store_entry(struct ftrace_graph_ent *trace, static __init void store_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops) { - const char *type = fgraph_store_type_name; + struct fgraph_fixture *fixture = container_of(gops, struct fgraph_fixture, gops); + const char *type = fixture->store_type_name; long long expect = 0; long long found = -1; int size; @@ -812,20 +817,18 @@ static __init void store_return(struct ftrace_graph_ret *trace, p = fgraph_retrieve_data(gops->idx, &size); if (!p) { - snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + snprintf(fixture->error_str_buf, ERRSTR_BUFLEN, "Failed to retrieve %s\n", type); - fgraph_error_str = fgraph_error_str_buf; return; } - if (fgraph_store_size > size) { - snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + if (fixture->store_size > size) { + snprintf(fixture->error_str_buf, ERRSTR_BUFLEN, "Retrieved size %d is smaller than expected %d\n", - size, (int)fgraph_store_size); - fgraph_error_str = fgraph_error_str_buf; + size, (int)fixture->store_size); return; } - switch (fgraph_store_size) { + switch (fixture->store_size) { case 1: expect = BYTE_NUMBER; found = *(char *)p; @@ -845,45 +848,44 @@ static __init void store_return(struct ftrace_graph_ret *trace, } if (found != expect) { - snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), + snprintf(fixture->error_str_buf, ERRSTR_BUFLEN, "%s returned not %lld but %lld\n", type, expect, found); - fgraph_error_str = fgraph_error_str_buf; return; } - fgraph_error_str = NULL; + fixture->error_str = NULL; } -static struct fgraph_ops store_bytes __initdata = { - .entryfunc = store_entry, - .retfunc = store_return, -}; - -static int __init test_graph_storage_type(const char *name, int size) +static int __init init_fgraph_fixture(struct fgraph_fixture *fixture) { char *func_name; int len; - int ret; - fgraph_store_type_name = name; - fgraph_store_size = size; - - snprintf(fgraph_error_str_buf, sizeof(fgraph_error_str_buf), - "Failed to execute storage %s\n", name); - fgraph_error_str = fgraph_error_str_buf; - - pr_cont("PASSED\n"); - pr_info("Testing fgraph storage of %d byte%s: ", size, size > 1 ? "s" : ""); + snprintf(fixture->error_str_buf, ERRSTR_BUFLEN, + "Failed to execute storage %s\n", fixture->store_type_name); + fixture->error_str = fixture->error_str_buf; func_name = "*" __stringify(DYN_FTRACE_TEST_NAME); len = strlen(func_name); - ret = ftrace_set_filter(&store_bytes.ops, func_name, len, 1); + return ftrace_set_filter(&fixture->gops.ops, func_name, len, 1); +} + +/* Test fgraph storage for each size */ +static int __init test_graph_storage_single(struct fgraph_fixture *fixture) +{ + int size = fixture->store_size; + int ret; + + pr_cont("PASSED\n"); + pr_info("Testing fgraph storage of %d byte%s: ", size, size > 1 ? "s" : ""); + + ret = init_fgraph_fixture(fixture); if (ret && ret != -ENODEV) { pr_cont("*Could not set filter* "); return -1; } - ret = register_ftrace_graph(&store_bytes); + ret = register_ftrace_graph(&fixture->gops); if (ret) { pr_warn("Failed to init store_bytes fgraph tracing\n"); return -1; @@ -891,30 +893,109 @@ static int __init test_graph_storage_type(const char *name, int size) DYN_FTRACE_TEST_NAME(); - unregister_ftrace_graph(&store_bytes); + unregister_ftrace_graph(&fixture->gops); - if (fgraph_error_str) { - pr_cont("*** %s ***", fgraph_error_str); + if (fixture->error_str) { + pr_cont("*** %s ***", fixture->error_str); return -1; } return 0; } + +static struct fgraph_fixture store_bytes[4] __initdata = { + [0] = { + .gops = { + .entryfunc = store_entry, + .retfunc = store_return, + }, + .store_size = 1, + .store_type_name = "byte", + }, + [1] = { + .gops = { + .entryfunc = store_entry, + .retfunc = store_return, + }, + .store_size = 2, + .store_type_name = "short", + }, + [2] = { + .gops = { + .entryfunc = store_entry, + .retfunc = store_return, + }, + .store_size = 4, + .store_type_name = "word", + }, + [3] = { + .gops = { + .entryfunc = store_entry, + .retfunc = store_return, + }, + .store_size = 8, + .store_type_name = "long long", + }, +}; + +static __init int test_graph_storage_multi(void) +{ + struct fgraph_fixture *fixture; + bool printed = false; + int i, ret; + + pr_cont("PASSED\n"); + pr_info("Testing multiple fgraph storage on a function: "); + + for (i = 0; i < ARRAY_SIZE(store_bytes); i++) { + fixture = &store_bytes[i]; + ret = init_fgraph_fixture(fixture); + if (ret && ret != -ENODEV) { + pr_cont("*Could not set filter* "); + printed = true; + goto out; + } + + ret = register_ftrace_graph(&fixture->gops); + if (ret) { + pr_warn("Failed to init store_bytes fgraph tracing\n"); + printed = true; + goto out; + } + } + + DYN_FTRACE_TEST_NAME(); +out: + while (--i >= 0) { + fixture = &store_bytes[i]; + unregister_ftrace_graph(&fixture->gops); + + if (fixture->error_str && !printed) { + pr_cont("*** %s ***", fixture->error_str); + printed = true; + } + } + return printed ? -1 : 0; +} + /* Test the storage passed across function_graph entry and return */ static __init int test_graph_storage(void) { int ret; - ret = test_graph_storage_type("byte", 1); + ret = test_graph_storage_single(&store_bytes[0]); if (ret) return ret; - ret = test_graph_storage_type("short", 2); + ret = test_graph_storage_single(&store_bytes[1]); if (ret) return ret; - ret = test_graph_storage_type("word", 4); + ret = test_graph_storage_single(&store_bytes[2]); if (ret) return ret; - ret = test_graph_storage_type("long long", 8); + ret = test_graph_storage_single(&store_bytes[3]); + if (ret) + return ret; + ret = test_graph_storage_multi(); if (ret) return ret; return 0; From 420e1354bcb6f006f183a1b6fe5dd21f60a457ef Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:26 -0400 Subject: [PATCH 047/280] function_graph: Use for_each_set_bit() in __ftrace_return_to_handler() Instead of iterating through the entire fgraph_array[] and seeing if one of the bitmap bits are set to know to call the array's retfunc() function, use for_each_set_bit() on the bitmap itself. This will only iterate for the number of set bits. Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.447448026@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index f207b7ae5f46..0827b67f746d 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -783,11 +783,10 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs #endif bitmap = get_bitmap_bits(current, offset); - for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { + + for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) { struct fgraph_ops *gops = fgraph_array[i]; - if (!(bitmap & BIT(i))) - continue; if (gops == &fgraph_stub) continue; From a5b6d4da0218a0539c36ad6794c624c2c6ca7b32 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:27 -0400 Subject: [PATCH 048/280] function_graph: Use bitmask to loop on fgraph entry Instead of looping through all the elements of fgraph_array[] to see if there's an gops attached to one and then calling its gops->func(). Create a fgraph_array_bitmask that sets bits when an index in the array is reserved (via the simple lru algorithm). Then only the bits set in this bitmask needs to be looked at where only elements in the array that have ops registered need to be looked at. Note, we do not care about races. If a bit is set before the gops is assigned, it only wastes time looking at the element and ignoring it (as it did before this bitmask is added). Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.604448781@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 0827b67f746d..4d566a0a741d 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -172,6 +172,7 @@ DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph); int ftrace_graph_active; static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE]; +static unsigned long fgraph_array_bitmask; /* LRU index table for fgraph_array */ static int fgraph_lru_table[FGRAPH_ARRAY_SIZE]; @@ -196,6 +197,8 @@ static int fgraph_lru_release_index(int idx) fgraph_lru_table[fgraph_lru_last] = idx; fgraph_lru_last = (fgraph_lru_last + 1) % FGRAPH_ARRAY_SIZE; + + clear_bit(idx, &fgraph_array_bitmask); return 0; } @@ -210,6 +213,8 @@ static int fgraph_lru_alloc_index(void) fgraph_lru_table[fgraph_lru_next] = -1; fgraph_lru_next = (fgraph_lru_next + 1) % FGRAPH_ARRAY_SIZE; + + set_bit(idx, &fgraph_array_bitmask); return idx; } @@ -631,7 +636,8 @@ int function_graph_enter(unsigned long ret, unsigned long func, if (offset < 0) goto out; - for (i = 0; i < FGRAPH_ARRAY_SIZE; i++) { + for_each_set_bit(i, &fgraph_array_bitmask, + sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { struct fgraph_ops *gops = fgraph_array[i]; int save_curr_ret_stack; From cc60ee813b50334b32343861057dc9e981e9c7f0 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:28 -0400 Subject: [PATCH 049/280] function_graph: Use static_call and branch to optimize entry function In most cases function graph is used by a single user. Instead of calling a loop to call function graph callbacks in this case, call the function entry callback directly. Add a static_key that will be used to set the function graph logic to either do the loop (when more than one callback is registered) or to call the callback directly if there is only one registered callback. Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.766858241@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 77 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 4d566a0a741d..7c3b0261b1bb 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -511,6 +512,10 @@ static struct fgraph_ops fgraph_stub = { .retfunc = ftrace_graph_ret_stub, }; +static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; +DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); +DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); + /** * ftrace_graph_stop - set to permanently disable function graph tracing * @@ -636,21 +641,34 @@ int function_graph_enter(unsigned long ret, unsigned long func, if (offset < 0) goto out; - for_each_set_bit(i, &fgraph_array_bitmask, - sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { - struct fgraph_ops *gops = fgraph_array[i]; - int save_curr_ret_stack; +#ifdef CONFIG_HAVE_STATIC_CALL + if (static_branch_likely(&fgraph_do_direct)) { + int save_curr_ret_stack = current->curr_ret_stack; - if (gops == &fgraph_stub) - continue; - - save_curr_ret_stack = current->curr_ret_stack; - if (ftrace_ops_test(&gops->ops, func, NULL) && - gops->entryfunc(&trace, gops)) - bitmap |= BIT(i); + if (static_call(fgraph_func)(&trace, fgraph_direct_gops)) + bitmap |= BIT(fgraph_direct_gops->idx); else /* Clear out any saved storage */ current->curr_ret_stack = save_curr_ret_stack; + } else +#endif + { + for_each_set_bit(i, &fgraph_array_bitmask, + sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { + struct fgraph_ops *gops = fgraph_array[i]; + int save_curr_ret_stack; + + if (gops == &fgraph_stub) + continue; + + save_curr_ret_stack = current->curr_ret_stack; + if (ftrace_ops_test(&gops->ops, func, NULL) && + gops->entryfunc(&trace, gops)) + bitmap |= BIT(i); + else + /* Clear out any saved storage */ + current->curr_ret_stack = save_curr_ret_stack; + } } if (!bitmap) @@ -1155,6 +1173,8 @@ void fgraph_update_pid_func(void) gops = container_of(op, struct fgraph_ops, ops); gops->entryfunc = ftrace_pids_enabled(op) ? fgraph_pid_func : gops->saved_func; + if (ftrace_graph_active == 1) + static_call_update(fgraph_func, gops->entryfunc); } } } @@ -1209,6 +1229,32 @@ static void init_task_vars(int idx) read_unlock(&tasklist_lock); } +static void ftrace_graph_enable_direct(bool enable_branch) +{ + trace_func_graph_ent_t func = NULL; + int i; + + for_each_set_bit(i, &fgraph_array_bitmask, + sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { + func = fgraph_array[i]->entryfunc; + fgraph_direct_gops = fgraph_array[i]; + } + if (WARN_ON_ONCE(!func)) + return; + + static_call_update(fgraph_func, func); + if (enable_branch) + static_branch_disable(&fgraph_do_direct); +} + +static void ftrace_graph_disable_direct(bool disable_branch) +{ + if (disable_branch) + static_branch_disable(&fgraph_do_direct); + static_call_update(fgraph_func, ftrace_graph_entry_stub); + fgraph_direct_gops = &fgraph_stub; +} + int register_ftrace_graph(struct fgraph_ops *gops) { int command = 0; @@ -1235,7 +1281,11 @@ int register_ftrace_graph(struct fgraph_ops *gops) ftrace_graph_active++; + if (ftrace_graph_active == 2) + ftrace_graph_disable_direct(true); + if (ftrace_graph_active == 1) { + ftrace_graph_enable_direct(false); register_pm_notifier(&ftrace_suspend_notifier); ret = start_graph_tracing(); if (ret) @@ -1292,6 +1342,11 @@ void unregister_ftrace_graph(struct fgraph_ops *gops) ftrace_shutdown_subops(&graph_ops, &gops->ops, command); + if (ftrace_graph_active == 1) + ftrace_graph_enable_direct(true); + else if (!ftrace_graph_active) + ftrace_graph_disable_direct(false); + if (!ftrace_graph_active) { ftrace_graph_return = ftrace_stub_graph; ftrace_graph_entry = ftrace_graph_entry_stub; From fe835e3ca40e172aa8ad12f4ed2898c181fafab0 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:29 -0400 Subject: [PATCH 050/280] function_graph: Use static_call and branch to optimize return function In most cases function graph is used by a single user. Instead of calling a loop to call function graph callbacks in this case, call the function return callback directly. Use the static_key that is set when the function graph tracer has less than 2 callbacks registered. It will do the direct call in that case, and will do the loop over all callers when there are 2 or more callbacks registered. Link: https://lore.kernel.org/linux-trace-kernel/20240603190824.921460797@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 7c3b0261b1bb..4bf91eebbb08 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -514,6 +514,7 @@ static struct fgraph_ops fgraph_stub = { static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); +DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub); DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); /** @@ -808,13 +809,21 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs bitmap = get_bitmap_bits(current, offset); - for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) { - struct fgraph_ops *gops = fgraph_array[i]; +#ifdef CONFIG_HAVE_STATIC_CALL + if (static_branch_likely(&fgraph_do_direct)) { + if (test_bit(fgraph_direct_gops->idx, &bitmap)) + static_call(fgraph_retfunc)(&trace, fgraph_direct_gops); + } else +#endif + { + for_each_set_bit(i, &bitmap, sizeof(bitmap) * BITS_PER_BYTE) { + struct fgraph_ops *gops = fgraph_array[i]; - if (gops == &fgraph_stub) - continue; + if (gops == &fgraph_stub) + continue; - gops->retfunc(&trace, gops); + gops->retfunc(&trace, gops); + } } /* @@ -1232,17 +1241,20 @@ static void init_task_vars(int idx) static void ftrace_graph_enable_direct(bool enable_branch) { trace_func_graph_ent_t func = NULL; + trace_func_graph_ret_t retfunc = NULL; int i; for_each_set_bit(i, &fgraph_array_bitmask, sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { func = fgraph_array[i]->entryfunc; + retfunc = fgraph_array[i]->retfunc; fgraph_direct_gops = fgraph_array[i]; } if (WARN_ON_ONCE(!func)) return; static_call_update(fgraph_func, func); + static_call_update(fgraph_retfunc, retfunc); if (enable_branch) static_branch_disable(&fgraph_do_direct); } @@ -1252,6 +1264,7 @@ static void ftrace_graph_disable_direct(bool disable_branch) if (disable_branch) static_branch_disable(&fgraph_do_direct); static_call_update(fgraph_func, ftrace_graph_entry_stub); + static_call_update(fgraph_retfunc, ftrace_graph_ret_stub); fgraph_direct_gops = &fgraph_stub; } From 35b944a997e25962122c3dea68b020e7fbb06cbd Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:30 -0400 Subject: [PATCH 051/280] selftests/ftrace: Add function_graph tracer to func-filter-pid test The function tracer is tested to see if pid filtering works. Add a test to test function_graph tracer as well, but only if the function_graph tracer is enabled for the top level or instance. Link: https://lore.kernel.org/linux-trace-kernel/20240603190825.083048115@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- .../ftrace/test.d/ftrace/func-filter-pid.tc | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc index 2f7211254529..c6fc9d31a496 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc @@ -14,6 +14,11 @@ if [ ! -f options/function-fork ]; then echo "no option for function-fork found. Option will not be tested." fi +if [ ! -f options/funcgraph-proc ]; then + do_funcgraph_proc=0 + echo "no option for function-fork found. Option will not be tested." +fi + read PID _ < /proc/self/stat if [ $do_function_fork -eq 1 ]; then @@ -21,12 +26,18 @@ if [ $do_function_fork -eq 1 ]; then orig_value=`grep function-fork trace_options` fi +if [ $do_funcgraph_proc -eq 1 ]; then + orig_value2=`cat options/funcgraph-proc` +fi + do_reset() { - if [ $do_function_fork -eq 0 ]; then - return + if [ $do_function_fork -eq 1 ]; then + echo $orig_value > trace_options fi - echo $orig_value > trace_options + if [ $do_funcgraph_proc -eq 1 ]; then + echo $orig_value2 > options/funcgraph-proc + fi } fail() { # msg @@ -36,13 +47,15 @@ fail() { # msg } do_test() { + TRACER=$1 + disable_tracing echo do_execve* > set_ftrace_filter echo $FUNCTION_FORK >> set_ftrace_filter echo $PID > set_ftrace_pid - echo function > current_tracer + echo $TRACER > current_tracer if [ $do_function_fork -eq 1 ]; then # don't allow children to be traced @@ -82,7 +95,11 @@ do_test() { fi } -do_test +do_test function +if grep -s function_graph available_tracers; then + do_test function_graph +fi + do_reset exit 0 From 8d4e21bd4cca8013d2c6f55d42df85838d4ebce6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 3 Jun 2024 15:07:31 -0400 Subject: [PATCH 052/280] selftests/ftrace: Add fgraph-multi.tc test Add a test that creates 3 instances and enables function_graph tracer in each as well as the top instance, where each will enable a filter (but one that traces all functions) and check that they are filtering properly. Link: https://lore.kernel.org/linux-trace-kernel/20240603190825.252845939@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Cc: Alexei Starovoitov Cc: Florent Revest Cc: Martin KaFai Lau Cc: bpf Cc: Sven Schnelle Cc: Alexei Starovoitov Cc: Jiri Olsa Cc: Arnaldo Carvalho de Melo Cc: Daniel Borkmann Cc: Alan Maguire Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Guo Ren Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- .../ftrace/test.d/ftrace/fgraph-multi.tc | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tools/testing/selftests/ftrace/test.d/ftrace/fgraph-multi.tc diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-multi.tc b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-multi.tc new file mode 100644 index 000000000000..ff88f97e41fb --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/fgraph-multi.tc @@ -0,0 +1,103 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: ftrace - function graph filters +# requires: set_ftrace_filter function_graph:tracer + +# Make sure that function graph filtering works + +INSTANCE1="instances/test1_$$" +INSTANCE2="instances/test2_$$" +INSTANCE3="instances/test3_$$" + +WD=`pwd` + +do_reset() { + cd $WD + if [ -d $INSTANCE1 ]; then + echo nop > $INSTANCE1/current_tracer + rmdir $INSTANCE1 + fi + if [ -d $INSTANCE2 ]; then + echo nop > $INSTANCE2/current_tracer + rmdir $INSTANCE2 + fi + if [ -d $INSTANCE3 ]; then + echo nop > $INSTANCE3/current_tracer + rmdir $INSTANCE3 + fi +} + +mkdir $INSTANCE1 +if ! grep -q function_graph $INSTANCE1/available_tracers; then + echo "function_graph not allowed with instances" + rmdir $INSTANCE1 + exit_unsupported +fi + +mkdir $INSTANCE2 +mkdir $INSTANCE3 + +fail() { # msg + do_reset + echo $1 + exit_fail +} + +disable_tracing +clear_trace + +do_test() { + REGEX=$1 + TEST=$2 + + # filter something, schedule is always good + if ! echo "$REGEX" > set_ftrace_filter; then + fail "can not enable filter $REGEX" + fi + + echo > trace + echo function_graph > current_tracer + enable_tracing + sleep 1 + # search for functions (has "{" or ";" on the line) + echo 0 > tracing_on + count=`cat trace | grep -v '^#' | grep -e '{' -e ';' | grep -v "$TEST" | wc -l` + echo 1 > tracing_on + if [ $count -ne 0 ]; then + fail "Graph filtering not working by itself against $TEST?" + fi + + # Make sure we did find something + echo 0 > tracing_on + count=`cat trace | grep -v '^#' | grep -e '{' -e ';' | grep "$TEST" | wc -l` + echo 1 > tracing_on + if [ $count -eq 0 ]; then + fail "No traces found with $TEST?" + fi +} + +do_test '*sched*' 'sched' +cd $INSTANCE1 +do_test '*lock*' 'lock' +cd $WD +cd $INSTANCE2 +do_test '*rcu*' 'rcu' +cd $WD +cd $INSTANCE3 +echo function_graph > current_tracer + +sleep 1 +count=`cat trace | grep -v '^#' | grep -e '{' -e ';' | grep "$TEST" | wc -l` +if [ $count -eq 0 ]; then + fail "No traces found with all tracing?" +fi + +cd $WD +echo nop > current_tracer +echo nop > $INSTANCE1/current_tracer +echo nop > $INSTANCE2/current_tracer +echo nop > $INSTANCE3/current_tracer + +do_reset + +exit 0 From 0770ceaff2f6a084d4d020295cfba6c5ef278cf4 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sun, 2 Jun 2024 01:07:09 +0100 Subject: [PATCH 053/280] perf hisi-ptt: remove unused struct 'hisi_ptt_queue' 'hisi_ptt_queue' has been unused since the original commit 5e91e57e6809 ("perf auxtrace arm64: Add support for parsing HiSilicon PCIe Trace packet"). Remove it. Signed-off-by: Dr. David Alan Gilbert Reviewed-by: Jonathan Cameron Reviewed-by: Ian Rogers Cc: yangyicong@hisilicon.com Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240602000709.213116-1-linux@treblig.org --- tools/perf/util/hisi-ptt.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c index 52d0ce302ca0..37ea987017f6 100644 --- a/tools/perf/util/hisi-ptt.c +++ b/tools/perf/util/hisi-ptt.c @@ -35,11 +35,6 @@ struct hisi_ptt { u32 pmu_type; }; -struct hisi_ptt_queue { - struct hisi_ptt *ptt; - struct auxtrace_buffer *buffer; -}; - static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf) { uint32_t head = *(uint32_t *)buf; From 721f4a6526daafca15634f30c9865e880da3e1d1 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Fri, 5 Apr 2024 01:58:21 +0000 Subject: [PATCH 054/280] mm/memblock: remove empty dummy entry The dummy entry is introduced in the initial implementation of lmb in commit 7c8c6b9776fb ("powerpc: Merge lmb.c and make MM initialization use it."). As the comment says the empty dummy entry is to simplify the code. /* Create a dummy zero size LMB which will get coalesced away later. * This simplifies the lmb_add() code below... */ While current code is reimplemented by Tejun in commit 784656f9c680 ("memblock: Reimplement memblock_add_region()"). This empty dummy entry seems not benefit the code any more. Let's remove it. Signed-off-by: Wei Yang CC: Paul Mackerras CC: Tejun Heo CC: Mike Rapoport Link: https://lore.kernel.org/r/20240405015821.13411-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/memblock.c | 7 ++----- tools/testing/memblock/tests/basic_api.c | 8 ++++---- tools/testing/memblock/tests/common.c | 4 ++-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/mm/memblock.c b/mm/memblock.c index d09136e040d3..98d25689cf10 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -114,12 +114,10 @@ static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS struct memblock memblock __initdata_memblock = { .memory.regions = memblock_memory_init_regions, - .memory.cnt = 1, /* empty dummy entry */ .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, .memory.name = "memory", .reserved.regions = memblock_reserved_init_regions, - .reserved.cnt = 1, /* empty dummy entry */ .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, .reserved.name = "reserved", @@ -130,7 +128,6 @@ struct memblock memblock __initdata_memblock = { #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP struct memblock_type physmem = { .regions = memblock_physmem_init_regions, - .cnt = 1, /* empty dummy entry */ .max = INIT_PHYSMEM_REGIONS, .name = "physmem", }; @@ -356,7 +353,6 @@ static void __init_memblock memblock_remove_region(struct memblock_type *type, u /* Special case for empty arrays */ if (type->cnt == 0) { WARN_ON(type->total_size != 0); - type->cnt = 1; type->regions[0].base = 0; type->regions[0].size = 0; type->regions[0].flags = 0; @@ -600,12 +596,13 @@ static int __init_memblock memblock_add_range(struct memblock_type *type, /* special case for empty array */ if (type->regions[0].size == 0) { - WARN_ON(type->cnt != 1 || type->total_size); + WARN_ON(type->cnt != 0 || type->total_size); type->regions[0].base = base; type->regions[0].size = size; type->regions[0].flags = flags; memblock_set_region_node(&type->regions[0], nid); type->total_size = size; + type->cnt = 1; return 0; } diff --git a/tools/testing/memblock/tests/basic_api.c b/tools/testing/memblock/tests/basic_api.c index 57bf2688edfd..f317fe691fc4 100644 --- a/tools/testing/memblock/tests/basic_api.c +++ b/tools/testing/memblock/tests/basic_api.c @@ -15,12 +15,12 @@ static int memblock_initialization_check(void) PREFIX_PUSH(); ASSERT_NE(memblock.memory.regions, NULL); - ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.cnt, 0); ASSERT_EQ(memblock.memory.max, EXPECTED_MEMBLOCK_REGIONS); ASSERT_EQ(strcmp(memblock.memory.name, "memory"), 0); ASSERT_NE(memblock.reserved.regions, NULL); - ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.cnt, 0); ASSERT_EQ(memblock.memory.max, EXPECTED_MEMBLOCK_REGIONS); ASSERT_EQ(strcmp(memblock.reserved.name, "reserved"), 0); @@ -1295,7 +1295,7 @@ static int memblock_remove_only_region_check(void) ASSERT_EQ(rgn->base, 0); ASSERT_EQ(rgn->size, 0); - ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.cnt, 0); ASSERT_EQ(memblock.memory.total_size, 0); test_pass_pop(); @@ -1723,7 +1723,7 @@ static int memblock_free_only_region_check(void) ASSERT_EQ(rgn->base, 0); ASSERT_EQ(rgn->size, 0); - ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.cnt, 0); ASSERT_EQ(memblock.reserved.total_size, 0); test_pass_pop(); diff --git a/tools/testing/memblock/tests/common.c b/tools/testing/memblock/tests/common.c index f43b6f414983..c2c569f12178 100644 --- a/tools/testing/memblock/tests/common.c +++ b/tools/testing/memblock/tests/common.c @@ -40,13 +40,13 @@ void reset_memblock_regions(void) { memset(memblock.memory.regions, 0, memblock.memory.cnt * sizeof(struct memblock_region)); - memblock.memory.cnt = 1; + memblock.memory.cnt = 0; memblock.memory.max = INIT_MEMBLOCK_REGIONS; memblock.memory.total_size = 0; memset(memblock.reserved.regions, 0, memblock.reserved.cnt * sizeof(struct memblock_region)); - memblock.reserved.cnt = 1; + memblock.reserved.cnt = 0; memblock.reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS; memblock.reserved.total_size = 0; } From 3d3165193776ddacf59f101f0fa05cfab9f1a9ba Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:27 +0000 Subject: [PATCH 055/280] memblock tests: add memblock_reserve_all_locations_check() Instead of adding 129th memory block at the last position, let's try all possible position. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-2-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- tools/testing/memblock/tests/basic_api.c | 107 +++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/tools/testing/memblock/tests/basic_api.c b/tools/testing/memblock/tests/basic_api.c index f317fe691fc4..bd3ebbf6b697 100644 --- a/tools/testing/memblock/tests/basic_api.c +++ b/tools/testing/memblock/tests/basic_api.c @@ -982,6 +982,112 @@ static int memblock_reserve_many_check(void) return 0; } + +/* + * A test that trying to reserve the 129th memory block at all locations. + * Expect to trigger memblock_double_array() to double the + * memblock.memory.max, find a new valid memory as reserved.regions. + * + * 0 1 2 128 + * +-------+ +-------+ +-------+ +-------+ + * | 32K | | 32K | | 32K | ... | 32K | + * +-------+-------+-------+-------+-------+ +-------+ + * |<-32K->| |<-32K->| + * + */ +/* Keep the gap so these memory region will not be merged. */ +#define MEMORY_BASE(idx) (SZ_128K + (MEM_SIZE * 2) * (idx)) +static int memblock_reserve_all_locations_check(void) +{ + int i, skip; + void *orig_region; + struct region r = { + .base = SZ_16K, + .size = SZ_16K, + }; + phys_addr_t new_reserved_regions_size; + + PREFIX_PUSH(); + + /* Reserve the 129th memory block for all possible positions*/ + for (skip = 0; skip < INIT_MEMBLOCK_REGIONS + 1; skip++) { + reset_memblock_regions(); + memblock_allow_resize(); + + /* Add a valid memory region used by double_array(). */ + dummy_physical_memory_init(); + memblock_add(dummy_physical_memory_base(), MEM_SIZE); + + for (i = 0; i < INIT_MEMBLOCK_REGIONS + 1; i++) { + if (i == skip) + continue; + + /* Reserve some fakes memory region to fulfill the memblock. */ + memblock_reserve(MEMORY_BASE(i), MEM_SIZE); + + if (i < skip) { + ASSERT_EQ(memblock.reserved.cnt, i + 1); + ASSERT_EQ(memblock.reserved.total_size, (i + 1) * MEM_SIZE); + } else { + ASSERT_EQ(memblock.reserved.cnt, i); + ASSERT_EQ(memblock.reserved.total_size, i * MEM_SIZE); + } + } + + orig_region = memblock.reserved.regions; + + /* This reserve the 129 memory_region, and makes it double array. */ + memblock_reserve(MEMORY_BASE(skip), MEM_SIZE); + + /* + * This is the memory region size used by the doubled reserved.regions, + * and it has been reserved due to it has been used. The size is used to + * calculate the total_size that the memblock.reserved have now. + */ + new_reserved_regions_size = PAGE_ALIGN((INIT_MEMBLOCK_REGIONS * 2) * + sizeof(struct memblock_region)); + /* + * The double_array() will find a free memory region as the new + * reserved.regions, and the used memory region will be reserved, so + * there will be one more region exist in the reserved memblock. And the + * one more reserved region's size is new_reserved_regions_size. + */ + ASSERT_EQ(memblock.reserved.cnt, INIT_MEMBLOCK_REGIONS + 2); + ASSERT_EQ(memblock.reserved.total_size, (INIT_MEMBLOCK_REGIONS + 1) * MEM_SIZE + + new_reserved_regions_size); + ASSERT_EQ(memblock.reserved.max, INIT_MEMBLOCK_REGIONS * 2); + + /* + * Now memblock_double_array() works fine. Let's check after the + * double_array(), the memblock_reserve() still works as normal. + */ + memblock_reserve(r.base, r.size); + ASSERT_EQ(memblock.reserved.regions[0].base, r.base); + ASSERT_EQ(memblock.reserved.regions[0].size, r.size); + + ASSERT_EQ(memblock.reserved.cnt, INIT_MEMBLOCK_REGIONS + 3); + ASSERT_EQ(memblock.reserved.total_size, (INIT_MEMBLOCK_REGIONS + 1) * MEM_SIZE + + new_reserved_regions_size + + r.size); + ASSERT_EQ(memblock.reserved.max, INIT_MEMBLOCK_REGIONS * 2); + + dummy_physical_memory_cleanup(); + + /* + * The current reserved.regions is occupying a range of memory that + * allocated from dummy_physical_memory_init(). After free the memory, + * we must not use it. So restore the origin memory region to make sure + * the tests can run as normal and not affected by the double array. + */ + memblock.reserved.regions = orig_region; + memblock.reserved.cnt = INIT_MEMBLOCK_RESERVED_REGIONS; + } + + test_pass_pop(); + + return 0; +} + static int memblock_reserve_checks(void) { prefix_reset(); @@ -997,6 +1103,7 @@ static int memblock_reserve_checks(void) memblock_reserve_between_check(); memblock_reserve_near_max_check(); memblock_reserve_many_check(); + memblock_reserve_all_locations_check(); prefix_pop(); From f6df89c3582a337090ae1f37c3648bdb35da29f7 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:28 +0000 Subject: [PATCH 056/280] memblock tests: add memblock_reserve_many_may_conflict_check() This may trigger the case fixed by commit 48c3b583bbdd ("mm/memblock: fix overlapping allocation when doubling reserved array"). This is done by adding the 129th reserve region into memblock.memory. If memblock_double_array() use this reserve region as new array, it fails. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-3-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- tools/testing/memblock/tests/basic_api.c | 151 +++++++++++++++++++++++ tools/testing/memblock/tests/common.c | 4 +- tools/testing/memblock/tests/common.h | 1 + 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/tools/testing/memblock/tests/basic_api.c b/tools/testing/memblock/tests/basic_api.c index bd3ebbf6b697..fdac82656d15 100644 --- a/tools/testing/memblock/tests/basic_api.c +++ b/tools/testing/memblock/tests/basic_api.c @@ -1088,6 +1088,156 @@ static int memblock_reserve_all_locations_check(void) return 0; } +/* + * A test that trying to reserve the 129th memory block at all locations. + * Expect to trigger memblock_double_array() to double the + * memblock.memory.max, find a new valid memory as reserved.regions. And make + * sure it doesn't conflict with the range we want to reserve. + * + * For example, we have 128 regions in reserved and now want to reserve + * the skipped one. Since reserved is full, memblock_double_array() would find + * an available range in memory for the new array. We intended to put two + * ranges in memory with one is the exact range of the skipped one. Before + * commit 48c3b583bbdd ("mm/memblock: fix overlapping allocation when doubling + * reserved array"), the new array would sits in the skipped range which is a + * conflict. The expected new array should be allocated from memory.regions[0]. + * + * 0 1 + * memory +-------+ +-------+ + * | 32K | | 32K | + * +-------+ ------+-------+-------+-------+ + * |<-32K->|<-32K->|<-32K->| + * + * 0 skipped 127 + * reserved +-------+ ......... +-------+ + * | 32K | . 32K . ... | 32K | + * +-------+-------+-------+ +-------+ + * |<-32K->| + * ^ + * | + * | + * skipped one + */ +/* Keep the gap so these memory region will not be merged. */ +#define MEMORY_BASE_OFFSET(idx, offset) ((offset) + (MEM_SIZE * 2) * (idx)) +static int memblock_reserve_many_may_conflict_check(void) +{ + int i, skip; + void *orig_region; + struct region r = { + .base = SZ_16K, + .size = SZ_16K, + }; + phys_addr_t new_reserved_regions_size; + + /* + * 0 1 129 + * +---+ +---+ +---+ + * |32K| |32K| .. |32K| + * +---+ +---+ +---+ + * + * Pre-allocate the range for 129 memory block + one range for double + * memblock.reserved.regions at idx 0. + */ + dummy_physical_memory_init(); + phys_addr_t memory_base = dummy_physical_memory_base(); + phys_addr_t offset = PAGE_ALIGN(memory_base); + + PREFIX_PUSH(); + + /* Reserve the 129th memory block for all possible positions*/ + for (skip = 1; skip <= INIT_MEMBLOCK_REGIONS + 1; skip++) { + reset_memblock_regions(); + memblock_allow_resize(); + + reset_memblock_attributes(); + /* Add a valid memory region used by double_array(). */ + memblock_add(MEMORY_BASE_OFFSET(0, offset), MEM_SIZE); + /* + * Add a memory region which will be reserved as 129th memory + * region. This is not expected to be used by double_array(). + */ + memblock_add(MEMORY_BASE_OFFSET(skip, offset), MEM_SIZE); + + for (i = 1; i <= INIT_MEMBLOCK_REGIONS + 1; i++) { + if (i == skip) + continue; + + /* Reserve some fakes memory region to fulfill the memblock. */ + memblock_reserve(MEMORY_BASE_OFFSET(i, offset), MEM_SIZE); + + if (i < skip) { + ASSERT_EQ(memblock.reserved.cnt, i); + ASSERT_EQ(memblock.reserved.total_size, i * MEM_SIZE); + } else { + ASSERT_EQ(memblock.reserved.cnt, i - 1); + ASSERT_EQ(memblock.reserved.total_size, (i - 1) * MEM_SIZE); + } + } + + orig_region = memblock.reserved.regions; + + /* This reserve the 129 memory_region, and makes it double array. */ + memblock_reserve(MEMORY_BASE_OFFSET(skip, offset), MEM_SIZE); + + /* + * This is the memory region size used by the doubled reserved.regions, + * and it has been reserved due to it has been used. The size is used to + * calculate the total_size that the memblock.reserved have now. + */ + new_reserved_regions_size = PAGE_ALIGN((INIT_MEMBLOCK_REGIONS * 2) * + sizeof(struct memblock_region)); + /* + * The double_array() will find a free memory region as the new + * reserved.regions, and the used memory region will be reserved, so + * there will be one more region exist in the reserved memblock. And the + * one more reserved region's size is new_reserved_regions_size. + */ + ASSERT_EQ(memblock.reserved.cnt, INIT_MEMBLOCK_REGIONS + 2); + ASSERT_EQ(memblock.reserved.total_size, (INIT_MEMBLOCK_REGIONS + 1) * MEM_SIZE + + new_reserved_regions_size); + ASSERT_EQ(memblock.reserved.max, INIT_MEMBLOCK_REGIONS * 2); + + /* + * The first reserved region is allocated for double array + * with the size of new_reserved_regions_size and the base to be + * MEMORY_BASE_OFFSET(0, offset) + SZ_32K - new_reserved_regions_size + */ + ASSERT_EQ(memblock.reserved.regions[0].base + memblock.reserved.regions[0].size, + MEMORY_BASE_OFFSET(0, offset) + SZ_32K); + ASSERT_EQ(memblock.reserved.regions[0].size, new_reserved_regions_size); + + /* + * Now memblock_double_array() works fine. Let's check after the + * double_array(), the memblock_reserve() still works as normal. + */ + memblock_reserve(r.base, r.size); + ASSERT_EQ(memblock.reserved.regions[0].base, r.base); + ASSERT_EQ(memblock.reserved.regions[0].size, r.size); + + ASSERT_EQ(memblock.reserved.cnt, INIT_MEMBLOCK_REGIONS + 3); + ASSERT_EQ(memblock.reserved.total_size, (INIT_MEMBLOCK_REGIONS + 1) * MEM_SIZE + + new_reserved_regions_size + + r.size); + ASSERT_EQ(memblock.reserved.max, INIT_MEMBLOCK_REGIONS * 2); + + /* + * The current reserved.regions is occupying a range of memory that + * allocated from dummy_physical_memory_init(). After free the memory, + * we must not use it. So restore the origin memory region to make sure + * the tests can run as normal and not affected by the double array. + */ + memblock.reserved.regions = orig_region; + memblock.reserved.cnt = INIT_MEMBLOCK_RESERVED_REGIONS; + } + + dummy_physical_memory_cleanup(); + + test_pass_pop(); + + return 0; +} + static int memblock_reserve_checks(void) { prefix_reset(); @@ -1104,6 +1254,7 @@ static int memblock_reserve_checks(void) memblock_reserve_near_max_check(); memblock_reserve_many_check(); memblock_reserve_all_locations_check(); + memblock_reserve_many_may_conflict_check(); prefix_pop(); diff --git a/tools/testing/memblock/tests/common.c b/tools/testing/memblock/tests/common.c index c2c569f12178..3250c8e5124b 100644 --- a/tools/testing/memblock/tests/common.c +++ b/tools/testing/memblock/tests/common.c @@ -61,7 +61,7 @@ void reset_memblock_attributes(void) static inline void fill_memblock(void) { - memset(memory_block.base, 1, MEM_SIZE); + memset(memory_block.base, 1, PHYS_MEM_SIZE); } void setup_memblock(void) @@ -103,7 +103,7 @@ void setup_numa_memblock(const unsigned int node_fracs[]) void dummy_physical_memory_init(void) { - memory_block.base = malloc(MEM_SIZE); + memory_block.base = malloc(PHYS_MEM_SIZE); assert(memory_block.base); fill_memblock(); } diff --git a/tools/testing/memblock/tests/common.h b/tools/testing/memblock/tests/common.h index b5ec59aa62d7..2f26405562b0 100644 --- a/tools/testing/memblock/tests/common.h +++ b/tools/testing/memblock/tests/common.h @@ -12,6 +12,7 @@ #include <../selftests/kselftest.h> #define MEM_SIZE SZ_32K +#define PHYS_MEM_SIZE SZ_16M #define NUMA_NODES 8 #define INIT_MEMBLOCK_REGIONS 128 From 3aca2cea907c647ee7720b7ba22734f9e8e7cfa3 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:29 +0000 Subject: [PATCH 057/280] mm/memblock: fix comment for memblock_isolate_range() The isolated range is [*@start_rgn, *@end_rgn - 1], while the comment says "the end region inside the range" is *@end_rgn. Let's correct it. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-4-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/memblock.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/memblock.c b/mm/memblock.c index 98d25689cf10..7f3cd96d6769 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -777,7 +777,8 @@ bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_byt * Walk @type and ensure that regions don't cross the boundaries defined by * [@base, @base + @size). Crossing regions are split at the boundaries, * which may create at most two more regions. The index of the first - * region inside the range is returned in *@start_rgn and end in *@end_rgn. + * region inside the range is returned in *@start_rgn and the index of the + * first region after the range is returned in *@end_rgn. * * Return: * 0 on success, -errno on failure. From 1a879671bdfd14698a839f30de8e6d76e1e858fd Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:30 +0000 Subject: [PATCH 058/280] memblock tests: add memblock_overlaps_region_checks Add a test case for memblock_overlaps_region(). Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-5-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- tools/testing/memblock/tests/basic_api.c | 48 ++++++++++++++++++++++++ tools/testing/memblock/tests/common.h | 3 ++ 2 files changed, 51 insertions(+) diff --git a/tools/testing/memblock/tests/basic_api.c b/tools/testing/memblock/tests/basic_api.c index fdac82656d15..67503089e6a0 100644 --- a/tools/testing/memblock/tests/basic_api.c +++ b/tools/testing/memblock/tests/basic_api.c @@ -2387,6 +2387,53 @@ static int memblock_trim_memory_checks(void) return 0; } +static int memblock_overlaps_region_check(void) +{ + struct region r = { + .base = SZ_1G, + .size = SZ_4M + }; + + PREFIX_PUSH(); + + reset_memblock_regions(); + memblock_add(r.base, r.size); + + /* Far Away */ + ASSERT_FALSE(memblock_overlaps_region(&memblock.memory, SZ_1M, SZ_1M)); + ASSERT_FALSE(memblock_overlaps_region(&memblock.memory, SZ_2G, SZ_1M)); + + /* Neighbor */ + ASSERT_FALSE(memblock_overlaps_region(&memblock.memory, SZ_1G - SZ_1M, SZ_1M)); + ASSERT_FALSE(memblock_overlaps_region(&memblock.memory, SZ_1G + SZ_4M, SZ_1M)); + + /* Partial Overlap */ + ASSERT_TRUE(memblock_overlaps_region(&memblock.memory, SZ_1G - SZ_1M, SZ_2M)); + ASSERT_TRUE(memblock_overlaps_region(&memblock.memory, SZ_1G + SZ_2M, SZ_2M)); + + /* Totally Overlap */ + ASSERT_TRUE(memblock_overlaps_region(&memblock.memory, SZ_1G, SZ_4M)); + ASSERT_TRUE(memblock_overlaps_region(&memblock.memory, SZ_1G - SZ_2M, SZ_8M)); + ASSERT_TRUE(memblock_overlaps_region(&memblock.memory, SZ_1G + SZ_1M, SZ_1M)); + + test_pass_pop(); + + return 0; +} + +static int memblock_overlaps_region_checks(void) +{ + prefix_reset(); + prefix_push("memblock_overlaps_region"); + test_print("Running memblock_overlaps_region tests...\n"); + + memblock_overlaps_region_check(); + + prefix_pop(); + + return 0; +} + int memblock_basic_checks(void) { memblock_initialization_check(); @@ -2396,6 +2443,7 @@ int memblock_basic_checks(void) memblock_free_checks(); memblock_bottom_up_checks(); memblock_trim_memory_checks(); + memblock_overlaps_region_checks(); return 0; } diff --git a/tools/testing/memblock/tests/common.h b/tools/testing/memblock/tests/common.h index 2f26405562b0..e1138e06c903 100644 --- a/tools/testing/memblock/tests/common.h +++ b/tools/testing/memblock/tests/common.h @@ -40,6 +40,9 @@ enum test_flags { assert((_expected) == (_seen)); \ } while (0) +#define ASSERT_TRUE(_seen) ASSERT_EQ(true, _seen) +#define ASSERT_FALSE(_seen) ASSERT_EQ(false, _seen) + /** * ASSERT_NE(): * Check the condition From 1eb0a28d039a479bb4adec0320592caf5bd5175b Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:31 +0000 Subject: [PATCH 059/280] mm/memblock: return true directly on finding overlap region Not necessary to break and check i against type->cnt again. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-6-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/memblock.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/memblock.c b/mm/memblock.c index 7f3cd96d6769..da9a6c862a69 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -194,8 +194,8 @@ bool __init_memblock memblock_overlaps_region(struct memblock_type *type, for (i = 0; i < type->cnt; i++) if (memblock_addrs_overlap(base, size, type->regions[i].base, type->regions[i].size)) - break; - return i < type->cnt; + return true; + return false; } /** From b73f6b98bbd0b4c1fdcebc0c5b926349455035bf Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Tue, 7 May 2024 07:58:32 +0000 Subject: [PATCH 060/280] mm/memblock: use PAGE_ALIGN_DOWN to get pgend in free_memmap Leverage the macro PAGE_ALIGN_DOWN to get pgend. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240507075833.6346-7-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/memblock.c | 2 +- tools/include/linux/mm.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/memblock.c b/mm/memblock.c index da9a6c862a69..33a8b6f7b626 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2039,7 +2039,7 @@ static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) * downwards. */ pg = PAGE_ALIGN(__pa(start_pg)); - pgend = __pa(end_pg) & PAGE_MASK; + pgend = PAGE_ALIGN_DOWN(__pa(end_pg)); /* * If there are free pages between these, free the section of the diff --git a/tools/include/linux/mm.h b/tools/include/linux/mm.h index dc0fc7125bc3..cad4f2927983 100644 --- a/tools/include/linux/mm.h +++ b/tools/include/linux/mm.h @@ -12,6 +12,7 @@ #define PHYS_ADDR_MAX (~(phys_addr_t)0) #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE) +#define PAGE_ALIGN_DOWN(addr) ALIGN_DOWN(addr, PAGE_SIZE) #define __va(x) ((void *)((unsigned long)(x))) #define __pa(x) ((unsigned long)(x)) From 3be381d11f872de066774317031ba8edd2d8797e Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 25 May 2024 02:30:38 +0000 Subject: [PATCH 061/280] mm/mm_init.c: use memblock_region_memory_base_pfn() to get startpfn Just like what it does in "if (mirrored_kernelcore)", we should use memblock_region_memory_base_pfn() to get the startpfn. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240525023040.13509-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/mm_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index f72b852bd5b8..2dfb87841fdb 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -363,7 +363,7 @@ static void __init find_zone_movable_pfns_for_nodes(void) nid = memblock_get_region_node(r); - usable_startpfn = PFN_DOWN(r->base); + usable_startpfn = memblock_region_memory_base_pfn(r); zone_movable_pfn[nid] = zone_movable_pfn[nid] ? min(usable_startpfn, zone_movable_pfn[nid]) : usable_startpfn; From 93bbbcb1e762a49fc18ee1272545b77371595f1e Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 25 May 2024 02:30:39 +0000 Subject: [PATCH 062/280] mm/memblock: fix a typo in description of for_each_mem_region() No functional change. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240525023040.13509-2-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- include/linux/memblock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index e2082240586d..6cf18dc2b4d0 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -565,7 +565,7 @@ static inline unsigned long memblock_region_reserved_end_pfn(const struct memblo } /** - * for_each_mem_region - itereate over memory regions + * for_each_mem_region - iterate over memory regions * @region: loop variable */ #define for_each_mem_region(region) \ From 922306a253e20ad5d0c4b8479d2dc5df9f325a04 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 25 May 2024 02:30:40 +0000 Subject: [PATCH 063/280] mm/mm_init.c: move nr_initialised reset down a bit We don't need to count nr_initialised in two cases: * for low zones that are always populated * after first_deferred_pfn is detected Let's move the nr_initialised reset down a bit to reduce some comparison of prev_end_pfn and end_pfn. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240525023040.13509-3-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/mm_init.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 2dfb87841fdb..bdbd800b0436 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -676,6 +676,14 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) if (early_page_ext_enabled()) return false; + + /* Always populate low zones for address-constrained allocations */ + if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) + return false; + + if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) + return true; + /* * prev_end_pfn static that contains the end of previous zone * No need to protect because called very early in boot before smp_init. @@ -685,12 +693,6 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) nr_initialised = 0; } - /* Always populate low zones for address-constrained allocations */ - if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) - return false; - - if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) - return true; /* * We start only with one section of pages, more pages are added as * needed until the rest of deferred pages are initialized. From 3f29f6537f54d74e64bac0a390fb2e26da25800d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:46 +0200 Subject: [PATCH 064/280] s390/uv: Don't call folio_wait_writeback() without a folio reference folio_wait_writeback() requires that no spinlocks are held and that a folio reference is held, as documented. After we dropped the PTL, the folio could get freed concurrently. So grab a temporary reference. Fixes: 214d9bbcd3a6 ("s390/mm: provide memory management functions for protected KVM guests") Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-2-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/uv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 265fea37e030..016993e9eb72 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -318,6 +318,13 @@ again: rc = make_folio_secure(folio, uvcb); folio_unlock(folio); } + + /* + * Once we drop the PTL, the folio may get unmapped and + * freed immediately. We need a temporary reference. + */ + if (rc == -EAGAIN) + folio_get(folio); } unlock: pte_unmap_unlock(ptep, ptelock); @@ -330,6 +337,7 @@ out: * completion, this is just a useless check, but it is safe. */ folio_wait_writeback(folio); + folio_put(folio); } else if (rc == -EBUSY) { /* * If we have tried a local drain and the folio refcount From 68ad4743beaa7187eb026f1f9bec8848ecf5ed43 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:47 +0200 Subject: [PATCH 065/280] s390/uv: gmap_make_secure() cleanups for further changes Let's factor out handling of LRU cache draining and convert the if-else chain to a switch-case. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-3-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/uv.c | 66 ++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 016993e9eb72..25fe28d189df 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -266,6 +266,36 @@ static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_str return atomic_read(&mm->context.protected_count) > 1; } +/* + * Drain LRU caches: the local one on first invocation and the ones of all + * CPUs on successive invocations. Returns "true" on the first invocation. + */ +static bool drain_lru(bool *drain_lru_called) +{ + /* + * If we have tried a local drain and the folio refcount + * still does not match our expected safe value, try with a + * system wide drain. This is needed if the pagevecs holding + * the page are on a different CPU. + */ + if (*drain_lru_called) { + lru_add_drain_all(); + /* We give up here, don't retry immediately. */ + return false; + } + /* + * We are here if the folio refcount does not match the + * expected safe value. The main culprits are usually + * pagevecs. With lru_add_drain() we drain the pagevecs + * on the local CPU so that hopefully the refcount will + * reach the expected safe value. + */ + lru_add_drain(); + *drain_lru_called = true; + /* The caller should try again immediately */ + return true; +} + /* * Requests the Ultravisor to make a page accessible to a guest. * If it's brought in the first time, it will be cleared. If @@ -275,7 +305,7 @@ static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_str int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb) { struct vm_area_struct *vma; - bool local_drain = false; + bool drain_lru_called = false; spinlock_t *ptelock; unsigned long uaddr; struct folio *folio; @@ -331,37 +361,21 @@ unlock: out: mmap_read_unlock(gmap->mm); - if (rc == -EAGAIN) { + switch (rc) { + case -EAGAIN: /* * If we are here because the UVC returned busy or partial * completion, this is just a useless check, but it is safe. */ folio_wait_writeback(folio); folio_put(folio); - } else if (rc == -EBUSY) { - /* - * If we have tried a local drain and the folio refcount - * still does not match our expected safe value, try with a - * system wide drain. This is needed if the pagevecs holding - * the page are on a different CPU. - */ - if (local_drain) { - lru_add_drain_all(); - /* We give up here, and let the caller try again */ - return -EAGAIN; - } - /* - * We are here if the folio refcount does not match the - * expected safe value. The main culprits are usually - * pagevecs. With lru_add_drain() we drain the pagevecs - * on the local CPU so that hopefully the refcount will - * reach the expected safe value. - */ - lru_add_drain(); - local_drain = true; - /* And now we try again immediately after draining */ - goto again; - } else if (rc == -ENXIO) { + return -EAGAIN; + case -EBUSY: + /* Additional folio references. */ + if (drain_lru(&drain_lru_called)) + goto again; + return -EAGAIN; + case -ENXIO: if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE)) return -EFAULT; return -EAGAIN; From eef88fe45ac9783e0dd108b033c1e933c38cab34 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:48 +0200 Subject: [PATCH 066/280] s390/uv: Split large folios in gmap_make_secure() While s390x makes sure to never have PMD-mapped THP in processes that use KVM -- by remapping them using PTEs in thp_split_walk_pmd_entry()->split_huge_pmd() -- there is still the possibility of having PTE-mapped THPs (large folios) mapped into guest memory. This would happen if user space allocates memory before calling KVM_CREATE_VM (which would call s390_enable_sie()). With upstream QEMU, this currently doesn't happen, because guest memory is setup and conditionally preallocated after KVM_CREATE_VM. Could it happen with shmem/file-backed memory when another process allocated memory in the pagecache? Likely, although currently not a common setup. Trying to split any PTE-mapped large folios sounds like the right and future-proof thing to do here. So let's call split_folio() and handle the return values accordingly. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-4-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/uv.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 25fe28d189df..3c6d86e3e828 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -338,11 +338,10 @@ again: goto out; if (pte_present(*ptep) && !(pte_val(*ptep) & _PAGE_INVALID) && pte_write(*ptep)) { folio = page_folio(pte_page(*ptep)); - rc = -EINVAL; - if (folio_test_large(folio)) - goto unlock; rc = -EAGAIN; - if (folio_trylock(folio)) { + if (folio_test_large(folio)) { + rc = -E2BIG; + } else if (folio_trylock(folio)) { if (should_export_before_import(uvcb, gmap->mm)) uv_convert_from_secure(PFN_PHYS(folio_pfn(folio))); rc = make_folio_secure(folio, uvcb); @@ -353,15 +352,35 @@ again: * Once we drop the PTL, the folio may get unmapped and * freed immediately. We need a temporary reference. */ - if (rc == -EAGAIN) + if (rc == -EAGAIN || rc == -E2BIG) folio_get(folio); } -unlock: pte_unmap_unlock(ptep, ptelock); out: mmap_read_unlock(gmap->mm); switch (rc) { + case -E2BIG: + folio_lock(folio); + rc = split_folio(folio); + folio_unlock(folio); + folio_put(folio); + + switch (rc) { + case 0: + /* Splitting succeeded, try again immediately. */ + goto again; + case -EAGAIN: + /* Additional folio references. */ + if (drain_lru(&drain_lru_called)) + goto again; + return -EAGAIN; + case -EBUSY: + /* Unexpected race. */ + return -EAGAIN; + } + WARN_ON_ONCE(1); + return -ENXIO; case -EAGAIN: /* * If we are here because the UVC returned busy or partial From 036c0e104bd2cf45304a3795fd8ec985ea01bfe8 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:49 +0200 Subject: [PATCH 067/280] s390/uv: Convert PG_arch_1 users to only work on small folios Now that make_folio_secure() may only set PG_arch_1 for small folios, let's convert relevant remaining UV code to only work on (small) folios and simply reject large folios early. This way, we'll never end up touching PG_arch_1 on tail pages of a large folio in UV code. The folio_get()/folio_put() for functions that are documented to already hold a folio reference look weird; likely they are required to make concurrent gmap_make_secure() back off because the caller might only hold an implicit reference due to the page mapping. So leave that alone for now. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-5-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/page.h | 2 ++ arch/s390/kernel/uv.c | 41 ++++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 224ff9d433ea..ecbf4b626f46 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -247,7 +247,9 @@ static inline unsigned long __phys_addr(unsigned long x, bool is_31bit) #define pfn_to_phys(pfn) ((pfn) << PAGE_SHIFT) #define phys_to_page(phys) pfn_to_page(phys_to_pfn(phys)) +#define phys_to_folio(phys) page_folio(phys_to_page(phys)) #define page_to_phys(page) pfn_to_phys(page_to_pfn(page)) +#define folio_to_phys(page) pfn_to_phys(folio_pfn(folio)) static inline void *pfn_to_virt(unsigned long pfn) { diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 3c6d86e3e828..914dcec27329 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -135,14 +135,18 @@ static int uv_destroy_page(unsigned long paddr) */ int uv_destroy_owned_page(unsigned long paddr) { - struct page *page = phys_to_page(paddr); + struct folio *folio = phys_to_folio(paddr); int rc; - get_page(page); + /* See gmap_make_secure(): large folios cannot be secure */ + if (unlikely(folio_test_large(folio))) + return 0; + + folio_get(folio); rc = uv_destroy_page(paddr); if (!rc) - clear_bit(PG_arch_1, &page->flags); - put_page(page); + clear_bit(PG_arch_1, &folio->flags); + folio_put(folio); return rc; } @@ -170,14 +174,18 @@ int uv_convert_from_secure(unsigned long paddr) */ int uv_convert_owned_from_secure(unsigned long paddr) { - struct page *page = phys_to_page(paddr); + struct folio *folio = phys_to_folio(paddr); int rc; - get_page(page); + /* See gmap_make_secure(): large folios cannot be secure */ + if (unlikely(folio_test_large(folio))) + return 0; + + folio_get(folio); rc = uv_convert_from_secure(paddr); if (!rc) - clear_bit(PG_arch_1, &page->flags); - put_page(page); + clear_bit(PG_arch_1, &folio->flags); + folio_put(folio); return rc; } @@ -479,33 +487,34 @@ EXPORT_SYMBOL_GPL(gmap_destroy_page); */ int arch_make_page_accessible(struct page *page) { + struct folio *folio = page_folio(page); int rc = 0; - /* Hugepage cannot be protected, so nothing to do */ - if (PageHuge(page)) + /* See gmap_make_secure(): large folios cannot be secure */ + if (unlikely(folio_test_large(folio))) return 0; /* * PG_arch_1 is used in 3 places: * 1. for kernel page tables during early boot * 2. for storage keys of huge pages and KVM - * 3. As an indication that this page might be secure. This can + * 3. As an indication that this small folio might be secure. This can * overindicate, e.g. we set the bit before calling * convert_to_secure. * As secure pages are never huge, all 3 variants can co-exists. */ - if (!test_bit(PG_arch_1, &page->flags)) + if (!test_bit(PG_arch_1, &folio->flags)) return 0; - rc = uv_pin_shared(page_to_phys(page)); + rc = uv_pin_shared(folio_to_phys(folio)); if (!rc) { - clear_bit(PG_arch_1, &page->flags); + clear_bit(PG_arch_1, &folio->flags); return 0; } - rc = uv_convert_from_secure(page_to_phys(page)); + rc = uv_convert_from_secure(folio_to_phys(folio)); if (!rc) { - clear_bit(PG_arch_1, &page->flags); + clear_bit(PG_arch_1, &folio->flags); return 0; } From 80cf817949264eff32642aa90da00f03e84e3c0f Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:50 +0200 Subject: [PATCH 068/280] s390/uv: Update PG_arch_1 comment We removed the usage of PG_arch_1 for page tables in commit a51324c430db ("s390/cmma: rework no-dat handling"). Let's update the comment in UV to reflect that. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-6-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/kernel/uv.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 914dcec27329..ecfc08902215 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -495,13 +495,12 @@ int arch_make_page_accessible(struct page *page) return 0; /* - * PG_arch_1 is used in 3 places: - * 1. for kernel page tables during early boot - * 2. for storage keys of huge pages and KVM - * 3. As an indication that this small folio might be secure. This can + * PG_arch_1 is used in 2 places: + * 1. for storage keys of hugetlb folios and KVM + * 2. As an indication that this small folio might be secure. This can * overindicate, e.g. we set the bit before calling * convert_to_secure. - * As secure pages are never huge, all 3 variants can co-exists. + * As secure pages are never large folios, both variants can co-exists. */ if (!test_bit(PG_arch_1, &folio->flags)) return 0; From e58623fbc178d6c074074c0107103c5d3c8041b1 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:51 +0200 Subject: [PATCH 069/280] s390/uv: Make uv_convert_from_secure() a static function It's not used outside of uv.c, so let's make it a static function. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-7-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/uv.h | 6 ------ arch/s390/kernel/uv.c | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index 0e7bd3873907..d2205ff97007 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -484,7 +484,6 @@ int uv_pin_shared(unsigned long paddr); int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb); int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr); int uv_destroy_owned_page(unsigned long paddr); -int uv_convert_from_secure(unsigned long paddr); int uv_convert_owned_from_secure(unsigned long paddr); int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr); @@ -503,11 +502,6 @@ static inline int uv_destroy_owned_page(unsigned long paddr) return 0; } -static inline int uv_convert_from_secure(unsigned long paddr) -{ - return 0; -} - static inline int uv_convert_owned_from_secure(unsigned long paddr) { return 0; diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index ecfc08902215..3d3250b406a6 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -156,7 +156,7 @@ int uv_destroy_owned_page(unsigned long paddr) * * @paddr: Absolute host address of page to be exported */ -int uv_convert_from_secure(unsigned long paddr) +static int uv_convert_from_secure(unsigned long paddr) { struct uv_cb_cfs uvcb = { .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR, From 7063150650d828ad3ca77b06531b37bb1aed82b6 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:52 +0200 Subject: [PATCH 070/280] s390/uv: Convert uv_destroy_owned_page() to uv_destroy_(folio|pte)() Let's have the following variants for destroying pages: (1) uv_destroy(): Like uv_pin_shared() and uv_convert_from_secure(), "low level" helper that operates on paddr and doesn't mess with folios. (2) uv_destroy_folio(): Consumes a folio to which we hold a reference. (3) uv_destroy_pte(): Consumes a PTE that holds a reference through the mapping. Unfortunately we need uv_destroy_pte(), because pfn_folio() and friends are not available in pgtable.h. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-8-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/pgtable.h | 2 +- arch/s390/include/asm/uv.h | 10 ++++++++-- arch/s390/kernel/uv.c | 24 +++++++++++++++++------- arch/s390/mm/gmap.c | 6 ++++-- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 70b6ee557eb2..e9eaa4281d94 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1217,7 +1217,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, * The notifier should have destroyed all protected vCPUs at this * point, so the destroy should be successful. */ - if (full && !uv_destroy_owned_page(pte_val(res) & PAGE_MASK)) + if (full && !uv_destroy_pte(res)) return res; /* * If something went wrong and the page could not be destroyed, or diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index d2205ff97007..a1bef30066ef 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -483,7 +483,8 @@ static inline int is_prot_virt_host(void) int uv_pin_shared(unsigned long paddr); int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb); int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr); -int uv_destroy_owned_page(unsigned long paddr); +int uv_destroy_folio(struct folio *folio); +int uv_destroy_pte(pte_t pte); int uv_convert_owned_from_secure(unsigned long paddr); int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr); @@ -497,7 +498,12 @@ static inline int uv_pin_shared(unsigned long paddr) return 0; } -static inline int uv_destroy_owned_page(unsigned long paddr) +static inline int uv_destroy_folio(struct folio *folio) +{ + return 0; +} + +static inline int uv_destroy_pte(pte_t pte) { return 0; } diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 3d3250b406a6..61c1ce51c883 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -110,7 +110,7 @@ EXPORT_SYMBOL_GPL(uv_pin_shared); * * @paddr: Absolute host address of page to be destroyed */ -static int uv_destroy_page(unsigned long paddr) +static int uv_destroy(unsigned long paddr) { struct uv_cb_cfs uvcb = { .header.cmd = UVC_CMD_DESTR_SEC_STOR, @@ -131,11 +131,10 @@ static int uv_destroy_page(unsigned long paddr) } /* - * The caller must already hold a reference to the page + * The caller must already hold a reference to the folio */ -int uv_destroy_owned_page(unsigned long paddr) +int uv_destroy_folio(struct folio *folio) { - struct folio *folio = phys_to_folio(paddr); int rc; /* See gmap_make_secure(): large folios cannot be secure */ @@ -143,13 +142,22 @@ int uv_destroy_owned_page(unsigned long paddr) return 0; folio_get(folio); - rc = uv_destroy_page(paddr); + rc = uv_destroy(folio_to_phys(folio)); if (!rc) clear_bit(PG_arch_1, &folio->flags); folio_put(folio); return rc; } +/* + * The present PTE still indirectly holds a folio reference through the mapping. + */ +int uv_destroy_pte(pte_t pte) +{ + VM_WARN_ON(!pte_present(pte)); + return uv_destroy_folio(pfn_folio(pte_pfn(pte))); +} + /* * Requests the Ultravisor to encrypt a guest page and make it * accessible to the host for paging (export). @@ -437,6 +445,7 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr) { struct vm_area_struct *vma; unsigned long uaddr; + struct folio *folio; struct page *page; int rc; @@ -460,7 +469,8 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr) page = follow_page(vma, uaddr, FOLL_WRITE | FOLL_GET); if (IS_ERR_OR_NULL(page)) goto out; - rc = uv_destroy_owned_page(page_to_phys(page)); + folio = page_folio(page); + rc = uv_destroy_folio(folio); /* * Fault handlers can race; it is possible that two CPUs will fault * on the same secure page. One CPU can destroy the page, reboot, @@ -472,7 +482,7 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr) */ if (rc) rc = uv_convert_owned_from_secure(page_to_phys(page)); - put_page(page); + folio_put(folio); out: mmap_read_unlock(gmap->mm); return rc; diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 474a25ca5c48..7537e7b4be39 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2841,13 +2841,15 @@ static const struct mm_walk_ops gather_pages_ops = { */ void s390_uv_destroy_pfns(unsigned long count, unsigned long *pfns) { + struct folio *folio; unsigned long i; for (i = 0; i < count; i++) { + folio = pfn_folio(pfns[i]); /* we always have an extra reference */ - uv_destroy_owned_page(pfn_to_phys(pfns[i])); + uv_destroy_folio(folio); /* get rid of the extra reference */ - put_page(pfn_to_page(pfns[i])); + folio_put(folio); cond_resched(); } } From 7d17143469879409692dc6279794e71be4bb1196 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:53 +0200 Subject: [PATCH 071/280] s390/uv: Convert uv_convert_owned_from_secure() to uv_convert_from_secure_(folio|pte)() Let's do the same as we did for uv_destroy_(folio|pte)() and have the following variants: (1) uv_convert_from_secure(): "low level" helper that operates on paddr and does not mess with folios. (2) uv_convert_from_secure_folio(): Consumes a folio to which we hold a reference. (3) uv_convert_from_secure_pte(): Consumes a PTE that holds a reference through the mapping. Unfortunately we need uv_convert_from_secure_pte(), because pfn_folio() and friends are not available in pgtable.h. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-9-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/pgtable.h | 6 +++--- arch/s390/include/asm/uv.h | 4 ++-- arch/s390/kernel/uv.c | 18 +++++++++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index e9eaa4281d94..0fd5a9c7e901 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1167,7 +1167,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); /* At this point the reference through the mapping is still present */ if (mm_is_protected(mm) && pte_present(res)) - uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK); + uv_convert_from_secure_pte(res); return res; } @@ -1185,7 +1185,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); /* At this point the reference through the mapping is still present */ if (mm_is_protected(vma->vm_mm) && pte_present(res)) - uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK); + uv_convert_from_secure_pte(res); return res; } @@ -1224,7 +1224,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, * if this is not a mm teardown, the slower export is used as * fallback instead. */ - uv_convert_owned_from_secure(pte_val(res) & PAGE_MASK); + uv_convert_from_secure_pte(res); return res; } diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index a1bef30066ef..0679445cac0b 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -485,7 +485,7 @@ int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb); int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr); int uv_destroy_folio(struct folio *folio); int uv_destroy_pte(pte_t pte); -int uv_convert_owned_from_secure(unsigned long paddr); +int uv_convert_from_secure_pte(pte_t pte); int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr); void setup_uv(void); @@ -508,7 +508,7 @@ static inline int uv_destroy_pte(pte_t pte) return 0; } -static inline int uv_convert_owned_from_secure(unsigned long paddr) +static inline int uv_convert_from_secure_pte(pte_t pte) { return 0; } diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 61c1ce51c883..b456066d72da 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -178,11 +178,10 @@ static int uv_convert_from_secure(unsigned long paddr) } /* - * The caller must already hold a reference to the page + * The caller must already hold a reference to the folio. */ -int uv_convert_owned_from_secure(unsigned long paddr) +static int uv_convert_from_secure_folio(struct folio *folio) { - struct folio *folio = phys_to_folio(paddr); int rc; /* See gmap_make_secure(): large folios cannot be secure */ @@ -190,13 +189,22 @@ int uv_convert_owned_from_secure(unsigned long paddr) return 0; folio_get(folio); - rc = uv_convert_from_secure(paddr); + rc = uv_convert_from_secure(folio_to_phys(folio)); if (!rc) clear_bit(PG_arch_1, &folio->flags); folio_put(folio); return rc; } +/* + * The present PTE still indirectly holds a folio reference through the mapping. + */ +int uv_convert_from_secure_pte(pte_t pte) +{ + VM_WARN_ON(!pte_present(pte)); + return uv_convert_from_secure_folio(pfn_folio(pte_pfn(pte))); +} + /* * Calculate the expected ref_count for a folio that would otherwise have no * further pins. This was cribbed from similar functions in other places in @@ -481,7 +489,7 @@ int gmap_destroy_page(struct gmap *gmap, unsigned long gaddr) * we instead try to export the page. */ if (rc) - rc = uv_convert_owned_from_secure(page_to_phys(page)); + rc = uv_convert_from_secure_folio(folio); folio_put(folio); out: mmap_read_unlock(gmap->mm); From 99b3f8f76f7a9eb2816d2af5b0b9de4a11dbc721 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:54 +0200 Subject: [PATCH 072/280] s390/uv: Implement HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE Let's also implement HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE, so we can convert arch_make_page_accessible() to be a simple wrapper around arch_make_folio_accessible(). Unfortunately, we cannot do that in the header. There are only two arch_make_page_accessible() calls remaining in gup.c. We can now drop HAVE_ARCH_MAKE_PAGE_ACCESSIBLE completely form core-MM. We'll handle that separately, once the s390x part landed. Suggested-by: Matthew Wilcox Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-10-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/page.h | 3 +++ arch/s390/kernel/uv.c | 18 +++++++++++------- arch/s390/mm/fault.c | 14 ++++++++------ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index ecbf4b626f46..5ec41ec3d761 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -162,6 +162,7 @@ static inline int page_reset_referenced(unsigned long addr) #define _PAGE_ACC_BITS 0xf0 /* HW access control bits */ struct page; +struct folio; void arch_free_page(struct page *page, int order); void arch_alloc_page(struct page *page, int order); @@ -174,6 +175,8 @@ static inline int devmem_is_allowed(unsigned long pfn) #define HAVE_ARCH_ALLOC_PAGE #if IS_ENABLED(CONFIG_PGSTE) +int arch_make_folio_accessible(struct folio *folio); +#define HAVE_ARCH_MAKE_FOLIO_ACCESSIBLE int arch_make_page_accessible(struct page *page); #define HAVE_ARCH_MAKE_PAGE_ACCESSIBLE #endif diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index b456066d72da..fa62fa0e369f 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -498,14 +498,13 @@ out: EXPORT_SYMBOL_GPL(gmap_destroy_page); /* - * To be called with the page locked or with an extra reference! This will - * prevent gmap_make_secure from touching the page concurrently. Having 2 - * parallel make_page_accessible is fine, as the UV calls will become a - * no-op if the page is already exported. + * To be called with the folio locked or with an extra reference! This will + * prevent gmap_make_secure from touching the folio concurrently. Having 2 + * parallel arch_make_folio_accessible is fine, as the UV calls will become a + * no-op if the folio is already exported. */ -int arch_make_page_accessible(struct page *page) +int arch_make_folio_accessible(struct folio *folio) { - struct folio *folio = page_folio(page); int rc = 0; /* See gmap_make_secure(): large folios cannot be secure */ @@ -537,8 +536,13 @@ int arch_make_page_accessible(struct page *page) return rc; } -EXPORT_SYMBOL_GPL(arch_make_page_accessible); +EXPORT_SYMBOL_GPL(arch_make_folio_accessible); +int arch_make_page_accessible(struct page *page) +{ + return arch_make_folio_accessible(page_folio(page)); +} +EXPORT_SYMBOL_GPL(arch_make_page_accessible); #endif #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 65747f15dbec..7cd50ad3b4ad 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -492,6 +492,7 @@ void do_secure_storage_access(struct pt_regs *regs) unsigned long addr = get_fault_address(regs); struct vm_area_struct *vma; struct mm_struct *mm; + struct folio *folio; struct page *page; struct gmap *gmap; int rc; @@ -539,17 +540,18 @@ void do_secure_storage_access(struct pt_regs *regs) mmap_read_unlock(mm); break; } - if (arch_make_page_accessible(page)) + folio = page_folio(page); + if (arch_make_folio_accessible(folio)) send_sig(SIGSEGV, current, 0); - put_page(page); + folio_put(folio); mmap_read_unlock(mm); break; case KERNEL_FAULT: - page = phys_to_page(addr); - if (unlikely(!try_get_page(page))) + folio = phys_to_folio(addr); + if (unlikely(!folio_try_get(folio))) break; - rc = arch_make_page_accessible(page); - put_page(page); + rc = arch_make_folio_accessible(folio); + folio_put(folio); if (rc) BUG(); break; From 1433b36e3ab67772a37db624fc1f8e66d443690d Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Wed, 8 May 2024 20:29:55 +0200 Subject: [PATCH 073/280] s390/hugetlb: Convert PG_arch_1 code to work on folio->flags Let's make it clearer that we are always working on folio flags and never page flags of tail pages by converting remaining PG_arch_1 users that modify page->flags to modify folio->flags instead. No functional change intended, because we would always have worked with the head page (where page->flags corresponds to folio->flags) and never with tail pages. Reviewed-by: Claudio Imbrenda Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20240508182955.358628-11-david@redhat.com Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/mm/gmap.c | 4 ++-- arch/s390/mm/hugetlbpage.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 7537e7b4be39..d5a5756dd69f 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2733,7 +2733,7 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, { pmd_t *pmd = (pmd_t *)pte; unsigned long start, end; - struct page *page = pmd_page(*pmd); + struct folio *folio = page_folio(pmd_page(*pmd)); /* * The write check makes sure we do not set a key on shared @@ -2748,7 +2748,7 @@ static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr, start = pmd_val(*pmd) & HPAGE_MASK; end = start + HPAGE_SIZE; __storage_key_init_range(start, end); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_arch_1, &folio->flags); cond_resched(); return 0; } diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 2675aab4acc7..34d558164f0d 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -121,7 +121,7 @@ static inline pte_t __rste_to_pte(unsigned long rste) static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste) { - struct page *page; + struct folio *folio; unsigned long size, paddr; if (!mm_uses_skeys(mm) || @@ -129,16 +129,16 @@ static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste) return; if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) { - page = pud_page(__pud(rste)); + folio = page_folio(pud_page(__pud(rste))); size = PUD_SIZE; paddr = rste & PUD_MASK; } else { - page = pmd_page(__pmd(rste)); + folio = page_folio(pmd_page(__pmd(rste))); size = PMD_SIZE; paddr = rste & PMD_MASK; } - if (!test_and_set_bit(PG_arch_1, &page->flags)) + if (!test_and_set_bit(PG_arch_1, &folio->flags)) __storage_key_init_range(paddr, paddr + size); } From 24311962484385fa3a06c4fc1f2864b613df5af6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 20:53:37 -0400 Subject: [PATCH 074/280] ftrace: Add back ftrace_update_trampoline() to ftrace_update_pid_func() The update to the ops trampoline done by the function ftrace_update_trampoline() was accidentally removed from ftrace_update_pid_func(). Add it back. Link: https://lore.kernel.org/linux-trace-kernel/20240605205337.6115e9a5@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Fixes: df3ec5da6a1e ("function_graph: Add pid tracing back to function graph tracer") Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index da7e6abf48b4..897d7541041c 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -402,6 +402,7 @@ static void ftrace_update_pid_func(void) if (op->flags & FTRACE_OPS_FL_PID) { op->func = ftrace_pids_enabled(op) ? ftrace_pid_func : op->saved_func; + ftrace_update_trampoline(op); } } while_for_each_ftrace_op(op); From 5efe9688f9488e8e95177d03be2678021233e877 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 4 Jun 2024 15:25:50 -0400 Subject: [PATCH 075/280] ftrace/selftests: Fix pid test with function graph not showing pids The pid filtering test will set the pid filters and make sure that both function and function_graph tracing honors the filters. But the function_graph tracer test was failing because the PID was not being filtered properly. That's because the funcgraph-proc option wasn't getting set. Without that option the PID is not shown. Instead we get: + cat trace # tracer: function_graph # # CPU DURATION FUNCTION CALLS # | | | | | | | 3) ! 143.685 us | kernel_clone(); 3) ! 127.055 us | kernel_clone(); 1) ! 127.170 us | kernel_clone(); 3) ! 126.840 us | kernel_clone(); When we should be getting: + cat trace # tracer: function_graph # # CPU TASK/PID DURATION FUNCTION CALLS # | | | | | | | | | 4) bash-939 | # 1070.009 us | kernel_clone(); 4) bash-939 | # 1116.903 us | kernel_clone(); 5) bash-939 | ! 976.133 us | kernel_clone(); 5) bash-939 | ! 954.012 us | kernel_clone(); The test looks for the pids it is filtering and will fail if it can not find them. Without fungraph-proc option set, it will not be displayed and the test will fail. Link: https://lore.kernel.org/all/Zl9JFnzKGuUM10X2@J2N7QTR9R3/ Link: https://lore.kernel.org/linux-trace-kernel/20240604152550.0c01d7cd@gandalf.local.home Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Fixes: 35b944a997e2 ("selftests/ftrace: Add function_graph tracer to func-filter-pid test") Reported-by: Mark Rutland Signed-off-by: Steven Rostedt (Google) Tested-by: Mark Rutland --- tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc index c6fc9d31a496..8dcce001881d 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-pid.tc @@ -8,6 +8,7 @@ # Also test it on an instance directory do_function_fork=1 +do_funcgraph_proc=1 if [ ! -f options/function-fork ]; then do_function_fork=0 @@ -28,6 +29,7 @@ fi if [ $do_funcgraph_proc -eq 1 ]; then orig_value2=`cat options/funcgraph-proc` + echo 1 > options/funcgraph-proc fi do_reset() { From ce8ebb95439459f7e24b02c6943e278f46d2d328 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 5 Jun 2024 07:13:37 +0000 Subject: [PATCH 076/280] mm/mm_init.c: get the highest zone directly We have recorded nr_zones in pgdat, just get it directly. Signed-off-by: Wei Yang Reviewed-by: Mike Rapoport (IBM) Link: https://lore.kernel.org/all/20240605071339.15330-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/mm_init.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index bdbd800b0436..6eda9367730b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2140,7 +2140,7 @@ static int __init deferred_init_memmap(void *data) unsigned long first_init_pfn, flags; unsigned long start = jiffies; struct zone *zone; - int zid, max_threads; + int max_threads; u64 i; /* Bind memory initialisation thread to a local node if possible */ @@ -2167,12 +2167,8 @@ static int __init deferred_init_memmap(void *data) */ pgdat_resize_unlock(pgdat, &flags); - /* Only the highest zone is deferred so find it */ - for (zid = 0; zid < MAX_NR_ZONES; zid++) { - zone = pgdat->node_zones + zid; - if (first_init_pfn < zone_end_pfn(zone)) - break; - } + /* Only the highest zone is deferred */ + zone = pgdat->node_zones + pgdat->nr_zones - 1; /* If the zone is empty somebody else may have cleared out the zone */ if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, @@ -2200,7 +2196,7 @@ static int __init deferred_init_memmap(void *data) } zone_empty: /* Sanity check that the next zone really is unpopulated */ - WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone)); + WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone)); pr_info("node %d deferred pages initialised in %ums\n", pgdat->node_id, jiffies_to_msecs(jiffies - start)); From 544b8e14c24b1f1927659e546d008d678e77e19c Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 5 Jun 2024 07:13:38 +0000 Subject: [PATCH 077/280] mm/mm_init.c: use deferred_init_mem_pfn_range_in_zone() to decide loop condition If deferred_init_mem_pfn_range_in_zone() return true, we know it finds some range in (spfn, epfn). Then we can use it directly for the loop condition. Signed-off-by: Wei Yang Reviewed-by: Mike Rapoport (IBM) Link: https://lore.kernel.org/all/20240605071339.15330-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/mm_init.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 6eda9367730b..c9c8c7458f27 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2170,20 +2170,15 @@ static int __init deferred_init_memmap(void *data) /* Only the highest zone is deferred */ zone = pgdat->node_zones + pgdat->nr_zones - 1; - /* If the zone is empty somebody else may have cleared out the zone */ - if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - first_init_pfn)) - goto zone_empty; - max_threads = deferred_page_init_max_threads(cpumask); - while (spfn < epfn) { - unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION); + while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_init_pfn)) { + first_init_pfn = ALIGN(epfn, PAGES_PER_SECTION); struct padata_mt_job job = { .thread_fn = deferred_init_memmap_chunk, .fn_arg = zone, .start = spfn, - .size = epfn_align - spfn, + .size = first_init_pfn - spfn, .align = PAGES_PER_SECTION, .min_chunk = PAGES_PER_SECTION, .max_threads = max_threads, @@ -2191,10 +2186,8 @@ static int __init deferred_init_memmap(void *data) }; padata_do_multithreaded(&job); - deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - epfn_align); } -zone_empty: + /* Sanity check that the next zone really is unpopulated */ WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone)); From f1180fd2a7c039691b64ebf404c746a74e40b7b0 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Wed, 5 Jun 2024 07:13:39 +0000 Subject: [PATCH 078/280] mm/mm_init.c: not always search next deferred_init_pfn from very beginning In function deferred_init_memmap(), we call deferred_init_mem_pfn_range_in_zone() to get the next deferred_init_pfn. But we always search it from the very beginning. Since we save the index in i, we can leverage this to search from i next time. [rppt refine the comment] Signed-off-by: Wei Yang Link: https://lore.kernel.org/all/20240605071339.15330-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- include/linux/memblock.h | 19 ------------------- mm/mm_init.c | 23 ++++++++++++++--------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 6cf18dc2b4d0..45cac33334c8 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -299,25 +299,6 @@ void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, void __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, unsigned long *out_spfn, unsigned long *out_epfn); -/** - * for_each_free_mem_pfn_range_in_zone - iterate through zone specific free - * memblock areas - * @i: u64 used as loop variable - * @zone: zone in which all of the memory blocks reside - * @p_start: ptr to phys_addr_t for start address of the range, can be %NULL - * @p_end: ptr to phys_addr_t for end address of the range, can be %NULL - * - * Walks over free (memory && !reserved) areas of memblock in a specific - * zone. Available once memblock and an empty zone is initialized. The main - * assumption is that the zone start, end, and pgdat have been associated. - * This way we can use the zone to determine NUMA node, and if a given part - * of the memblock is valid for the zone. - */ -#define for_each_free_mem_pfn_range_in_zone(i, zone, p_start, p_end) \ - for (i = 0, \ - __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end); \ - i != U64_MAX; \ - __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end)) /** * for_each_free_mem_pfn_range_in_zone_from - iterate through zone specific diff --git a/mm/mm_init.c b/mm/mm_init.c index c9c8c7458f27..5a4c846393b7 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2021,24 +2021,29 @@ static unsigned long __init deferred_init_pages(struct zone *zone, } /* - * This function is meant to pre-load the iterator for the zone init. - * Specifically it walks through the ranges until we are caught up to the - * first_init_pfn value and exits there. If we never encounter the value we - * return false indicating there are no valid ranges left. + * This function is meant to pre-load the iterator for the zone init from + * a given point. + * Specifically it walks through the ranges starting with initial index + * passed to it until we are caught up to the first_init_pfn value and + * exits there. If we never encounter the value we return false indicating + * there are no valid ranges left. */ static bool __init deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, unsigned long *spfn, unsigned long *epfn, unsigned long first_init_pfn) { - u64 j; + u64 j = *i; + + if (j == 0) + __next_mem_pfn_range_in_zone(&j, zone, spfn, epfn); /* * Start out by walking through the ranges in this zone that have * already been initialized. We don't need to do anything with them * so we just need to flush them out of the system. */ - for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) { + for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) { if (*epfn <= first_init_pfn) continue; if (*spfn < first_init_pfn) @@ -2110,7 +2115,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, { unsigned long spfn, epfn; struct zone *zone = arg; - u64 i; + u64 i = 0; deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); @@ -2141,7 +2146,7 @@ static int __init deferred_init_memmap(void *data) unsigned long start = jiffies; struct zone *zone; int max_threads; - u64 i; + u64 i = 0; /* Bind memory initialisation thread to a local node if possible */ if (!cpumask_empty(cpumask)) @@ -2216,7 +2221,7 @@ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; unsigned long spfn, epfn, flags; unsigned long nr_pages = 0; - u64 i; + u64 i = 0; /* Only the last zone may have deferred pages */ if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) From 9aa61d8ecb7f6f176ff8247a41a4f6eea8376112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Le=20Goffic?= Date: Wed, 5 Jun 2024 16:04:53 +0200 Subject: [PATCH 079/280] perf: parse-events: Fix compilation error while defining DEBUG_PARSER MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compiling perf tool with 'DEBUG_PARSER=1' leads to errors: $> make -C tools/perf PARSER_DEBUG=1 NO_LIBTRACEEVENT=1 ... CC util/expr-flex.o CC util/expr.o util/parse-events.c:33:12: error: redundant redeclaration of ‘parse_events_debug’ [-Werror=redundant-decls] 33 | extern int parse_events_debug; | ^~~~~~~~~~~~~~~~~~ In file included from util/parse-events.c:18: util/parse-events-bison.h:43:12: note: previous declaration of ‘parse_events_debug’ with type ‘int’ 43 | extern int parse_events_debug; | ^~~~~~~~~~~~~~~~~~ util/expr.c:27:12: error: redundant redeclaration of ‘expr_debug’ [-Werror=redundant-decls] 27 | extern int expr_debug; | ^~~~~~~~~~ In file included from util/expr.c:11: util/expr-bison.h:43:12: note: previous declaration of ‘expr_debug’ with type ‘int’ 43 | extern int expr_debug; | ^~~~~~~~~~ cc-1: all warnings being treated as errors Remove extern declaration from the parse-envents.c file as there is a conflict with the ones generated using bison and yacc tools from the file parse-events.[ly]. Signed-off-by: Clément Le Goffic Reviewed-by: Ian Rogers Cc: James Clark Cc: John Garry Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240605140453.614862-1-clement.legoffic@foss.st.com --- tools/perf/util/expr.c | 4 ---- tools/perf/util/parse-events.c | 3 --- 2 files changed, 7 deletions(-) diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index b8875aac8f87..b2536a59c44e 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -25,10 +25,6 @@ #include #include "pmu.h" -#ifdef PARSER_DEBUG -extern int expr_debug; -#endif - struct expr_id_data { union { struct { diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 6ed0f9c5581d..8d5fb05f20c2 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -31,9 +31,6 @@ #define MAX_NAME_LEN 100 -#ifdef PARSER_DEBUG -extern int parse_events_debug; -#endif static int get_config_terms(const struct parse_events_terms *head_config, struct list_head *head_terms); static int parse_events_terms__copy(const struct parse_events_terms *src, From 22b639253ec046d66c69c54b9d28bb1bd30f3a7a Mon Sep 17 00:00:00 2001 From: Jeff Johnson Date: Mon, 6 May 2024 07:22:33 -0700 Subject: [PATCH 080/280] tracing: Fix trace_pid_list_free() kernel-doc make C=1 reports: kernel/trace/pid_list.c:458: warning: Function parameter or struct member 'pid_list' not described in 'trace_pid_list_free' Add the missing parameter to the trace_pid_list_free() kernel-doc. Link: https://lore.kernel.org/linux-trace-kernel/20240506-trace_pid_list_free-kdoc-v1-1-c70f0ae29144@quicinc.com Cc: Masami Hiramatsu Cc: Mathieu Desnoyers Signed-off-by: Jeff Johnson Signed-off-by: Steven Rostedt (Google) --- kernel/trace/pid_list.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/trace/pid_list.c b/kernel/trace/pid_list.c index 95106d02b32d..19b271a12c99 100644 --- a/kernel/trace/pid_list.c +++ b/kernel/trace/pid_list.c @@ -451,6 +451,7 @@ struct trace_pid_list *trace_pid_list_alloc(void) /** * trace_pid_list_free - Frees an allocated pid_list. + * @pid_list: The pid list to free. * * Frees the memory for a pid_list that was allocated. */ From 3afd801f4264c7a6c84957cd3ab653cfd8f61d3e Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 14:03:35 -0400 Subject: [PATCH 081/280] ftrace: Rename dup_hash() and comment it The name "dup_hash()" is a misnomer as it does not duplicate the hash that is passed in, but instead moves its entities from that hash to a newly allocated one. Rename it to "__move_hash()" (using starting underscores as it is an internal function), and add some comments about what it does. Link: https://lore.kernel.org/linux-trace-kernel/20240605180408.537723591@goodmis.org Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Acked-by: Mark Rutland Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 897d7541041c..f4b253d20df8 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1392,7 +1392,11 @@ ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, struct ftrace_hash *new_hash); -static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size) +/* + * Allocate a new hash and remove entries from @src and move them to the new hash. + * On success, the @src hash will be empty and should be freed. + */ +static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) { struct ftrace_func_entry *entry; struct ftrace_hash *new_hash; @@ -1439,7 +1443,7 @@ __ftrace_hash_move(struct ftrace_hash *src) if (ftrace_hash_empty(src)) return EMPTY_HASH; - return dup_hash(src, size); + return __move_hash(src, size); } static int From 07bbe0833ed62f48785dffa8e429f35c1d906415 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 14:03:36 -0400 Subject: [PATCH 082/280] ftrace: Remove "filter_hash" parameter from __ftrace_hash_rec_update() While adding comments to the function __ftrace_hash_rec_update() and trying to describe in detail what the parameter for "filter_hash" does, I realized that it basically does exactly the same thing (but differently) if it is set or not! If it is set, the idea was the ops->filter_hash was being updated, and the code should focus on the functions that are in the ops->filter_hash and add them. But it still had to pay attention to the functions in the ops->notrace_hash, to ignore them. If it was cleared, it focused on the ops->notrace_hash, and would add functions that were not in the ops->notrace_hash but would still keep functions in the "ops->filter_hash". Basically doing the same thing. In reality, the __ftrace_hash_rec_update() only needs to either remove the functions associated to the give ops (if "inc" is set) or remove them (if "inc" is cleared). It has to pay attention to both the filter_hash and notrace_hash regardless. Remove the "filter_hash" parameter from __filter_hash_rec_update() and comment the function for what it really is doing. Link: https://lore.kernel.org/linux-trace-kernel/20240605180408.691995506@goodmis.org Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Reviewed-by: Mark Rutland Tested-by: Mark Rutland Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 102 ++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 64 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f4b253d20df8..abcdffa4d572 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1384,10 +1384,8 @@ alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) return NULL; } -static void -ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash); -static void -ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash); +static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops); +static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops); static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, struct ftrace_hash *new_hash); @@ -1475,11 +1473,11 @@ ftrace_hash_move(struct ftrace_ops *ops, int enable, * Remove the current set, update the hash and add * them back. */ - ftrace_hash_rec_disable_modify(ops, enable); + ftrace_hash_rec_disable_modify(ops); rcu_assign_pointer(*dst, new_hash); - ftrace_hash_rec_enable_modify(ops, enable); + ftrace_hash_rec_enable_modify(ops); return 0; } @@ -1702,12 +1700,21 @@ static bool skip_record(struct dyn_ftrace *rec) !(rec->flags & FTRACE_FL_ENABLED); } +/* + * This is the main engine to the ftrace updates to the dyn_ftrace records. + * + * It will iterate through all the available ftrace functions + * (the ones that ftrace can have callbacks to) and set the flags + * in the associated dyn_ftrace records. + * + * @inc: If true, the functions associated to @ops are added to + * the dyn_ftrace records, otherwise they are removed. + */ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, - int filter_hash, bool inc) { struct ftrace_hash *hash; - struct ftrace_hash *other_hash; + struct ftrace_hash *notrace_hash; struct ftrace_page *pg; struct dyn_ftrace *rec; bool update = false; @@ -1719,35 +1726,16 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, return false; /* - * In the filter_hash case: * If the count is zero, we update all records. * Otherwise we just update the items in the hash. - * - * In the notrace_hash case: - * We enable the update in the hash. - * As disabling notrace means enabling the tracing, - * and enabling notrace means disabling, the inc variable - * gets inversed. */ - if (filter_hash) { - hash = ops->func_hash->filter_hash; - other_hash = ops->func_hash->notrace_hash; - if (ftrace_hash_empty(hash)) - all = true; - } else { - inc = !inc; - hash = ops->func_hash->notrace_hash; - other_hash = ops->func_hash->filter_hash; - /* - * If the notrace hash has no items, - * then there's nothing to do. - */ - if (ftrace_hash_empty(hash)) - return false; - } + hash = ops->func_hash->filter_hash; + notrace_hash = ops->func_hash->notrace_hash; + if (ftrace_hash_empty(hash)) + all = true; do_for_each_ftrace_rec(pg, rec) { - int in_other_hash = 0; + int in_notrace_hash = 0; int in_hash = 0; int match = 0; @@ -1759,26 +1747,17 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, * Only the filter_hash affects all records. * Update if the record is not in the notrace hash. */ - if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) + if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip)) match = 1; } else { in_hash = !!ftrace_lookup_ip(hash, rec->ip); - in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); + in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip); /* - * If filter_hash is set, we want to match all functions - * that are in the hash but not in the other hash. - * - * If filter_hash is not set, then we are decrementing. - * That means we match anything that is in the hash - * and also in the other_hash. That is, we need to turn - * off functions in the other hash because they are disabled - * by this hash. + * We want to match all functions that are in the hash but + * not in the other hash. */ - if (filter_hash && in_hash && !in_other_hash) - match = 1; - else if (!filter_hash && in_hash && - (in_other_hash || ftrace_hash_empty(other_hash))) + if (in_hash && !in_notrace_hash) match = 1; } if (!match) @@ -1884,24 +1863,21 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, return update; } -static bool ftrace_hash_rec_disable(struct ftrace_ops *ops, - int filter_hash) +static bool ftrace_hash_rec_disable(struct ftrace_ops *ops) { - return __ftrace_hash_rec_update(ops, filter_hash, 0); + return __ftrace_hash_rec_update(ops, 0); } -static bool ftrace_hash_rec_enable(struct ftrace_ops *ops, - int filter_hash) +static bool ftrace_hash_rec_enable(struct ftrace_ops *ops) { - return __ftrace_hash_rec_update(ops, filter_hash, 1); + return __ftrace_hash_rec_update(ops, 1); } -static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, - int filter_hash, int inc) +static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, int inc) { struct ftrace_ops *op; - __ftrace_hash_rec_update(ops, filter_hash, inc); + __ftrace_hash_rec_update(ops, inc); if (ops->func_hash != &global_ops.local_hash) return; @@ -1915,20 +1891,18 @@ static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, if (op == ops) continue; if (op->func_hash == &global_ops.local_hash) - __ftrace_hash_rec_update(op, filter_hash, inc); + __ftrace_hash_rec_update(op, inc); } while_for_each_ftrace_op(op); } -static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, - int filter_hash) +static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops) { - ftrace_hash_rec_update_modify(ops, filter_hash, 0); + ftrace_hash_rec_update_modify(ops, 0); } -static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, - int filter_hash) +static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops) { - ftrace_hash_rec_update_modify(ops, filter_hash, 1); + ftrace_hash_rec_update_modify(ops, 1); } /* @@ -3051,7 +3025,7 @@ int ftrace_startup(struct ftrace_ops *ops, int command) return ret; } - if (ftrace_hash_rec_enable(ops, 1)) + if (ftrace_hash_rec_enable(ops)) command |= FTRACE_UPDATE_CALLS; ftrace_startup_enable(command); @@ -3093,7 +3067,7 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) /* Disabling ipmodify never fails */ ftrace_hash_ipmodify_disable(ops); - if (ftrace_hash_rec_disable(ops, 1)) + if (ftrace_hash_rec_disable(ops)) command |= FTRACE_UPDATE_CALLS; ops->flags &= ~FTRACE_OPS_FL_ENABLED; From da73f6d49007d1b6889d5cb3320dc9f90cf3ccbe Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 14:03:37 -0400 Subject: [PATCH 083/280] ftrace: Add comments to ftrace_hash_rec_disable/enable() Add comments to describe what the functions ftrace_hash_rec_disable() and ftrace_hash_rec_enable() do. Also change the passing of the "inc" variable to __ftrace_hash_rec_update() to a boolean value as that is what it is supposed to take. Link: https://lore.kernel.org/linux-trace-kernel/20240605180408.857333430@goodmis.org Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Mark Rutland Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index abcdffa4d572..4140d0ce25f1 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1863,14 +1863,24 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, return update; } +/* + * This is called when an ops is removed from tracing. It will decrement + * the counters of the dyn_ftrace records for all the functions that + * the @ops attached to. + */ static bool ftrace_hash_rec_disable(struct ftrace_ops *ops) { - return __ftrace_hash_rec_update(ops, 0); + return __ftrace_hash_rec_update(ops, false); } +/* + * This is called when an ops is added to tracing. It will increment + * the counters of the dyn_ftrace records for all the functions that + * the @ops attached to. + */ static bool ftrace_hash_rec_enable(struct ftrace_ops *ops) { - return __ftrace_hash_rec_update(ops, 1); + return __ftrace_hash_rec_update(ops, true); } static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, int inc) From 1a88c071679496907ba3e62ec1190d198c20ace8 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 14:03:38 -0400 Subject: [PATCH 084/280] ftrace: Convert "inc" parameter to bool in ftrace_hash_rec_update_modify() The parameter "inc" in the function ftrace_hash_rec_update_modify() is boolean. Change it to be such. Also add documentation to what the function does. Link: https://lore.kernel.org/linux-trace-kernel/20240605180409.021080462@goodmis.org Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Acked-by: Mark Rutland Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4140d0ce25f1..256b5e07c39a 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1883,7 +1883,24 @@ static bool ftrace_hash_rec_enable(struct ftrace_ops *ops) return __ftrace_hash_rec_update(ops, true); } -static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, int inc) +/* + * This function will update what functions @ops traces when its filter + * changes. + * + * The @inc states if the @ops callbacks are going to be added or removed. + * When one of the @ops hashes are updated to a "new_hash" the dyn_ftrace + * records are update via: + * + * ftrace_hash_rec_disable_modify(ops); + * ops->hash = new_hash + * ftrace_hash_rec_enable_modify(ops); + * + * Where the @ops is removed from all the records it is tracing using + * its old hash. The @ops hash is updated to the new hash, and then + * the @ops is added back to the records so that it is tracing all + * the new functions. + */ +static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, bool inc) { struct ftrace_ops *op; @@ -1907,12 +1924,12 @@ static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, int inc) static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops) { - ftrace_hash_rec_update_modify(ops, 0); + ftrace_hash_rec_update_modify(ops, false); } static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops) { - ftrace_hash_rec_update_modify(ops, 1); + ftrace_hash_rec_update_modify(ops, true); } /* From d66bb33479e594ba7cf72fa2c0f848e3d0bc9eb1 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 14:03:39 -0400 Subject: [PATCH 085/280] ftrace: Add comments to ftrace_hash_move() and friends Describe what ftrace_hash_move() does and add some more comments to some other functions to make it easier to understand. Link: https://lore.kernel.org/linux-trace-kernel/20240605180409.179520305@goodmis.org Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Mark Rutland Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 256b5e07c39a..9c4d01b1bb68 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -169,6 +169,7 @@ static inline void ftrace_ops_init(struct ftrace_ops *ops) #endif } +/* Call this function for when a callback filters on set_ftrace_pid */ static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { @@ -1318,7 +1319,7 @@ static struct ftrace_hash *alloc_ftrace_hash(int size_bits) return hash; } - +/* Used to save filters on functions for modules not loaded yet */ static int ftrace_add_mod(struct trace_array *tr, const char *func, const char *module, int enable) @@ -1430,6 +1431,7 @@ static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) return new_hash; } +/* Move the @src entries to a newly allocated hash */ static struct ftrace_hash * __ftrace_hash_move(struct ftrace_hash *src) { @@ -1444,6 +1446,26 @@ __ftrace_hash_move(struct ftrace_hash *src) return __move_hash(src, size); } +/** + * ftrace_hash_move - move a new hash to a filter and do updates + * @ops: The ops with the hash that @dst points to + * @enable: True if for the filter hash, false for the notrace hash + * @dst: Points to the @ops hash that should be updated + * @src: The hash to update @dst with + * + * This is called when an ftrace_ops hash is being updated and the + * the kernel needs to reflect this. Note, this only updates the kernel + * function callbacks if the @ops is enabled (not to be confused with + * @enable above). If the @ops is enabled, its hash determines what + * callbacks get called. This function gets called when the @ops hash + * is updated and it requires new callbacks. + * + * On success the elements of @src is moved to @dst, and @dst is updated + * properly, as well as the functions determined by the @ops hashes + * are now calling the @ops callback function. + * + * Regardless of return type, @src should be freed with free_ftrace_hash(). + */ static int ftrace_hash_move(struct ftrace_ops *ops, int enable, struct ftrace_hash **dst, struct ftrace_hash *src) From 1d5f0222944fe723b9bfaaeb27e368363644ccab Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:45 -0400 Subject: [PATCH 086/280] ftrace: Declare function_trace_op in header to quiet sparse warning Sparse complains that function_trace_op is not static but is not declared in a header file. It is used only in assembly code. But add it to a header so that sparse no longer complains: kernel/trace/ftrace.c:99:19: warning: symbol 'function_trace_op' was not declared. Should it be static? Link: https://lore.kernel.org/linux-trace-kernel/20240605202708.289105647@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/ftrace.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 9f61556a9491..4135dc171447 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1131,6 +1131,9 @@ extern void ftrace_graph_init_task(struct task_struct *t); extern void ftrace_graph_exit_task(struct task_struct *t); extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu); +/* Used by assembly, but to quiet sparse warnings */ +extern struct ftrace_ops *function_trace_op; + static inline void pause_graph_tracing(void) { atomic_inc(¤t->tracing_graph_pause); From 1f51ba905e792a4f6d7b313756a4f99386990ddf Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:46 -0400 Subject: [PATCH 087/280] ftrace: Assign ftrace_list_end to ftrace_ops_list type cast to RCU Use a type cast to convert ftrace_list_end to RCU when assigning ftrace_ops_list. This will quiet the sparse warning: kernel/trace/ftrace.c:125:59: warning: incorrect type in initializer (different address spaces) kernel/trace/ftrace.c:125:59: expected struct ftrace_ops [noderef] __rcu *[addressable] [toplevel] ftrace_ops_list kernel/trace/ftrace.c:125:59: got struct ftrace_ops * Link: https://lore.kernel.org/linux-trace-kernel/20240605202708.450784356@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 9c4d01b1bb68..034242675e7b 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -122,7 +122,7 @@ static int ftrace_disabled __read_mostly; DEFINE_MUTEX(ftrace_lock); -struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; +struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = (struct ftrace_ops __rcu *)&ftrace_list_end; ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; struct ftrace_ops global_ops; From 0ddef5d601ff992430b7874074c3549aa66f2c85 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:47 -0400 Subject: [PATCH 088/280] ftrace: Assign RCU list variable with rcu_assign_ptr() Use rcu_assign_ptr() to assign the list pointer as it is marked as RCU, and this quiets the sparse warning: kernel/trace/ftrace.c:313:23: warning: incorrect type in assignment (different address spaces) kernel/trace/ftrace.c:313:23: expected struct ftrace_ops [noderef] __rcu * kernel/trace/ftrace.c:313:23: got struct ftrace_ops * Link: https://lore.kernel.org/linux-trace-kernel/20240605202708.613471310@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 034242675e7b..4aeb1183ea9f 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -311,7 +311,7 @@ static int remove_ftrace_ops(struct ftrace_ops __rcu **list, lockdep_is_held(&ftrace_lock)) == ops && rcu_dereference_protected(ops->next, lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { - *list = &ftrace_list_end; + rcu_assign_pointer(*list, &ftrace_list_end); return 0; } From 86b49970e071921490ebab3d982d93dd616cd967 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:48 -0400 Subject: [PATCH 089/280] ftrace: Fix prototypes for ftrace_startup/shutdown_subops() The ftrace_startup_subops() was in the wrong header, and both functions were not defined on !CONFIG_DYNAMIC_FTRACE. Link: https://lore.kernel.org/linux-trace-kernel/20240605202708.773583114@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Fixes: 5fccc7552ccbc ("ftrace: Add subops logic to allow one ops to manage many") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202406051524.a12JqLqx-lkp@intel.com/ Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace_internal.h | 9 +++++++++ kernel/trace/trace.h | 1 - 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h index bfba10c2fcf1..4bb1e881154a 100644 --- a/kernel/trace/ftrace_internal.h +++ b/kernel/trace/ftrace_internal.h @@ -15,6 +15,7 @@ extern struct ftrace_ops global_ops; int ftrace_startup(struct ftrace_ops *ops, int command); int ftrace_shutdown(struct ftrace_ops *ops, int command); int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs); +int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command); int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command); #else /* !CONFIG_DYNAMIC_FTRACE */ @@ -39,6 +40,14 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) { return 1; } +static inline int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) +{ + return -EINVAL; +} +static inline int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) +{ + return -EINVAL; +} #endif /* CONFIG_DYNAMIC_FTRACE */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index b37402e3f0c9..8783bebd0562 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1186,7 +1186,6 @@ extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, int len, int reset); extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, int len, int reset); -extern int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command); #else struct ftrace_func_command; From 0c4d8cbb2ce20df3a72b5b3189d93b5c0cc3e968 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:49 -0400 Subject: [PATCH 090/280] function_graph: Make fgraph_do_direct static key static The static branch key "fgraph_do_direct" was not declared static but is only used in one file. Change it to a static variable. Link: https://lore.kernel.org/linux-trace-kernel/20240605202708.936515302@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Fixes: cc60ee813b503 ("function_graph: Use static_call and branch to optimize entry function") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202406051711.dS1sQZ9n-lkp@intel.com/ Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 4bf91eebbb08..63d828054c79 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -515,7 +515,7 @@ static struct fgraph_ops fgraph_stub = { static struct fgraph_ops *fgraph_direct_gops = &fgraph_stub; DEFINE_STATIC_CALL(fgraph_func, ftrace_graph_entry_stub); DEFINE_STATIC_CALL(fgraph_retfunc, ftrace_graph_ret_stub); -DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); +static DEFINE_STATIC_KEY_TRUE(fgraph_do_direct); /** * ftrace_graph_stop - set to permanently disable function graph tracing From 4057fd2cddaea0d6f6c5f8af4a455741f5632259 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 5 Jun 2024 16:26:50 -0400 Subject: [PATCH 091/280] function_graph: Do not update pid func if CONFIG_DYNAMIC_FTRACE not enabled The ftrace subops is only defined if CONFIG_DYNAMIC_FTRACE is enabled. If it is not, function tracing is extremely limited, and the subops in the ftrace_ops structure is not defined (and will fail to compile). If DYNAMIC_FTRACE is not enabled, then function graph filtering will not work (as it shouldn't). Link: https://lore.kernel.org/linux-trace-kernel/20240605202709.096020676@goodmis.org Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Andrew Morton Fixes: df3ec5da6a1e7 ("function_graph: Add pid tracing back to function graph tracer") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202406051855.9VIYXbTB-lkp@intel.com/ Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 63d828054c79..c0e428c87ea5 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -1177,6 +1177,7 @@ void fgraph_update_pid_func(void) if (!(graph_ops.flags & FTRACE_OPS_FL_INITIALIZED)) return; +#ifdef CONFIG_DYNAMIC_FTRACE list_for_each_entry(op, &graph_ops.subop_list, list) { if (op->flags & FTRACE_OPS_FL_PID) { gops = container_of(op, struct fgraph_ops, ops); @@ -1186,6 +1187,7 @@ void fgraph_update_pid_func(void) static_call_update(fgraph_func, gops->entryfunc); } } +#endif } /* Allocate a return stack for each task */ From 6c1f7f0acabd35ad73525bd2766da4669c18a9e6 Mon Sep 17 00:00:00 2001 From: Tatsuya S Date: Tue, 28 May 2024 12:24:14 +0900 Subject: [PATCH 092/280] ftrace: Hide one more entry in stack trace when ftrace_pid is enabled On setting set_ftrace_pid, a extra entry generated by ftrace_pid_func() is shown on stack trace(CONFIG_UNWINDER_FRAME_POINTER=y). [004] ..... 68.459382: => 0xffffffffa00090af => ksys_read => __x64_sys_read => x64_sys_call => do_syscall_64 => entry_SYSCALL_64_after_hwframe To resolve this issue, increment skip count in function_stack_trace_call() if pids are set. Link: https://lore.kernel.org/linux-trace-kernel/20240528032604.6813-3-tatsuya.s2862@gmail.com Signed-off-by: Tatsuya S [ Rebased to current tree ] Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_functions.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 13bf2415245d..3b0cea37e029 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -231,6 +231,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, long disabled; int cpu; unsigned int trace_ctx; + int skip = STACK_SKIP; if (unlikely(!tr->function_enabled)) return; @@ -247,7 +248,11 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, if (likely(disabled == 1)) { trace_ctx = tracing_gen_ctx_flags(flags); trace_function(tr, ip, parent_ip, trace_ctx); - __trace_stack(tr, trace_ctx, STACK_SKIP); +#ifdef CONFIG_UNWINDER_FRAME_POINTER + if (ftrace_pids_enabled(op)) + skip++; +#endif + __trace_stack(tr, trace_ctx, skip); } atomic_dec(&data->disabled); From 9a2a3aab7329f1e62982531559866b90d5a5208b Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 6 Jun 2024 10:10:53 +0800 Subject: [PATCH 093/280] fgraph: Remove some unused functions These functions are defined in the fgraph.c file, but not called elsewhere, so delete these unused functions. kernel/trace/fgraph.c:273:1: warning: unused function 'set_bitmap_bits'. kernel/trace/fgraph.c:259:19: warning: unused function 'get_fgraph_type'. Link: https://lore.kernel.org/linux-trace-kernel/20240606021053.27783-1-jiapeng.chong@linux.alibaba.com Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=9289 Signed-off-by: Jiapeng Chong Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index c0e428c87ea5..a13551a023aa 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -255,12 +255,6 @@ static inline int get_frame_offset(struct task_struct *t, int offset) return __get_offset(t->ret_stack[offset]); } -/* Get FGRAPH_TYPE from the word from the @offset at ret_stack */ -static inline int get_fgraph_type(struct task_struct *t, int offset) -{ - return __get_type(t->ret_stack[offset]); -} - /* For BITMAP type: get the bitmask from the @offset at ret_stack */ static inline unsigned long get_bitmap_bits(struct task_struct *t, int offset) @@ -268,13 +262,6 @@ get_bitmap_bits(struct task_struct *t, int offset) return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK; } -/* For BITMAP type: set the bits in the bitmap bitmask at @offset on ret_stack */ -static inline void -set_bitmap_bits(struct task_struct *t, int offset, unsigned long bitmap) -{ - t->ret_stack[offset] |= (bitmap << FGRAPH_INDEX_SHIFT); -} - /* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */ static inline void set_bitmap(struct task_struct *t, int offset, unsigned long bitmap) From 2f6b884dfcc55065b76d2bf1e2424b93991ae92d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 6 Jun 2024 08:18:46 -0400 Subject: [PATCH 094/280] function_graph: Rename BYTE_NUMBER to CHAR_NUMBER in selftests The function_graph selftests checks various size variables to pass from the entry of the function to the exit. It tests 1, 2, 4 and 8 byte words. The 1 byte macro was called BYTE_NUMBER but that is used in the sh architecture: arch/sh/include/asm/bitops-op32.h Just rename the macro to CHAR_NUMBER. Link: https://lore.kernel.org/linux-trace-kernel/20240606081846.4cb82dc4@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Fixes: 47c3c70aa3697 ("function_graph: Add selftest for passing local variables") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202406061744.rZDXfRrG-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_selftest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 369efc569238..adf0f436d84b 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c @@ -758,7 +758,7 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr) #ifdef CONFIG_DYNAMIC_FTRACE -#define BYTE_NUMBER 123 +#define CHAR_NUMBER 123 #define SHORT_NUMBER 12345 #define WORD_NUMBER 1234567890 #define LONG_NUMBER 1234567890123456789LL @@ -789,7 +789,7 @@ static __init int store_entry(struct ftrace_graph_ent *trace, switch (size) { case 1: - *(char *)p = BYTE_NUMBER; + *(char *)p = CHAR_NUMBER; break; case 2: *(short *)p = SHORT_NUMBER; @@ -830,7 +830,7 @@ static __init void store_return(struct ftrace_graph_ret *trace, switch (fixture->store_size) { case 1: - expect = BYTE_NUMBER; + expect = CHAR_NUMBER; found = *(char *)p; break; case 2: From a9700511fd50b9203a9a9d61b4874eb28571d5da Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Wed, 5 Jun 2024 16:44:42 +0200 Subject: [PATCH 095/280] perf script: netdev-times: add location parameter to consume_skb dd1b527831a3 ("net: add location to trace_consume_skb()") added a new parameter to the consume_skb tracepoint. Adapt the script to match. Signed-off-by: Lucas Stach Acked-by: Arnaldo Carvalho de Melo Cc: kernel@pengutronix.de Cc: patchwork-lst@pengutronix.de Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240605144442.1985270-1-l.stach@pengutronix.de --- tools/perf/scripts/python/netdev-times.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/perf/scripts/python/netdev-times.py b/tools/perf/scripts/python/netdev-times.py index 00552eeb7178..30c4bccee5b2 100644 --- a/tools/perf/scripts/python/netdev-times.py +++ b/tools/perf/scripts/python/netdev-times.py @@ -293,7 +293,8 @@ def skb__kfree_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr, location, protocol, reason) all_event_list.append(event_info) -def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, callchain, skbaddr): +def skb__consume_skb(name, context, cpu, sec, nsec, pid, comm, callchain, + skbaddr, location): event_info = (name, context, cpu, nsecs(sec, nsec), pid, comm, skbaddr) all_event_list.append(event_info) From 0b90dfda222e38b7ca8dad6e098e36f5186f0b94 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 21 May 2024 09:51:07 -0700 Subject: [PATCH 096/280] perf maps: Fix use after free in __maps__fixup_overlap_and_insert In the case 'before' and 'after' are broken out from pos, maps_by_address may be changed by __maps__insert, as such it needs re-reading. Don't ignore the return value from __maps_insert. Fixes: 659ad3492b91 ("perf maps: Switch from rbtree to lazily sorted array for addresses") Signed-off-by: Ian Rogers Reviewed-by: James Clark Cc: Steinar H . Gunderson Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240521165109.708593-2-irogers@google.com --- tools/perf/util/maps.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 16b39db594f4..eaada3e0f5b4 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -741,7 +741,6 @@ static unsigned int first_ending_after(struct maps *maps, const struct map *map) */ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) { - struct map **maps_by_address; int err = 0; FILE *fp = debug_file(); @@ -749,12 +748,12 @@ sort_again: if (!maps__maps_by_address_sorted(maps)) __maps__sort_by_address(maps); - maps_by_address = maps__maps_by_address(maps); /* * Iterate through entries where the end of the existing entry is * greater-than the new map's start. */ for (unsigned int i = first_ending_after(maps, new); i < maps__nr_maps(maps); ) { + struct map **maps_by_address = maps__maps_by_address(maps); struct map *pos = maps_by_address[i]; struct map *before = NULL, *after = NULL; @@ -821,8 +820,10 @@ sort_again: /* Maps are still ordered, go to next one. */ i++; if (after) { - __maps__insert(maps, after); + err = __maps__insert(maps, after); map__put(after); + if (err) + goto out_err; if (!maps__maps_by_address_sorted(maps)) { /* * Sorting broken so invariants don't @@ -851,7 +852,7 @@ sort_again: check_invariants(maps); } /* Add the map. */ - __maps__insert(maps, new); + err = __maps__insert(maps, new); out_err: return err; } From aeefb04393f7525c0d5163f966f60d070b03ab99 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 21 May 2024 09:51:08 -0700 Subject: [PATCH 097/280] perf maps: Reduce sorting for overlapping mappings When an 'after' map is generated the 'new' map must be before it so terminate iterating and don't resort. If the entry 'pos' is entirely overlapped by the 'new' mapping then don't remove and insert the mapping, just replace - again to remove sorting. For a perf report on a perf.data file containing overlapping mappings the time numbers are: Before: real 0m9.856s user 0m9.637s sys 0m0.204s After: real 0m5.894s user 0m5.650s sys 0m0.231s Signed-off-by: Ian Rogers Reviewed-by: James Clark Cc: Steinar H . Gunderson Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240521165109.708593-3-irogers@google.com --- tools/perf/util/maps.c | 55 +++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index eaada3e0f5b4..f6b6df82f4cf 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -744,7 +744,6 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) int err = 0; FILE *fp = debug_file(); -sort_again: if (!maps__maps_by_address_sorted(maps)) __maps__sort_by_address(maps); @@ -820,36 +819,54 @@ sort_again: /* Maps are still ordered, go to next one. */ i++; if (after) { - err = __maps__insert(maps, after); - map__put(after); - if (err) - goto out_err; - if (!maps__maps_by_address_sorted(maps)) { - /* - * Sorting broken so invariants don't - * hold, sort and go again. - */ - goto sort_again; - } /* - * Maps are still ordered, skip after and go to - * next one (terminate loop). + * 'before' and 'after' mean 'new' split the + * 'pos' mapping and therefore there are no + * later mappings. */ - i++; + err = __maps__insert(maps, new); + if (!err) + err = __maps__insert(maps, after); + map__put(after); + check_invariants(maps); + return err; } + check_invariants(maps); } else if (after) { + /* + * 'after' means 'new' split 'pos' and there are no + * later mappings. + */ map__put(maps_by_address[i]); - maps_by_address[i] = after; - /* Maps are ordered, go to next one. */ - i++; + maps_by_address[i] = map__get(new); + err = __maps__insert(maps, after); + map__put(after); + check_invariants(maps); + return err; } else { + struct map *next = NULL; + + if (i + 1 < maps__nr_maps(maps)) + next = maps_by_address[i + 1]; + + if (!next || map__start(next) >= map__end(new)) { + /* + * Replace existing mapping and end knowing + * there aren't later overlapping or any + * mappings. + */ + map__put(maps_by_address[i]); + maps_by_address[i] = map__get(new); + check_invariants(maps); + return err; + } __maps__remove(maps, pos); + check_invariants(maps); /* * Maps are ordered but no need to increase `i` as the * later maps were moved down. */ } - check_invariants(maps); } /* Add the map. */ err = __maps__insert(maps, new); From d2307fd4f9895b44361d491f8bf474866b8351a2 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 21 May 2024 09:51:09 -0700 Subject: [PATCH 098/280] perf maps: Add/use a sorted insert for fixup overlap and insert Data may have lots of overlapping mmaps. The regular insert adds at the end and relies on a later sort. For data with overlapping mappings the sort will happen during a subsequent maps__find or __maps__fixup_overlap_and_insert, there's never a period where the inserted maps buffer up and a single sort happens. To avoid back to back sorts, maintain the sort order when fixing up and inserting. Previously the first_ending_after search was O(log n) where n is the size of maps, and the insert was O(1) but because of the continuous sorting was becoming O(n*log(n)). With maintaining sort order, the insert now becomes O(n) for a memmove. For a perf report on a perf.data file containing overlapping mappings the time numbers are: Before: real 0m5.894s user 0m5.650s sys 0m0.231s After: real 0m0.675s user 0m0.454s sys 0m0.196s Signed-off-by: Ian Rogers Reviewed-by: James Clark Cc: Steinar H . Gunderson Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240521165109.708593-4-irogers@google.com --- tools/perf/util/maps.c | 65 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index f6b6df82f4cf..432399cbe5dd 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -735,6 +735,60 @@ static unsigned int first_ending_after(struct maps *maps, const struct map *map) return first; } +static int __maps__insert_sorted(struct maps *maps, unsigned int first_after_index, + struct map *new1, struct map *new2) +{ + struct map **maps_by_address = maps__maps_by_address(maps); + struct map **maps_by_name = maps__maps_by_name(maps); + unsigned int nr_maps = maps__nr_maps(maps); + unsigned int nr_allocate = RC_CHK_ACCESS(maps)->nr_maps_allocated; + unsigned int to_add = new2 ? 2 : 1; + + assert(maps__maps_by_address_sorted(maps)); + assert(first_after_index == nr_maps || + map__end(new1) <= map__start(maps_by_address[first_after_index])); + assert(!new2 || map__end(new1) <= map__start(new2)); + assert(first_after_index == nr_maps || !new2 || + map__end(new2) <= map__start(maps_by_address[first_after_index])); + + if (nr_maps + to_add > nr_allocate) { + nr_allocate = !nr_allocate ? 32 : nr_allocate * 2; + + maps_by_address = realloc(maps_by_address, nr_allocate * sizeof(new1)); + if (!maps_by_address) + return -ENOMEM; + + maps__set_maps_by_address(maps, maps_by_address); + if (maps_by_name) { + maps_by_name = realloc(maps_by_name, nr_allocate * sizeof(new1)); + if (!maps_by_name) { + /* + * If by name fails, just disable by name and it will + * recompute next time it is required. + */ + __maps__free_maps_by_name(maps); + } + maps__set_maps_by_name(maps, maps_by_name); + } + RC_CHK_ACCESS(maps)->nr_maps_allocated = nr_allocate; + } + memmove(&maps_by_address[first_after_index+to_add], + &maps_by_address[first_after_index], + (nr_maps - first_after_index) * sizeof(new1)); + maps_by_address[first_after_index] = map__get(new1); + if (maps_by_name) + maps_by_name[nr_maps] = map__get(new1); + if (new2) { + maps_by_address[first_after_index + 1] = map__get(new2); + if (maps_by_name) + maps_by_name[nr_maps + 1] = map__get(new2); + } + RC_CHK_ACCESS(maps)->nr_maps = nr_maps + to_add; + maps__set_maps_by_name_sorted(maps, false); + check_invariants(maps); + return 0; +} + /* * Adds new to maps, if new overlaps existing entries then the existing maps are * adjusted or removed so that new fits without overlapping any entries. @@ -743,6 +797,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) { int err = 0; FILE *fp = debug_file(); + unsigned int i; if (!maps__maps_by_address_sorted(maps)) __maps__sort_by_address(maps); @@ -751,7 +806,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) * Iterate through entries where the end of the existing entry is * greater-than the new map's start. */ - for (unsigned int i = first_ending_after(maps, new); i < maps__nr_maps(maps); ) { + for (i = first_ending_after(maps, new); i < maps__nr_maps(maps); ) { struct map **maps_by_address = maps__maps_by_address(maps); struct map *pos = maps_by_address[i]; struct map *before = NULL, *after = NULL; @@ -824,9 +879,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) * 'pos' mapping and therefore there are no * later mappings. */ - err = __maps__insert(maps, new); - if (!err) - err = __maps__insert(maps, after); + err = __maps__insert_sorted(maps, i, new, after); map__put(after); check_invariants(maps); return err; @@ -839,7 +892,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) */ map__put(maps_by_address[i]); maps_by_address[i] = map__get(new); - err = __maps__insert(maps, after); + err = __maps__insert_sorted(maps, i + 1, after, NULL); map__put(after); check_invariants(maps); return err; @@ -869,7 +922,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) } } /* Add the map. */ - err = __maps__insert(maps, new); + err = __maps__insert_sorted(maps, i, new, NULL); out_err: return err; } From 47d911b02cbe61494bb066ad84cc66d25091d506 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Fri, 7 Jun 2024 16:40:12 +0800 Subject: [PATCH 099/280] slab: make check_object() more consistent Now check_object() calls check_bytes_and_report() multiple times to check every section of the object it cares about, like left and right redzones, object poison, paddings poison and freepointer. It will abort the checking process and return 0 once it finds an error. There are two inconsistencies in check_object(), which are alignment padding checking and object padding checking. We only print the error messages but don't return 0 to tell callers that something is wrong and needs to be handled. Please see alloc_debug_processing() and free_debug_processing() for details. We want to do all checks without skipping, so use a local variable "ret" to save each check result and change check_bytes_and_report() to only report specific error findings. Then at end of check_object(), print the trailer once if any found an error. Suggested-by: Vlastimil Babka Signed-off-by: Chengming Zhou Reviewed-by: Vlastimil Babka Signed-off-by: Vlastimil Babka --- mm/slub.c | 62 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index b8ba068ca079..5df8c8302784 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -788,8 +788,24 @@ static bool slab_add_kunit_errors(void) kunit_put_resource(resource); return true; } + +static bool slab_in_kunit_test(void) +{ + struct kunit_resource *resource; + + if (!kunit_get_current_test()) + return false; + + resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); + if (!resource) + return false; + + kunit_put_resource(resource); + return true; +} #else static inline bool slab_add_kunit_errors(void) { return false; } +static inline bool slab_in_kunit_test(void) { return false; } #endif static inline unsigned int size_from_object(struct kmem_cache *s) @@ -1190,8 +1206,6 @@ static int check_bytes_and_report(struct kmem_cache *s, struct slab *slab, pr_err("0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", fault, end - 1, fault - addr, fault[0], value); - print_trailer(s, slab, object); - add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); skip_bug_print: restore_bytes(s, what, value, fault, end); @@ -1300,15 +1314,16 @@ static int check_object(struct kmem_cache *s, struct slab *slab, u8 *p = object; u8 *endobject = object + s->object_size; unsigned int orig_size, kasan_meta_size; + int ret = 1; if (s->flags & SLAB_RED_ZONE) { if (!check_bytes_and_report(s, slab, object, "Left Redzone", object - s->red_left_pad, val, s->red_left_pad)) - return 0; + ret = 0; if (!check_bytes_and_report(s, slab, object, "Right Redzone", endobject, val, s->inuse - s->object_size)) - return 0; + ret = 0; if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { orig_size = get_orig_size(s, object); @@ -1317,14 +1332,15 @@ static int check_object(struct kmem_cache *s, struct slab *slab, !check_bytes_and_report(s, slab, object, "kmalloc Redzone", p + orig_size, val, s->object_size - orig_size)) { - return 0; + ret = 0; } } } else { if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { - check_bytes_and_report(s, slab, p, "Alignment padding", + if (!check_bytes_and_report(s, slab, p, "Alignment padding", endobject, POISON_INUSE, - s->inuse - s->object_size); + s->inuse - s->object_size)) + ret = 0; } } @@ -1340,27 +1356,25 @@ static int check_object(struct kmem_cache *s, struct slab *slab, !check_bytes_and_report(s, slab, p, "Poison", p + kasan_meta_size, POISON_FREE, s->object_size - kasan_meta_size - 1)) - return 0; + ret = 0; if (kasan_meta_size < s->object_size && !check_bytes_and_report(s, slab, p, "End Poison", p + s->object_size - 1, POISON_END, 1)) - return 0; + ret = 0; } /* * check_pad_bytes cleans up on its own. */ - check_pad_bytes(s, slab, p); + if (!check_pad_bytes(s, slab, p)) + ret = 0; } - if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE) - /* - * Object and freepointer overlap. Cannot check - * freepointer while object is allocated. - */ - return 1; - - /* Check free pointer validity */ - if (!check_valid_pointer(s, slab, get_freepointer(s, p))) { + /* + * Cannot check freepointer while object is allocated if + * object and freepointer overlap. + */ + if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) && + !check_valid_pointer(s, slab, get_freepointer(s, p))) { object_err(s, slab, p, "Freepointer corrupt"); /* * No choice but to zap it and thus lose the remainder @@ -1368,9 +1382,15 @@ static int check_object(struct kmem_cache *s, struct slab *slab, * another error because the object count is now wrong. */ set_freepointer(s, p, NULL); - return 0; + ret = 0; } - return 1; + + if (!ret && !slab_in_kunit_test()) { + print_trailer(s, slab, object); + add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); + } + + return ret; } static int check_slab(struct kmem_cache *s, struct slab *slab) From adef2aeaa2b936c97865d56c59be2cb7266acbb7 Mon Sep 17 00:00:00 2001 From: Chengming Zhou Date: Fri, 7 Jun 2024 16:40:13 +0800 Subject: [PATCH 100/280] slab: don't put freepointer outside of object if only orig_size The commit 946fa0dbf2d8 ("mm/slub: extend redzone check to extra allocated kmalloc space than requested") will extend right redzone when allocating for orig_size < object_size. So we can't overlay the freepointer in the object space in this case. But the code looks like it forgot to check SLAB_RED_ZONE, since there won't be extended right redzone if only orig_size enabled. As we are here, make this complex conditional expressions a little prettier and add some comments about extending right redzone when slub_debug_orig_size() enabled. Reviewed-by: Feng Tang Reviewed-by: Vlastimil Babka Signed-off-by: Chengming Zhou Signed-off-by: Vlastimil Babka --- mm/slub.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 5df8c8302784..5fc052a4f38f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5167,10 +5167,9 @@ static int calculate_sizes(struct kmem_cache *s) */ s->inuse = size; - if (slub_debug_orig_size(s) || - (flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) || - ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) || - s->ctor) { + if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) || s->ctor || + ((flags & SLAB_RED_ZONE) && + (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) { /* * Relocate free pointer after the object if it is not * permitted to overwrite the first word of the object on @@ -5178,7 +5177,9 @@ static int calculate_sizes(struct kmem_cache *s) * * This is the case if we do RCU, have a constructor or * destructor, are poisoning the objects, or are - * redzoning an object smaller than sizeof(void *). + * redzoning an object smaller than sizeof(void *) or are + * redzoning an object with slub_debug_orig_size() enabled, + * in which case the right redzone may be extended. * * The assumption that s->offset >= s->inuse means free * pointer is outside of the object is used in the From fb412c6241dccc530416917efe8e9fea5fa1fda2 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 29 Feb 2024 15:13:48 +0100 Subject: [PATCH 101/280] s390/pai_crypto: Enable concurrent system-wide counting/sampling event The PMU for PAI crypto counters enforces the following restriction: - No system wide counting while system wide sampling is active. This restriction is removed. One or more system wide counting events can now be active at the same time while at most one system wide sampling event is active. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_crypto.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 4ad472d130a3..d3a64f041819 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -97,6 +97,8 @@ static void paicrypt_event_destroy(struct perf_event *event) event->attr.config, event->cpu, cpump->active_events, cpump->mode, refcount_read(&cpump->refcnt)); + if (event->attr.sample_period) + cpump->mode &= ~PAI_MODE_SAMPLING; free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", @@ -160,9 +162,7 @@ static u64 paicrypt_getall(struct perf_event *event) * sampling for crypto events * * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is - * allowed and when this event is running, no counting event is allowed. - * Several counting events are allowed in parallel, but no sampling event - * is allowed while one (or more) counting events are running. + * allowed. Several counting events are allowed in parallel. * * This function is called in process context and it is save to block. * When the event initialization functions fails, no other call back will @@ -196,12 +196,12 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event) } if (a->sample_period) { /* Sampling requested */ - if (cpump->mode != PAI_MODE_NONE) - rc = -EBUSY; /* ... sampling/counting active */ - } else { /* Counting requested */ - if (cpump->mode == PAI_MODE_SAMPLING) + if (cpump->mode & PAI_MODE_SAMPLING) rc = -EBUSY; /* ... and sampling active */ + else + cpump->mode |= PAI_MODE_SAMPLING; } + /* * This error case triggers when there is a conflict: * Either sampling requested and counting already active, or visa @@ -235,7 +235,6 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event) /* Set mode and reference count */ rc = 0; refcount_set(&cpump->refcnt, 1); - cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; mp->mapptr = cpump; debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" " mode %d refcnt %u page %#lx save %p rc %d\n", @@ -249,7 +248,6 @@ free_paicrypt_map: mp->mapptr = NULL; free_root: paicrypt_root_free(); - unlock: mutex_unlock(&pai_reserve_mutex); return rc ? ERR_PTR(rc) : cpump; @@ -332,6 +330,7 @@ static void paicrypt_start(struct perf_event *event, int flags) local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ cpump->event = event; + memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); perf_sched_cb_inc(event->pmu); } } @@ -480,7 +479,7 @@ static int paicrypt_have_sample(void) static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) { /* We started with a clean page on event installation. So read out - * results on schedule_out and if page was dirty, clear values. + * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) paicrypt_have_sample(); From 92ea686840ac5e2e74ad5b1a569b07123d753e5e Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 29 Feb 2024 15:45:04 +0100 Subject: [PATCH 102/280] s390/pai_crypto: Enable per-task counting event The PMU for PAI crypto counters enforces the following restriction: - No per-task context for PAI crypto counters events. This restriction is removed. One or more per-task/system-wide counting events can now be active at the same time while at most one system wide sampling event is active. Example for per-task context of a PAI crypto counter event: # perf stat -e pai_crypto/KM_AES_128/ -- true Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/pai.h | 1 + arch/s390/kernel/perf_pai_crypto.c | 81 ++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 3f609565734b..7da1cec42016 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -82,4 +82,5 @@ enum paievt_mode { }; #define PAI_SAVE_AREA(x) ((x)->hw.event_base) +#define PAI_CPU_MASK(x) ((x)->hw.addr_filters) #endif diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index d3a64f041819..0e296a1482bc 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -84,13 +84,11 @@ static DEFINE_MUTEX(pai_reserve_mutex); /* Adjust usage counters and remove allocated memory when all users are * gone. */ -static void paicrypt_event_destroy(struct perf_event *event) +static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu) { - struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, - event->cpu); + struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, cpu); struct paicrypt_map *cpump = mp->mapptr; - static_branch_dec(&pai_key); mutex_lock(&pai_reserve_mutex); debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" " mode %d refcnt %u\n", __func__, @@ -99,7 +97,6 @@ static void paicrypt_event_destroy(struct perf_event *event) refcount_read(&cpump->refcnt)); if (event->attr.sample_period) cpump->mode &= ~PAI_MODE_SAMPLING; - free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, @@ -113,6 +110,23 @@ static void paicrypt_event_destroy(struct perf_event *event) mutex_unlock(&pai_reserve_mutex); } +static void paicrypt_event_destroy(struct perf_event *event) +{ + int cpu; + + static_branch_dec(&pai_key); + free_page(PAI_SAVE_AREA(event)); + if (event->cpu == -1) { + struct cpumask *mask = PAI_CPU_MASK(event); + + for_each_cpu(cpu, mask) + paicrypt_event_destroy_cpu(event, cpu); + kfree(mask); + } else { + paicrypt_event_destroy_cpu(event, event->cpu); + } +} + static u64 paicrypt_getctr(unsigned long *page, int nr, bool kernel) { if (kernel) @@ -170,7 +184,7 @@ static u64 paicrypt_getall(struct perf_event *event) * * Allocate the memory for the event. */ -static struct paicrypt_map *paicrypt_busy(struct perf_event *event) +static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) { struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump = NULL; @@ -185,7 +199,7 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event) goto unlock; /* Allocate node for this event */ - mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); + mp = per_cpu_ptr(paicrypt_root.mapptr, cpu); cpump = mp->mapptr; if (!cpump) { /* Paicrypt_map allocated? */ cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); @@ -253,6 +267,40 @@ unlock: return rc ? ERR_PTR(rc) : cpump; } +static int paicrypt_event_init_all(struct perf_event *event) +{ + struct paicrypt_map *cpump; + struct cpumask *maskptr; + int cpu, rc = -ENOMEM; + + maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL); + if (!maskptr) + goto out; + + for_each_online_cpu(cpu) { + cpump = paicrypt_busy(event, cpu); + if (IS_ERR(cpump)) { + for_each_cpu(cpu, maskptr) + paicrypt_event_destroy_cpu(event, cpu); + kfree(maskptr); + rc = PTR_ERR(cpump); + goto out; + } + cpumask_set_cpu(cpu, maskptr); + } + + /* + * On error all cpumask are freed and all events have been destroyed. + * Save of which CPUs data structures have been allocated for. + * Release them in paicrypt_event_destroy call back function + * for this event. + */ + PAI_CPU_MASK(event) = maskptr; + rc = 0; +out: + return rc; +} + /* Might be called on different CPU than the one the event is intended for. */ static int paicrypt_event_init(struct perf_event *event) { @@ -267,8 +315,9 @@ static int paicrypt_event_init(struct perf_event *event) if (a->config < PAI_CRYPTO_BASE || a->config > PAI_CRYPTO_BASE + paicrypt_cnt) return -EINVAL; - /* Allow only CPU wide operation, no process context for now. */ - if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) + /* Allow only CPU wide operation for sampling */ + if (a->sample_period && + ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)) return -ENOENT; /* Allow only CRYPTO_ALL for sampling. */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) @@ -282,13 +331,17 @@ static int paicrypt_event_init(struct perf_event *event) } } - cpump = paicrypt_busy(event); - if (IS_ERR(cpump)) { + if (event->cpu >= 0) { + cpump = paicrypt_busy(event, event->cpu); + if (IS_ERR(cpump)) + rc = PTR_ERR(cpump); + } else { + rc = paicrypt_event_init_all(event); + } + if (rc) { free_page(PAI_SAVE_AREA(event)); - rc = PTR_ERR(cpump); goto out; } - event->destroy = paicrypt_event_destroy; if (a->sample_period) { @@ -526,7 +579,7 @@ static const struct attribute_group *paicrypt_attr_groups[] = { /* Performance monitoring unit for mapped counters */ static struct pmu paicrypt = { - .task_ctx_nr = perf_invalid_context, + .task_ctx_nr = perf_hw_context, .event_init = paicrypt_event_init, .add = paicrypt_add, .del = paicrypt_del, From 9f66572f2889a5e72a9d7e17787e52f03b1f7bd8 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 11 Apr 2024 13:15:47 +0200 Subject: [PATCH 103/280] s390/pai_crypto: Enable per-task and system-wide sampling event The PMU for PAI crypto counters enforces the following restrictions: - No per-task context for PAI crypto sampling event CRYPTO_ALL - No multiple system-wide PAI crypto sampling event CRYPTO_ALL Both restrictions are removed. One or more per-task sampling events are supported. Also one or more system-wide sampling events are supported. Example for per-task context of sampling event CRYPTO_ALL: # perf record -e pai_crypto/CRYPTO_ALL/ -- true Example for system-wide context of sampling event CRYPTO_ALL: # perf record -e pai_crypto/CRYPTO_ALL/ -a -- sleep 4 Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_crypto.c | 107 +++++++++++++---------------- 1 file changed, 49 insertions(+), 58 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 0e296a1482bc..c0b7f5422e5e 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -24,6 +24,8 @@ static debug_info_t *cfm_dbg; static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ /* extracted with QPACI instruction */ +#define PAI_SWLIST(x) (&(x)->hw.tp_list) + DEFINE_STATIC_KEY_FALSE(pai_key); struct pai_userdata { @@ -36,8 +38,8 @@ struct paicrypt_map { struct pai_userdata *save; /* Page to store no-zero counters */ unsigned int active_events; /* # of PAI crypto users */ refcount_t refcnt; /* Reference count mapped buffers */ - enum paievt_mode mode; /* Type of event */ struct perf_event *event; /* Perf event for sampling */ + struct list_head syswide_list; /* List system-wide sampling events */ }; struct paicrypt_mapptr { @@ -90,13 +92,10 @@ static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu) struct paicrypt_map *cpump = mp->mapptr; mutex_lock(&pai_reserve_mutex); - debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d" - " mode %d refcnt %u\n", __func__, - event->attr.config, event->cpu, - cpump->active_events, cpump->mode, + debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d " + "refcnt %u\n", __func__, event->attr.config, + event->cpu, cpump->active_events, refcount_read(&cpump->refcnt)); - if (event->attr.sample_period) - cpump->mode &= ~PAI_MODE_SAMPLING; if (refcount_dec_and_test(&cpump->refcnt)) { debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n", __func__, (unsigned long)cpump->page, @@ -172,12 +171,7 @@ static u64 paicrypt_getall(struct perf_event *event) return sum; } -/* Used to avoid races in checking concurrent access of counting and - * sampling for crypto events - * - * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is - * allowed. Several counting events are allowed in parallel. - * +/* Check concurrent access of counting and sampling for crypto events. * This function is called in process context and it is save to block. * When the event initialization functions fails, no other call back will * be invoked. @@ -186,7 +180,6 @@ static u64 paicrypt_getall(struct perf_event *event) */ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) { - struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump = NULL; struct paicrypt_mapptr *mp; int rc; @@ -207,25 +200,9 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) rc = -ENOMEM; goto free_root; } + INIT_LIST_HEAD(&cpump->syswide_list); } - if (a->sample_period) { /* Sampling requested */ - if (cpump->mode & PAI_MODE_SAMPLING) - rc = -EBUSY; /* ... and sampling active */ - else - cpump->mode |= PAI_MODE_SAMPLING; - } - - /* - * This error case triggers when there is a conflict: - * Either sampling requested and counting already active, or visa - * versa. Therefore the struct paicrypto_map for this CPU is - * needed or the error could not have occurred. Only adjust root - * node refcount. - */ - if (rc) - goto free_root; - /* Allocate memory for counter page and counter extraction. * Only the first counting event has to allocate a page. */ @@ -250,14 +227,14 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu) rc = 0; refcount_set(&cpump->refcnt, 1); mp->mapptr = cpump; - debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" - " mode %d refcnt %u page %#lx save %p rc %d\n", - __func__, a->sample_period, cpump->active_events, - cpump->mode, refcount_read(&cpump->refcnt), + debug_sprintf_event(cfm_dbg, 5, "%s users %d refcnt %u page %#lx " + "save %p rc %d\n", __func__, cpump->active_events, + refcount_read(&cpump->refcnt), (unsigned long)cpump->page, cpump->save, rc); goto unlock; free_paicrypt_map: + /* Undo memory allocation */ kfree(cpump); mp->mapptr = NULL; free_root: @@ -315,11 +292,7 @@ static int paicrypt_event_init(struct perf_event *event) if (a->config < PAI_CRYPTO_BASE || a->config > PAI_CRYPTO_BASE + paicrypt_cnt) return -EINVAL; - /* Allow only CPU wide operation for sampling */ - if (a->sample_period && - ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)) - return -ENOENT; - /* Allow only CRYPTO_ALL for sampling. */ + /* Allow only CRYPTO_ALL for sampling */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; /* Get a page to store last counter values for sampling */ @@ -382,9 +355,14 @@ static void paicrypt_start(struct perf_event *event, int flags) sum = paicrypt_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ - cpump->event = event; memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE); - perf_sched_cb_inc(event->pmu); + /* Enable context switch callback for system-wide sampling */ + if (!(event->attach_state & PERF_ATTACH_TASK)) { + list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); + perf_sched_cb_inc(event->pmu); + } else { + cpump->event = event; + } } } @@ -405,6 +383,7 @@ static int paicrypt_add(struct perf_event *event, int flags) return 0; } +static void paicrypt_have_sample(struct perf_event *, struct paicrypt_map *); static void paicrypt_stop(struct perf_event *event, int flags) { struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); @@ -413,8 +392,13 @@ static void paicrypt_stop(struct perf_event *event, int flags) if (!event->attr.sample_period) { /* Counting */ paicrypt_read(event); } else { /* Sampling */ - perf_sched_cb_dec(event->pmu); - cpump->event = NULL; + if (!(event->attach_state & PERF_ATTACH_TASK)) { + perf_sched_cb_dec(event->pmu); + list_del(PAI_SWLIST(event)); + } else { + paicrypt_have_sample(event, cpump); + cpump->event = NULL; + } } event->hw.state = PERF_HES_STOPPED; } @@ -507,23 +491,30 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump, } /* Check if there is data to be saved on schedule out of a task. */ -static int paicrypt_have_sample(void) +static void paicrypt_have_sample(struct perf_event *event, + struct paicrypt_map *cpump) +{ + size_t rawsize; + + if (!event) /* No event active */ + return; + rawsize = paicrypt_copy(cpump->save, cpump->page, + (unsigned long *)PAI_SAVE_AREA(event), + event->attr.exclude_user, + event->attr.exclude_kernel); + if (rawsize) /* No incremented counters */ + paicrypt_push_sample(rawsize, cpump, event); +} + +/* Check if there is data to be saved on schedule out of a task. */ +static void paicrypt_have_samples(void) { struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); struct paicrypt_map *cpump = mp->mapptr; - struct perf_event *event = cpump->event; - size_t rawsize; - int rc = 0; + struct perf_event *event; - if (!event) /* No event active */ - return 0; - rawsize = paicrypt_copy(cpump->save, cpump->page, - (unsigned long *)PAI_SAVE_AREA(event), - cpump->event->attr.exclude_user, - cpump->event->attr.exclude_kernel); - if (rawsize) /* No incremented counters */ - rc = paicrypt_push_sample(rawsize, cpump, event); - return rc; + list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) + paicrypt_have_sample(event, cpump); } /* Called on schedule-in and schedule-out. No access to event structure, @@ -535,7 +526,7 @@ static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sch * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) - paicrypt_have_sample(); + paicrypt_have_samples(); } /* Attribute definitions for paicrypt interface. As with other CPU From 14e3768435da557079216f51d774165c70aad9b1 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 15 Apr 2024 14:40:03 +0200 Subject: [PATCH 104/280] s390/pai_ext: Enable concurrent system-wide counting/sampling The PMU for PAI NNPA counters enforces the following restriction: - No system wide counting while system wide sampling is active. This restriction is removed. One or more system wide counting events can now be active at the same time while at most one system wide sampling event is active. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_ext.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index a6da7e0cc7a6..5d00f77a64a5 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -120,8 +120,10 @@ static void paiext_event_destroy(struct perf_event *event) struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); struct paiext_map *cpump = mp->mapptr; - free_page(PAI_SAVE_AREA(event)); mutex_lock(&paiext_reserve_mutex); + if (event->attr.sample_period) + cpump->mode &= ~PAI_MODE_SAMPLING; + free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ paiext_free(mp); paiext_root_free(); @@ -186,21 +188,19 @@ static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) goto undo; } refcount_set(&cpump->refcnt, 1); - cpump->mode = a->sample_period ? PAI_MODE_SAMPLING - : PAI_MODE_COUNTING; } else { /* Multiple invocation, check what is active. - * Supported are multiple counter events or only one sampling + * Supported are multiple counter events and only one sampling * event concurrently at any one time. */ - if (cpump->mode == PAI_MODE_SAMPLING || - (cpump->mode == PAI_MODE_COUNTING && a->sample_period)) { + if (a->sample_period && (cpump->mode & PAI_MODE_SAMPLING)) { rc = -EBUSY; goto undo; } refcount_inc(&cpump->refcnt); } - + if (a->sample_period) + cpump->mode |= PAI_MODE_SAMPLING; rc = 0; undo: @@ -335,6 +335,8 @@ static void paiext_start(struct perf_event *event, int flags) local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ cpump->event = event; + memcpy((void *)PAI_SAVE_AREA(event), cpump->area, + PAIE1_CTRBLOCK_SZ); perf_sched_cb_inc(event->pmu); } } @@ -493,7 +495,7 @@ static int paiext_have_sample(void) static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) { /* We started with a clean page on event installation. So read out - * results on schedule_out and if page was dirty, clear values. + * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) paiext_have_sample(); From 3f9ff4c5a073294131818a26d1ac2b77b7104695 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 15 Apr 2024 14:50:52 +0200 Subject: [PATCH 105/280] s390/pai_ext: Enable per-task counting event The PMU for PAI NNPA counters enforces the following restriction: - No per-task context for PAI NNPA counters. This restriction is removed. One or more per-task/system-wide counting events can now be active at the same time while one system wide sampling event is active. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/kernel/perf_pai_ext.c | 79 ++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 5d00f77a64a5..20fc12dd9d0d 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -115,22 +115,36 @@ static void paiext_free(struct paiext_mapptr *mp) } /* Release the PMU if event is the last perf event */ -static void paiext_event_destroy(struct perf_event *event) +static void paiext_event_destroy_cpu(struct perf_event *event, int cpu) { - struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); + struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, cpu); struct paiext_map *cpump = mp->mapptr; mutex_lock(&paiext_reserve_mutex); if (event->attr.sample_period) cpump->mode &= ~PAI_MODE_SAMPLING; - free_page(PAI_SAVE_AREA(event)); if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ paiext_free(mp); paiext_root_free(); mutex_unlock(&paiext_reserve_mutex); - debug_sprintf_event(paiext_dbg, 4, "%s cpu %d mapptr %p\n", __func__, - event->cpu, mp->mapptr); +} +static void paiext_event_destroy(struct perf_event *event) +{ + int cpu; + + free_page(PAI_SAVE_AREA(event)); + if (event->cpu == -1) { + struct cpumask *mask = PAI_CPU_MASK(event); + + for_each_cpu(cpu, mask) + paiext_event_destroy_cpu(event, cpu); + kfree(mask); + } else { + paiext_event_destroy_cpu(event, event->cpu); + } + debug_sprintf_event(paiext_dbg, 4, "%s cpu %d\n", __func__, + event->cpu); } /* Used to avoid races in checking concurrent access of counting and @@ -147,19 +161,18 @@ static void paiext_event_destroy(struct perf_event *event) * * Allocate the memory for the event. */ -static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) +static int paiext_alloc_cpu(struct perf_event_attr *a, int cpu) { struct paiext_mapptr *mp; struct paiext_map *cpump; int rc; mutex_lock(&paiext_reserve_mutex); - rc = paiext_root_alloc(); if (rc) goto unlock; - mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); + mp = per_cpu_ptr(paiext_root.mapptr, cpu); cpump = mp->mapptr; if (!cpump) { /* Paiext_map allocated? */ rc = -ENOMEM; @@ -217,6 +230,40 @@ unlock: return rc; } +static int paiext_alloc(struct perf_event *event) +{ + struct cpumask *maskptr; + int cpu, rc = -ENOMEM; + + maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL); + if (!maskptr) + goto out; + + for_each_online_cpu(cpu) { + rc = paiext_alloc_cpu(&event->attr, cpu); + if (rc) { + for_each_cpu(cpu, maskptr) + paiext_event_destroy_cpu(event, cpu); + kfree(maskptr); + goto out; + } + cpumask_set_cpu(cpu, maskptr); + } + + /* + * On error all cpumask are freed and all events have been destroyed. + * Save of which CPUs data structures have been allocated for. + * Release them in paicrypt_event_destroy call back function + * for this event. + */ + PAI_CPU_MASK(event) = maskptr; + rc = 0; +out: + debug_sprintf_event(paiext_dbg, 5, "%s cpu %u rc %d\n", __func__, + cpu, rc); + return rc; +} + /* The PAI extension 1 control block supports up to 128 entries. Return * the index within PAIE1_CB given the event number. Also validate event * number. @@ -246,8 +293,9 @@ static int paiext_event_init(struct perf_event *event) rc = paiext_event_valid(event); if (rc) return rc; - /* Allow only CPU wide operation, no process context for now. */ - if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) + /* Allow only CPU wide operation for sampling */ + if (a->sample_period && + ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)) return -ENOENT; /* Allow only event NNPA_ALL for sampling. */ if (a->sample_period && a->config != PAI_NNPA_BASE) @@ -262,7 +310,10 @@ static int paiext_event_init(struct perf_event *event) return -ENOMEM; } - rc = paiext_alloc(a, event); + if (event->cpu >= 0) + rc = paiext_alloc_cpu(a, event->cpu); + else + rc = paiext_alloc(event); if (rc) { free_page(PAI_SAVE_AREA(event)); return rc; @@ -352,8 +403,6 @@ static int paiext_add(struct perf_event *event, int flags) pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); - debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", - __func__, S390_lowcore.aicd, pcb->acc); } if (flags & PERF_EF_START) paiext_start(event, PERF_EF_RELOAD); @@ -387,8 +436,6 @@ static void paiext_del(struct perf_event *event, int flags) local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); pcb->acc = 0; S390_lowcore.aicd = 0; - debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", - __func__, S390_lowcore.aicd, pcb->acc); } } @@ -544,7 +591,7 @@ static const struct attribute_group *paiext_attr_groups[] = { /* Performance monitoring unit for mapped counters */ static struct pmu paiext = { - .task_ctx_nr = perf_invalid_context, + .task_ctx_nr = perf_hw_context, .event_init = paiext_event_init, .add = paiext_add, .del = paiext_del, From 582cc1b28e8c8cb25f9a5f2595ca4b6d90737d03 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 15 Apr 2024 16:41:23 +0200 Subject: [PATCH 106/280] s390/pai_ext: Enable per-task and system-wide sampling event The PMU for PAI NNPA counters enforces the following restriction: - No per-task context for PAI sampling event NNPA_ALL - No multiple system-wide PAI sampling event NNPA_ALL Both restrictions are removed. One or more per-task sampling events are supported. Also one or more system-wide sampling events are supported. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Alexander Gordeev --- arch/s390/include/asm/pai.h | 8 +--- arch/s390/kernel/perf_pai_crypto.c | 2 - arch/s390/kernel/perf_pai_ext.c | 75 ++++++++++++++++-------------- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 7da1cec42016..8e1dd389ae88 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -75,12 +75,8 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) WRITE_ONCE(S390_lowcore.ccd, S390_lowcore.ccd & ~PAI_CRYPTO_KERNEL_OFFSET); } -enum paievt_mode { - PAI_MODE_NONE, - PAI_MODE_SAMPLING, - PAI_MODE_COUNTING, -}; - #define PAI_SAVE_AREA(x) ((x)->hw.event_base) #define PAI_CPU_MASK(x) ((x)->hw.addr_filters) +#define PAI_SWLIST(x) (&(x)->hw.tp_list) + #endif diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index c0b7f5422e5e..95079a289109 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -24,8 +24,6 @@ static debug_info_t *cfm_dbg; static unsigned int paicrypt_cnt; /* Size of the mapped counter sets */ /* extracted with QPACI instruction */ -#define PAI_SWLIST(x) (&(x)->hw.tp_list) - DEFINE_STATIC_KEY_FALSE(pai_key); struct pai_userdata { diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 20fc12dd9d0d..6040f3c2b942 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -47,11 +47,11 @@ struct paiext_cb { /* PAI extension 1 control block */ struct paiext_map { unsigned long *area; /* Area for CPU to store counters */ struct pai_userdata *save; /* Area to store non-zero counters */ - enum paievt_mode mode; /* Type of event */ unsigned int active_events; /* # of PAI Extension users */ refcount_t refcnt; struct perf_event *event; /* Perf event for sampling */ struct paiext_cb *paiext_cb; /* PAI extension control block area */ + struct list_head syswide_list; /* List system-wide sampling events */ }; struct paiext_mapptr { @@ -70,6 +70,8 @@ static void paiext_root_free(void) free_percpu(paiext_root.mapptr); paiext_root.mapptr = NULL; } + debug_sprintf_event(paiext_dbg, 5, "%s root.refcount %d\n", __func__, + refcount_read(&paiext_root.refcnt)); } /* On initialization of first event also allocate per CPU data dynamically. @@ -121,8 +123,6 @@ static void paiext_event_destroy_cpu(struct perf_event *event, int cpu) struct paiext_map *cpump = mp->mapptr; mutex_lock(&paiext_reserve_mutex); - if (event->attr.sample_period) - cpump->mode &= ~PAI_MODE_SAMPLING; if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */ paiext_free(mp); paiext_root_free(); @@ -161,7 +161,7 @@ static void paiext_event_destroy(struct perf_event *event) * * Allocate the memory for the event. */ -static int paiext_alloc_cpu(struct perf_event_attr *a, int cpu) +static int paiext_alloc_cpu(struct perf_event *event, int cpu) { struct paiext_mapptr *mp; struct paiext_map *cpump; @@ -200,21 +200,12 @@ static int paiext_alloc_cpu(struct perf_event_attr *a, int cpu) paiext_free(mp); goto undo; } + INIT_LIST_HEAD(&cpump->syswide_list); refcount_set(&cpump->refcnt, 1); + rc = 0; } else { - /* Multiple invocation, check what is active. - * Supported are multiple counter events and only one sampling - * event concurrently at any one time. - */ - if (a->sample_period && (cpump->mode & PAI_MODE_SAMPLING)) { - rc = -EBUSY; - goto undo; - } refcount_inc(&cpump->refcnt); } - if (a->sample_period) - cpump->mode |= PAI_MODE_SAMPLING; - rc = 0; undo: if (rc) { @@ -240,7 +231,7 @@ static int paiext_alloc(struct perf_event *event) goto out; for_each_online_cpu(cpu) { - rc = paiext_alloc_cpu(&event->attr, cpu); + rc = paiext_alloc_cpu(event, cpu); if (rc) { for_each_cpu(cpu, maskptr) paiext_event_destroy_cpu(event, cpu); @@ -259,8 +250,6 @@ static int paiext_alloc(struct perf_event *event) PAI_CPU_MASK(event) = maskptr; rc = 0; out: - debug_sprintf_event(paiext_dbg, 5, "%s cpu %u rc %d\n", __func__, - cpu, rc); return rc; } @@ -293,10 +282,6 @@ static int paiext_event_init(struct perf_event *event) rc = paiext_event_valid(event); if (rc) return rc; - /* Allow only CPU wide operation for sampling */ - if (a->sample_period && - ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1)) - return -ENOENT; /* Allow only event NNPA_ALL for sampling. */ if (a->sample_period && a->config != PAI_NNPA_BASE) return -EINVAL; @@ -311,7 +296,7 @@ static int paiext_event_init(struct perf_event *event) } if (event->cpu >= 0) - rc = paiext_alloc_cpu(a, event->cpu); + rc = paiext_alloc_cpu(event, event->cpu); else rc = paiext_alloc(event); if (rc) { @@ -385,10 +370,15 @@ static void paiext_start(struct perf_event *event, int flags) sum = paiext_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); } else { /* Sampling */ - cpump->event = event; memcpy((void *)PAI_SAVE_AREA(event), cpump->area, PAIE1_CTRBLOCK_SZ); - perf_sched_cb_inc(event->pmu); + /* Enable context switch callback for system-wide sampling */ + if (!(event->attach_state & PERF_ATTACH_TASK)) { + list_add_tail(PAI_SWLIST(event), &cpump->syswide_list); + perf_sched_cb_inc(event->pmu); + } else { + cpump->event = event; + } } } @@ -410,6 +400,7 @@ static int paiext_add(struct perf_event *event, int flags) return 0; } +static void paiext_have_sample(struct perf_event *, struct paiext_map *); static void paiext_stop(struct perf_event *event, int flags) { struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); @@ -418,8 +409,13 @@ static void paiext_stop(struct perf_event *event, int flags) if (!event->attr.sample_period) { /* Counting */ paiext_read(event); } else { /* Sampling */ - perf_sched_cb_dec(event->pmu); - cpump->event = NULL; + if (!(event->attach_state & PERF_ATTACH_TASK)) { + list_del(PAI_SWLIST(event)); + perf_sched_cb_dec(event->pmu); + } else { + paiext_have_sample(event, cpump); + cpump->event = NULL; + } } event->hw.state = PERF_HES_STOPPED; } @@ -519,21 +515,28 @@ static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump, } /* Check if there is data to be saved on schedule out of a task. */ -static int paiext_have_sample(void) +static void paiext_have_sample(struct perf_event *event, + struct paiext_map *cpump) { - struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); - struct paiext_map *cpump = mp->mapptr; - struct perf_event *event = cpump->event; size_t rawsize; - int rc = 0; if (!event) - return 0; + return; rawsize = paiext_copy(cpump->save, cpump->area, (unsigned long *)PAI_SAVE_AREA(event)); if (rawsize) /* Incremented counters */ - rc = paiext_push_sample(rawsize, cpump, event); - return rc; + paiext_push_sample(rawsize, cpump, event); +} + +/* Check if there is data to be saved on schedule out of a task. */ +static void paiext_have_samples(void) +{ + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; + struct perf_event *event; + + list_for_each_entry(event, &cpump->syswide_list, hw.tp_list) + paiext_have_sample(event, cpump); } /* Called on schedule-in and schedule-out. No access to event structure, @@ -545,7 +548,7 @@ static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched * results on schedule_out and if page was dirty, save old values. */ if (!sched_in) - paiext_have_sample(); + paiext_have_samples(); } /* Attribute definitions for pai extension1 interface. As with other CPU From 0dddd91ab68fed4fe12b69fe225cb00a4ea16126 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 4 Jun 2024 23:38:27 -0700 Subject: [PATCH 107/280] perf stat: Make options local Reduce the scope of stat_options to cmd_stat, and pass as an argument to __cmd_record. This is done to make more localized changes to the options in later patches. A side-effect of the change is to reduce the size of a stripped PIE perf binary by 5952 bytes. The savings come mainly in the dynamic relocation section. Signed-off-by: Ian Rogers Cc: Stephane Eranian Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240605063828.195700-1-irogers@google.com --- tools/perf/builtin-stat.c | 291 +++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 146 deletions(-) diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 35f79b48e8dc..6227b25d1446 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -1140,150 +1140,6 @@ out: return 0; } -static struct option stat_options[] = { - OPT_BOOLEAN('T', "transaction", &transaction_run, - "hardware transaction statistics"), - OPT_CALLBACK('e', "event", &parse_events_option_args, "event", - "event selector. use 'perf list' to list available events", - parse_events_option), - OPT_CALLBACK(0, "filter", &evsel_list, "filter", - "event filter", parse_filter), - OPT_BOOLEAN('i', "no-inherit", &stat_config.no_inherit, - "child tasks do not inherit counters"), - OPT_STRING('p', "pid", &target.pid, "pid", - "stat events on existing process id"), - OPT_STRING('t', "tid", &target.tid, "tid", - "stat events on existing thread id"), -#ifdef HAVE_BPF_SKEL - OPT_STRING('b', "bpf-prog", &target.bpf_str, "bpf-prog-id", - "stat events on existing bpf program id"), - OPT_BOOLEAN(0, "bpf-counters", &target.use_bpf, - "use bpf program to count events"), - OPT_STRING(0, "bpf-attr-map", &target.attr_map, "attr-map-path", - "path to perf_event_attr map"), -#endif - OPT_BOOLEAN('a', "all-cpus", &target.system_wide, - "system-wide collection from all CPUs"), - OPT_BOOLEAN(0, "scale", &stat_config.scale, - "Use --no-scale to disable counter scaling for multiplexing"), - OPT_INCR('v', "verbose", &verbose, - "be more verbose (show counter open errors, etc)"), - OPT_INTEGER('r', "repeat", &stat_config.run_count, - "repeat command and print average + stddev (max: 100, forever: 0)"), - OPT_BOOLEAN(0, "table", &stat_config.walltime_run_table, - "display details about each run (only with -r option)"), - OPT_BOOLEAN('n', "null", &stat_config.null_run, - "null run - dont start any counters"), - OPT_INCR('d', "detailed", &detailed_run, - "detailed run - start a lot of events"), - OPT_BOOLEAN('S', "sync", &sync_run, - "call sync() before starting a run"), - OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL, - "print large numbers with thousands\' separators", - stat__set_big_num), - OPT_STRING('C', "cpu", &target.cpu_list, "cpu", - "list of cpus to monitor in system-wide"), - OPT_SET_UINT('A', "no-aggr", &stat_config.aggr_mode, - "disable aggregation across CPUs or PMUs", AGGR_NONE), - OPT_SET_UINT(0, "no-merge", &stat_config.aggr_mode, - "disable aggregation the same as -A or -no-aggr", AGGR_NONE), - OPT_BOOLEAN(0, "hybrid-merge", &stat_config.hybrid_merge, - "Merge identical named hybrid events"), - OPT_STRING('x', "field-separator", &stat_config.csv_sep, "separator", - "print counts with custom separator"), - OPT_BOOLEAN('j', "json-output", &stat_config.json_output, - "print counts in JSON format"), - OPT_CALLBACK('G', "cgroup", &evsel_list, "name", - "monitor event in cgroup name only", parse_stat_cgroups), - OPT_STRING(0, "for-each-cgroup", &stat_config.cgroup_list, "name", - "expand events for each cgroup"), - OPT_STRING('o', "output", &output_name, "file", "output file name"), - OPT_BOOLEAN(0, "append", &append_file, "append to the output file"), - OPT_INTEGER(0, "log-fd", &output_fd, - "log output to fd, instead of stderr"), - OPT_STRING(0, "pre", &pre_cmd, "command", - "command to run prior to the measured command"), - OPT_STRING(0, "post", &post_cmd, "command", - "command to run after to the measured command"), - OPT_UINTEGER('I', "interval-print", &stat_config.interval, - "print counts at regular interval in ms " - "(overhead is possible for values <= 100ms)"), - OPT_INTEGER(0, "interval-count", &stat_config.times, - "print counts for fixed number of times"), - OPT_BOOLEAN(0, "interval-clear", &stat_config.interval_clear, - "clear screen in between new interval"), - OPT_UINTEGER(0, "timeout", &stat_config.timeout, - "stop workload and print counts after a timeout period in ms (>= 10ms)"), - OPT_SET_UINT(0, "per-socket", &stat_config.aggr_mode, - "aggregate counts per processor socket", AGGR_SOCKET), - OPT_SET_UINT(0, "per-die", &stat_config.aggr_mode, - "aggregate counts per processor die", AGGR_DIE), - OPT_SET_UINT(0, "per-cluster", &stat_config.aggr_mode, - "aggregate counts per processor cluster", AGGR_CLUSTER), - OPT_CALLBACK_OPTARG(0, "per-cache", &stat_config.aggr_mode, &stat_config.aggr_level, - "cache level", "aggregate count at this cache level (Default: LLC)", - parse_cache_level), - OPT_SET_UINT(0, "per-core", &stat_config.aggr_mode, - "aggregate counts per physical processor core", AGGR_CORE), - OPT_SET_UINT(0, "per-thread", &stat_config.aggr_mode, - "aggregate counts per thread", AGGR_THREAD), - OPT_SET_UINT(0, "per-node", &stat_config.aggr_mode, - "aggregate counts per numa node", AGGR_NODE), - OPT_INTEGER('D', "delay", &target.initial_delay, - "ms to wait before starting measurement after program start (-1: start with events disabled)"), - OPT_CALLBACK_NOOPT(0, "metric-only", &stat_config.metric_only, NULL, - "Only print computed metrics. No raw values", enable_metric_only), - OPT_BOOLEAN(0, "metric-no-group", &stat_config.metric_no_group, - "don't group metric events, impacts multiplexing"), - OPT_BOOLEAN(0, "metric-no-merge", &stat_config.metric_no_merge, - "don't try to share events between metrics in a group"), - OPT_BOOLEAN(0, "metric-no-threshold", &stat_config.metric_no_threshold, - "disable adding events for the metric threshold calculation"), - OPT_BOOLEAN(0, "topdown", &topdown_run, - "measure top-down statistics"), - OPT_UINTEGER(0, "td-level", &stat_config.topdown_level, - "Set the metrics level for the top-down statistics (0: max level)"), - OPT_BOOLEAN(0, "smi-cost", &smi_cost, - "measure SMI cost"), - OPT_CALLBACK('M', "metrics", &evsel_list, "metric/metric group list", - "monitor specified metrics or metric groups (separated by ,)", - append_metric_groups), - OPT_BOOLEAN_FLAG(0, "all-kernel", &stat_config.all_kernel, - "Configure all used events to run in kernel space.", - PARSE_OPT_EXCLUSIVE), - OPT_BOOLEAN_FLAG(0, "all-user", &stat_config.all_user, - "Configure all used events to run in user space.", - PARSE_OPT_EXCLUSIVE), - OPT_BOOLEAN(0, "percore-show-thread", &stat_config.percore_show_thread, - "Use with 'percore' event qualifier to show the event " - "counts of one hardware thread by sum up total hardware " - "threads of same physical core"), - OPT_BOOLEAN(0, "summary", &stat_config.summary, - "print summary for interval mode"), - OPT_BOOLEAN(0, "no-csv-summary", &stat_config.no_csv_summary, - "don't print 'summary' for CSV summary output"), - OPT_BOOLEAN(0, "quiet", &quiet, - "don't print any output, messages or warnings (useful with record)"), - OPT_CALLBACK(0, "cputype", &evsel_list, "hybrid cpu type", - "Only enable events on applying cpu with this type " - "for hybrid platform (e.g. core or atom)", - parse_cputype), -#ifdef HAVE_LIBPFM - OPT_CALLBACK(0, "pfm-events", &evsel_list, "event", - "libpfm4 event selector. use 'perf list' to list available events", - parse_libpfm_events_option), -#endif - OPT_CALLBACK(0, "control", &stat_config, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]", - "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events).\n" - "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n" - "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.", - parse_control_option), - OPT_CALLBACK_OPTARG(0, "iostat", &evsel_list, &stat_config, "default", - "measure I/O performance metrics provided by arch/platform", - iostat_parse), - OPT_END() -}; - /** * Calculate the cache instance ID from the map in * /sys/devices/system/cpu/cpuX/cache/indexY/shared_cpu_list @@ -2245,7 +2101,7 @@ static void init_features(struct perf_session *session) perf_header__clear_feat(&session->header, HEADER_AUXTRACE); } -static int __cmd_record(int argc, const char **argv) +static int __cmd_record(const struct option stat_options[], int argc, const char **argv) { struct perf_session *session; struct perf_data *data = &perf_stat.data; @@ -2494,6 +2350,149 @@ static void setup_system_wide(int forks) int cmd_stat(int argc, const char **argv) { + struct option stat_options[] = { + OPT_BOOLEAN('T', "transaction", &transaction_run, + "hardware transaction statistics"), + OPT_CALLBACK('e', "event", &parse_events_option_args, "event", + "event selector. use 'perf list' to list available events", + parse_events_option), + OPT_CALLBACK(0, "filter", &evsel_list, "filter", + "event filter", parse_filter), + OPT_BOOLEAN('i', "no-inherit", &stat_config.no_inherit, + "child tasks do not inherit counters"), + OPT_STRING('p', "pid", &target.pid, "pid", + "stat events on existing process id"), + OPT_STRING('t', "tid", &target.tid, "tid", + "stat events on existing thread id"), +#ifdef HAVE_BPF_SKEL + OPT_STRING('b', "bpf-prog", &target.bpf_str, "bpf-prog-id", + "stat events on existing bpf program id"), + OPT_BOOLEAN(0, "bpf-counters", &target.use_bpf, + "use bpf program to count events"), + OPT_STRING(0, "bpf-attr-map", &target.attr_map, "attr-map-path", + "path to perf_event_attr map"), +#endif + OPT_BOOLEAN('a', "all-cpus", &target.system_wide, + "system-wide collection from all CPUs"), + OPT_BOOLEAN(0, "scale", &stat_config.scale, + "Use --no-scale to disable counter scaling for multiplexing"), + OPT_INCR('v', "verbose", &verbose, + "be more verbose (show counter open errors, etc)"), + OPT_INTEGER('r', "repeat", &stat_config.run_count, + "repeat command and print average + stddev (max: 100, forever: 0)"), + OPT_BOOLEAN(0, "table", &stat_config.walltime_run_table, + "display details about each run (only with -r option)"), + OPT_BOOLEAN('n', "null", &stat_config.null_run, + "null run - dont start any counters"), + OPT_INCR('d', "detailed", &detailed_run, + "detailed run - start a lot of events"), + OPT_BOOLEAN('S', "sync", &sync_run, + "call sync() before starting a run"), + OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL, + "print large numbers with thousands\' separators", + stat__set_big_num), + OPT_STRING('C', "cpu", &target.cpu_list, "cpu", + "list of cpus to monitor in system-wide"), + OPT_SET_UINT('A', "no-aggr", &stat_config.aggr_mode, + "disable aggregation across CPUs or PMUs", AGGR_NONE), + OPT_SET_UINT(0, "no-merge", &stat_config.aggr_mode, + "disable aggregation the same as -A or -no-aggr", AGGR_NONE), + OPT_BOOLEAN(0, "hybrid-merge", &stat_config.hybrid_merge, + "Merge identical named hybrid events"), + OPT_STRING('x', "field-separator", &stat_config.csv_sep, "separator", + "print counts with custom separator"), + OPT_BOOLEAN('j', "json-output", &stat_config.json_output, + "print counts in JSON format"), + OPT_CALLBACK('G', "cgroup", &evsel_list, "name", + "monitor event in cgroup name only", parse_stat_cgroups), + OPT_STRING(0, "for-each-cgroup", &stat_config.cgroup_list, "name", + "expand events for each cgroup"), + OPT_STRING('o', "output", &output_name, "file", "output file name"), + OPT_BOOLEAN(0, "append", &append_file, "append to the output file"), + OPT_INTEGER(0, "log-fd", &output_fd, + "log output to fd, instead of stderr"), + OPT_STRING(0, "pre", &pre_cmd, "command", + "command to run prior to the measured command"), + OPT_STRING(0, "post", &post_cmd, "command", + "command to run after to the measured command"), + OPT_UINTEGER('I', "interval-print", &stat_config.interval, + "print counts at regular interval in ms " + "(overhead is possible for values <= 100ms)"), + OPT_INTEGER(0, "interval-count", &stat_config.times, + "print counts for fixed number of times"), + OPT_BOOLEAN(0, "interval-clear", &stat_config.interval_clear, + "clear screen in between new interval"), + OPT_UINTEGER(0, "timeout", &stat_config.timeout, + "stop workload and print counts after a timeout period in ms (>= 10ms)"), + OPT_SET_UINT(0, "per-socket", &stat_config.aggr_mode, + "aggregate counts per processor socket", AGGR_SOCKET), + OPT_SET_UINT(0, "per-die", &stat_config.aggr_mode, + "aggregate counts per processor die", AGGR_DIE), + OPT_SET_UINT(0, "per-cluster", &stat_config.aggr_mode, + "aggregate counts per processor cluster", AGGR_CLUSTER), + OPT_CALLBACK_OPTARG(0, "per-cache", &stat_config.aggr_mode, &stat_config.aggr_level, + "cache level", "aggregate count at this cache level (Default: LLC)", + parse_cache_level), + OPT_SET_UINT(0, "per-core", &stat_config.aggr_mode, + "aggregate counts per physical processor core", AGGR_CORE), + OPT_SET_UINT(0, "per-thread", &stat_config.aggr_mode, + "aggregate counts per thread", AGGR_THREAD), + OPT_SET_UINT(0, "per-node", &stat_config.aggr_mode, + "aggregate counts per numa node", AGGR_NODE), + OPT_INTEGER('D', "delay", &target.initial_delay, + "ms to wait before starting measurement after program start (-1: start with events disabled)"), + OPT_CALLBACK_NOOPT(0, "metric-only", &stat_config.metric_only, NULL, + "Only print computed metrics. No raw values", enable_metric_only), + OPT_BOOLEAN(0, "metric-no-group", &stat_config.metric_no_group, + "don't group metric events, impacts multiplexing"), + OPT_BOOLEAN(0, "metric-no-merge", &stat_config.metric_no_merge, + "don't try to share events between metrics in a group"), + OPT_BOOLEAN(0, "metric-no-threshold", &stat_config.metric_no_threshold, + "disable adding events for the metric threshold calculation"), + OPT_BOOLEAN(0, "topdown", &topdown_run, + "measure top-down statistics"), + OPT_UINTEGER(0, "td-level", &stat_config.topdown_level, + "Set the metrics level for the top-down statistics (0: max level)"), + OPT_BOOLEAN(0, "smi-cost", &smi_cost, + "measure SMI cost"), + OPT_CALLBACK('M', "metrics", &evsel_list, "metric/metric group list", + "monitor specified metrics or metric groups (separated by ,)", + append_metric_groups), + OPT_BOOLEAN_FLAG(0, "all-kernel", &stat_config.all_kernel, + "Configure all used events to run in kernel space.", + PARSE_OPT_EXCLUSIVE), + OPT_BOOLEAN_FLAG(0, "all-user", &stat_config.all_user, + "Configure all used events to run in user space.", + PARSE_OPT_EXCLUSIVE), + OPT_BOOLEAN(0, "percore-show-thread", &stat_config.percore_show_thread, + "Use with 'percore' event qualifier to show the event " + "counts of one hardware thread by sum up total hardware " + "threads of same physical core"), + OPT_BOOLEAN(0, "summary", &stat_config.summary, + "print summary for interval mode"), + OPT_BOOLEAN(0, "no-csv-summary", &stat_config.no_csv_summary, + "don't print 'summary' for CSV summary output"), + OPT_BOOLEAN(0, "quiet", &quiet, + "don't print any output, messages or warnings (useful with record)"), + OPT_CALLBACK(0, "cputype", &evsel_list, "hybrid cpu type", + "Only enable events on applying cpu with this type " + "for hybrid platform (e.g. core or atom)", + parse_cputype), +#ifdef HAVE_LIBPFM + OPT_CALLBACK(0, "pfm-events", &evsel_list, "event", + "libpfm4 event selector. use 'perf list' to list available events", + parse_libpfm_events_option), +#endif + OPT_CALLBACK(0, "control", &stat_config, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]", + "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events).\n" + "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n" + "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.", + parse_control_option), + OPT_CALLBACK_OPTARG(0, "iostat", &evsel_list, &stat_config, "default", + "measure I/O performance metrics provided by arch/platform", + iostat_parse), + OPT_END() + }; const char * const stat_usage[] = { "perf stat [] []", NULL @@ -2530,7 +2529,7 @@ int cmd_stat(int argc, const char **argv) stat_config.csv_sep = DEFAULT_SEPARATOR; if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) { - argc = __cmd_record(argc, argv); + argc = __cmd_record(stat_options, argc, argv); if (argc < 0) return -1; } else if (argc && strlen(argv[0]) > 2 && strstarts("report", argv[0])) From f5803651b4a4b67d9521994d501619a0d8ff3b11 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 4 Jun 2024 23:38:28 -0700 Subject: [PATCH 108/280] perf stat: Choose the most disaggregate command line option When multiple aggregation options are passed to perf stat the behavior isn't clear. Consider "perf stat -A --per-socket .." and "perf stat --per-socket -A ..", the first won't aggregate at all while the second will do per-socket aggregation, even though the same options were passed. Rather than set an enum value, gather the options in a struct and process them from most to least aggregate. This ensures the least aggregate option always applies, so no aggregation if "-A" is passed. Signed-off-by: Ian Rogers Cc: Stephane Eranian Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240605063828.195700-2-irogers@google.com --- tools/perf/builtin-stat.c | 73 ++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 6227b25d1446..603a9684153d 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -164,6 +164,35 @@ static struct perf_stat_config stat_config = { .iostat_run = false, }; +/* Options set from the command line. */ +struct opt_aggr_mode { + bool node, socket, die, cluster, cache, core, thread, no_aggr; +}; + +/* Turn command line option into most generic aggregation mode setting. */ +static enum aggr_mode opt_aggr_mode_to_aggr_mode(struct opt_aggr_mode *opt_mode) +{ + enum aggr_mode mode = AGGR_GLOBAL; + + if (opt_mode->node) + mode = AGGR_NODE; + if (opt_mode->socket) + mode = AGGR_SOCKET; + if (opt_mode->die) + mode = AGGR_DIE; + if (opt_mode->cluster) + mode = AGGR_CLUSTER; + if (opt_mode->cache) + mode = AGGR_CACHE; + if (opt_mode->core) + mode = AGGR_CORE; + if (opt_mode->thread) + mode = AGGR_THREAD; + if (opt_mode->no_aggr) + mode = AGGR_NONE; + return mode; +} + static void evlist__check_cpu_maps(struct evlist *evlist) { struct evsel *evsel, *warned_leader = NULL; @@ -1096,7 +1125,7 @@ static int parse_cache_level(const struct option *opt, int unset __maybe_unused) { int level; - u32 *aggr_mode = (u32 *)opt->value; + struct opt_aggr_mode *opt_aggr_mode = (struct opt_aggr_mode *)opt->value; u32 *aggr_level = (u32 *)opt->data; /* @@ -1135,7 +1164,7 @@ static int parse_cache_level(const struct option *opt, return -EINVAL; } out: - *aggr_mode = AGGR_CACHE; + opt_aggr_mode->cache = true; *aggr_level = level; return 0; } @@ -2101,13 +2130,15 @@ static void init_features(struct perf_session *session) perf_header__clear_feat(&session->header, HEADER_AUXTRACE); } -static int __cmd_record(const struct option stat_options[], int argc, const char **argv) +static int __cmd_record(const struct option stat_options[], struct opt_aggr_mode *opt_mode, + int argc, const char **argv) { struct perf_session *session; struct perf_data *data = &perf_stat.data; argc = parse_options(argc, argv, stat_options, stat_record_usage, PARSE_OPT_STOP_AT_NON_OPTION); + stat_config.aggr_mode = opt_aggr_mode_to_aggr_mode(opt_mode); if (output_name) data->path = output_name; @@ -2350,6 +2381,7 @@ static void setup_system_wide(int forks) int cmd_stat(int argc, const char **argv) { + struct opt_aggr_mode opt_mode = {}; struct option stat_options[] = { OPT_BOOLEAN('T', "transaction", &transaction_run, "hardware transaction statistics"), @@ -2393,10 +2425,10 @@ int cmd_stat(int argc, const char **argv) stat__set_big_num), OPT_STRING('C', "cpu", &target.cpu_list, "cpu", "list of cpus to monitor in system-wide"), - OPT_SET_UINT('A', "no-aggr", &stat_config.aggr_mode, - "disable aggregation across CPUs or PMUs", AGGR_NONE), - OPT_SET_UINT(0, "no-merge", &stat_config.aggr_mode, - "disable aggregation the same as -A or -no-aggr", AGGR_NONE), + OPT_BOOLEAN('A', "no-aggr", &opt_mode.no_aggr, + "disable aggregation across CPUs or PMUs"), + OPT_BOOLEAN(0, "no-merge", &opt_mode.no_aggr, + "disable aggregation the same as -A or -no-aggr"), OPT_BOOLEAN(0, "hybrid-merge", &stat_config.hybrid_merge, "Merge identical named hybrid events"), OPT_STRING('x', "field-separator", &stat_config.csv_sep, "separator", @@ -2424,21 +2456,18 @@ int cmd_stat(int argc, const char **argv) "clear screen in between new interval"), OPT_UINTEGER(0, "timeout", &stat_config.timeout, "stop workload and print counts after a timeout period in ms (>= 10ms)"), - OPT_SET_UINT(0, "per-socket", &stat_config.aggr_mode, - "aggregate counts per processor socket", AGGR_SOCKET), - OPT_SET_UINT(0, "per-die", &stat_config.aggr_mode, - "aggregate counts per processor die", AGGR_DIE), - OPT_SET_UINT(0, "per-cluster", &stat_config.aggr_mode, - "aggregate counts per processor cluster", AGGR_CLUSTER), - OPT_CALLBACK_OPTARG(0, "per-cache", &stat_config.aggr_mode, &stat_config.aggr_level, + OPT_BOOLEAN(0, "per-socket", &opt_mode.socket, + "aggregate counts per processor socket"), + OPT_BOOLEAN(0, "per-die", &opt_mode.die, "aggregate counts per processor die"), + OPT_BOOLEAN(0, "per-cluster", &opt_mode.cluster, + "aggregate counts per processor cluster"), + OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode, &stat_config.aggr_level, "cache level", "aggregate count at this cache level (Default: LLC)", parse_cache_level), - OPT_SET_UINT(0, "per-core", &stat_config.aggr_mode, - "aggregate counts per physical processor core", AGGR_CORE), - OPT_SET_UINT(0, "per-thread", &stat_config.aggr_mode, - "aggregate counts per thread", AGGR_THREAD), - OPT_SET_UINT(0, "per-node", &stat_config.aggr_mode, - "aggregate counts per numa node", AGGR_NODE), + OPT_BOOLEAN(0, "per-core", &opt_mode.core, + "aggregate counts per physical processor core"), + OPT_BOOLEAN(0, "per-thread", &opt_mode.thread, "aggregate counts per thread"), + OPT_BOOLEAN(0, "per-node", &opt_mode.node, "aggregate counts per numa node"), OPT_INTEGER('D', "delay", &target.initial_delay, "ms to wait before starting measurement after program start (-1: start with events disabled)"), OPT_CALLBACK_NOOPT(0, "metric-only", &stat_config.metric_only, NULL, @@ -2521,6 +2550,8 @@ int cmd_stat(int argc, const char **argv) (const char **) stat_usage, PARSE_OPT_STOP_AT_NON_OPTION); + stat_config.aggr_mode = opt_aggr_mode_to_aggr_mode(&opt_mode); + if (stat_config.csv_sep) { stat_config.csv_output = true; if (!strcmp(stat_config.csv_sep, "\\t")) @@ -2529,7 +2560,7 @@ int cmd_stat(int argc, const char **argv) stat_config.csv_sep = DEFAULT_SEPARATOR; if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) { - argc = __cmd_record(stat_options, argc, argv); + argc = __cmd_record(stat_options, &opt_mode, argc, argv); if (argc < 0) return -1; } else if (argc && strlen(argv[0]) > 2 && strstarts("report", argv[0])) From 658a8805cb604260ae9f35ae5e50012e3d1ed5e3 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 7 Jun 2024 07:43:52 +0200 Subject: [PATCH 109/280] perf test: Speed up test case 70 annotate basic tests On some s390 linux machine (mostly older models) and with debug packages installed, the test case 'perf annotate basic tests' runs for some longer time. Speed up the test and save the output of command perf annotate in a temporary file. This is used to perform pattern matching via grep command. This saves on invocation of perf annotate which runs for some time. Output before: # time bash -x tests/shell/annotate.sh >/dev/null 2>&1; echo EXIT CODE $? real 4m35.543s user 3m19.442s sys 1m14.322s EXIT CODE 0 # Output after: # time bash -x tests/shell/annotate.sh >/dev/null 2>&1; echo EXIT CODE $? real 2m2.881s user 1m30.980s sys 0m30.684s EXIT CODE 0 # Signed-off-by: Thomas Richter Acked-by: Namhyung Kim Cc: gor@linux.ibm.com Cc: hca@linux.ibm.com Cc: sumanthk@linux.ibm.com Cc: svens@linux.ibm.com Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607054352.2774936-1-tmricht@linux.ibm.com --- tools/perf/tests/shell/annotate.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/perf/tests/shell/annotate.sh b/tools/perf/tests/shell/annotate.sh index 1db1e8113d99..b072d9b97387 100755 --- a/tools/perf/tests/shell/annotate.sh +++ b/tools/perf/tests/shell/annotate.sh @@ -15,12 +15,13 @@ skip_test_missing_symbol ${testsym} err=0 perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +perfout=$(mktemp /tmp/__perf_test.perf.out.XXXXX) testprog="perf test -w noploop" # disassembly format: "percent : offset: instruction (operands ...)" disasm_regex="[0-9]*\.[0-9]* *: *\w*: *\w*" cleanup() { - rm -rf "${perfdata}" + rm -rf "${perfdata}" "${perfout}" rm -rf "${perfdata}".old trap - EXIT TERM INT @@ -41,8 +42,11 @@ test_basic() { return fi + # Generate the annotated output file + perf annotate -i "${perfdata}" --stdio 2> /dev/null > "${perfout}" + # check if it has the target symbol - if ! perf annotate -i "${perfdata}" 2> /dev/null | grep "${testsym}" + if ! grep "${testsym}" "${perfout}" then echo "Basic annotate [Failed: missing target symbol]" err=1 @@ -50,7 +54,7 @@ test_basic() { fi # check if it has the disassembly lines - if ! perf annotate -i "${perfdata}" 2> /dev/null | grep "${disasm_regex}" + if ! grep "${disasm_regex}" "${perfout}" then echo "Basic annotate [Failed: missing disasm output from default disassembler]" err=1 From 0e9899feed9cbb7d33c01ad849dc307b1560b0ab Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Mon, 10 Jun 2024 14:37:42 +0000 Subject: [PATCH 110/280] mm/mm_init.c: don't initialize page->lru again Current page initialization call flow looks like this with some simplification: setup_arch() paging_init() free_area_init() memmap_init() memmap_init_zone_range() memmap_init_range() defer_init() __init_single_page() mm_core_init() mem_init() memblock_free_all() free_low_memory_core_early() memmap_init_reserved_pages() reserve_bootmem_region() init_reserved_page() __init_single_page() There two cases depends on CONFIG_DEFERRED_STRUCT_PAGE_INIT. * If CONFIG_DEFERRED_STRUCT_PAGE_INIT, pages after first_init_pfn is skipped at defer_init(). Then init_reserved_page() is defined to call __init_single_page() for them. * If !CONFIG_DEFERRED_STRUCT_PAGE_INIT, pages are all initialized by memmap_init_range(). In both cases, after init_reserved_page(), we expect __init_single_page() has done its work to the page, which already initialize page->lru properly. We don't need to do it again. Signed-off-by: Wei Yang Link: https://lore.kernel.org/r/20240610143742.26401-1-richard.weiyang@gmail.com Signed-off-by: Mike Rapoport (IBM) --- mm/mm_init.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 5a4c846393b7..ba993305e8c2 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -760,9 +760,6 @@ void __meminit reserve_bootmem_region(phys_addr_t start, init_reserved_page(start_pfn, nid); - /* Avoid false-positive PageTail() */ - INIT_LIST_HEAD(&page->lru); - /* * no need for atomic set_bit because the struct * page is not visible yet so nobody should From 4267fda4afd9fcf1f92b6bfa0259295f140c8ecd Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Fri, 7 Jun 2024 09:48:33 -0400 Subject: [PATCH 111/280] function_graph: Make fgraph_update_pid_func() a stub for !DYNAMIC_FTRACE When CONFIG_DYNAMIC_FTRACE is not set, the function fgraph_update_pid_func() doesn't do anything. Currently, most of its logic is within a "#ifdef CONFIG_DYNAMIC_FTRACE" block, but its variables were declared outside that, and when DYNAMIC_FTRACE is not set, it produces unused variable warnings. Instead, just place it (and the helper function fgraph_pid_func()) within the #ifdef block and have the header file use a empty stub function for when DYNAMIC_FTRACE is not defined. Link: https://lore.kernel.org/linux-trace-kernel/20240607094833.6a787d73@rorschach.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202406071806.BRjaC5FF-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 4 ++-- kernel/trace/ftrace_internal.h | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index a13551a023aa..63d0c2f84ce1 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -1138,6 +1138,7 @@ void ftrace_graph_exit_task(struct task_struct *t) kfree(ret_stack); } +#ifdef CONFIG_DYNAMIC_FTRACE static int fgraph_pid_func(struct ftrace_graph_ent *trace, struct fgraph_ops *gops) { @@ -1164,7 +1165,6 @@ void fgraph_update_pid_func(void) if (!(graph_ops.flags & FTRACE_OPS_FL_INITIALIZED)) return; -#ifdef CONFIG_DYNAMIC_FTRACE list_for_each_entry(op, &graph_ops.subop_list, list) { if (op->flags & FTRACE_OPS_FL_PID) { gops = container_of(op, struct fgraph_ops, ops); @@ -1174,8 +1174,8 @@ void fgraph_update_pid_func(void) static_call_update(fgraph_func, gops->entryfunc); } } -#endif } +#endif /* Allocate a return stack for each task */ static int start_graph_tracing(void) diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h index 4bb1e881154a..3235470e61b3 100644 --- a/kernel/trace/ftrace_internal.h +++ b/kernel/trace/ftrace_internal.h @@ -52,7 +52,11 @@ static inline int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_o #ifdef CONFIG_FUNCTION_GRAPH_TRACER extern int ftrace_graph_active; +# ifdef CONFIG_DYNAMIC_FTRACE extern void fgraph_update_pid_func(void); +# else +static inline void fgraph_update_pid_func(void) {} +# endif #else /* !CONFIG_FUNCTION_GRAPH_TRACER */ # define ftrace_graph_active 0 static inline void fgraph_update_pid_func(void) {} From 6828d6929b763499b7a8768c623891f5d4fca258 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 3 May 2024 16:28:49 -0700 Subject: [PATCH 112/280] perf evsel: Refactor tool events Tool events unnecessarily open a dummy perf event which is useless even with `perf record` which will still open a dummy event. Change the behavior of tool events so: - duration_time - call `rdclock` on open and then report the count as a delta since the start in evsel__read_counter. This moves code out of builtin-stat making it more general purpose. - user_time/system_time - open the fd as either `/proc/pid/stat` or `/proc/stat` for cases like system wide. evsel__read_counter will read the appropriate field out of the procfs file. These values were previously supplied by wait4, if the procfs read fails then the wait4 values are used, assuming the process/thread terminated. By reading user_time and system_time this way, interval mode, per PID and per CPU can be supported although there are restrictions given what the files provide (e.g. per PID can't be combined with per CPU). Opening any of the tool events for `perf record` is changed to return invalid. Signed-off-by: Ian Rogers Tested-by: Weilin Wang Cc: Ravi Bangoria Cc: James Clark Cc: Dmitrii Dolgov <9erthalion6@gmail.com> Cc: Ze Gao Cc: Song Liu Cc: Leo Yan Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240503232849.17752-1-irogers@google.com --- tools/perf/builtin-stat.c | 73 +++++------ tools/perf/util/evsel.c | 226 ++++++++++++++++++++++++++++++++- tools/perf/util/evsel.h | 14 ++ tools/perf/util/parse-events.c | 7 +- 4 files changed, 276 insertions(+), 44 deletions(-) diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 603a9684153d..661832756a24 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -284,45 +284,38 @@ static int evsel__write_stat_event(struct evsel *counter, int cpu_map_idx, u32 t process_synthesized_event, NULL); } -static int read_single_counter(struct evsel *counter, int cpu_map_idx, - int thread, struct timespec *rs) +static int read_single_counter(struct evsel *counter, int cpu_map_idx, int thread) { - switch(counter->tool_event) { - case PERF_TOOL_DURATION_TIME: { - u64 val = rs->tv_nsec + rs->tv_sec*1000000000ULL; - struct perf_counts_values *count = - perf_counts(counter->counts, cpu_map_idx, thread); - count->ena = count->run = val; - count->val = val; - return 0; - } - case PERF_TOOL_USER_TIME: - case PERF_TOOL_SYSTEM_TIME: { - u64 val; - struct perf_counts_values *count = - perf_counts(counter->counts, cpu_map_idx, thread); - if (counter->tool_event == PERF_TOOL_USER_TIME) - val = ru_stats.ru_utime_usec_stat.mean; - else - val = ru_stats.ru_stime_usec_stat.mean; - count->ena = count->run = val; - count->val = val; - return 0; - } - default: - case PERF_TOOL_NONE: - return evsel__read_counter(counter, cpu_map_idx, thread); - case PERF_TOOL_MAX: - /* This should never be reached */ - return 0; + int err = evsel__read_counter(counter, cpu_map_idx, thread); + + /* + * Reading user and system time will fail when the process + * terminates. Use the wait4 values in that case. + */ + if (err && cpu_map_idx == 0 && + (counter->tool_event == PERF_TOOL_USER_TIME || + counter->tool_event == PERF_TOOL_SYSTEM_TIME)) { + u64 val, *start_time; + struct perf_counts_values *count = + perf_counts(counter->counts, cpu_map_idx, thread); + + start_time = xyarray__entry(counter->start_times, cpu_map_idx, thread); + if (counter->tool_event == PERF_TOOL_USER_TIME) + val = ru_stats.ru_utime_usec_stat.mean; + else + val = ru_stats.ru_stime_usec_stat.mean; + count->ena = count->run = *start_time + val; + count->val = val; + return 0; } + return err; } /* * Read out the results of a single counter: * do not aggregate counts across CPUs in system-wide mode */ -static int read_counter_cpu(struct evsel *counter, struct timespec *rs, int cpu_map_idx) +static int read_counter_cpu(struct evsel *counter, int cpu_map_idx) { int nthreads = perf_thread_map__nr(evsel_list->core.threads); int thread; @@ -340,7 +333,7 @@ static int read_counter_cpu(struct evsel *counter, struct timespec *rs, int cpu_ * (via evsel__read_counter()) and sets their count->loaded. */ if (!perf_counts__is_loaded(counter->counts, cpu_map_idx, thread) && - read_single_counter(counter, cpu_map_idx, thread, rs)) { + read_single_counter(counter, cpu_map_idx, thread)) { counter->counts->scaled = -1; perf_counts(counter->counts, cpu_map_idx, thread)->ena = 0; perf_counts(counter->counts, cpu_map_idx, thread)->run = 0; @@ -369,7 +362,7 @@ static int read_counter_cpu(struct evsel *counter, struct timespec *rs, int cpu_ return 0; } -static int read_affinity_counters(struct timespec *rs) +static int read_affinity_counters(void) { struct evlist_cpu_iterator evlist_cpu_itr; struct affinity saved_affinity, *affinity; @@ -390,10 +383,8 @@ static int read_affinity_counters(struct timespec *rs) if (evsel__is_bpf(counter)) continue; - if (!counter->err) { - counter->err = read_counter_cpu(counter, rs, - evlist_cpu_itr.cpu_map_idx); - } + if (!counter->err) + counter->err = read_counter_cpu(counter, evlist_cpu_itr.cpu_map_idx); } if (affinity) affinity__cleanup(&saved_affinity); @@ -417,11 +408,11 @@ static int read_bpf_map_counters(void) return 0; } -static int read_counters(struct timespec *rs) +static int read_counters(void) { if (!stat_config.stop_read_counter) { if (read_bpf_map_counters() || - read_affinity_counters(rs)) + read_affinity_counters()) return -1; } return 0; @@ -452,7 +443,7 @@ static void process_interval(void) evlist__reset_aggr_stats(evsel_list); - if (read_counters(&rs) == 0) + if (read_counters() == 0) process_counters(); if (STAT_RECORD) { @@ -940,7 +931,7 @@ try_again_reset: * avoid arbitrary skew, we must read all counters before closing any * group leaders. */ - if (read_counters(&(struct timespec) { .tv_nsec = t1-t0 }) == 0) + if (read_counters() == 0) process_counters(); /* diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 4f818ab6b662..28c54897a97e 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include "counts.h" #include "event.h" #include "evsel.h" +#include "time-utils.h" #include "util/env.h" #include "util/evsel_config.h" #include "util/evsel_fprintf.h" @@ -1493,6 +1495,9 @@ void evsel__exit(struct evsel *evsel) evsel->per_pkg_mask = NULL; zfree(&evsel->metric_events); perf_evsel__object.fini(evsel); + if (evsel->tool_event == PERF_TOOL_SYSTEM_TIME || + evsel->tool_event == PERF_TOOL_USER_TIME) + xyarray__delete(evsel->start_times); } void evsel__delete(struct evsel *evsel) @@ -1606,11 +1611,173 @@ static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread) return evsel__process_group_data(leader, cpu_map_idx, thread, data); } +static bool read_until_char(struct io *io, char e) +{ + char c; + + do { + c = io__get_char(io); + if (c == -1) + return false; + } while (c != e); + return true; +} + +static int read_stat_field(int fd, struct perf_cpu cpu, int field, __u64 *val) +{ + char buf[256]; + struct io io; + int i; + + io__init(&io, fd, buf, sizeof(buf)); + + /* Skip lines to relevant CPU. */ + for (i = -1; i < cpu.cpu; i++) { + if (!read_until_char(&io, '\n')) + return -EINVAL; + } + /* Skip to "cpu". */ + if (io__get_char(&io) != 'c') return -EINVAL; + if (io__get_char(&io) != 'p') return -EINVAL; + if (io__get_char(&io) != 'u') return -EINVAL; + + /* Skip N of cpuN. */ + if (!read_until_char(&io, ' ')) + return -EINVAL; + + i = 1; + while (true) { + if (io__get_dec(&io, val) != ' ') + break; + if (field == i) + return 0; + i++; + } + return -EINVAL; +} + +static int read_pid_stat_field(int fd, int field, __u64 *val) +{ + char buf[256]; + struct io io; + int c, i; + + io__init(&io, fd, buf, sizeof(buf)); + if (io__get_dec(&io, val) != ' ') + return -EINVAL; + if (field == 1) + return 0; + + /* Skip comm. */ + if (io__get_char(&io) != '(' || !read_until_char(&io, ')')) + return -EINVAL; + if (field == 2) + return -EINVAL; /* String can't be returned. */ + + /* Skip state */ + if (io__get_char(&io) != ' ' || io__get_char(&io) == -1) + return -EINVAL; + if (field == 3) + return -EINVAL; /* String can't be returned. */ + + /* Loop over numeric fields*/ + if (io__get_char(&io) != ' ') + return -EINVAL; + + i = 4; + while (true) { + c = io__get_dec(&io, val); + if (c == -1) + return -EINVAL; + if (c == -2) { + /* Assume a -ve was read */ + c = io__get_dec(&io, val); + *val *= -1; + } + if (c != ' ') + return -EINVAL; + if (field == i) + return 0; + i++; + } + return -EINVAL; +} + +static int evsel__read_tool(struct evsel *evsel, int cpu_map_idx, int thread) +{ + __u64 *start_time, cur_time, delta_start; + int fd, err = 0; + struct perf_counts_values *count; + bool adjust = false; + + count = perf_counts(evsel->counts, cpu_map_idx, thread); + + switch (evsel->tool_event) { + case PERF_TOOL_DURATION_TIME: + /* + * Pretend duration_time is only on the first CPU and thread, or + * else aggregation will scale duration_time by the number of + * CPUs/threads. + */ + start_time = &evsel->start_time; + if (cpu_map_idx == 0 && thread == 0) + cur_time = rdclock(); + else + cur_time = *start_time; + break; + case PERF_TOOL_USER_TIME: + case PERF_TOOL_SYSTEM_TIME: { + bool system = evsel->tool_event == PERF_TOOL_SYSTEM_TIME; + + start_time = xyarray__entry(evsel->start_times, cpu_map_idx, thread); + fd = FD(evsel, cpu_map_idx, thread); + lseek(fd, SEEK_SET, 0); + if (evsel->pid_stat) { + /* The event exists solely on 1 CPU. */ + if (cpu_map_idx == 0) + err = read_pid_stat_field(fd, system ? 15 : 14, &cur_time); + else + cur_time = 0; + } else { + /* The event is for all threads. */ + if (thread == 0) { + struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, + cpu_map_idx); + + err = read_stat_field(fd, cpu, system ? 3 : 1, &cur_time); + } else { + cur_time = 0; + } + } + adjust = true; + break; + } + case PERF_TOOL_NONE: + case PERF_TOOL_MAX: + default: + err = -EINVAL; + } + if (err) + return err; + + delta_start = cur_time - *start_time; + if (adjust) { + __u64 ticks_per_sec = sysconf(_SC_CLK_TCK); + + delta_start *= 1000000000 / ticks_per_sec; + } + count->val = delta_start; + count->ena = count->run = delta_start; + count->lost = 0; + return 0; +} + int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread) { - u64 read_format = evsel->core.attr.read_format; + if (evsel__is_tool(evsel)) + return evsel__read_tool(evsel, cpu_map_idx, thread); - if (read_format & PERF_FORMAT_GROUP) + if (evsel->core.attr.read_format & PERF_FORMAT_GROUP) return evsel__read_group(evsel, cpu_map_idx, thread); return evsel__read_one(evsel, cpu_map_idx, thread); @@ -1829,6 +1996,14 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) return -ENOMEM; + if ((evsel->tool_event == PERF_TOOL_SYSTEM_TIME || + evsel->tool_event == PERF_TOOL_USER_TIME) && + !evsel->start_times) { + evsel->start_times = xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); + if (!evsel->start_times) + return -ENOMEM; + } + evsel->open_flags = PERF_FLAG_FD_CLOEXEC; if (evsel->cgrp) evsel->open_flags |= PERF_FLAG_PID_CGROUP; @@ -2011,6 +2186,13 @@ static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int pid = -1, err, old_errno; enum rlimit_action set_rlimit = NO_CHANGE; + if (evsel->tool_event == PERF_TOOL_DURATION_TIME) { + if (evsel->core.attr.sample_period) /* no sampling */ + return -EINVAL; + evsel->start_time = rdclock(); + return 0; + } + err = __evsel__prepare_open(evsel, cpus, threads); if (err) return err; @@ -2043,6 +2225,46 @@ retry_open: if (!evsel->cgrp && !evsel->core.system_wide) pid = perf_thread_map__pid(threads, thread); + if (evsel->tool_event == PERF_TOOL_USER_TIME || + evsel->tool_event == PERF_TOOL_SYSTEM_TIME) { + bool system = evsel->tool_event == PERF_TOOL_SYSTEM_TIME; + __u64 *start_time = NULL; + + if (evsel->core.attr.sample_period) { + /* no sampling */ + err = -EINVAL; + goto out_close; + } + if (pid > -1) { + char buf[64]; + + snprintf(buf, sizeof(buf), "/proc/%d/stat", pid); + fd = open(buf, O_RDONLY); + evsel->pid_stat = true; + } else { + fd = open("/proc/stat", O_RDONLY); + } + FD(evsel, idx, thread) = fd; + if (fd < 0) { + err = -errno; + goto out_close; + } + start_time = xyarray__entry(evsel->start_times, idx, thread); + if (pid > -1) { + err = read_pid_stat_field(fd, system ? 15 : 14, + start_time); + } else { + struct perf_cpu cpu; + + cpu = perf_cpu_map__cpu(evsel->core.cpus, idx); + err = read_stat_field(fd, cpu, system ? 3 : 1, + start_time); + } + if (err) + goto out_close; + continue; + } + group_fd = get_group_fd(evsel, idx, thread); if (group_fd == -2) { diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 375a38e15cd9..80b5f6dd868e 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -170,6 +170,20 @@ struct evsel { /* for missing_features */ struct perf_pmu *pmu; + + /* For tool events */ + /* Beginning time subtracted when the counter is read. */ + union { + /* duration_time is a single global time. */ + __u64 start_time; + /* + * user_time and system_time read an initial value potentially + * per-CPU or per-pid. + */ + struct xyarray *start_times; + }; + /* Is the tool's fd for /proc/pid/stat or /proc/stat. */ + bool pid_stat; }; struct perf_missing_features { diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 8d5fb05f20c2..321586fb5556 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -305,11 +305,16 @@ static int add_event_tool(struct list_head *list, int *idx, .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_DUMMY, }; + const char *cpu_list = NULL; + if (tool_event == PERF_TOOL_DURATION_TIME) { + /* Duration time is gathered globally, pretend it is only on CPU0. */ + cpu_list = "0"; + } evsel = __add_event(list, idx, &attr, /*init_attr=*/true, /*name=*/NULL, /*metric_id=*/NULL, /*pmu=*/NULL, /*config_terms=*/NULL, /*auto_merge_stats=*/false, - /*cpu_list=*/"0"); + cpu_list); if (!evsel) return -ENOMEM; evsel->tool_event = tool_event; From 29c1c24a2707a579bba078d8b73b70fef4451b34 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 10 Jun 2024 23:09:35 -0400 Subject: [PATCH 113/280] function_graph: Fix up ftrace_graph_ret_addr() Yang Li sent a patch to fix the kerneldoc of ftrace_graph_ret_addr(). While reviewing it, I realized that the comments in the entire function header needed a rewrite. When doing that, I realized that @idx parameter was being ignored. Every time this was called by the unwinder, it would start the loop at the top of the shadow stack and look for the matching stack pointer. When it found it, it would return it. When the unwinder asked for the next function, it would search from the beginning again. In reality, it should start from where it left off. That was the reason for the @idx parameter in the first place. The first time the unwinder calls this function, the @idx pointer would contain zero. That would mean to start from the top of the stack. The function was supposed to update the @idx with the index where it found the return address, so that the next time the unwinder calls this function it doesn't have to search through the previous addresses it found (making it O(n^2)!). This speeds up the unwinder's use of ftrace_graph_ret_addr() by an order of magnitude. Link: https://lore.kernel.org/linux-trace-kernel/20240610181746.656e3759@gandalf.local.home/ Link: https://lore.kernel.org/linux-trace-kernel/20240611031737.821995106@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Jonathan Corbet Cc: Catalin Marinas Cc: Will Deacon Cc: Guo Ren Cc: Huacai Chen Cc: WANG Xuerui Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy Cc: "Naveen N. Rao" Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Reported-by: Yang Li Fixes: 7aa1eaef9f428 ("function_graph: Allow multiple users to attach to function graph") Signed-off-by: Steven Rostedt (Google) --- kernel/trace/fgraph.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 63d0c2f84ce1..91f1eef256af 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -870,18 +870,24 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int idx) } /** - * ftrace_graph_ret_addr - convert a potentially modified stack return address - * to its original value + * ftrace_graph_ret_addr - return the original value of the return address + * @task: The task the unwinder is being executed on + * @idx: An initialized pointer to the next stack index to use + * @ret: The current return address (likely pointing to return_handler) + * @retp: The address on the stack of the current return location * * This function can be called by stack unwinding code to convert a found stack - * return address ('ret') to its original value, in case the function graph + * return address (@ret) to its original value, in case the function graph * tracer has modified it to be 'return_to_handler'. If the address hasn't - * been modified, the unchanged value of 'ret' is returned. + * been modified, the unchanged value of @ret is returned. * - * 'idx' is a state variable which should be initialized by the caller to zero - * before the first call. + * @idx holds the last index used to know where to start from. It should be + * initialized to zero for the first iteration as that will mean to start + * at the top of the shadow stack. If the location is found, this pointer + * will be assigned that location so that if called again, it will continue + * where it left off. * - * 'retp' is a pointer to the return address on the stack. It's ignored if + * @retp is a pointer to the return address on the stack. It's ignored if * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined. */ #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR @@ -895,6 +901,10 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, if (ret != return_handler) return ret; + if (!idx) + return ret; + + i = *idx ? : task->curr_ret_stack; while (i > 0) { ret_stack = get_ret_stack(current, i, &i); if (!ret_stack) @@ -908,8 +918,10 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, * Thus we will continue to find real return address. */ if (ret_stack->retp == retp && - ret_stack->ret != return_handler) + ret_stack->ret != return_handler) { + *idx = i; return ret_stack->ret; + } } return ret; From 5f7fb89a115d53b4a10bf7ba2733e78df281e98d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 10 Jun 2024 23:09:36 -0400 Subject: [PATCH 114/280] function_graph: Everyone uses HAVE_FUNCTION_GRAPH_RET_ADDR_PTR, remove it All architectures that implement function graph also implements HAVE_FUNCTION_GRAPH_RET_ADDR_PTR. Remove it, as it is no longer a differentiator. Link: https://lore.kernel.org/linux-trace-kernel/20240611031737.982047614@goodmis.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Jonathan Corbet Cc: Catalin Marinas Cc: Will Deacon Cc: Guo Ren Cc: Huacai Chen Cc: WANG Xuerui Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Christophe Leroy Cc: "Naveen N. Rao" Cc: Paul Walmsley Cc: Palmer Dabbelt Cc: Albert Ou Cc: Heiko Carstens Cc: Vasily Gorbik Cc: Alexander Gordeev Cc: Christian Borntraeger Cc: Sven Schnelle Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: "H. Peter Anvin" Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/ftrace-design.rst | 12 --------- arch/arm64/include/asm/ftrace.h | 11 --------- arch/csky/include/asm/ftrace.h | 2 -- arch/loongarch/include/asm/ftrace.h | 1 - arch/powerpc/include/asm/ftrace.h | 2 -- arch/riscv/include/asm/ftrace.h | 1 - arch/s390/include/asm/ftrace.h | 1 - arch/x86/include/asm/ftrace.h | 2 -- include/linux/ftrace.h | 2 -- kernel/trace/fgraph.c | 35 +-------------------------- 10 files changed, 1 insertion(+), 68 deletions(-) diff --git a/Documentation/trace/ftrace-design.rst b/Documentation/trace/ftrace-design.rst index 6893399157f0..dc82d64b3a44 100644 --- a/Documentation/trace/ftrace-design.rst +++ b/Documentation/trace/ftrace-design.rst @@ -217,18 +217,6 @@ along to ftrace_push_return_trace() instead of a stub value of 0. Similarly, when you call ftrace_return_to_handler(), pass it the frame pointer. -HAVE_FUNCTION_GRAPH_RET_ADDR_PTR --------------------------------- - -An arch may pass in a pointer to the return address on the stack. This -prevents potential stack unwinding issues where the unwinder gets out of -sync with ret_stack and the wrong addresses are reported by -ftrace_graph_ret_addr(). - -Adding support for it is easy: just define the macro in asm/ftrace.h and -pass the return address pointer as the 'retp' argument to -ftrace_push_return_trace(). - HAVE_SYSCALL_TRACEPOINTS ------------------------ diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index ab158196480c..dc9cf0bd2a4c 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -12,17 +12,6 @@ #define HAVE_FUNCTION_GRAPH_FP_TEST -/* - * HAVE_FUNCTION_GRAPH_RET_ADDR_PTR means that the architecture can provide a - * "return address pointer" which can be used to uniquely identify a return - * address which has been overwritten. - * - * On arm64 we use the address of the caller's frame record, which remains the - * same for the lifetime of the instrumented function, unlike the return - * address in the LR. - */ -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR - #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS #define ARCH_SUPPORTS_FTRACE_OPS 1 #else diff --git a/arch/csky/include/asm/ftrace.h b/arch/csky/include/asm/ftrace.h index fd215c38ef27..00f9f7647e3f 100644 --- a/arch/csky/include/asm/ftrace.h +++ b/arch/csky/include/asm/ftrace.h @@ -7,8 +7,6 @@ #define HAVE_FUNCTION_GRAPH_FP_TEST -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR - #define ARCH_SUPPORTS_FTRACE_OPS 1 #define MCOUNT_ADDR ((unsigned long)_mcount) diff --git a/arch/loongarch/include/asm/ftrace.h b/arch/loongarch/include/asm/ftrace.h index de891c2c83d4..c0a682808e07 100644 --- a/arch/loongarch/include/asm/ftrace.h +++ b/arch/loongarch/include/asm/ftrace.h @@ -28,7 +28,6 @@ struct dyn_ftrace; struct dyn_arch_ftrace { }; #define ARCH_SUPPORTS_FTRACE_OPS 1 -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #define ftrace_init_nop ftrace_init_nop int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec); diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h index 107fc5a48456..559560286e6d 100644 --- a/arch/powerpc/include/asm/ftrace.h +++ b/arch/powerpc/include/asm/ftrace.h @@ -8,8 +8,6 @@ #define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR - /* Ignore unused weak functions which will have larger offsets */ #if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) #define FTRACE_MCOUNT_MAX_OFFSET 16 diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 9eb31a7ea0aa..2cddd79ff21b 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -11,7 +11,6 @@ #if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER) #define HAVE_FUNCTION_GRAPH_FP_TEST #endif -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #define ARCH_SUPPORTS_FTRACE_OPS 1 #ifndef __ASSEMBLY__ diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 77e479d44f1e..fbadca645af7 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -2,7 +2,6 @@ #ifndef _ASM_S390_FTRACE_H #define _ASM_S390_FTRACE_H -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #define ARCH_SUPPORTS_FTRACE_OPS 1 #define MCOUNT_INSN_SIZE 6 diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index 897cf02c20b1..0152a81d9b4a 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -20,8 +20,6 @@ #define ARCH_SUPPORTS_FTRACE_OPS 1 #endif -#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR - #ifndef __ASSEMBLY__ extern void __fentry__(void); diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 4135dc171447..845c2ab0bc1c 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1071,9 +1071,7 @@ struct ftrace_ret_stack { #ifdef HAVE_FUNCTION_GRAPH_FP_TEST unsigned long fp; #endif -#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR unsigned long *retp; -#endif }; /* diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 91f1eef256af..8317d1a7f43a 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -593,9 +593,7 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, #ifdef HAVE_FUNCTION_GRAPH_FP_TEST ret_stack->fp = frame_pointer; #endif -#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR ret_stack->retp = retp; -#endif return offset; } @@ -887,10 +885,8 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int idx) * will be assigned that location so that if called again, it will continue * where it left off. * - * @retp is a pointer to the return address on the stack. It's ignored if - * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined. + * @retp is a pointer to the return address on the stack. */ -#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, unsigned long *retp) { @@ -926,35 +922,6 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, return ret; } -#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ -unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, - unsigned long ret, unsigned long *retp) -{ - struct ftrace_ret_stack *ret_stack; - unsigned long return_handler = (unsigned long)dereference_kernel_function_descriptor(return_to_handler); - int offset = task->curr_ret_stack; - int i; - - if (ret != return_handler) - return ret; - - if (!idx) - return ret; - - i = *idx; - do { - ret_stack = get_ret_stack(task, offset, &offset); - if (ret_stack && ret_stack->ret == return_handler) - continue; - i--; - } while (i >= 0 && ret_stack); - - if (ret_stack) - return ret_stack->ret; - - return ret; -} -#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ static struct ftrace_ops graph_ops = { .func = ftrace_graph_func, From 9b5a45eb639c46c0374b5e040e6e6db386909676 Mon Sep 17 00:00:00 2001 From: Marilene A Garcia Date: Thu, 6 Jun 2024 10:25:20 -0300 Subject: [PATCH 115/280] ftrace: Add missing kerneldoc parameters to unregister_ftrace_direct() Add the description to the parameters addr and free_filters of the function unregister_ftrace_direct(). Link: https://lore.kernel.org/linux-trace-kernel/20240606132520.1397567-1-marilene.agarcia@gmail.com Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Mathieu Desnoyers Cc: Shuah Khan Cc: Javier Carrasco Signed-off-by: Marilene A Garcia Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4aeb1183ea9f..f44229294e9d 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5988,6 +5988,8 @@ EXPORT_SYMBOL_GPL(register_ftrace_direct); * unregister_ftrace_direct - Remove calls to custom trampoline * previously registered by register_ftrace_direct for @ops object. * @ops: The address of the struct ftrace_ops object + * @addr: The address of the direct function that is called by the @ops functions + * @free_filters: Set to true to remove all filters for the ftrace_ops, false otherwise * * This is used to remove a direct calls to @addr from the nop locations * of the functions registered in @ops (with by ftrace_set_filter_ip From 6c1785cd75ef55a308701813330a162002ffe192 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 10 Jun 2024 22:06:26 -0700 Subject: [PATCH 116/280] perf record: Ensure space for lost samples Previous allocation didn't account for sample ID written after the lost samples event. Switch from malloc/free to a stack allocation. Reported-by: Milian Wolff Closes: https://lore.kernel.org/linux-perf-users/23879991.0LEYPuXRzz@milian-workstation/ Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240611050626.1223155-1-irogers@google.com --- tools/lib/perf/include/perf/event.h | 6 +++++ tools/perf/builtin-record.c | 34 ++++++++--------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index ae64090184d3..37bb7771d914 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -77,6 +77,12 @@ struct perf_record_lost_samples { __u64 lost; }; +#define MAX_ID_HDR_ENTRIES 6 +struct perf_record_lost_samples_and_ids { + struct perf_record_lost_samples lost; + __u64 sample_ids[MAX_ID_HDR_ENTRIES]; +}; + /* * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID | PERF_FORMAT_LOST */ diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 66a3de8ac661..019305b94e5f 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -1926,7 +1926,7 @@ static void __record__save_lost_samples(struct record *rec, struct evsel *evsel, static void record__read_lost_samples(struct record *rec) { struct perf_session *session = rec->session; - struct perf_record_lost_samples *lost = NULL; + struct perf_record_lost_samples_and_ids lost; struct evsel *evsel; /* there was an error during record__open */ @@ -1951,20 +1951,13 @@ static void record__read_lost_samples(struct record *rec) if (perf_evsel__read(&evsel->core, x, y, &count) < 0) { pr_debug("read LOST count failed\n"); - goto out; + return; } if (count.lost) { - if (!lost) { - lost = zalloc(sizeof(*lost) + - session->machines.host.id_hdr_size); - if (!lost) { - pr_debug("Memory allocation failed\n"); - return; - } - lost->header.type = PERF_RECORD_LOST_SAMPLES; - } - __record__save_lost_samples(rec, evsel, lost, + memset(&lost.lost, 0, sizeof(lost)); + lost.lost.header.type = PERF_RECORD_LOST_SAMPLES; + __record__save_lost_samples(rec, evsel, &lost.lost, x, y, count.lost, 0); } } @@ -1972,21 +1965,12 @@ static void record__read_lost_samples(struct record *rec) lost_count = perf_bpf_filter__lost_count(evsel); if (lost_count) { - if (!lost) { - lost = zalloc(sizeof(*lost) + - session->machines.host.id_hdr_size); - if (!lost) { - pr_debug("Memory allocation failed\n"); - return; - } - lost->header.type = PERF_RECORD_LOST_SAMPLES; - } - __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count, + memset(&lost.lost, 0, sizeof(lost)); + lost.lost.header.type = PERF_RECORD_LOST_SAMPLES; + __record__save_lost_samples(rec, evsel, &lost.lost, 0, 0, lost_count, PERF_RECORD_MISC_LOST_SAMPLES_BPF); } } -out: - free(lost); } static volatile sig_atomic_t workload_exec_errno; @@ -3198,7 +3182,7 @@ static int switch_output_setup(struct record *rec) unsigned long val; /* - * If we're using --switch-output-events, then we imply its + * If we're using --switch-output-events, then we imply its * --switch-output=signal, as we'll send a SIGUSR2 from the side band * thread to its parent. */ From 183373586743a73da51b505ae3f4e5eb6300f208 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 7 Jun 2024 10:13:52 +0530 Subject: [PATCH 117/280] tools/perf: Fix perf bench futex to enable the run when some CPU's are offline Perf bench futex fails as below when attempted to run on on a powerpc system: ./perf bench futex all Running futex/hash benchmark... Run summary [PID 626307]: 80 threads, each operating on 1024 [private] futexes for 10 secs. perf: pthread_create: No such file or directory In the setup where this perf bench was ran, difference was that partition had 640 CPU's, but not all CPUs were online. 80 CPUs were online. While blocking the threads with futex_wait, code sets the affinity using cpumask. The cpumask size used is 80 which is picked from "nrcpus = perf_cpu_map__nr(cpu)". Here the benchmark reports fail while setting affinity for cpu number which is greater than 80 or higher, because it attempts to set a bit position which is not allocated on the cpumask. Fix this by changing the size of cpumask to number of possible cpus and not the number of online cpus. Signed-off-by: Athira Rajeev Reviewed-by: Ian Rogers Tested-by: Disha Goel Cc: akanksha@linux.ibm.com Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607044354.82225-1-atrajeev@linux.vnet.ibm.com --- tools/perf/bench/futex-hash.c | 2 +- tools/perf/bench/futex-lock-pi.c | 2 +- tools/perf/bench/futex-requeue.c | 2 +- tools/perf/bench/futex-wake-parallel.c | 2 +- tools/perf/bench/futex-wake.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/perf/bench/futex-hash.c b/tools/perf/bench/futex-hash.c index 0c69d20efa32..b472eded521b 100644 --- a/tools/perf/bench/futex-hash.c +++ b/tools/perf/bench/futex-hash.c @@ -174,7 +174,7 @@ int bench_futex_hash(int argc, const char **argv) pthread_attr_init(&thread_attr); gettimeofday(&bench__start, NULL); - nrcpus = perf_cpu_map__nr(cpu); + nrcpus = cpu__max_cpu().cpu; cpuset = CPU_ALLOC(nrcpus); BUG_ON(!cpuset); size = CPU_ALLOC_SIZE(nrcpus); diff --git a/tools/perf/bench/futex-lock-pi.c b/tools/perf/bench/futex-lock-pi.c index 7a4973346180..0416120c091b 100644 --- a/tools/perf/bench/futex-lock-pi.c +++ b/tools/perf/bench/futex-lock-pi.c @@ -122,7 +122,7 @@ static void create_threads(struct worker *w, struct perf_cpu_map *cpu) { cpu_set_t *cpuset; unsigned int i; - int nrcpus = perf_cpu_map__nr(cpu); + int nrcpus = cpu__max_cpu().cpu; size_t size; threads_starting = params.nthreads; diff --git a/tools/perf/bench/futex-requeue.c b/tools/perf/bench/futex-requeue.c index d9ad736c1a3e..aad5bfc4fe18 100644 --- a/tools/perf/bench/futex-requeue.c +++ b/tools/perf/bench/futex-requeue.c @@ -125,7 +125,7 @@ static void block_threads(pthread_t *w, struct perf_cpu_map *cpu) { cpu_set_t *cpuset; unsigned int i; - int nrcpus = perf_cpu_map__nr(cpu); + int nrcpus = cpu__max_cpu().cpu; size_t size; threads_starting = params.nthreads; diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c index b66df553e561..90a5b91bf139 100644 --- a/tools/perf/bench/futex-wake-parallel.c +++ b/tools/perf/bench/futex-wake-parallel.c @@ -149,7 +149,7 @@ static void block_threads(pthread_t *w, struct perf_cpu_map *cpu) { cpu_set_t *cpuset; unsigned int i; - int nrcpus = perf_cpu_map__nr(cpu); + int nrcpus = cpu__max_cpu().cpu; size_t size; threads_starting = params.nthreads; diff --git a/tools/perf/bench/futex-wake.c b/tools/perf/bench/futex-wake.c index 690fd6d3da13..49b3c89b0b35 100644 --- a/tools/perf/bench/futex-wake.c +++ b/tools/perf/bench/futex-wake.c @@ -100,7 +100,7 @@ static void block_threads(pthread_t *w, struct perf_cpu_map *cpu) cpu_set_t *cpuset; unsigned int i; size_t size; - int nrcpus = perf_cpu_map__nr(cpu); + int nrcpus = cpu__max_cpu().cpu; threads_starting = params.nthreads; cpuset = CPU_ALLOC(nrcpus); From 3638e44542a56e8adc2018df4894eaf31d387c54 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 7 Jun 2024 10:13:53 +0530 Subject: [PATCH 118/280] tools/perf: Fix perf bench epoll to enable the run when some CPU's are offline Perf bench epoll fails as below when attempted to run on on a powerpc system: ./perf bench epoll wait Running 'epoll/wait' benchmark: Run summary [PID 627653]: 79 threads monitoring on 64 file-descriptors for 8 secs. perf: pthread_create: No such file or directory In the setup where this perf bench was ran, difference was that partition had 640 CPU's, but not all CPUs were online. 80 CPUs were online. While creating threads and using epoll_wait , code sets the affinity using cpumask. The cpumask size used is 80 which is picked from "nrcpus = perf_cpu_map__nr(cpu)". Here the benchmark reports fail while setting affinity for cpu number which is greater than 80 or higher, because it attempts to set a bit position which is not allocated on the cpumask. Fix this by changing the size of cpumask to number of possible cpus and not the number of online cpus. Signed-off-by: Athira Rajeev Reviewed-by: Ian Rogers Tested-by: Disha Goel Cc: akanksha@linux.ibm.com Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607044354.82225-2-atrajeev@linux.vnet.ibm.com --- tools/perf/bench/epoll-ctl.c | 2 +- tools/perf/bench/epoll-wait.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/bench/epoll-ctl.c b/tools/perf/bench/epoll-ctl.c index d3db73dac66a..d66d852b90e4 100644 --- a/tools/perf/bench/epoll-ctl.c +++ b/tools/perf/bench/epoll-ctl.c @@ -232,7 +232,7 @@ static int do_threads(struct worker *worker, struct perf_cpu_map *cpu) if (!noaffinity) pthread_attr_init(&thread_attr); - nrcpus = perf_cpu_map__nr(cpu); + nrcpus = cpu__max_cpu().cpu; cpuset = CPU_ALLOC(nrcpus); BUG_ON(!cpuset); size = CPU_ALLOC_SIZE(nrcpus); diff --git a/tools/perf/bench/epoll-wait.c b/tools/perf/bench/epoll-wait.c index 06bb3187660a..ef5c4257844d 100644 --- a/tools/perf/bench/epoll-wait.c +++ b/tools/perf/bench/epoll-wait.c @@ -309,7 +309,7 @@ static int do_threads(struct worker *worker, struct perf_cpu_map *cpu) if (!noaffinity) pthread_attr_init(&thread_attr); - nrcpus = perf_cpu_map__nr(cpu); + nrcpus = cpu__max_cpu().cpu; cpuset = CPU_ALLOC(nrcpus); BUG_ON(!cpuset); size = CPU_ALLOC_SIZE(nrcpus); From 245b0edf4838874801cd35b0507ed9ec38742e8e Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 7 Jun 2024 10:13:54 +0530 Subject: [PATCH 119/280] tools/perf: Fix timing issue with parallel threads in perf bench wake-up-parallel perf bench futex fails as below and hangs intermittently when attempted to run on on a powerpc system: ./perf bench futex wake-parallel Running 'futex/wake-parallel' benchmark: Run summary [PID 88588]: blocking on 640 threads (at [private] futex 0x10464b8c), 640 threads waking up 1 at a time. [Run 1]: Avg per-thread latency (waking 1/640 threads) in 0.1309 ms (+-53.27%) [Run 2]: Avg per-thread latency (waking 1/640 threads) in 0.0120 ms (+-31.16%) [Run 3]: Avg per-thread latency (waking 1/640 threads) in 0.1474 ms (+-92.47%) [Run 4]: Avg per-thread latency (waking 1/640 threads) in 0.2883 ms (+-67.75%) [Run 5]: Avg per-thread latency (waking 1/640 threads) in 0.4108 ms (+-39.60%) [Run 6]: Avg per-thread latency (waking 1/640 threads) in 0.7843 ms (+-78.98%) perf: couldn't wakeup all tasks (0/1) perf: couldn't wakeup all tasks (0/1) perf: couldn't wakeup all tasks (0/1) perf: couldn't wakeup all tasks (0/1) perf: couldn't wakeup all tasks (0/1) perf: couldn't wakeup all tasks (0/1) In the system, where perf bench wake-up-parallel is has system configuration of 640 cpus. After debugging, this turned out to be a timing issue. The benchmark creates threads equal to number of cpus and issues a futex_wait. Then it does a usleep for .1 second before initiating futex_wake. In system configuration with more threads, the usleep time is not enough. Patch changes the usleep from 100000 to 200000 With the patch, ran multiple iterations and there were no issues further seen Reported-by: Disha Goel Signed-off-by: Athira Rajeev Reviewed-by: Ian Rogers Tested-by: Disha Goel Cc: akanksha@linux.ibm.com Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607044354.82225-3-atrajeev@linux.vnet.ibm.com --- tools/perf/bench/futex-wake-parallel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c index 90a5b91bf139..4352e318631e 100644 --- a/tools/perf/bench/futex-wake-parallel.c +++ b/tools/perf/bench/futex-wake-parallel.c @@ -318,7 +318,7 @@ int bench_futex_wake_parallel(int argc, const char **argv) cond_broadcast(&thread_worker); mutex_unlock(&thread_lock); - usleep(100000); + usleep(200000); /* Ok, all threads are patiently blocked, start waking folks up */ wakeup_threads(waking_worker); From 65b37df8c61ceea863ac0ac188fe3464c2ee3ac0 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 12 Jun 2024 05:40:27 -0700 Subject: [PATCH 120/280] perf test pmu: Warn don't fail for legacy mixed case event names PowerPC has mixed case events matching legacy hardware cache events. Warn but don't fail in this case. Event parsing will still work in this case by matching the legacy case. Signed-off-by: Ian Rogers Tested-by: Kajol Jain Cc: James Clark Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240612124027.2712643-1-irogers@google.com --- tools/perf/tests/pmu.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index cc88b5920c3e..201df4446f7c 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -260,26 +260,42 @@ err_out: static bool permitted_event_name(const char *name) { bool has_lower = false, has_upper = false; + __u64 config; for (size_t i = 0; i < strlen(name); i++) { char c = name[i]; if (islower(c)) { if (has_upper) - return false; + goto check_legacy; has_lower = true; continue; } if (isupper(c)) { if (has_lower) - return false; + goto check_legacy; has_upper = true; continue; } if (!isdigit(c) && c != '.' && c != '_' && c != '-') - return false; + goto check_legacy; } return true; +check_legacy: + /* + * If the event name matches a legacy cache name the legacy encoding + * will still be used. This isn't quite WAI as sysfs events should take + * priority, but this case happens on PowerPC and matches the behavior + * in older perf tools where legacy events were the priority. Be + * permissive and assume later PMU drivers will use all lower or upper + * case names. + */ + if (parse_events__decode_legacy_cache(name, /*extended_pmu_type=*/0, &config) == 0) { + pr_warning("sysfs event '%s' should be all lower/upper case, it will be matched using legacy encoding.", + name); + return true; + } + return false; } static int test__pmu_event_names(struct test_suite *test __maybe_unused, From 63a8dfb889112ab4a065aa60a9a1b590b410d055 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 13 Jun 2024 09:52:23 -0400 Subject: [PATCH 121/280] function_graph: Add READ_ONCE() when accessing fgraph_array[] In function_graph_enter() there's a loop that looks at fgraph_array[] elements which are fgraph_ops. It first tests if it is a fgraph_stub op, and if so skips it, as that's just there as a place holder. Then it checks the fgraph_ops filters to see if the ops wants to trace the current function. But if the compiler reloads the fgraph_array[] after the check against fgraph_stub, it could race with the fgraph_array[] being updated with the fgraph_stub. That would cause the stub to be processed. But the stub has a null "func_hash" field which will cause a NULL pointer dereference. Add a READ_ONCE() so that the gops that is compared against the fgraph_stub is also the gops that is processed later. Link: https://lore.kernel.org/all/CA+G9fYsSVJQZH=nM=1cjTc94PgSnMF9y65BnOv6XSoCG_b6wmw@mail.gmail.com/ Link: https://lore.kernel.org/linux-trace-kernel/20240613095223.1f07e3a4@rorschach.local.home Cc: Mark Rutland Cc: Mathieu Desnoyers Fixes: cc60ee813b503 ("function_graph: Use static_call and branch to optimize entry function") Reported-by: Naresh Kamboju Signed-off-by: Steven Rostedt (Google) Acked-by: Masami Hiramatsu (Google) --- kernel/trace/fgraph.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 8317d1a7f43a..fc205ad167a9 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -641,7 +641,7 @@ int function_graph_enter(unsigned long ret, unsigned long func, { for_each_set_bit(i, &fgraph_array_bitmask, sizeof(fgraph_array_bitmask) * BITS_PER_BYTE) { - struct fgraph_ops *gops = fgraph_array[i]; + struct fgraph_ops *gops = READ_ONCE(fgraph_array[i]); int save_curr_ret_stack; if (gops == &fgraph_stub) From 231295a186b60e779152ee88a2647a392307431a Mon Sep 17 00:00:00 2001 From: Fernand Sieber Date: Fri, 14 Jun 2024 09:35:17 +0200 Subject: [PATCH 122/280] perf: sched map skips redundant lines with cpu filters perf sched map supports cpu filter. However, even with cpu filters active, any context switch currently corresponds to a separate line. As result, context switches on irrelevant cpus result to redundant lines, which makes the output particlularly difficult to read on wide architectures. Fix it by skipping printing for irrelevant CPUs. Example snippet of output before fix: *B0 1.461147 secs B0 B0 B0 *G0 1.517139 secs After fix: *B0 1.461147 secs *G0 1.517139 secs Signed-off-by: Fernand Sieber Acked-by: Namhyung Kim Reviewed-and-tested-by: Madadi Vineeth Reddy Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240614073517.94974-1-sieberf@amazon.com --- tools/perf/builtin-sched.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 5977c49ae2c7..8cdf18139a7e 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -1594,8 +1594,6 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, sched->curr_thread[this_cpu.cpu] = thread__get(sched_in); - printf(" "); - new_shortname = 0; if (!tr->shortname[0]) { if (!strcmp(thread__comm_str(sched_in), "swapper")) { @@ -1622,6 +1620,11 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, new_shortname = 1; } + if (sched->map.cpus && !perf_cpu_map__has(sched->map.cpus, this_cpu)) + goto out; + + printf(" "); + for (i = 0; i < cpus_nr; i++) { struct perf_cpu cpu = { .cpu = sched->map.comp ? sched->map.comp_cpus[i].cpu : i, @@ -1656,9 +1659,6 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, color_fprintf(stdout, color, " "); } - if (sched->map.cpus && !perf_cpu_map__has(sched->map.cpus, this_cpu)) - goto out; - timestamp__scnprintf_usec(timestamp, stimestamp, sizeof(stimestamp)); color_fprintf(stdout, color, " %12s secs ", stimestamp); if (new_shortname || tr->comm_changed || (verbose > 0 && thread__tid(sched_in))) { @@ -1675,9 +1675,9 @@ static int map_switch_event(struct perf_sched *sched, struct evsel *evsel, if (sched->map.comp && new_cpu) color_fprintf(stdout, color, " (CPU %d)", this_cpu); -out: color_fprintf(stdout, color, "\n"); +out: thread__put(sched_in); return 0; From cc2621cecd0f9e248eb12e2b512804e1a1d38cbf Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 7 Jun 2024 13:29:15 -0700 Subject: [PATCH 123/280] perf hist: Factor out __hpp__fmt_print() Split the logic to print the histogram values according to the format string. This was used in 3 different places so it's better to move out the logic into a function. Tested-by: Arnaldo Carvalho de Melo Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607202918.2357459-2-namhyung@kernel.org --- tools/perf/ui/hist.c | 83 +++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 47 deletions(-) diff --git a/tools/perf/ui/hist.c b/tools/perf/ui/hist.c index 685ba2a54fd8..e30fcb1e87e7 100644 --- a/tools/perf/ui/hist.c +++ b/tools/perf/ui/hist.c @@ -23,6 +23,29 @@ __ret; \ }) +static int __hpp__fmt_print(struct perf_hpp *hpp, struct hists *hists, u64 val, + int nr_samples, const char *fmt, int len, + hpp_snprint_fn print_fn, enum perf_hpp_fmt_type fmtype) +{ + if (fmtype == PERF_HPP_FMT_TYPE__PERCENT) { + double percent = 0.0; + u64 total = hists__total_period(hists); + + if (total) + percent = 100.0 * val / total; + + return hpp__call_print_fn(hpp, print_fn, fmt, len, percent); + } + + if (fmtype == PERF_HPP_FMT_TYPE__AVERAGE) { + double avg = nr_samples ? (1.0 * val / nr_samples) : 0; + + return hpp__call_print_fn(hpp, print_fn, fmt, len, avg); + } + + return hpp__call_print_fn(hpp, print_fn, fmt, len, val); +} + static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, hpp_field_fn get_field, const char *fmt, int len, hpp_snprint_fn print_fn, enum perf_hpp_fmt_type fmtype) @@ -33,24 +56,8 @@ static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, char *buf = hpp->buf; size_t size = hpp->size; - if (fmtype == PERF_HPP_FMT_TYPE__PERCENT) { - double percent = 0.0; - u64 total = hists__total_period(hists); - - if (total) - percent = 100.0 * get_field(he) / total; - - ret = hpp__call_print_fn(hpp, print_fn, fmt, len, percent); - } else if (fmtype == PERF_HPP_FMT_TYPE__AVERAGE) { - double average = 0; - - if (he->stat.nr_events) - average = 1.0 * get_field(he) / he->stat.nr_events; - - ret = hpp__call_print_fn(hpp, print_fn, fmt, len, average); - } else { - ret = hpp__call_print_fn(hpp, print_fn, fmt, len, get_field(he)); - } + ret = __hpp__fmt_print(hpp, hists, get_field(he), he->stat.nr_events, + fmt, len, print_fn, fmtype); if (evsel__is_group_event(evsel)) { int prev_idx, idx_delta; @@ -72,30 +79,16 @@ static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, while (idx_delta--) { /* - * zero-fill group members in the middle which - * have no sample + * zero-fill group members in the middle which have + * no samples, pair->hists is not correct but it's + * fine since the value is 0. */ - if (fmtype != PERF_HPP_FMT_TYPE__RAW) { - ret += hpp__call_print_fn(hpp, print_fn, - fmt, len, 0.0); - } else { - ret += hpp__call_print_fn(hpp, print_fn, - fmt, len, 0ULL); - } + ret += __hpp__fmt_print(hpp, pair->hists, 0, 0, + fmt, len, print_fn, fmtype); } - if (fmtype == PERF_HPP_FMT_TYPE__PERCENT) { - ret += hpp__call_print_fn(hpp, print_fn, fmt, len, - 100.0 * period / total); - } else if (fmtype == PERF_HPP_FMT_TYPE__AVERAGE) { - double avg = nr_samples ? (period / nr_samples) : 0; - - ret += hpp__call_print_fn(hpp, print_fn, fmt, - len, avg); - } else { - ret += hpp__call_print_fn(hpp, print_fn, fmt, - len, period); - } + ret += __hpp__fmt_print(hpp, pair->hists, period, nr_samples, + fmt, len, print_fn, fmtype); prev_idx = evsel__group_idx(evsel); } @@ -104,15 +97,11 @@ static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, while (idx_delta--) { /* - * zero-fill group members at last which have no sample + * zero-fill group members at last which have no sample. + * the hists is not correct but it's fine like above. */ - if (fmtype != PERF_HPP_FMT_TYPE__RAW) { - ret += hpp__call_print_fn(hpp, print_fn, - fmt, len, 0.0); - } else { - ret += hpp__call_print_fn(hpp, print_fn, - fmt, len, 0ULL); - } + ret += __hpp__fmt_print(hpp, evsel__hists(evsel), 0, 0, + fmt, len, print_fn, fmtype); } } From 8f6071a3dce40e6991877873699b84c4fb570ab3 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 7 Jun 2024 13:29:16 -0700 Subject: [PATCH 124/280] perf hist: Simplify __hpp_fmt() using hpp_fmt_data The struct hpp_fmt_data is to keep the values for each group members so it doesn't need to check the event index in the group. Tested-by: Arnaldo Carvalho de Melo Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607202918.2357459-3-namhyung@kernel.org --- tools/perf/ui/hist.c | 77 +++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/tools/perf/ui/hist.c b/tools/perf/ui/hist.c index e30fcb1e87e7..7e863cd92781 100644 --- a/tools/perf/ui/hist.c +++ b/tools/perf/ui/hist.c @@ -46,65 +46,62 @@ static int __hpp__fmt_print(struct perf_hpp *hpp, struct hists *hists, u64 val, return hpp__call_print_fn(hpp, print_fn, fmt, len, val); } +struct hpp_fmt_value { + struct hists *hists; + u64 val; + int samples; +}; + static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, hpp_field_fn get_field, const char *fmt, int len, hpp_snprint_fn print_fn, enum perf_hpp_fmt_type fmtype) { - int ret; + int ret = 0; struct hists *hists = he->hists; struct evsel *evsel = hists_to_evsel(hists); + struct evsel *pos; char *buf = hpp->buf; size_t size = hpp->size; + int i, nr_members = 1; + struct hpp_fmt_value *values; - ret = __hpp__fmt_print(hpp, hists, get_field(he), he->stat.nr_events, - fmt, len, print_fn, fmtype); + if (evsel__is_group_event(evsel)) + nr_members = evsel->core.nr_members; + + values = calloc(nr_members, sizeof(*values)); + if (values == NULL) + return 0; + + i = 0; + for_each_group_evsel(pos, evsel) + values[i++].hists = evsel__hists(pos); + + values[0].val = get_field(he); + values[0].samples = he->stat.nr_events; if (evsel__is_group_event(evsel)) { - int prev_idx, idx_delta; struct hist_entry *pair; - int nr_members = evsel->core.nr_members; - - prev_idx = evsel__group_idx(evsel); list_for_each_entry(pair, &he->pairs.head, pairs.node) { - u64 period = get_field(pair); - u64 total = hists__total_period(pair->hists); - int nr_samples = pair->stat.nr_events; + for (i = 0; i < nr_members; i++) { + if (values[i].hists != pair->hists) + continue; - if (!total) - continue; - - evsel = hists_to_evsel(pair->hists); - idx_delta = evsel__group_idx(evsel) - prev_idx - 1; - - while (idx_delta--) { - /* - * zero-fill group members in the middle which have - * no samples, pair->hists is not correct but it's - * fine since the value is 0. - */ - ret += __hpp__fmt_print(hpp, pair->hists, 0, 0, - fmt, len, print_fn, fmtype); + values[i].val = get_field(pair); + values[i].samples = pair->stat.nr_events; + break; } - - ret += __hpp__fmt_print(hpp, pair->hists, period, nr_samples, - fmt, len, print_fn, fmtype); - - prev_idx = evsel__group_idx(evsel); - } - - idx_delta = nr_members - prev_idx - 1; - - while (idx_delta--) { - /* - * zero-fill group members at last which have no sample. - * the hists is not correct but it's fine like above. - */ - ret += __hpp__fmt_print(hpp, evsel__hists(evsel), 0, 0, - fmt, len, print_fn, fmtype); } } + for (i = 0; i < nr_members; i++) { + ret += __hpp__fmt_print(hpp, values[i].hists, values[i].val, + values[i].samples, fmt, len, + print_fn, fmtype); + } + + free(values); + /* * Restore original buf and size as it's where caller expects * the result will be saved. From 411ee13598ef322c1a7f4a4022a84d995873f235 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 7 Jun 2024 13:29:17 -0700 Subject: [PATCH 125/280] perf hist: Add symbol_conf.skip_empty Add the skip_empty flag to symbol_conf and set the value from the report command to preserve the existing behavior. This makes the code simpler and will be needed other code which is hard to add a new argument. Tested-by: Arnaldo Carvalho de Melo Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607202918.2357459-4-namhyung@kernel.org --- tools/perf/builtin-annotate.c | 4 ++-- tools/perf/builtin-report.c | 12 ++++++------ tools/perf/ui/stdio/hist.c | 5 ++--- tools/perf/util/events_stats.h | 3 +-- tools/perf/util/hist.c | 6 +++--- tools/perf/util/hist.h | 3 +-- tools/perf/util/session.c | 5 ++--- tools/perf/util/session.h | 3 +-- tools/perf/util/symbol_conf.h | 3 ++- 9 files changed, 20 insertions(+), 24 deletions(-) diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c index 50d2fb222d48..b10b7f005658 100644 --- a/tools/perf/builtin-annotate.c +++ b/tools/perf/builtin-annotate.c @@ -571,8 +571,8 @@ static int __cmd_annotate(struct perf_annotate *ann) goto out; if (dump_trace) { - perf_session__fprintf_nr_events(session, stdout, false); - evlist__fprintf_nr_events(session->evlist, stdout, false); + perf_session__fprintf_nr_events(session, stdout); + evlist__fprintf_nr_events(session->evlist, stdout); goto out; } diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 69618fb0110b..9718770facb5 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -810,8 +810,8 @@ static int stats_print(struct report *rep) { struct perf_session *session = rep->session; - perf_session__fprintf_nr_events(session, stdout, rep->skip_empty); - evlist__fprintf_nr_events(session->evlist, stdout, rep->skip_empty); + perf_session__fprintf_nr_events(session, stdout); + evlist__fprintf_nr_events(session->evlist, stdout); return 0; } @@ -1089,10 +1089,8 @@ static int __cmd_report(struct report *rep) perf_session__fprintf_dsos(session, stdout); if (dump_trace) { - perf_session__fprintf_nr_events(session, stdout, - rep->skip_empty); - evlist__fprintf_nr_events(session->evlist, stdout, - rep->skip_empty); + perf_session__fprintf_nr_events(session, stdout); + evlist__fprintf_nr_events(session->evlist, stdout); return 0; } } @@ -1562,6 +1560,8 @@ int cmd_report(int argc, const char **argv) data.path = input_name; data.force = symbol_conf.force; + symbol_conf.skip_empty = report.skip_empty; + repeat: session = perf_session__new(&data, &report.tool); if (IS_ERR(session)) { diff --git a/tools/perf/ui/stdio/hist.c b/tools/perf/ui/stdio/hist.c index b849caace398..9372e8904d22 100644 --- a/tools/perf/ui/stdio/hist.c +++ b/tools/perf/ui/stdio/hist.c @@ -897,8 +897,7 @@ out: return ret; } -size_t events_stats__fprintf(struct events_stats *stats, FILE *fp, - bool skip_empty) +size_t events_stats__fprintf(struct events_stats *stats, FILE *fp) { int i; size_t ret = 0; @@ -910,7 +909,7 @@ size_t events_stats__fprintf(struct events_stats *stats, FILE *fp, name = perf_event__name(i); if (!strcmp(name, "UNKNOWN")) continue; - if (skip_empty && !stats->nr_events[i]) + if (symbol_conf.skip_empty && !stats->nr_events[i]) continue; if (i && total) { diff --git a/tools/perf/util/events_stats.h b/tools/perf/util/events_stats.h index 8fecc9fbaecc..f43e5b1a366a 100644 --- a/tools/perf/util/events_stats.h +++ b/tools/perf/util/events_stats.h @@ -52,7 +52,6 @@ struct hists_stats { void events_stats__inc(struct events_stats *stats, u32 type); -size_t events_stats__fprintf(struct events_stats *stats, FILE *fp, - bool skip_empty); +size_t events_stats__fprintf(struct events_stats *stats, FILE *fp); #endif /* __PERF_EVENTS_STATS_ */ diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 2e9e193179dd..f028f113c4fd 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -2706,8 +2706,7 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, } } -size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp, - bool skip_empty) +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp) { struct evsel *pos; size_t ret = 0; @@ -2715,7 +2714,8 @@ size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp, evlist__for_each_entry(evlist, pos) { struct hists *hists = evsel__hists(pos); - if (skip_empty && !hists->stats.nr_samples && !hists->stats.nr_lost_samples) + if (symbol_conf.skip_empty && !hists->stats.nr_samples && + !hists->stats.nr_lost_samples) continue; ret += fprintf(fp, "%s stats:\n", evsel__name(pos)); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index 8fb3bdd29188..5273f5c37050 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -375,8 +375,7 @@ void hists__inc_nr_lost_samples(struct hists *hists, u32 lost); size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows, int max_cols, float min_pcnt, FILE *fp, bool ignore_callchains); -size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp, - bool skip_empty); +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp); void hists__filter_by_dso(struct hists *hists); void hists__filter_by_thread(struct hists *hists); diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index a10343b9dcd4..0ec92d47373c 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -2696,8 +2696,7 @@ size_t perf_session__fprintf_dsos_buildid(struct perf_session *session, FILE *fp return machines__fprintf_dsos_buildid(&session->machines, fp, skip, parm); } -size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp, - bool skip_empty) +size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp) { size_t ret; const char *msg = ""; @@ -2707,7 +2706,7 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp, ret = fprintf(fp, "\nAggregated stats:%s\n", msg); - ret += events_stats__fprintf(&session->evlist->stats, fp, skip_empty); + ret += events_stats__fprintf(&session->evlist->stats, fp); return ret; } diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h index 3b0256e977a6..4c29dc86956f 100644 --- a/tools/perf/util/session.h +++ b/tools/perf/util/session.h @@ -130,8 +130,7 @@ size_t perf_session__fprintf_dsos(struct perf_session *session, FILE *fp); size_t perf_session__fprintf_dsos_buildid(struct perf_session *session, FILE *fp, bool (fn)(struct dso *dso, int parm), int parm); -size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp, - bool skip_empty); +size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp); void perf_session__dump_kmaps(struct perf_session *session); diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h index c114bbceef40..657cfa5af43c 100644 --- a/tools/perf/util/symbol_conf.h +++ b/tools/perf/util/symbol_conf.h @@ -46,7 +46,8 @@ struct symbol_conf { lazy_load_kernel_maps, keep_exited_threads, annotate_data_member, - annotate_data_sample; + annotate_data_sample, + skip_empty; const char *vmlinux_name, *kallsyms_name, *source_prefix, From eae7044b67a606f10d245ff2866ee04f235e1722 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 7 Jun 2024 13:29:18 -0700 Subject: [PATCH 126/280] perf hist: Honor symbol_conf.skip_empty So that it can skip events with no sample according to the config value. This can omit the dummy event in the output of perf report --group. An example output: $ sudo perf mem record -a sleep 1 $ sudo perf report --group Before) # # Samples: 232 of events 'cpu/mem-loads,ldlat=30/P, cpu/mem-stores/P, dummy:u' # Event count (approx.): 3089861 # # Overhead Command Shared Object Symbol # ........................ ........... ................. ..................................... # 9.29% 0.00% 0.00% swapper [kernel.kallsyms] [k] update_blocked_averages 5.26% 0.15% 0.00% swapper [kernel.kallsyms] [k] __update_load_avg_se 4.15% 0.00% 0.00% perf-exec [kernel.kallsyms] [k] slab_update_freelist.isra.0 3.87% 0.00% 0.00% perf-exec [kernel.kallsyms] [k] memcg_slab_post_alloc_hook 3.79% 0.17% 0.00% swapper [kernel.kallsyms] [k] enqueue_task_fair 3.63% 0.00% 0.00% sleep [kernel.kallsyms] [k] next_uptodate_page 2.86% 0.00% 0.00% swapper [kernel.kallsyms] [k] __update_load_avg_cfs_rq 2.78% 0.00% 0.00% swapper [kernel.kallsyms] [k] __schedule 2.34% 0.00% 0.00% swapper [kernel.kallsyms] [k] intel_idle 2.32% 0.97% 0.00% swapper [kernel.kallsyms] [k] psi_group_change After) # # Samples: 232 of events 'cpu/mem-loads,ldlat=30/P, cpu/mem-stores/P' # Event count (approx.): 3089861 # # Overhead Command Shared Object Symbol # ................ ........... ................. ..................................... # 9.29% 0.00% swapper [kernel.kallsyms] [k] update_blocked_averages 5.26% 0.15% swapper [kernel.kallsyms] [k] __update_load_avg_se 4.15% 0.00% perf-exec [kernel.kallsyms] [k] slab_update_freelist.isra.0 3.87% 0.00% perf-exec [kernel.kallsyms] [k] memcg_slab_post_alloc_hook 3.79% 0.17% swapper [kernel.kallsyms] [k] enqueue_task_fair 3.63% 0.00% sleep [kernel.kallsyms] [k] next_uptodate_page 2.86% 0.00% swapper [kernel.kallsyms] [k] __update_load_avg_cfs_rq 2.78% 0.00% swapper [kernel.kallsyms] [k] __schedule 2.34% 0.00% swapper [kernel.kallsyms] [k] intel_idle 2.32% 0.97% swapper [kernel.kallsyms] [k] psi_group_change Now it doesn't have a column for the dummy event. Tested-by: Arnaldo Carvalho de Melo Signed-off-by: Namhyung Kim Link: https://lore.kernel.org/r/20240607202918.2357459-5-namhyung@kernel.org --- tools/perf/ui/hist.c | 18 ++++++++++++++++-- tools/perf/util/evsel.c | 13 ++++++++++--- tools/perf/util/python.c | 3 +++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tools/perf/ui/hist.c b/tools/perf/ui/hist.c index 7e863cd92781..5d1f04f66a5a 100644 --- a/tools/perf/ui/hist.c +++ b/tools/perf/ui/hist.c @@ -95,6 +95,10 @@ static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, } for (i = 0; i < nr_members; i++) { + if (symbol_conf.skip_empty && + values[i].hists->stats.nr_samples == 0) + continue; + ret += __hpp__fmt_print(hpp, values[i].hists, values[i].val, values[i].samples, fmt, len, print_fn, fmtype); @@ -296,8 +300,18 @@ static int hpp__width_fn(struct perf_hpp_fmt *fmt, int len = fmt->user_len ?: fmt->len; struct evsel *evsel = hists_to_evsel(hists); - if (symbol_conf.event_group) - len = max(len, evsel->core.nr_members * fmt->len); + if (symbol_conf.event_group) { + int nr = 0; + struct evsel *pos; + + for_each_group_evsel(pos, evsel) { + if (!symbol_conf.skip_empty || + evsel__hists(pos)->stats.nr_samples) + nr++; + } + + len = max(len, nr * fmt->len); + } if (len < (int)strlen(fmt->name)) len = strlen(fmt->name); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 28c54897a97e..25857894c047 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -55,6 +55,7 @@ #include "../perf-sys.h" #include "util/parse-branch-options.h" #include "util/bpf-filter.h" +#include "util/hist.h" #include #include #include @@ -832,16 +833,22 @@ const char *evsel__group_name(struct evsel *evsel) int evsel__group_desc(struct evsel *evsel, char *buf, size_t size) { int ret = 0; + bool first = true; struct evsel *pos; const char *group_name = evsel__group_name(evsel); if (!evsel->forced_leader) ret = scnprintf(buf, size, "%s { ", group_name); - ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel)); + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; - for_each_group_member(pos, evsel) - ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos)); + ret += scnprintf(buf + ret, size - ret, "%s%s", + first ? "" : ", ", evsel__name(pos)); + first = false; + } if (!evsel->forced_leader) ret += scnprintf(buf + ret, size - ret, " }"); diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index 0aeb97c11c03..88f98f2772fb 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -23,6 +23,7 @@ #include "util/env.h" #include "util/pmu.h" #include "util/pmus.h" +#include "util/symbol_conf.h" #include #include "util.h" @@ -50,6 +51,8 @@ #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif +struct symbol_conf symbol_conf; + /* * Avoid bringing in event parsing. */ From 7e8f89e5e05983089be1d431f1c82e5abc37574a Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:24 +0200 Subject: [PATCH 127/280] s390: Add get_lowcore() function Add a get_lowcore() function which returns the address of lowcore (currently always NULL). This function will be used as a replacement of the S390_lowcore macro. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/lowcore.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 8c5f16857539..94f954250ae0 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -215,6 +215,11 @@ struct lowcore { #define S390_lowcore (*((struct lowcore *) 0)) +static __always_inline struct lowcore *get_lowcore(void) +{ + return NULL; +} + extern struct lowcore *lowcore_ptr[]; static inline void set_prefix(__u32 address) From 208da1d5fc3c67d8ae5d34e844fd67cc47a136f0 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:25 +0200 Subject: [PATCH 128/280] s390: Replace S390_lowcore by get_lowcore() Replace all S390_lowcore usages in arch/s390/ by get_lowcore(). Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/current.h | 2 +- arch/s390/include/asm/facility.h | 4 +- arch/s390/include/asm/hardirq.h | 6 +-- arch/s390/include/asm/mmu_context.h | 8 +-- arch/s390/include/asm/pai.h | 8 +-- arch/s390/include/asm/percpu.h | 2 +- arch/s390/include/asm/preempt.h | 30 +++++------ arch/s390/include/asm/processor.h | 8 +-- arch/s390/include/asm/setup.h | 34 ++++++------ arch/s390/include/asm/smp.h | 2 +- arch/s390/include/asm/softirq_stack.h | 2 +- arch/s390/include/asm/spinlock.h | 2 +- arch/s390/include/asm/timex.h | 10 ++-- arch/s390/include/asm/vtime.h | 12 ++--- arch/s390/kernel/dumpstack.c | 8 +-- arch/s390/kernel/early.c | 36 ++++++------- arch/s390/kernel/idle.c | 10 ++-- arch/s390/kernel/irq.c | 18 +++---- arch/s390/kernel/machine_kexec.c | 4 +- arch/s390/kernel/nmi.c | 29 +++++------ arch/s390/kernel/perf_cpum_sf.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 4 +- arch/s390/kernel/perf_pai_ext.c | 4 +- arch/s390/kernel/process.c | 6 +-- arch/s390/kernel/setup.c | 24 ++++----- arch/s390/kernel/smp.c | 30 +++++------ arch/s390/kernel/syscall.c | 4 +- arch/s390/kernel/time.c | 22 ++++---- arch/s390/kernel/traps.c | 24 ++++----- arch/s390/kernel/vtime.c | 74 +++++++++++++-------------- arch/s390/kvm/kvm-s390.c | 2 +- arch/s390/lib/spinlock.c | 4 +- arch/s390/lib/test_unwind.c | 2 +- arch/s390/lib/uaccess.c | 4 +- arch/s390/mm/dump_pagetables.c | 2 +- arch/s390/mm/fault.c | 16 +++--- arch/s390/mm/gmap.c | 6 +-- arch/s390/mm/pageattr.c | 2 +- arch/s390/mm/pgalloc.c | 4 +- arch/s390/pci/pci.c | 2 +- 40 files changed, 236 insertions(+), 237 deletions(-) diff --git a/arch/s390/include/asm/current.h b/arch/s390/include/asm/current.h index 68f84315277c..d03a922c641e 100644 --- a/arch/s390/include/asm/current.h +++ b/arch/s390/include/asm/current.h @@ -14,6 +14,6 @@ struct task_struct; -#define current ((struct task_struct *const)S390_lowcore.current_task) +#define current ((struct task_struct *const)get_lowcore()->current_task) #endif /* !(_S390_CURRENT_H) */ diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index 796007125dff..d46cc725f024 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -92,8 +92,8 @@ static inline void __stfle(u64 *stfle_fac_list, int size) asm volatile( " stfl 0(0)\n" - : "=m" (S390_lowcore.stfl_fac_list)); - stfl_fac_list = S390_lowcore.stfl_fac_list; + : "=m" (get_lowcore()->stfl_fac_list)); + stfl_fac_list = get_lowcore()->stfl_fac_list; memcpy(stfle_fac_list, &stfl_fac_list, 4); nr = 4; /* bytes stored by stfl */ if (stfl_fac_list & 0x01000000) { diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h index 58668ffb5488..a5b45388c91f 100644 --- a/arch/s390/include/asm/hardirq.h +++ b/arch/s390/include/asm/hardirq.h @@ -13,9 +13,9 @@ #include -#define local_softirq_pending() (S390_lowcore.softirq_pending) -#define set_softirq_pending(x) (S390_lowcore.softirq_pending = (x)) -#define or_softirq_pending(x) (S390_lowcore.softirq_pending |= (x)) +#define local_softirq_pending() (get_lowcore()->softirq_pending) +#define set_softirq_pending(x) (get_lowcore()->softirq_pending = (x)) +#define or_softirq_pending(x) (get_lowcore()->softirq_pending |= (x)) #define __ARCH_IRQ_STAT #define __ARCH_IRQ_EXIT_IRQS_DISABLED diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index a7789a9f6218..d56eb0a1f37b 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -76,9 +76,9 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * int cpu = smp_processor_id(); if (next == &init_mm) - S390_lowcore.user_asce = s390_invalid_asce; + get_lowcore()->user_asce = s390_invalid_asce; else - S390_lowcore.user_asce.val = next->context.asce; + get_lowcore()->user_asce.val = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ local_ctl_load(7, &s390_invalid_asce); @@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - local_ctl_load(7, &S390_lowcore.user_asce); + local_ctl_load(7, &get_lowcore()->user_asce); } #define activate_mm activate_mm @@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - local_ctl_load(7, &S390_lowcore.user_asce); + local_ctl_load(7, &get_lowcore()->user_asce); } #include diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 8e1dd389ae88..25f2077ba3c9 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -55,11 +55,11 @@ static __always_inline void pai_kernel_enter(struct pt_regs *regs) return; if (!static_branch_unlikely(&pai_key)) return; - if (!S390_lowcore.ccd) + if (!get_lowcore()->ccd) return; if (!user_mode(regs)) return; - WRITE_ONCE(S390_lowcore.ccd, S390_lowcore.ccd | PAI_CRYPTO_KERNEL_OFFSET); + WRITE_ONCE(get_lowcore()->ccd, get_lowcore()->ccd | PAI_CRYPTO_KERNEL_OFFSET); } static __always_inline void pai_kernel_exit(struct pt_regs *regs) @@ -68,11 +68,11 @@ static __always_inline void pai_kernel_exit(struct pt_regs *regs) return; if (!static_branch_unlikely(&pai_key)) return; - if (!S390_lowcore.ccd) + if (!get_lowcore()->ccd) return; if (!user_mode(regs)) return; - WRITE_ONCE(S390_lowcore.ccd, S390_lowcore.ccd & ~PAI_CRYPTO_KERNEL_OFFSET); + WRITE_ONCE(get_lowcore()->ccd, get_lowcore()->ccd & ~PAI_CRYPTO_KERNEL_OFFSET); } #define PAI_SAVE_AREA(x) ((x)->hw.event_base) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 264095dd84bc..89a28740b6ab 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -9,7 +9,7 @@ * s390 uses its own implementation for per cpu data, the offset of * the cpu local data area is cached in the cpu's lowcore memory. */ -#define __my_cpu_offset S390_lowcore.percpu_offset +#define __my_cpu_offset get_lowcore()->percpu_offset /* * For 64 bit module code, the module may be more than 4G above the diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h index 0e3da500e98c..3ae5f31c665d 100644 --- a/arch/s390/include/asm/preempt.h +++ b/arch/s390/include/asm/preempt.h @@ -14,7 +14,7 @@ static __always_inline int preempt_count(void) { - return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED; + return READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED; } static __always_inline void preempt_count_set(int pc) @@ -22,26 +22,26 @@ static __always_inline void preempt_count_set(int pc) int old, new; do { - old = READ_ONCE(S390_lowcore.preempt_count); + old = READ_ONCE(get_lowcore()->preempt_count); new = (old & PREEMPT_NEED_RESCHED) | (pc & ~PREEMPT_NEED_RESCHED); - } while (__atomic_cmpxchg(&S390_lowcore.preempt_count, + } while (__atomic_cmpxchg(&get_lowcore()->preempt_count, old, new) != old); } static __always_inline void set_preempt_need_resched(void) { - __atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); + __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); } static __always_inline void clear_preempt_need_resched(void) { - __atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count); + __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count); } static __always_inline bool test_preempt_need_resched(void) { - return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED); + return !(READ_ONCE(get_lowcore()->preempt_count) & PREEMPT_NEED_RESCHED); } static __always_inline void __preempt_count_add(int val) @@ -52,11 +52,11 @@ static __always_inline void __preempt_count_add(int val) */ if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES)) { if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) { - __atomic_add_const(val, &S390_lowcore.preempt_count); + __atomic_add_const(val, &get_lowcore()->preempt_count); return; } } - __atomic_add(val, &S390_lowcore.preempt_count); + __atomic_add(val, &get_lowcore()->preempt_count); } static __always_inline void __preempt_count_sub(int val) @@ -66,12 +66,12 @@ static __always_inline void __preempt_count_sub(int val) static __always_inline bool __preempt_count_dec_and_test(void) { - return __atomic_add(-1, &S390_lowcore.preempt_count) == 1; + return __atomic_add(-1, &get_lowcore()->preempt_count) == 1; } static __always_inline bool should_resched(int preempt_offset) { - return unlikely(READ_ONCE(S390_lowcore.preempt_count) == + return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset); } @@ -81,12 +81,12 @@ static __always_inline bool should_resched(int preempt_offset) static __always_inline int preempt_count(void) { - return READ_ONCE(S390_lowcore.preempt_count); + return READ_ONCE(get_lowcore()->preempt_count); } static __always_inline void preempt_count_set(int pc) { - S390_lowcore.preempt_count = pc; + get_lowcore()->preempt_count = pc; } static __always_inline void set_preempt_need_resched(void) @@ -104,17 +104,17 @@ static __always_inline bool test_preempt_need_resched(void) static __always_inline void __preempt_count_add(int val) { - S390_lowcore.preempt_count += val; + get_lowcore()->preempt_count += val; } static __always_inline void __preempt_count_sub(int val) { - S390_lowcore.preempt_count -= val; + get_lowcore()->preempt_count -= val; } static __always_inline bool __preempt_count_dec_and_test(void) { - return !--S390_lowcore.preempt_count && tif_need_resched(); + return !--get_lowcore()->preempt_count && tif_need_resched(); } static __always_inline bool should_resched(int preempt_offset) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 07ad5a1df878..c87cf2b8e81a 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -46,17 +46,17 @@ typedef long (*sys_call_ptr_t)(struct pt_regs *regs); static __always_inline void set_cpu_flag(int flag) { - S390_lowcore.cpu_flags |= (1UL << flag); + get_lowcore()->cpu_flags |= (1UL << flag); } static __always_inline void clear_cpu_flag(int flag) { - S390_lowcore.cpu_flags &= ~(1UL << flag); + get_lowcore()->cpu_flags &= ~(1UL << flag); } static __always_inline bool test_cpu_flag(int flag) { - return S390_lowcore.cpu_flags & (1UL << flag); + return get_lowcore()->cpu_flags & (1UL << flag); } static __always_inline bool test_and_set_cpu_flag(int flag) @@ -269,7 +269,7 @@ static __always_inline unsigned long __current_stack_pointer(void) static __always_inline bool on_thread_stack(void) { - unsigned long ksp = S390_lowcore.kernel_stack; + unsigned long ksp = get_lowcore()->kernel_stack; return !((ksp ^ current_stack_pointer) & ~(THREAD_SIZE - 1)); } diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 32f70873e2b7..8505737712ee 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -77,24 +77,24 @@ extern unsigned long max_mappable; /* The Write Back bit position in the physaddr is given by the SLPC PCI */ extern unsigned long mio_wb_bit_mask; -#define MACHINE_IS_VM (S390_lowcore.machine_flags & MACHINE_FLAG_VM) -#define MACHINE_IS_KVM (S390_lowcore.machine_flags & MACHINE_FLAG_KVM) -#define MACHINE_IS_LPAR (S390_lowcore.machine_flags & MACHINE_FLAG_LPAR) +#define MACHINE_IS_VM (get_lowcore()->machine_flags & MACHINE_FLAG_VM) +#define MACHINE_IS_KVM (get_lowcore()->machine_flags & MACHINE_FLAG_KVM) +#define MACHINE_IS_LPAR (get_lowcore()->machine_flags & MACHINE_FLAG_LPAR) -#define MACHINE_HAS_DIAG9C (S390_lowcore.machine_flags & MACHINE_FLAG_DIAG9C) -#define MACHINE_HAS_ESOP (S390_lowcore.machine_flags & MACHINE_FLAG_ESOP) -#define MACHINE_HAS_IDTE (S390_lowcore.machine_flags & MACHINE_FLAG_IDTE) -#define MACHINE_HAS_EDAT1 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT1) -#define MACHINE_HAS_EDAT2 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT2) -#define MACHINE_HAS_TOPOLOGY (S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY) -#define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE) -#define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC) -#define MACHINE_HAS_TLB_GUEST (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_GUEST) -#define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX) -#define MACHINE_HAS_GS (S390_lowcore.machine_flags & MACHINE_FLAG_GS) -#define MACHINE_HAS_SCC (S390_lowcore.machine_flags & MACHINE_FLAG_SCC) -#define MACHINE_HAS_PCI_MIO (S390_lowcore.machine_flags & MACHINE_FLAG_PCI_MIO) -#define MACHINE_HAS_RDP (S390_lowcore.machine_flags & MACHINE_FLAG_RDP) +#define MACHINE_HAS_DIAG9C (get_lowcore()->machine_flags & MACHINE_FLAG_DIAG9C) +#define MACHINE_HAS_ESOP (get_lowcore()->machine_flags & MACHINE_FLAG_ESOP) +#define MACHINE_HAS_IDTE (get_lowcore()->machine_flags & MACHINE_FLAG_IDTE) +#define MACHINE_HAS_EDAT1 (get_lowcore()->machine_flags & MACHINE_FLAG_EDAT1) +#define MACHINE_HAS_EDAT2 (get_lowcore()->machine_flags & MACHINE_FLAG_EDAT2) +#define MACHINE_HAS_TOPOLOGY (get_lowcore()->machine_flags & MACHINE_FLAG_TOPOLOGY) +#define MACHINE_HAS_TE (get_lowcore()->machine_flags & MACHINE_FLAG_TE) +#define MACHINE_HAS_TLB_LC (get_lowcore()->machine_flags & MACHINE_FLAG_TLB_LC) +#define MACHINE_HAS_TLB_GUEST (get_lowcore()->machine_flags & MACHINE_FLAG_TLB_GUEST) +#define MACHINE_HAS_NX (get_lowcore()->machine_flags & MACHINE_FLAG_NX) +#define MACHINE_HAS_GS (get_lowcore()->machine_flags & MACHINE_FLAG_GS) +#define MACHINE_HAS_SCC (get_lowcore()->machine_flags & MACHINE_FLAG_SCC) +#define MACHINE_HAS_PCI_MIO (get_lowcore()->machine_flags & MACHINE_FLAG_PCI_MIO) +#define MACHINE_HAS_RDP (get_lowcore()->machine_flags & MACHINE_FLAG_RDP) /* * Console mode. Override with conmode= diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 6e5b1b4b19a9..0b1ed637bfd6 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -11,7 +11,7 @@ #include #include -#define raw_smp_processor_id() (S390_lowcore.cpu_nr) +#define raw_smp_processor_id() (get_lowcore()->cpu_nr) extern struct mutex smp_cpu_state_mutex; extern unsigned int smp_cpu_mt_shift; diff --git a/arch/s390/include/asm/softirq_stack.h b/arch/s390/include/asm/softirq_stack.h index 1ac5115d3115..42d61296bbad 100644 --- a/arch/s390/include/asm/softirq_stack.h +++ b/arch/s390/include/asm/softirq_stack.h @@ -8,7 +8,7 @@ #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK static inline void do_softirq_own_stack(void) { - call_on_stack(0, S390_lowcore.async_stack, void, __do_softirq); + call_on_stack(0, get_lowcore()->async_stack, void, __do_softirq); } #endif #endif /* __ASM_S390_SOFTIRQ_STACK_H */ diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 37127cd7749e..3e43c90ff135 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -16,7 +16,7 @@ #include #include -#define SPINLOCK_LOCKVAL (S390_lowcore.spinlock_lockval) +#define SPINLOCK_LOCKVAL (get_lowcore()->spinlock_lockval) extern int spin_retry; diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 4d646659a5f5..640901f2fbc3 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -161,16 +161,16 @@ static inline unsigned long local_tick_disable(void) { unsigned long old; - old = S390_lowcore.clock_comparator; - S390_lowcore.clock_comparator = clock_comparator_max; - set_clock_comparator(S390_lowcore.clock_comparator); + old = get_lowcore()->clock_comparator; + get_lowcore()->clock_comparator = clock_comparator_max; + set_clock_comparator(get_lowcore()->clock_comparator); return old; } static inline void local_tick_enable(unsigned long comp) { - S390_lowcore.clock_comparator = comp; - set_clock_comparator(S390_lowcore.clock_comparator); + get_lowcore()->clock_comparator = comp; + set_clock_comparator(get_lowcore()->clock_comparator); } #define CLOCK_TICK_RATE 1193180 /* Underlying HZ */ diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index 561c91c1a87c..ef4dd7d057a2 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -4,16 +4,16 @@ static inline void update_timer_sys(void) { - S390_lowcore.system_timer += S390_lowcore.last_update_timer - S390_lowcore.exit_timer; - S390_lowcore.user_timer += S390_lowcore.exit_timer - S390_lowcore.sys_enter_timer; - S390_lowcore.last_update_timer = S390_lowcore.sys_enter_timer; + get_lowcore()->system_timer += get_lowcore()->last_update_timer - get_lowcore()->exit_timer; + get_lowcore()->user_timer += get_lowcore()->exit_timer - get_lowcore()->sys_enter_timer; + get_lowcore()->last_update_timer = get_lowcore()->sys_enter_timer; } static inline void update_timer_mcck(void) { - S390_lowcore.system_timer += S390_lowcore.last_update_timer - S390_lowcore.exit_timer; - S390_lowcore.user_timer += S390_lowcore.exit_timer - S390_lowcore.mcck_enter_timer; - S390_lowcore.last_update_timer = S390_lowcore.mcck_enter_timer; + get_lowcore()->system_timer += get_lowcore()->last_update_timer - get_lowcore()->exit_timer; + get_lowcore()->user_timer += get_lowcore()->exit_timer - get_lowcore()->mcck_enter_timer; + get_lowcore()->last_update_timer = get_lowcore()->mcck_enter_timer; } #endif /* _S390_VTIME_H */ diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index d2012635b093..1ecd0580561f 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -61,28 +61,28 @@ static bool in_task_stack(unsigned long sp, struct task_struct *task, static bool in_irq_stack(unsigned long sp, struct stack_info *info) { - unsigned long stack = S390_lowcore.async_stack - STACK_INIT_OFFSET; + unsigned long stack = get_lowcore()->async_stack - STACK_INIT_OFFSET; return in_stack(sp, info, STACK_TYPE_IRQ, stack); } static bool in_nodat_stack(unsigned long sp, struct stack_info *info) { - unsigned long stack = S390_lowcore.nodat_stack - STACK_INIT_OFFSET; + unsigned long stack = get_lowcore()->nodat_stack - STACK_INIT_OFFSET; return in_stack(sp, info, STACK_TYPE_NODAT, stack); } static bool in_mcck_stack(unsigned long sp, struct stack_info *info) { - unsigned long stack = S390_lowcore.mcck_stack - STACK_INIT_OFFSET; + unsigned long stack = get_lowcore()->mcck_stack - STACK_INIT_OFFSET; return in_stack(sp, info, STACK_TYPE_MCCK, stack); } static bool in_restart_stack(unsigned long sp, struct stack_info *info) { - unsigned long stack = S390_lowcore.restart_stack - STACK_INIT_OFFSET; + unsigned long stack = get_lowcore()->restart_stack - STACK_INIT_OFFSET; return in_stack(sp, info, STACK_TYPE_RESTART, stack); } diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index c666271433fb..467ed4dba817 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -72,7 +72,7 @@ static void __init reset_tod_clock(void) memset(&tod_clock_base, 0, sizeof(tod_clock_base)); tod_clock_base.tod = TOD_UNIX_EPOCH; - S390_lowcore.last_update_clock = TOD_UNIX_EPOCH; + get_lowcore()->last_update_clock = TOD_UNIX_EPOCH; } /* @@ -99,7 +99,7 @@ static noinline __init void detect_machine_type(void) /* Check current-configuration-level */ if (stsi(NULL, 0, 0, 0) <= 2) { - S390_lowcore.machine_flags |= MACHINE_FLAG_LPAR; + get_lowcore()->machine_flags |= MACHINE_FLAG_LPAR; return; } /* Get virtual-machine cpu information. */ @@ -108,9 +108,9 @@ static noinline __init void detect_machine_type(void) /* Detect known hypervisors */ if (!memcmp(vmms->vm[0].cpi, "\xd2\xe5\xd4", 3)) - S390_lowcore.machine_flags |= MACHINE_FLAG_KVM; + get_lowcore()->machine_flags |= MACHINE_FLAG_KVM; else if (!memcmp(vmms->vm[0].cpi, "\xa9\x61\xe5\xd4", 4)) - S390_lowcore.machine_flags |= MACHINE_FLAG_VM; + get_lowcore()->machine_flags |= MACHINE_FLAG_VM; } /* Remove leading, trailing and double whitespace. */ @@ -166,7 +166,7 @@ static __init void setup_topology(void) if (!test_facility(11)) return; - S390_lowcore.machine_flags |= MACHINE_FLAG_TOPOLOGY; + get_lowcore()->machine_flags |= MACHINE_FLAG_TOPOLOGY; for (max_mnest = 6; max_mnest > 1; max_mnest--) { if (stsi(&sysinfo_page, 15, 1, max_mnest) == 0) break; @@ -186,8 +186,8 @@ static noinline __init void setup_lowcore_early(void) psw.addr = (unsigned long)early_pgm_check_handler; psw.mask = PSW_KERNEL_BITS; - S390_lowcore.program_new_psw = psw; - S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; + get_lowcore()->program_new_psw = psw; + get_lowcore()->preempt_count = INIT_PREEMPT_COUNT; } static noinline __init void setup_facility_list(void) @@ -211,43 +211,43 @@ static __init void detect_diag9c(void) EX_TABLE(0b,1b) : "=d" (rc) : "0" (-EOPNOTSUPP), "d" (cpu_address) : "cc"); if (!rc) - S390_lowcore.machine_flags |= MACHINE_FLAG_DIAG9C; + get_lowcore()->machine_flags |= MACHINE_FLAG_DIAG9C; } static __init void detect_machine_facilities(void) { if (test_facility(8)) { - S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; + get_lowcore()->machine_flags |= MACHINE_FLAG_EDAT1; system_ctl_set_bit(0, CR0_EDAT_BIT); } if (test_facility(78)) - S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; + get_lowcore()->machine_flags |= MACHINE_FLAG_EDAT2; if (test_facility(3)) - S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; + get_lowcore()->machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { - S390_lowcore.machine_flags |= MACHINE_FLAG_TE; + get_lowcore()->machine_flags |= MACHINE_FLAG_TE; system_ctl_set_bit(0, CR0_TRANSACTIONAL_EXECUTION_BIT); } if (test_facility(51)) - S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; + get_lowcore()->machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) system_ctl_set_bit(0, CR0_VECTOR_BIT); if (test_facility(130)) - S390_lowcore.machine_flags |= MACHINE_FLAG_NX; + get_lowcore()->machine_flags |= MACHINE_FLAG_NX; if (test_facility(133)) - S390_lowcore.machine_flags |= MACHINE_FLAG_GS; + get_lowcore()->machine_flags |= MACHINE_FLAG_GS; if (test_facility(139) && (tod_clock_base.tod >> 63)) { /* Enabled signed clock comparator comparisons */ - S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; + get_lowcore()->machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; system_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SIGN_BIT); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { - S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; + get_lowcore()->machine_flags |= MACHINE_FLAG_PCI_MIO; /* the control bit is set during PCI initialization */ } if (test_facility(194)) - S390_lowcore.machine_flags |= MACHINE_FLAG_RDP; + get_lowcore()->machine_flags |= MACHINE_FLAG_RDP; } static inline void save_vector_registers(void) diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index af9c97c0ad73..2c34e02ae64b 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -34,13 +34,13 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } - idle_time = S390_lowcore.int_clock - idle->clock_idle_enter; + idle_time = get_lowcore()->int_clock - idle->clock_idle_enter; - S390_lowcore.steal_timer += idle->clock_idle_enter - S390_lowcore.last_update_clock; - S390_lowcore.last_update_clock = S390_lowcore.int_clock; + get_lowcore()->steal_timer += idle->clock_idle_enter - get_lowcore()->last_update_clock; + get_lowcore()->last_update_clock = get_lowcore()->int_clock; - S390_lowcore.system_timer += S390_lowcore.last_update_timer - idle->timer_idle_enter; - S390_lowcore.last_update_timer = S390_lowcore.sys_enter_timer; + get_lowcore()->system_timer += get_lowcore()->last_update_timer - idle->timer_idle_enter; + get_lowcore()->last_update_timer = get_lowcore()->sys_enter_timer; /* Account time spent with enabled wait psw loaded as idle time. */ WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 9acc6630abd3..1af5a08d72ab 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -100,8 +100,8 @@ static const struct irq_class irqclass_sub_desc[] = { static void do_IRQ(struct pt_regs *regs, int irq) { - if (tod_after_eq(S390_lowcore.int_clock, - S390_lowcore.clock_comparator)) + if (tod_after_eq(get_lowcore()->int_clock, + get_lowcore()->clock_comparator)) /* Serve timer interrupts first. */ clock_comparator_work(); generic_handle_irq(irq); @@ -111,7 +111,7 @@ static int on_async_stack(void) { unsigned long frame = current_frame_address(); - return ((S390_lowcore.async_stack ^ frame) & ~(THREAD_SIZE - 1)) == 0; + return ((get_lowcore()->async_stack ^ frame) & ~(THREAD_SIZE - 1)) == 0; } static void do_irq_async(struct pt_regs *regs, int irq) @@ -119,7 +119,7 @@ static void do_irq_async(struct pt_regs *regs, int irq) if (on_async_stack()) { do_IRQ(regs, irq); } else { - call_on_stack(2, S390_lowcore.async_stack, void, do_IRQ, + call_on_stack(2, get_lowcore()->async_stack, void, do_IRQ, struct pt_regs *, regs, int, irq); } } @@ -153,8 +153,8 @@ void noinstr do_io_irq(struct pt_regs *regs) set_cpu_flag(CIF_NOHZ_DELAY); do { - regs->tpi_info = S390_lowcore.tpi_info; - if (S390_lowcore.tpi_info.adapter_IO) + regs->tpi_info = get_lowcore()->tpi_info; + if (get_lowcore()->tpi_info.adapter_IO) do_irq_async(regs, THIN_INTERRUPT); else do_irq_async(regs, IO_INTERRUPT); @@ -183,9 +183,9 @@ void noinstr do_ext_irq(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - regs->int_code = S390_lowcore.ext_int_code_addr; - regs->int_parm = S390_lowcore.ext_params; - regs->int_parm_long = S390_lowcore.ext_params2; + regs->int_code = get_lowcore()->ext_int_code_addr; + regs->int_parm = get_lowcore()->ext_params; + regs->int_parm_long = get_lowcore()->ext_params2; from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 3aee98efc374..f4cf65da6d49 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -52,7 +52,7 @@ static void __do_machine_kdump(void *data) purgatory = (purgatory_t)image->start; /* store_status() saved the prefix register to lowcore */ - prefix = (unsigned long) S390_lowcore.prefixreg_save_area; + prefix = (unsigned long)get_lowcore()->prefixreg_save_area; /* Now do the reset */ s390_reset_system(); @@ -91,7 +91,7 @@ static noinline void __machine_kdump(void *image) continue; } /* Store status of the boot CPU */ - mcesa = __va(S390_lowcore.mcesad & MCESA_ORIGIN_MASK); + mcesa = __va(get_lowcore()->mcesad & MCESA_ORIGIN_MASK); if (cpu_has_vx()) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 230d010bac9b..db562416d728 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -125,7 +125,7 @@ static notrace void s390_handle_damage(void) smp_emergency_stop(); diag_amode31_ops.diag308_reset(); ptr = nmi_puts(message, "System stopped due to unrecoverable machine check, code: 0x"); - u64_to_hex(ptr, S390_lowcore.mcck_interruption_code); + u64_to_hex(ptr, get_lowcore()->mcck_interruption_code); /* * Disable low address protection and make machine check new PSW a @@ -135,17 +135,17 @@ static notrace void s390_handle_damage(void) cr0_new = cr0; cr0_new.lap = 0; local_ctl_load(0, &cr0_new.reg); - psw_save = S390_lowcore.mcck_new_psw; - psw_bits(S390_lowcore.mcck_new_psw).io = 0; - psw_bits(S390_lowcore.mcck_new_psw).ext = 0; - psw_bits(S390_lowcore.mcck_new_psw).wait = 1; + psw_save = get_lowcore()->mcck_new_psw; + psw_bits(get_lowcore()->mcck_new_psw).io = 0; + psw_bits(get_lowcore()->mcck_new_psw).ext = 0; + psw_bits(get_lowcore()->mcck_new_psw).wait = 1; sclp_emergency_printk(message); /* * Restore machine check new PSW and control register 0 to original * values. This makes possible system dump analysis easier. */ - S390_lowcore.mcck_new_psw = psw_save; + get_lowcore()->mcck_new_psw = psw_save; local_ctl_load(0, &cr0.reg); disabled_wait(); while (1); @@ -226,7 +226,7 @@ static bool notrace nmi_registers_valid(union mci mci) /* * Set the clock comparator register to the next expected value. */ - set_clock_comparator(S390_lowcore.clock_comparator); + set_clock_comparator(get_lowcore()->clock_comparator); if (!mci.gr || !mci.fp || !mci.fc) return false; /* @@ -252,7 +252,7 @@ static bool notrace nmi_registers_valid(union mci mci) * check handling must take care of this. The host values are saved by * KVM and are not affected. */ - cr2.reg = S390_lowcore.cregs_save_area[2]; + cr2.reg = get_lowcore()->cregs_save_area[2]; if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) return false; if (!mci.ms || !mci.pm || !mci.ia) @@ -278,11 +278,10 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = S390_lowcore.mcck_interruption_code & + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); - mcck_backup->ext_damage_code = S390_lowcore.external_damage_code; - mcck_backup->failing_storage_address - = S390_lowcore.failing_storage_address; + mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; + mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } NOKPROBE_SYMBOL(s390_backup_mcck_info); @@ -314,7 +313,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (user_mode(regs)) update_timer_mcck(); inc_irq_stat(NMI_NMI); - mci.val = S390_lowcore.mcck_interruption_code; + mci.val = get_lowcore()->mcck_interruption_code; mcck = this_cpu_ptr(&cpu_mcck); /* @@ -382,9 +381,9 @@ void notrace s390_do_machine_check(struct pt_regs *regs) } if (mci.ed && mci.ec) { /* External damage */ - if (S390_lowcore.external_damage_code & (1U << ED_STP_SYNC)) + if (get_lowcore()->external_damage_code & (1U << ED_STP_SYNC)) mcck->stp_queue |= stp_sync_check(); - if (S390_lowcore.external_damage_code & (1U << ED_STP_ISLAND)) + if (get_lowcore()->external_damage_code & (1U << ED_STP_ISLAND)) mcck->stp_queue |= stp_island_check(); mcck_pending = 1; } diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index 06efad5b4f93..736c1d9632dd 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1022,7 +1022,7 @@ static void cpumsf_pmu_enable(struct pmu *pmu) } /* Load current program parameter */ - lpp(&S390_lowcore.lpp); + lpp(&get_lowcore()->lpp); debug_sprintf_event(sfdbg, 6, "%s: es %i cs %i ed %i cd %i " "interval %#lx tear %#lx dear %#lx\n", __func__, diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 95079a289109..2f5a20e300f6 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -372,7 +372,7 @@ static int paicrypt_add(struct perf_event *event, int flags) if (++cpump->active_events == 1) { ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; - WRITE_ONCE(S390_lowcore.ccd, ccd); + WRITE_ONCE(get_lowcore()->ccd, ccd); local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); } if (flags & PERF_EF_START) @@ -409,7 +409,7 @@ static void paicrypt_del(struct perf_event *event, int flags) paicrypt_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); - WRITE_ONCE(S390_lowcore.ccd, 0); + WRITE_ONCE(get_lowcore()->ccd, 0); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 6040f3c2b942..6295531b39a2 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -389,7 +389,7 @@ static int paiext_add(struct perf_event *event, int flags) struct paiext_cb *pcb = cpump->paiext_cb; if (++cpump->active_events == 1) { - S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); + get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); @@ -431,7 +431,7 @@ static void paiext_del(struct perf_event *event, int flags) /* Disable CPU instruction lookup for PAIE1 control block */ local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); pcb->acc = 0; - S390_lowcore.aicd = 0; + get_lowcore()->aicd = 0; } } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index d8740631df4b..9637aee43c40 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -71,10 +71,10 @@ void flush_thread(void) void arch_setup_new_exec(void) { - if (S390_lowcore.current_pid != current->pid) { - S390_lowcore.current_pid = current->pid; + if (get_lowcore()->current_pid != current->pid) { + get_lowcore()->current_pid = current->pid; if (test_facility(40)) - lpp(&S390_lowcore.lpp); + lpp(&get_lowcore()->lpp); } } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 90c2c786bb35..3993f4caf224 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -421,16 +421,16 @@ static void __init setup_lowcore(void) lc->clock_comparator = clock_comparator_max; lc->current_task = (unsigned long)&init_task; lc->lpp = LPP_MAGIC; - lc->machine_flags = S390_lowcore.machine_flags; - lc->preempt_count = S390_lowcore.preempt_count; + lc->machine_flags = get_lowcore()->machine_flags; + lc->preempt_count = get_lowcore()->preempt_count; nmi_alloc_mcesa_early(&lc->mcesad); - lc->sys_enter_timer = S390_lowcore.sys_enter_timer; - lc->exit_timer = S390_lowcore.exit_timer; - lc->user_timer = S390_lowcore.user_timer; - lc->system_timer = S390_lowcore.system_timer; - lc->steal_timer = S390_lowcore.steal_timer; - lc->last_update_timer = S390_lowcore.last_update_timer; - lc->last_update_clock = S390_lowcore.last_update_clock; + lc->sys_enter_timer = get_lowcore()->sys_enter_timer; + lc->exit_timer = get_lowcore()->exit_timer; + lc->user_timer = get_lowcore()->user_timer; + lc->system_timer = get_lowcore()->system_timer; + lc->steal_timer = get_lowcore()->steal_timer; + lc->last_update_timer = get_lowcore()->last_update_timer; + lc->last_update_clock = get_lowcore()->last_update_clock; /* * Allocate the global restart stack which is the same for * all CPUs in case *one* of them does a PSW restart. @@ -439,7 +439,7 @@ static void __init setup_lowcore(void) lc->mcck_stack = stack_alloc_early() + STACK_INIT_OFFSET; lc->async_stack = stack_alloc_early() + STACK_INIT_OFFSET; lc->nodat_stack = stack_alloc_early() + STACK_INIT_OFFSET; - lc->kernel_stack = S390_lowcore.kernel_stack; + lc->kernel_stack = get_lowcore()->kernel_stack; /* * Set up PSW restart to call ipl.c:do_restart(). Copy the relevant * restart data to the absolute zero lowcore. This is necessary if @@ -455,8 +455,8 @@ static void __init setup_lowcore(void) lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW); lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW); lc->preempt_count = PREEMPT_DISABLED; - lc->kernel_asce = S390_lowcore.kernel_asce; - lc->user_asce = S390_lowcore.user_asce; + lc->kernel_asce = get_lowcore()->kernel_asce; + lc->user_asce = get_lowcore()->user_asce; system_ctlreg_init_save_area(lc); abs_lc = get_abs_lowcore(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 0324649aae0a..ebe4bc326a6b 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -203,7 +203,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) mcck_stack = stack_alloc(); if (!lc || !nodat_stack || !async_stack || !mcck_stack) goto out; - memcpy(lc, &S390_lowcore, 512); + memcpy(lc, get_lowcore(), 512); memset((char *) lc + 512, 0, sizeof(*lc) - 512); lc->async_stack = async_stack + STACK_INIT_OFFSET; lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET; @@ -265,9 +265,9 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) lc->spinlock_lockval = arch_spin_lockval(cpu); lc->spinlock_index = 0; lc->percpu_offset = __per_cpu_offset[cpu]; - lc->kernel_asce = S390_lowcore.kernel_asce; + lc->kernel_asce = get_lowcore()->kernel_asce; lc->user_asce = s390_invalid_asce; - lc->machine_flags = S390_lowcore.machine_flags; + lc->machine_flags = get_lowcore()->machine_flags; lc->user_timer = lc->system_timer = lc->steal_timer = lc->avg_steal_timer = 0; abs_lc = get_abs_lowcore(); @@ -407,7 +407,7 @@ void smp_call_ipl_cpu(void (*func)(void *), void *data) struct lowcore *lc = lowcore_ptr[0]; if (pcpu_devices[0].address == stap()) - lc = &S390_lowcore; + lc = get_lowcore(); pcpu_delegate(&pcpu_devices[0], func, data, lc->nodat_stack); @@ -844,13 +844,13 @@ static void smp_start_secondary(void *cpuvoid) { int cpu = raw_smp_processor_id(); - S390_lowcore.last_update_clock = get_tod_clock(); - S390_lowcore.restart_stack = (unsigned long)restart_stack; - S390_lowcore.restart_fn = (unsigned long)do_restart; - S390_lowcore.restart_data = 0; - S390_lowcore.restart_source = -1U; - S390_lowcore.restart_flags = 0; - restore_access_regs(S390_lowcore.access_regs_save_area); + get_lowcore()->last_update_clock = get_tod_clock(); + get_lowcore()->restart_stack = (unsigned long)restart_stack; + get_lowcore()->restart_fn = (unsigned long)do_restart; + get_lowcore()->restart_data = 0; + get_lowcore()->restart_source = -1U; + get_lowcore()->restart_flags = 0; + restore_access_regs(get_lowcore()->access_regs_save_area); cpu_init(); rcutree_report_cpu_starting(cpu); init_cpu_timer(); @@ -981,16 +981,16 @@ void __init smp_prepare_boot_cpu(void) WARN_ON(!cpu_present(0) || !cpu_online(0)); pcpu->state = CPU_STATE_CONFIGURED; - S390_lowcore.percpu_offset = __per_cpu_offset[0]; + get_lowcore()->percpu_offset = __per_cpu_offset[0]; smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN); } void __init smp_setup_processor_id(void) { pcpu_devices[0].address = stap(); - S390_lowcore.cpu_nr = 0; - S390_lowcore.spinlock_lockval = arch_spin_lockval(0); - S390_lowcore.spinlock_index = 0; + get_lowcore()->cpu_nr = 0; + get_lowcore()->spinlock_lockval = arch_spin_lockval(0); + get_lowcore()->spinlock_index = 0; } /* diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c index dc2355c623d6..7b9bf0ad364d 100644 --- a/arch/s390/kernel/syscall.c +++ b/arch/s390/kernel/syscall.c @@ -151,8 +151,8 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap) { add_random_kstack_offset(); enter_from_user_mode(regs); - regs->psw = S390_lowcore.svc_old_psw; - regs->int_code = S390_lowcore.svc_int_code; + regs->psw = get_lowcore()->svc_old_psw; + regs->int_code = get_lowcore()->svc_int_code; update_timer_sys(); if (static_branch_likely(&cpu_has_bear)) current->thread.last_break = regs->last_break; diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index fb9f31f36628..b713effe0579 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -131,7 +131,7 @@ void clock_comparator_work(void) { struct clock_event_device *cd; - S390_lowcore.clock_comparator = clock_comparator_max; + get_lowcore()->clock_comparator = clock_comparator_max; cd = this_cpu_ptr(&comparators); cd->event_handler(cd); } @@ -139,8 +139,8 @@ void clock_comparator_work(void) static int s390_next_event(unsigned long delta, struct clock_event_device *evt) { - S390_lowcore.clock_comparator = get_tod_clock() + delta; - set_clock_comparator(S390_lowcore.clock_comparator); + get_lowcore()->clock_comparator = get_tod_clock() + delta; + set_clock_comparator(get_lowcore()->clock_comparator); return 0; } @@ -153,8 +153,8 @@ void init_cpu_timer(void) struct clock_event_device *cd; int cpu; - S390_lowcore.clock_comparator = clock_comparator_max; - set_clock_comparator(S390_lowcore.clock_comparator); + get_lowcore()->clock_comparator = clock_comparator_max; + set_clock_comparator(get_lowcore()->clock_comparator); cpu = smp_processor_id(); cd = &per_cpu(comparators, cpu); @@ -184,8 +184,8 @@ static void clock_comparator_interrupt(struct ext_code ext_code, unsigned long param64) { inc_irq_stat(IRQEXT_CLK); - if (S390_lowcore.clock_comparator == clock_comparator_max) - set_clock_comparator(S390_lowcore.clock_comparator); + if (get_lowcore()->clock_comparator == clock_comparator_max) + set_clock_comparator(get_lowcore()->clock_comparator); } static void stp_timing_alert(struct stp_irq_parm *); @@ -408,12 +408,12 @@ static void clock_sync_global(long delta) static void clock_sync_local(long delta) { /* Add the delta to the clock comparator. */ - if (S390_lowcore.clock_comparator != clock_comparator_max) { - S390_lowcore.clock_comparator += delta; - set_clock_comparator(S390_lowcore.clock_comparator); + if (get_lowcore()->clock_comparator != clock_comparator_max) { + get_lowcore()->clock_comparator += delta; + set_clock_comparator(get_lowcore()->clock_comparator); } /* Adjust the last_update_clock time-stamp. */ - S390_lowcore.last_update_clock += delta; + get_lowcore()->last_update_clock += delta; } /* Single threaded workqueue used for stp sync events */ diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 52578b5cecbd..8b904f7efb0e 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -293,10 +293,10 @@ void __init trap_init(void) local_irq_save(flags); cr0 = local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); - psw_bits(S390_lowcore.external_new_psw).mcheck = 1; - psw_bits(S390_lowcore.program_new_psw).mcheck = 1; - psw_bits(S390_lowcore.svc_new_psw).mcheck = 1; - psw_bits(S390_lowcore.io_new_psw).mcheck = 1; + psw_bits(get_lowcore()->external_new_psw).mcheck = 1; + psw_bits(get_lowcore()->program_new_psw).mcheck = 1; + psw_bits(get_lowcore()->svc_new_psw).mcheck = 1; + psw_bits(get_lowcore()->io_new_psw).mcheck = 1; local_ctl_load(0, &cr0); local_irq_restore(flags); local_mcck_enable(); @@ -310,8 +310,8 @@ void noinstr __do_pgm_check(struct pt_regs *regs) unsigned int trapnr; irqentry_state_t state; - regs->int_code = S390_lowcore.pgm_int_code; - regs->int_parm_long = S390_lowcore.trans_exc_code; + regs->int_code = get_lowcore()->pgm_int_code; + regs->int_parm_long = get_lowcore()->trans_exc_code; state = irqentry_enter(regs); @@ -324,19 +324,19 @@ void noinstr __do_pgm_check(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - if (S390_lowcore.pgm_code & 0x0200) { + if (get_lowcore()->pgm_code & 0x0200) { /* transaction abort */ - current->thread.trap_tdb = S390_lowcore.pgm_tdb; + current->thread.trap_tdb = get_lowcore()->pgm_tdb; } - if (S390_lowcore.pgm_code & PGM_INT_CODE_PER) { + if (get_lowcore()->pgm_code & PGM_INT_CODE_PER) { if (user_mode(regs)) { struct per_event *ev = ¤t->thread.per_event; set_thread_flag(TIF_PER_TRAP); - ev->address = S390_lowcore.per_address; - ev->cause = S390_lowcore.per_code_combined; - ev->paid = S390_lowcore.per_access_id; + ev->address = get_lowcore()->per_address; + ev->cause = get_lowcore()->per_code_combined; + ev->paid = get_lowcore()->per_access_id; } else { /* PER event in kernel is kprobes */ __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index ffc1db0cbf9c..7d8991c3cd3a 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -41,8 +41,8 @@ static inline void set_vtimer(u64 expires) " stpt %0\n" /* Store current cpu timer value */ " spt %1" /* Set new value imm. afterwards */ : "=Q" (timer) : "Q" (expires)); - S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; - S390_lowcore.last_update_timer = expires; + get_lowcore()->system_timer += get_lowcore()->last_update_timer - timer; + get_lowcore()->last_update_timer = expires; } static inline int virt_timer_forward(u64 elapsed) @@ -118,21 +118,21 @@ static int do_account_vtime(struct task_struct *tsk) { u64 timer, clock, user, guest, system, hardirq, softirq; - timer = S390_lowcore.last_update_timer; - clock = S390_lowcore.last_update_clock; + timer = get_lowcore()->last_update_timer; + clock = get_lowcore()->last_update_clock; asm volatile( " stpt %0\n" /* Store current cpu timer value */ " stckf %1" /* Store current tod clock value */ - : "=Q" (S390_lowcore.last_update_timer), - "=Q" (S390_lowcore.last_update_clock) + : "=Q" (get_lowcore()->last_update_timer), + "=Q" (get_lowcore()->last_update_clock) : : "cc"); - clock = S390_lowcore.last_update_clock - clock; - timer -= S390_lowcore.last_update_timer; + clock = get_lowcore()->last_update_clock - clock; + timer -= get_lowcore()->last_update_timer; if (hardirq_count()) - S390_lowcore.hardirq_timer += timer; + get_lowcore()->hardirq_timer += timer; else - S390_lowcore.system_timer += timer; + get_lowcore()->system_timer += timer; /* Update MT utilization calculation */ if (smp_cpu_mtid && @@ -141,16 +141,16 @@ static int do_account_vtime(struct task_struct *tsk) /* Calculate cputime delta */ user = update_tsk_timer(&tsk->thread.user_timer, - READ_ONCE(S390_lowcore.user_timer)); + READ_ONCE(get_lowcore()->user_timer)); guest = update_tsk_timer(&tsk->thread.guest_timer, - READ_ONCE(S390_lowcore.guest_timer)); + READ_ONCE(get_lowcore()->guest_timer)); system = update_tsk_timer(&tsk->thread.system_timer, - READ_ONCE(S390_lowcore.system_timer)); + READ_ONCE(get_lowcore()->system_timer)); hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, - READ_ONCE(S390_lowcore.hardirq_timer)); + READ_ONCE(get_lowcore()->hardirq_timer)); softirq = update_tsk_timer(&tsk->thread.softirq_timer, - READ_ONCE(S390_lowcore.softirq_timer)); - S390_lowcore.steal_timer += + READ_ONCE(get_lowcore()->softirq_timer)); + get_lowcore()->steal_timer += clock - user - guest - system - hardirq - softirq; /* Push account value */ @@ -177,16 +177,16 @@ static int do_account_vtime(struct task_struct *tsk) void vtime_task_switch(struct task_struct *prev) { do_account_vtime(prev); - prev->thread.user_timer = S390_lowcore.user_timer; - prev->thread.guest_timer = S390_lowcore.guest_timer; - prev->thread.system_timer = S390_lowcore.system_timer; - prev->thread.hardirq_timer = S390_lowcore.hardirq_timer; - prev->thread.softirq_timer = S390_lowcore.softirq_timer; - S390_lowcore.user_timer = current->thread.user_timer; - S390_lowcore.guest_timer = current->thread.guest_timer; - S390_lowcore.system_timer = current->thread.system_timer; - S390_lowcore.hardirq_timer = current->thread.hardirq_timer; - S390_lowcore.softirq_timer = current->thread.softirq_timer; + prev->thread.user_timer = get_lowcore()->user_timer; + prev->thread.guest_timer = get_lowcore()->guest_timer; + prev->thread.system_timer = get_lowcore()->system_timer; + prev->thread.hardirq_timer = get_lowcore()->hardirq_timer; + prev->thread.softirq_timer = get_lowcore()->softirq_timer; + get_lowcore()->user_timer = current->thread.user_timer; + get_lowcore()->guest_timer = current->thread.guest_timer; + get_lowcore()->system_timer = current->thread.system_timer; + get_lowcore()->hardirq_timer = current->thread.hardirq_timer; + get_lowcore()->softirq_timer = current->thread.softirq_timer; } /* @@ -201,23 +201,23 @@ void vtime_flush(struct task_struct *tsk) if (do_account_vtime(tsk)) virt_timer_expire(); - steal = S390_lowcore.steal_timer; - avg_steal = S390_lowcore.avg_steal_timer; + steal = get_lowcore()->steal_timer; + avg_steal = get_lowcore()->avg_steal_timer; if ((s64) steal > 0) { - S390_lowcore.steal_timer = 0; + get_lowcore()->steal_timer = 0; account_steal_time(cputime_to_nsecs(steal)); avg_steal += steal; } - S390_lowcore.avg_steal_timer = avg_steal / 2; + get_lowcore()->avg_steal_timer = avg_steal / 2; } static u64 vtime_delta(void) { - u64 timer = S390_lowcore.last_update_timer; + u64 timer = get_lowcore()->last_update_timer; - S390_lowcore.last_update_timer = get_cpu_timer(); + get_lowcore()->last_update_timer = get_cpu_timer(); - return timer - S390_lowcore.last_update_timer; + return timer - get_lowcore()->last_update_timer; } /* @@ -229,9 +229,9 @@ void vtime_account_kernel(struct task_struct *tsk) u64 delta = vtime_delta(); if (tsk->flags & PF_VCPU) - S390_lowcore.guest_timer += delta; + get_lowcore()->guest_timer += delta; else - S390_lowcore.system_timer += delta; + get_lowcore()->system_timer += delta; virt_timer_forward(delta); } @@ -241,7 +241,7 @@ void vtime_account_softirq(struct task_struct *tsk) { u64 delta = vtime_delta(); - S390_lowcore.softirq_timer += delta; + get_lowcore()->softirq_timer += delta; virt_timer_forward(delta); } @@ -250,7 +250,7 @@ void vtime_account_hardirq(struct task_struct *tsk) { u64 delta = vtime_delta(); - S390_lowcore.hardirq_timer += delta; + get_lowcore()->hardirq_timer += delta; virt_timer_forward(delta); } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 82e9631cd9ef..50b77b759042 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4079,7 +4079,7 @@ static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { /* do not poll with more than halt_poll_max_steal percent of steal time */ - if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >= + if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >= READ_ONCE(halt_poll_max_steal)) { vcpu->stat.halt_no_poll_steal++; return true; diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 81c53440b3e6..0c9a73a18826 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -119,7 +119,7 @@ static inline void arch_spin_lock_queued(arch_spinlock_t *lp) struct spin_wait *node, *next; int lockval, ix, node_id, tail_id, old, new, owner, count; - ix = S390_lowcore.spinlock_index++; + ix = get_lowcore()->spinlock_index++; barrier(); lockval = SPINLOCK_LOCKVAL; /* cpu + 1 */ node = this_cpu_ptr(&spin_wait[ix]); @@ -205,7 +205,7 @@ static inline void arch_spin_lock_queued(arch_spinlock_t *lp) } out: - S390_lowcore.spinlock_index--; + get_lowcore()->spinlock_index--; } static inline void arch_spin_lock_classic(arch_spinlock_t *lp) diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c index 2848e3fb2ff5..768898dacb92 100644 --- a/arch/s390/lib/test_unwind.c +++ b/arch/s390/lib/test_unwind.c @@ -356,7 +356,7 @@ static noinline int unwindme_func2(struct unwindme *u) if (u->flags & UWM_SWITCH_STACK) { local_irq_save(flags); local_mcck_save(mflags); - rc = call_on_stack(1, S390_lowcore.nodat_stack, + rc = call_on_stack(1, get_lowcore()->nodat_stack, int, unwindme_func3, struct unwindme *, u); local_mcck_restore(mflags); local_irq_restore(flags); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 61d8dcd95bbc..c7c269d5c491 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -21,13 +21,13 @@ void debug_user_asce(int exit) local_ctl_store(1, &cr1); local_ctl_store(7, &cr7); - if (cr1.val == S390_lowcore.kernel_asce.val && cr7.val == S390_lowcore.user_asce.val) + if (cr1.val == get_lowcore()->kernel_asce.val && cr7.val == get_lowcore()->user_asce.val) return; panic("incorrect ASCE on kernel %s\n" "cr1: %016lx cr7: %016lx\n" "kernel: %016lx user: %016lx\n", exit ? "exit" : "entry", cr1.val, cr7.val, - S390_lowcore.kernel_asce.val, S390_lowcore.user_asce.val); + get_lowcore()->kernel_asce.val, get_lowcore()->user_asce.val); } #endif /*CONFIG_DEBUG_ENTRY */ diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index ffd07ed7b4af..45db5f47b22d 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -288,7 +288,7 @@ static int pt_dump_init(void) * kernel ASCE. We need this to keep the page table walker functions * from accessing non-existent entries. */ - max_addr = (S390_lowcore.kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; + max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 7cd50ad3b4ad..6b19a33c49c2 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -74,7 +74,7 @@ static enum fault_type get_fault_type(struct pt_regs *regs) return USER_FAULT; if (!IS_ENABLED(CONFIG_PGSTE)) return KERNEL_FAULT; - gmap = (struct gmap *)S390_lowcore.gmap; + gmap = (struct gmap *)get_lowcore()->gmap; if (gmap && gmap->asce == regs->cr1) return GMAP_FAULT; return KERNEL_FAULT; @@ -182,15 +182,15 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("mode while using "); switch (get_fault_type(regs)) { case USER_FAULT: - asce = S390_lowcore.user_asce.val; + asce = get_lowcore()->user_asce.val; pr_cont("user "); break; case GMAP_FAULT: - asce = ((struct gmap *)S390_lowcore.gmap)->asce; + asce = ((struct gmap *)get_lowcore()->gmap)->asce; pr_cont("gmap "); break; case KERNEL_FAULT: - asce = S390_lowcore.kernel_asce.val; + asce = get_lowcore()->kernel_asce.val; pr_cont("kernel "); break; default: @@ -351,7 +351,7 @@ lock_mmap: mmap_read_lock(mm); gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { - gmap = (struct gmap *)S390_lowcore.gmap; + gmap = (struct gmap *)get_lowcore()->gmap; current->thread.gmap_addr = address; current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); current->thread.gmap_int_code = regs->int_code & 0xffff; @@ -522,7 +522,7 @@ void do_secure_storage_access(struct pt_regs *regs) switch (get_fault_type(regs)) { case GMAP_FAULT: mm = current->mm; - gmap = (struct gmap *)S390_lowcore.gmap; + gmap = (struct gmap *)get_lowcore()->gmap; mmap_read_lock(mm); addr = __gmap_translate(gmap, addr); mmap_read_unlock(mm); @@ -563,7 +563,7 @@ NOKPROBE_SYMBOL(do_secure_storage_access); void do_non_secure_storage_access(struct pt_regs *regs) { - struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + struct gmap *gmap = (struct gmap *)get_lowcore()->gmap; unsigned long gaddr = get_fault_address(regs); if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT)) @@ -575,7 +575,7 @@ NOKPROBE_SYMBOL(do_non_secure_storage_access); void do_secure_storage_violation(struct pt_regs *regs) { - struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + struct gmap *gmap = (struct gmap *)get_lowcore()->gmap; unsigned long gaddr = get_fault_address(regs); /* diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index d5a5756dd69f..eb0b51a36be0 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(gmap_remove); */ void gmap_enable(struct gmap *gmap) { - S390_lowcore.gmap = (unsigned long) gmap; + get_lowcore()->gmap = (unsigned long)gmap; } EXPORT_SYMBOL_GPL(gmap_enable); @@ -297,7 +297,7 @@ EXPORT_SYMBOL_GPL(gmap_enable); */ void gmap_disable(struct gmap *gmap) { - S390_lowcore.gmap = 0UL; + get_lowcore()->gmap = 0UL; } EXPORT_SYMBOL_GPL(gmap_disable); @@ -308,7 +308,7 @@ EXPORT_SYMBOL_GPL(gmap_disable); */ struct gmap *gmap_get_enabled(void) { - return (struct gmap *) S390_lowcore.gmap; + return (struct gmap *)get_lowcore()->gmap; } EXPORT_SYMBOL_GPL(gmap_get_enabled); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 01bc8fad64d6..5f805ad42d4c 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -75,7 +75,7 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, break; } table = (unsigned long *)((unsigned long)old & mask); - crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce.val); + crdte(*old, new, table, dtt, addr, get_lowcore()->kernel_asce.val); } else if (MACHINE_HAS_IDTE) { cspg(old, *old, new); } else { diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index abb629d7e131..07d0fe197dad 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -64,8 +64,8 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { - S390_lowcore.user_asce.val = mm->context.asce; - local_ctl_load(7, &S390_lowcore.user_asce); + get_lowcore()->user_asce.val = mm->context.asce; + local_ctl_load(7, &get_lowcore()->user_asce); } __tlb_flush_local(); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 0de0f6e405b5..cff4838fad21 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -1064,7 +1064,7 @@ char * __init pcibios_setup(char *str) return NULL; } if (!strcmp(str, "nomio")) { - S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO; + get_lowcore()->machine_flags &= ~MACHINE_FLAG_PCI_MIO; return NULL; } if (!strcmp(str, "force_floating")) { From eafcd205b1a6f32879c347d717b1ce3bdfdcc7c6 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:26 +0200 Subject: [PATCH 129/280] s390/drivers: Replace S390_lowcore by get_lowcore() Replace all S390_lowcore usages in drivers/s390 by get_lowcore(). Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp_early.c | 4 ++-- drivers/s390/char/sclp_early_core.c | 12 ++++++------ drivers/s390/cio/qdio_main.c | 2 +- drivers/s390/cio/qdio_thinint.c | 2 +- drivers/s390/cio/trace.h | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 60a247fdb2a7..42986284cc78 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -50,9 +50,9 @@ static void __init sclp_early_facilities_detect(void) sclp.has_aisi = !!(sccb->fac118 & 0x10); sclp.has_zpci_lsi = !!(sccb->fac118 & 0x01); if (sccb->fac85 & 0x02) - S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP; + get_lowcore()->machine_flags |= MACHINE_FLAG_ESOP; if (sccb->fac91 & 0x40) - S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_GUEST; + get_lowcore()->machine_flags |= MACHINE_FLAG_TLB_GUEST; if (sccb->cpuoff > 134) { sclp.has_diag318 = !!(sccb->byte_134 & 0x80); sclp.has_diag320 = !!(sccb->byte_134 & 0x04); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index 9f6165cafdc3..5a5383cceb6f 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -38,11 +38,11 @@ void sclp_early_wait_irq(void) cr0_new.sssm = 1; local_ctl_load(0, &cr0_new.reg); - psw_ext_save = S390_lowcore.external_new_psw; + psw_ext_save = get_lowcore()->external_new_psw; psw_mask = __extract_psw(); - S390_lowcore.external_new_psw.mask = psw_mask; + get_lowcore()->external_new_psw.mask = psw_mask; psw_wait.mask = psw_mask | PSW_MASK_EXT | PSW_MASK_WAIT; - S390_lowcore.ext_int_code = 0; + get_lowcore()->ext_int_code = 0; do { asm volatile( @@ -53,12 +53,12 @@ void sclp_early_wait_irq(void) "0:\n" : [addr] "=&d" (addr), [psw_wait_addr] "=Q" (psw_wait.addr), - [psw_ext_addr] "=Q" (S390_lowcore.external_new_psw.addr) + [psw_ext_addr] "=Q" (get_lowcore()->external_new_psw.addr) : [psw_wait] "Q" (psw_wait) : "cc", "memory"); - } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); + } while (get_lowcore()->ext_int_code != EXT_IRQ_SERVICE_SIG); - S390_lowcore.external_new_psw = psw_ext_save; + get_lowcore()->external_new_psw = psw_ext_save; local_ctl_load(0, &cr0.reg); } diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index a1cb39f4b7a2..b711bb17f9da 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -695,7 +695,7 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) return; qdio_deliver_irq(irq_ptr); - irq_ptr->last_data_irq_time = S390_lowcore.int_clock; + irq_ptr->last_data_irq_time = get_lowcore()->int_clock; } static void qdio_handle_activate_check(struct qdio_irq *irq_ptr, diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c index ccd4ed93bd92..f931954910c5 100644 --- a/drivers/s390/cio/qdio_thinint.c +++ b/drivers/s390/cio/qdio_thinint.c @@ -99,7 +99,7 @@ static inline u32 clear_shared_ind(void) static void tiqdio_thinint_handler(struct airq_struct *airq, struct tpi_info *tpi_info) { - u64 irq_time = S390_lowcore.int_clock; + u64 irq_time = get_lowcore()->int_clock; u32 si_used = clear_shared_ind(); struct qdio_irq *irq; diff --git a/drivers/s390/cio/trace.h b/drivers/s390/cio/trace.h index a4c5c6736b31..6bca5315ee2a 100644 --- a/drivers/s390/cio/trace.h +++ b/drivers/s390/cio/trace.h @@ -169,7 +169,7 @@ TRACE_EVENT(s390_cio_tpi, else if (addr) __entry->tpi_info = *addr; else - __entry->tpi_info = S390_lowcore.tpi_info; + __entry->tpi_info = get_lowcore()->tpi_info; __entry->cssid = __entry->tpi_info.schid.cssid; __entry->ssid = __entry->tpi_info.schid.ssid; __entry->schno = __entry->tpi_info.schid.sch_no; From bbf786061dbe38135b76f2cde9e72aec624eb710 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:27 +0200 Subject: [PATCH 130/280] s390/boot: Replace S390_lowcore by get_lowcore() Replace all S390_lowcore usages in arch/s390/boot by get_lowcore(). Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/boot/ipl_parm.c | 4 ++-- arch/s390/boot/ipl_report.c | 2 +- arch/s390/boot/pgm_check_info.c | 18 +++++++++--------- arch/s390/boot/physmem_info.c | 8 ++++---- arch/s390/boot/startup.c | 12 ++++++------ arch/s390/boot/vmem.c | 12 ++++++------ 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index b24de9aabf7d..a21f301acd29 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -51,11 +51,11 @@ static inline int __diag308(unsigned long subcode, void *addr) : [r1] "+&d" (r1.pair), [reg1] "=&d" (reg1), [reg2] "=&a" (reg2), - "+Q" (S390_lowcore.program_new_psw), + "+Q" (get_lowcore()->program_new_psw), "=Q" (old) : [subcode] "d" (subcode), [psw_old] "a" (&old), - [psw_pgm] "a" (&S390_lowcore.program_new_psw) + [psw_pgm] "a" (&get_lowcore()->program_new_psw) : "cc", "memory"); return r1.odd; } diff --git a/arch/s390/boot/ipl_report.c b/arch/s390/boot/ipl_report.c index 1803035e68d2..d00898852a88 100644 --- a/arch/s390/boot/ipl_report.c +++ b/arch/s390/boot/ipl_report.c @@ -106,7 +106,7 @@ int read_ipl_report(void) * the IPL parameter list, then align the address to a double * word boundary. */ - tmp = (unsigned long) S390_lowcore.ipl_parmblock_ptr; + tmp = (unsigned long)get_lowcore()->ipl_parmblock_ptr; pl_hdr = (struct ipl_pl_hdr *) tmp; tmp = (tmp + pl_hdr->len + 7) & -8UL; rl_hdr = (struct ipl_rl_hdr *) tmp; diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index ea96275b0380..5352b3d356da 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -145,22 +145,22 @@ void print_stacktrace(unsigned long sp) void print_pgm_check_info(void) { - unsigned long *gpregs = (unsigned long *)S390_lowcore.gpregs_save_area; - struct psw_bits *psw = &psw_bits(S390_lowcore.psw_save_area); + unsigned long *gpregs = (unsigned long *)get_lowcore()->gpregs_save_area; + struct psw_bits *psw = &psw_bits(get_lowcore()->psw_save_area); decompressor_printk("Linux version %s\n", kernel_version); if (!is_prot_virt_guest() && early_command_line[0]) decompressor_printk("Kernel command line: %s\n", early_command_line); decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n", - S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1); + get_lowcore()->pgm_code, get_lowcore()->pgm_ilc >> 1); if (kaslr_enabled()) { decompressor_printk("Kernel random base: %lx\n", __kaslr_offset); decompressor_printk("Kernel random base phys: %lx\n", __kaslr_offset_phys); } decompressor_printk("PSW : %016lx %016lx (%pS)\n", - S390_lowcore.psw_save_area.mask, - S390_lowcore.psw_save_area.addr, - (void *)S390_lowcore.psw_save_area.addr); + get_lowcore()->psw_save_area.mask, + get_lowcore()->psw_save_area.addr, + (void *)get_lowcore()->psw_save_area.addr); decompressor_printk( " R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n", psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck, @@ -174,8 +174,8 @@ void print_pgm_check_info(void) gpregs[8], gpregs[9], gpregs[10], gpregs[11]); decompressor_printk(" %016lx %016lx %016lx %016lx\n", gpregs[12], gpregs[13], gpregs[14], gpregs[15]); - print_stacktrace(S390_lowcore.gpregs_save_area[15]); + print_stacktrace(get_lowcore()->gpregs_save_area[15]); decompressor_printk("Last Breaking-Event-Address:\n"); - decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)S390_lowcore.pgm_last_break, - (void *)S390_lowcore.pgm_last_break); + decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)get_lowcore()->pgm_last_break, + (void *)get_lowcore()->pgm_last_break); } diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 0cf79826eef9..4c9ad8258f7e 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -81,11 +81,11 @@ static int __diag260(unsigned long rx1, unsigned long rx2) [reg2] "=&a" (reg2), [rc] "+&d" (rc), [ry] "+&d" (ry), - "+Q" (S390_lowcore.program_new_psw), + "+Q" (get_lowcore()->program_new_psw), "=Q" (old) : [rx] "d" (rx.pair), [psw_old] "a" (&old), - [psw_pgm] "a" (&S390_lowcore.program_new_psw) + [psw_pgm] "a" (&get_lowcore()->program_new_psw) : "cc", "memory"); return rc == 0 ? ry : -1; } @@ -129,10 +129,10 @@ static int tprot(unsigned long addr) : [reg1] "=&d" (reg1), [reg2] "=&a" (reg2), [rc] "+&d" (rc), - "=Q" (S390_lowcore.program_new_psw.addr), + "=Q" (get_lowcore()->program_new_psw.addr), "=Q" (old) : [psw_old] "a" (&old), - [psw_pgm] "a" (&S390_lowcore.program_new_psw), + [psw_pgm] "a" (&get_lowcore()->program_new_psw), [addr] "a" (addr) : "cc", "memory"); return rc; diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 182aac6a0f77..4d04ab84c20e 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -78,10 +78,10 @@ static int cmma_test_essa(void) [reg2] "=&a" (reg2), [rc] "+&d" (rc), [tmp] "=&d" (tmp), - "+Q" (S390_lowcore.program_new_psw), + "+Q" (get_lowcore()->program_new_psw), "=Q" (old) : [psw_old] "a" (&old), - [psw_pgm] "a" (&S390_lowcore.program_new_psw), + [psw_pgm] "a" (&get_lowcore()->program_new_psw), [cmd] "i" (ESSA_GET_STATE) : "cc", "memory"); return rc; @@ -101,10 +101,10 @@ static void cmma_init(void) static void setup_lpp(void) { - S390_lowcore.current_pid = 0; - S390_lowcore.lpp = LPP_MAGIC; + get_lowcore()->current_pid = 0; + get_lowcore()->lpp = LPP_MAGIC; if (test_facility(40)) - lpp(&S390_lowcore.lpp); + lpp(&get_lowcore()->lpp); } #ifdef CONFIG_KERNEL_UNCOMPRESSED @@ -477,7 +477,7 @@ void startup_kernel(void) * Save KASLR offset for early dumps, before vmcore_info is set. * Mark as uneven to distinguish from real vmcore_info pointer. */ - S390_lowcore.vmcore_info = __kaslr_offset_phys ? __kaslr_offset_phys | 0x1UL : 0; + get_lowcore()->vmcore_info = __kaslr_offset_phys ? __kaslr_offset_phys | 0x1UL : 0; /* * Jump to the decompressed kernel entry point and switch DAT mode on. diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 96d48b7112d4..07d697eb279d 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -470,13 +470,13 @@ void setup_vmem(unsigned long kernel_start, unsigned long kernel_end, unsigned l kasan_populate_shadow(kernel_start, kernel_end); - S390_lowcore.kernel_asce.val = swapper_pg_dir | asce_bits; - S390_lowcore.user_asce = s390_invalid_asce; + get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits; + get_lowcore()->user_asce = s390_invalid_asce; - local_ctl_load(1, &S390_lowcore.kernel_asce); - local_ctl_load(7, &S390_lowcore.user_asce); - local_ctl_load(13, &S390_lowcore.kernel_asce); + local_ctl_load(1, &get_lowcore()->kernel_asce); + local_ctl_load(7, &get_lowcore()->user_asce); + local_ctl_load(13, &get_lowcore()->kernel_asce); - init_mm.context.asce = S390_lowcore.kernel_asce.val; + init_mm.context.asce = get_lowcore()->kernel_asce.val; init_mm.pgd = init_mm_pgd; } From d7c3ebc49eb72e9c8142fed49e9c9ec21815810c Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:28 +0200 Subject: [PATCH 131/280] s390/nmi: Remove duplicate get_lowcore() calls Assign the output from get_lowcore() to a local variable, so the code is easier to read. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/nmi.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index db562416d728..fbd218b6fc8e 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -117,6 +117,7 @@ static __always_inline char *u64_to_hex(char *dest, u64 val) static notrace void s390_handle_damage(void) { + struct lowcore *lc = get_lowcore(); union ctlreg0 cr0, cr0_new; char message[100]; psw_t psw_save; @@ -125,7 +126,7 @@ static notrace void s390_handle_damage(void) smp_emergency_stop(); diag_amode31_ops.diag308_reset(); ptr = nmi_puts(message, "System stopped due to unrecoverable machine check, code: 0x"); - u64_to_hex(ptr, get_lowcore()->mcck_interruption_code); + u64_to_hex(ptr, lc->mcck_interruption_code); /* * Disable low address protection and make machine check new PSW a @@ -135,17 +136,17 @@ static notrace void s390_handle_damage(void) cr0_new = cr0; cr0_new.lap = 0; local_ctl_load(0, &cr0_new.reg); - psw_save = get_lowcore()->mcck_new_psw; - psw_bits(get_lowcore()->mcck_new_psw).io = 0; - psw_bits(get_lowcore()->mcck_new_psw).ext = 0; - psw_bits(get_lowcore()->mcck_new_psw).wait = 1; + psw_save = lc->mcck_new_psw; + psw_bits(lc->mcck_new_psw).io = 0; + psw_bits(lc->mcck_new_psw).ext = 0; + psw_bits(lc->mcck_new_psw).wait = 1; sclp_emergency_printk(message); /* * Restore machine check new PSW and control register 0 to original * values. This makes possible system dump analysis easier. */ - get_lowcore()->mcck_new_psw = psw_save; + lc->mcck_new_psw = psw_save; local_ctl_load(0, &cr0.reg); disabled_wait(); while (1); @@ -301,6 +302,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; + struct lowcore *lc = get_lowcore(); struct mcck_struct *mcck; unsigned long long tmp; irqentry_state_t irq_state; @@ -313,7 +315,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (user_mode(regs)) update_timer_mcck(); inc_irq_stat(NMI_NMI); - mci.val = get_lowcore()->mcck_interruption_code; + mci.val = lc->mcck_interruption_code; mcck = this_cpu_ptr(&cpu_mcck); /* @@ -381,9 +383,9 @@ void notrace s390_do_machine_check(struct pt_regs *regs) } if (mci.ed && mci.ec) { /* External damage */ - if (get_lowcore()->external_damage_code & (1U << ED_STP_SYNC)) + if (lc->external_damage_code & (1U << ED_STP_SYNC)) mcck->stp_queue |= stp_sync_check(); - if (get_lowcore()->external_damage_code & (1U << ED_STP_ISLAND)) + if (lc->external_damage_code & (1U << ED_STP_ISLAND)) mcck->stp_queue |= stp_island_check(); mcck_pending = 1; } From eb28ec2b2e55c58148fa07eb994af11e6099e748 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:29 +0200 Subject: [PATCH 132/280] s390/smp: Remove duplicate get_lowcore() calls Assign the output from get_lowcore() to a local variable, so the code is easier to read. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/smp.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index ebe4bc326a6b..4c2faab56eb6 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -842,15 +842,16 @@ void __init smp_detect_cpus(void) */ static void smp_start_secondary(void *cpuvoid) { + struct lowcore *lc = get_lowcore(); int cpu = raw_smp_processor_id(); - get_lowcore()->last_update_clock = get_tod_clock(); - get_lowcore()->restart_stack = (unsigned long)restart_stack; - get_lowcore()->restart_fn = (unsigned long)do_restart; - get_lowcore()->restart_data = 0; - get_lowcore()->restart_source = -1U; - get_lowcore()->restart_flags = 0; - restore_access_regs(get_lowcore()->access_regs_save_area); + lc->last_update_clock = get_tod_clock(); + lc->restart_stack = (unsigned long)restart_stack; + lc->restart_fn = (unsigned long)do_restart; + lc->restart_data = 0; + lc->restart_source = -1U; + lc->restart_flags = 0; + restore_access_regs(lc->access_regs_save_area); cpu_init(); rcutree_report_cpu_starting(cpu); init_cpu_timer(); @@ -987,10 +988,12 @@ void __init smp_prepare_boot_cpu(void) void __init smp_setup_processor_id(void) { + struct lowcore *lc = get_lowcore(); + pcpu_devices[0].address = stap(); - get_lowcore()->cpu_nr = 0; - get_lowcore()->spinlock_lockval = arch_spin_lockval(0); - get_lowcore()->spinlock_index = 0; + lc->cpu_nr = 0; + lc->spinlock_lockval = arch_spin_lockval(0); + lc->spinlock_index = 0; } /* From 46c3031108e83a63007cf2f4c9a379875825e972 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:30 +0200 Subject: [PATCH 133/280] s390/vtime: Remove duplicate get_lowcore() calls Assign the output from get_lowcore() to a local variable, so the code is easier to read. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/vtime.h | 16 ++++--- arch/s390/kernel/vtime.c | 78 +++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/arch/s390/include/asm/vtime.h b/arch/s390/include/asm/vtime.h index ef4dd7d057a2..9d25fb35a042 100644 --- a/arch/s390/include/asm/vtime.h +++ b/arch/s390/include/asm/vtime.h @@ -4,16 +4,20 @@ static inline void update_timer_sys(void) { - get_lowcore()->system_timer += get_lowcore()->last_update_timer - get_lowcore()->exit_timer; - get_lowcore()->user_timer += get_lowcore()->exit_timer - get_lowcore()->sys_enter_timer; - get_lowcore()->last_update_timer = get_lowcore()->sys_enter_timer; + struct lowcore *lc = get_lowcore(); + + lc->system_timer += lc->last_update_timer - lc->exit_timer; + lc->user_timer += lc->exit_timer - lc->sys_enter_timer; + lc->last_update_timer = lc->sys_enter_timer; } static inline void update_timer_mcck(void) { - get_lowcore()->system_timer += get_lowcore()->last_update_timer - get_lowcore()->exit_timer; - get_lowcore()->user_timer += get_lowcore()->exit_timer - get_lowcore()->mcck_enter_timer; - get_lowcore()->last_update_timer = get_lowcore()->mcck_enter_timer; + struct lowcore *lc = get_lowcore(); + + lc->system_timer += lc->last_update_timer - lc->exit_timer; + lc->user_timer += lc->exit_timer - lc->mcck_enter_timer; + lc->last_update_timer = lc->mcck_enter_timer; } #endif /* _S390_VTIME_H */ diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 7d8991c3cd3a..234a0ba30510 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -35,14 +35,15 @@ static DEFINE_PER_CPU(u64, mt_scaling_jiffies); static inline void set_vtimer(u64 expires) { + struct lowcore *lc = get_lowcore(); u64 timer; asm volatile( " stpt %0\n" /* Store current cpu timer value */ " spt %1" /* Set new value imm. afterwards */ : "=Q" (timer) : "Q" (expires)); - get_lowcore()->system_timer += get_lowcore()->last_update_timer - timer; - get_lowcore()->last_update_timer = expires; + lc->system_timer += lc->last_update_timer - timer; + lc->last_update_timer = expires; } static inline int virt_timer_forward(u64 elapsed) @@ -117,22 +118,23 @@ static void account_system_index_scaled(struct task_struct *p, u64 cputime, static int do_account_vtime(struct task_struct *tsk) { u64 timer, clock, user, guest, system, hardirq, softirq; + struct lowcore *lc = get_lowcore(); - timer = get_lowcore()->last_update_timer; - clock = get_lowcore()->last_update_clock; + timer = lc->last_update_timer; + clock = lc->last_update_clock; asm volatile( " stpt %0\n" /* Store current cpu timer value */ " stckf %1" /* Store current tod clock value */ - : "=Q" (get_lowcore()->last_update_timer), - "=Q" (get_lowcore()->last_update_clock) + : "=Q" (lc->last_update_timer), + "=Q" (lc->last_update_clock) : : "cc"); - clock = get_lowcore()->last_update_clock - clock; - timer -= get_lowcore()->last_update_timer; + clock = lc->last_update_clock - clock; + timer -= lc->last_update_timer; if (hardirq_count()) - get_lowcore()->hardirq_timer += timer; + lc->hardirq_timer += timer; else - get_lowcore()->system_timer += timer; + lc->system_timer += timer; /* Update MT utilization calculation */ if (smp_cpu_mtid && @@ -141,16 +143,16 @@ static int do_account_vtime(struct task_struct *tsk) /* Calculate cputime delta */ user = update_tsk_timer(&tsk->thread.user_timer, - READ_ONCE(get_lowcore()->user_timer)); + READ_ONCE(lc->user_timer)); guest = update_tsk_timer(&tsk->thread.guest_timer, - READ_ONCE(get_lowcore()->guest_timer)); + READ_ONCE(lc->guest_timer)); system = update_tsk_timer(&tsk->thread.system_timer, - READ_ONCE(get_lowcore()->system_timer)); + READ_ONCE(lc->system_timer)); hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, - READ_ONCE(get_lowcore()->hardirq_timer)); + READ_ONCE(lc->hardirq_timer)); softirq = update_tsk_timer(&tsk->thread.softirq_timer, - READ_ONCE(get_lowcore()->softirq_timer)); - get_lowcore()->steal_timer += + READ_ONCE(lc->softirq_timer)); + lc->steal_timer += clock - user - guest - system - hardirq - softirq; /* Push account value */ @@ -176,17 +178,19 @@ static int do_account_vtime(struct task_struct *tsk) void vtime_task_switch(struct task_struct *prev) { + struct lowcore *lc = get_lowcore(); + do_account_vtime(prev); - prev->thread.user_timer = get_lowcore()->user_timer; - prev->thread.guest_timer = get_lowcore()->guest_timer; - prev->thread.system_timer = get_lowcore()->system_timer; - prev->thread.hardirq_timer = get_lowcore()->hardirq_timer; - prev->thread.softirq_timer = get_lowcore()->softirq_timer; - get_lowcore()->user_timer = current->thread.user_timer; - get_lowcore()->guest_timer = current->thread.guest_timer; - get_lowcore()->system_timer = current->thread.system_timer; - get_lowcore()->hardirq_timer = current->thread.hardirq_timer; - get_lowcore()->softirq_timer = current->thread.softirq_timer; + prev->thread.user_timer = lc->user_timer; + prev->thread.guest_timer = lc->guest_timer; + prev->thread.system_timer = lc->system_timer; + prev->thread.hardirq_timer = lc->hardirq_timer; + prev->thread.softirq_timer = lc->softirq_timer; + lc->user_timer = current->thread.user_timer; + lc->guest_timer = current->thread.guest_timer; + lc->system_timer = current->thread.system_timer; + lc->hardirq_timer = current->thread.hardirq_timer; + lc->softirq_timer = current->thread.softirq_timer; } /* @@ -196,28 +200,29 @@ void vtime_task_switch(struct task_struct *prev) */ void vtime_flush(struct task_struct *tsk) { + struct lowcore *lc = get_lowcore(); u64 steal, avg_steal; if (do_account_vtime(tsk)) virt_timer_expire(); - steal = get_lowcore()->steal_timer; - avg_steal = get_lowcore()->avg_steal_timer; + steal = lc->steal_timer; + avg_steal = lc->avg_steal_timer; if ((s64) steal > 0) { - get_lowcore()->steal_timer = 0; + lc->steal_timer = 0; account_steal_time(cputime_to_nsecs(steal)); avg_steal += steal; } - get_lowcore()->avg_steal_timer = avg_steal / 2; + lc->avg_steal_timer = avg_steal / 2; } static u64 vtime_delta(void) { - u64 timer = get_lowcore()->last_update_timer; + struct lowcore *lc = get_lowcore(); + u64 timer = lc->last_update_timer; - get_lowcore()->last_update_timer = get_cpu_timer(); - - return timer - get_lowcore()->last_update_timer; + lc->last_update_timer = get_cpu_timer(); + return timer - lc->last_update_timer; } /* @@ -226,12 +231,13 @@ static u64 vtime_delta(void) */ void vtime_account_kernel(struct task_struct *tsk) { + struct lowcore *lc = get_lowcore(); u64 delta = vtime_delta(); if (tsk->flags & PF_VCPU) - get_lowcore()->guest_timer += delta; + lc->guest_timer += delta; else - get_lowcore()->system_timer += delta; + lc->system_timer += delta; virt_timer_forward(delta); } From 15428734e1da88c40924e6788403171c8dff7cf8 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:31 +0200 Subject: [PATCH 134/280] s390/idle: Remove duplicate get_lowcore() calls Assign the output from get_lowcore() to a local variable, so the code is easier to read. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/idle.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 2c34e02ae64b..39cb8d0ae348 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -24,6 +24,7 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle); void account_idle_time_irq(void) { struct s390_idle_data *idle = this_cpu_ptr(&s390_idle); + struct lowcore *lc = get_lowcore(); unsigned long idle_time; u64 cycles_new[8]; int i; @@ -34,13 +35,13 @@ void account_idle_time_irq(void) this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]); } - idle_time = get_lowcore()->int_clock - idle->clock_idle_enter; + idle_time = lc->int_clock - idle->clock_idle_enter; - get_lowcore()->steal_timer += idle->clock_idle_enter - get_lowcore()->last_update_clock; - get_lowcore()->last_update_clock = get_lowcore()->int_clock; + lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock; + lc->last_update_clock = lc->int_clock; - get_lowcore()->system_timer += get_lowcore()->last_update_timer - idle->timer_idle_enter; - get_lowcore()->last_update_timer = get_lowcore()->sys_enter_timer; + lc->system_timer += lc->last_update_timer - idle->timer_idle_enter; + lc->last_update_timer = lc->sys_enter_timer; /* Account time spent with enabled wait psw loaded as idle time. */ WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time); From 81f907b246f35b4cc5cb38b86b633601b76d17e8 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:32 +0200 Subject: [PATCH 135/280] s390/mm: Remove duplicate get_lowcore() calls Assign the output from get_lowcore() to a local variable, so the code is easier to read. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/traps.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 8b904f7efb0e..a7c211a3a0c9 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -288,15 +288,16 @@ static void __init test_monitor_call(void) void __init trap_init(void) { + struct lowcore *lc = get_lowcore(); unsigned long flags; struct ctlreg cr0; local_irq_save(flags); cr0 = local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); - psw_bits(get_lowcore()->external_new_psw).mcheck = 1; - psw_bits(get_lowcore()->program_new_psw).mcheck = 1; - psw_bits(get_lowcore()->svc_new_psw).mcheck = 1; - psw_bits(get_lowcore()->io_new_psw).mcheck = 1; + psw_bits(lc->external_new_psw).mcheck = 1; + psw_bits(lc->program_new_psw).mcheck = 1; + psw_bits(lc->svc_new_psw).mcheck = 1; + psw_bits(lc->io_new_psw).mcheck = 1; local_ctl_load(0, &cr0); local_irq_restore(flags); local_mcck_enable(); @@ -307,11 +308,12 @@ static void (*pgm_check_table[128])(struct pt_regs *regs); void noinstr __do_pgm_check(struct pt_regs *regs) { - unsigned int trapnr; + struct lowcore *lc = get_lowcore(); irqentry_state_t state; + unsigned int trapnr; - regs->int_code = get_lowcore()->pgm_int_code; - regs->int_parm_long = get_lowcore()->trans_exc_code; + regs->int_code = lc->pgm_int_code; + regs->int_parm_long = lc->trans_exc_code; state = irqentry_enter(regs); @@ -324,19 +326,19 @@ void noinstr __do_pgm_check(struct pt_regs *regs) current->thread.last_break = regs->last_break; } - if (get_lowcore()->pgm_code & 0x0200) { + if (lc->pgm_code & 0x0200) { /* transaction abort */ - current->thread.trap_tdb = get_lowcore()->pgm_tdb; + current->thread.trap_tdb = lc->pgm_tdb; } - if (get_lowcore()->pgm_code & PGM_INT_CODE_PER) { + if (lc->pgm_code & PGM_INT_CODE_PER) { if (user_mode(regs)) { struct per_event *ev = ¤t->thread.per_event; set_thread_flag(TIF_PER_TRAP); - ev->address = get_lowcore()->per_address; - ev->cause = get_lowcore()->per_code_combined; - ev->paid = get_lowcore()->per_access_id; + ev->address = lc->per_address; + ev->cause = lc->per_code_combined; + ev->paid = lc->per_access_id; } else { /* PER event in kernel is kprobes */ __arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER); From 39976f1278a97d79e758ceb59fe613764844c9c3 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Jun 2024 13:45:33 +0200 Subject: [PATCH 136/280] s390: Remove S390_lowcore With all users gone, remove S390_lowcore. Acked-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/lowcore.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 94f954250ae0..c724e71e1785 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -213,8 +213,6 @@ struct lowcore { __u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */ } __packed __aligned(8192); -#define S390_lowcore (*((struct lowcore *) 0)) - static __always_inline struct lowcore *get_lowcore(void) { return NULL; From 447360d75cf473d4b1048a7204443262f3f60758 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Mon, 10 Jun 2024 17:10:48 +0200 Subject: [PATCH 137/280] s390/sclp: Define commands for storage (un)assignment Replace immediate values with SCLP_CMDW_UN|ASSIGN_STORAGE defines. Acked-by: Heiko Carstens Link: https://lore.kernel.org/r/20240610151048.2548428-1-agordeev@linux.ibm.com Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp_cmd.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 7815e9bea69a..f905a6643a0f 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -31,6 +31,9 @@ #include "sclp.h" +#define SCLP_CMDW_ASSIGN_STORAGE 0x000d0001 +#define SCLP_CMDW_UNASSIGN_STORAGE 0x000c0001 + static void sclp_sync_callback(struct sclp_req *req, void *data) { struct completion *completion = data; @@ -225,7 +228,7 @@ static int sclp_assign_storage(u16 rn) unsigned long long start; int rc; - rc = do_assign_storage(0x000d0001, rn); + rc = do_assign_storage(SCLP_CMDW_ASSIGN_STORAGE, rn); if (rc) return rc; start = rn2addr(rn); @@ -235,7 +238,7 @@ static int sclp_assign_storage(u16 rn) static int sclp_unassign_storage(u16 rn) { - return do_assign_storage(0x000c0001, rn); + return do_assign_storage(SCLP_CMDW_UNASSIGN_STORAGE, rn); } struct attach_storage_sccb { From 1e4c64b71c9bf230b25fde12cbcceacfdc8b3332 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 13 Jun 2024 11:55:07 -0400 Subject: [PATCH 138/280] mm/memblock: Add "reserve_mem" to reserved named memory at boot up In order to allow for requesting a memory region that can be used for things like pstore on multiple machines where the memory layout is not the same, add a new option to the kernel command line called "reserve_mem". The format is: reserve_mem=nn:align:name Where it will find nn amount of memory at the given alignment of align. The name field is to allow another subsystem to retrieve where the memory was found. For example: reserve_mem=12M:4096:oops ramoops.mem_name=oops Where ramoops.mem_name will tell ramoops that memory was reserved for it via the reserve_mem option and it can find it by calling: if (reserve_mem_find_by_name("oops", &start, &size)) { // start holds the start address and size holds the size given This is typically used for systems that do not wipe the RAM, and this command line will try to reserve the same physical memory on soft reboots. Note, it is not guaranteed to be the same location. For example, if KASLR places the kernel at the location of where the RAM reservation was from a previous boot, the new reservation will be at a different location. Any subsystem using this feature must add a way to verify that the contents of the physical memory is from a previous boot, as there may be cases where the memory will not be located at the same location. Not all systems may work either. There could be bit flips if the reboot goes through the BIOS. Using kexec to reboot the machine is likely to have better results in such cases. Link: https://lore.kernel.org/all/ZjJVnZUX3NZiGW6q@kernel.org/ Suggested-by: Mike Rapoport Tested-by: Guilherme G. Piccoli Signed-off-by: Steven Rostedt (Google) Link: https://lore.kernel.org/r/20240613155527.437020271@goodmis.org Signed-off-by: Mike Rapoport (IBM) --- .../admin-guide/kernel-parameters.txt | 22 ++++ include/linux/mm.h | 2 + mm/memblock.c | 117 ++++++++++++++++++ 3 files changed, 141 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b600df82669d..56e18b1a520d 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5710,6 +5710,28 @@ them. If is less than 0x10000, the region is assumed to be I/O ports; otherwise it is memory. + reserve_mem= [RAM] + Format: nn[KNG]::