diff --git a/Documentation/arch/x86/xstate.rst b/Documentation/arch/x86/xstate.rst index 5cec7fb558d6..ae5c69e48b11 100644 --- a/Documentation/arch/x86/xstate.rst +++ b/Documentation/arch/x86/xstate.rst @@ -11,6 +11,22 @@ are enabled by XCR0 as well, but the first use of related instruction is trapped by the kernel because by default the required large XSTATE buffers are not allocated automatically. +The purpose for dynamic features +-------------------------------- + +Legacy userspace libraries often have hard-coded, static sizes for +alternate signal stacks, often using MINSIGSTKSZ which is typically 2KB. +That stack must be able to store at *least* the signal frame that the +kernel sets up before jumping into the signal handler. That signal frame +must include an XSAVE buffer defined by the CPU. + +However, that means that the size of signal stacks is dynamic, not static, +because different CPUs have differently-sized XSAVE buffers. A compiled-in +size of 2KB with existing applications is too small for new CPU features +like AMX. Instead of universally requiring larger stack, with the dynamic +enabling, the kernel can enforce userspace applications to have +properly-sized altstacks. + Using dynamically enabled XSTATE features in user space applications -------------------------------------------------------------------- @@ -64,6 +80,61 @@ the handler allocates a larger xstate buffer for the task so the large state can be context switched. In the unlikely cases that the allocation fails, the kernel sends SIGSEGV. +AMX TILE_DATA enabling example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Below is the example of how userspace applications enable +TILE_DATA dynamically: + + 1. The application first needs to query the kernel for AMX + support:: + + #include + #include + #include + #include + + #ifndef ARCH_GET_XCOMP_SUPP + #define ARCH_GET_XCOMP_SUPP 0x1021 + #endif + + #ifndef ARCH_XCOMP_TILECFG + #define ARCH_XCOMP_TILECFG 17 + #endif + + #ifndef ARCH_XCOMP_TILEDATA + #define ARCH_XCOMP_TILEDATA 18 + #endif + + #define MASK_XCOMP_TILE ((1 << ARCH_XCOMP_TILECFG) | \ + (1 << ARCH_XCOMP_TILEDATA)) + + unsigned long features; + long rc; + + ... + + rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_SUPP, &features); + + if (!rc && (features & MASK_XCOMP_TILE) == MASK_XCOMP_TILE) + printf("AMX is available.\n"); + + 2. After that, determining support for AMX, an application must + explicitly ask permission to use it:: + + #ifndef ARCH_REQ_XCOMP_PERM + #define ARCH_REQ_XCOMP_PERM 0x1023 + #endif + + ... + + rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, ARCH_XCOMP_TILEDATA); + + if (!rc) + printf("AMX is ready for use.\n"); + +Note this example does not include the sigaltstack preparation. + Dynamic features in signal frames --------------------------------- @@ -72,3 +143,32 @@ entry if the feature is in its initial configuration. This differs from non-dynamic features which are always written regardless of their configuration. Signal handlers can examine the XSAVE buffer's XSTATE_BV field to determine if a features was written. + +Dynamic features for virtual machines +------------------------------------- + +The permission for the guest state component needs to be managed separately +from the host, as they are exclusive to each other. A coupled of options +are extended to control the guest permission: + +-ARCH_GET_XCOMP_GUEST_PERM + + arch_prctl(ARCH_GET_XCOMP_GUEST_PERM, &features); + + ARCH_GET_XCOMP_GUEST_PERM is a variant of ARCH_GET_XCOMP_PERM. So it + provides the same semantics and functionality but for the guest + components. + +-ARCH_REQ_XCOMP_GUEST_PERM + + arch_prctl(ARCH_REQ_XCOMP_GUEST_PERM, feature_nr); + + ARCH_REQ_XCOMP_GUEST_PERM is a variant of ARCH_REQ_XCOMP_PERM. It has the + same semantics for the guest permission. While providing a similar + functionality, this comes with a constraint. Permission is frozen when the + first VCPU is created. Any attempt to change permission after that point + is going to be rejected. So, the permission has to be requested before the + first VCPU creation. + +Note that some VMMs may have already established a set of supported state +components. These options are not presumed to support any particular VMM. diff --git a/Documentation/livepatch/reliable-stacktrace.rst b/Documentation/livepatch/reliable-stacktrace.rst index 67459d2ca2af..d56bb706172f 100644 --- a/Documentation/livepatch/reliable-stacktrace.rst +++ b/Documentation/livepatch/reliable-stacktrace.rst @@ -183,7 +183,7 @@ trampoline or return trampoline. For example, considering the x86_64 .. code-block:: none SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED subq $24, %rsp /* Save the return values */ diff --git a/MAINTAINERS b/MAINTAINERS index 1327acdba6f9..5a979d19d294 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15187,8 +15187,8 @@ OBJTOOL M: Josh Poimboeuf M: Peter Zijlstra S: Supported +F: include/linux/objtool*.h F: tools/objtool/ -F: include/linux/objtool.h OCELOT ETHERNET SWITCH DRIVER M: Vladimir Oltean diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index e9cf7193eb81..582d96548385 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -9,6 +9,7 @@ * This file handles the architecture-dependent parts of process handling. */ +#include #include #include #include @@ -59,9 +60,10 @@ void arch_cpu_idle(void) wtint(0); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { wtint(INT_MAX); + BUG(); } #endif /* ALPHA_WTINT */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index b615cbc60af1..87111616ba77 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -324,7 +324,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); @@ -386,6 +386,8 @@ void arch_cpu_idle_dead(void) : "r" (task_stack_page(current) + THREAD_SIZE - 8), "r" (current) : "r0"); + + unreachable(); } #endif /* CONFIG_HOTPLUG_CPU */ @@ -781,7 +783,7 @@ void smp_send_stop(void) * kdump fails. So split out the panic_smp_self_stop() and add * set_cpu_online(smp_processor_id(), false). */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n", smp_processor_id()); diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 92963f98afec..e73af709cb7a 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -31,7 +31,7 @@ static inline unsigned long disr_to_esr(u64 disr) return esr; } -asmlinkage void handle_bad_stack(struct pt_regs *regs); +asmlinkage void __noreturn handle_bad_stack(struct pt_regs *regs); asmlinkage void el1t_64_sync_handler(struct pt_regs *regs); asmlinkage void el1t_64_irq_handler(struct pt_regs *regs); @@ -80,5 +80,5 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr); void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); -void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); +void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far); #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 72dbd6400549..56911691bef0 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -288,6 +288,12 @@ void post_ttbr_update_workaround(void); unsigned long arm64_mm_context_get(struct mm_struct *mm); void arm64_mm_context_put(struct mm_struct *mm); +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> 8; +} + #include #endif /* !__ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index b93637756f91..8e29f46b6288 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -102,10 +102,10 @@ static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask) extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); -extern void cpu_die(void); -extern void cpu_die_early(void); +extern void __noreturn cpu_die(void); +extern void __noreturn cpu_die_early(void); -static inline void cpu_park_loop(void) +static inline void __noreturn cpu_park_loop(void) { for (;;) { wfe(); @@ -125,7 +125,7 @@ static inline void update_cpu_boot_status(int val) * which calls for a kernel panic. Update the boot status and park the calling * CPU. */ -static inline void cpu_panic_kernel(void) +static inline void __noreturn cpu_panic_kernel(void) { update_cpu_boot_status(CPU_PANIC_KERNEL); cpu_park_loop(); @@ -145,7 +145,6 @@ bool cpus_are_stuck_in_kernel(void); extern void crash_smp_send_stop(void); extern bool smp_crash_stop_failed(void); -extern void panic_smp_self_stop(void); #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index cce1167199e3..3af3c01c93a6 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -840,7 +840,7 @@ UNHANDLED(el0t, 32, error) #endif /* CONFIG_COMPAT */ #ifdef CONFIG_VMAP_STACK -asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) +asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) { unsigned long esr = read_sysreg(esr_el1); unsigned long far = read_sysreg(far_el1); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index e73c5dcb4e49..33e510d192f4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -70,7 +70,7 @@ void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index f8bebad61101..bb85975c2d64 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -366,7 +366,7 @@ void __cpu_die(unsigned int cpu) * Called from the idle thread for the CPU which has been shutdown. * */ -void cpu_die(void) +void __noreturn cpu_die(void) { unsigned int cpu = smp_processor_id(); const struct cpu_operations *ops = get_cpu_ops(cpu); @@ -403,7 +403,7 @@ static void __cpu_try_die(int cpu) * Kill the calling secondary CPU, early in bringup before it is turned * online. */ -void cpu_die_early(void) +void __noreturn cpu_die_early(void) { int cpu = smp_processor_id(); @@ -821,7 +821,7 @@ void arch_irq_work_raise(void) } #endif -static void local_cpu_stop(void) +static void __noreturn local_cpu_stop(void) { set_cpu_online(smp_processor_id(), false); @@ -835,7 +835,7 @@ static void local_cpu_stop(void) * that cpu_online_mask gets correctly updated and smp_send_stop() can skip * CPUs that have already stopped themselves. */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { local_cpu_stop(); } @@ -844,7 +844,7 @@ void panic_smp_self_stop(void) static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0); #endif -static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) +static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) { #ifdef CONFIG_KEXEC_CORE crash_save_cpu(regs, cpu); @@ -859,6 +859,8 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) /* just in case */ cpu_park_loop(); +#else + BUG(); #endif } diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 4a79ba100799..4bb1b8f47298 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -863,7 +863,7 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr) DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); -void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far) +void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far) { unsigned long tsk_stk = (unsigned long)current->stack; unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); @@ -905,7 +905,6 @@ void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr) nmi_panic(regs, "Asynchronous SError Interrupt"); cpu_park_loop(); - unreachable(); } bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr) diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index b45d1073307f..9c7a20b73ac6 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -300,7 +300,7 @@ void __cpu_die(unsigned int cpu) pr_notice("CPU%u: shutdown\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); @@ -317,5 +317,7 @@ void arch_cpu_idle_dead(void) "jmpi csky_start_secondary" : : "r" (secondary_stack)); + + BUG(); } #endif diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index f6195a0a00ae..9a5cd9fad3a9 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -201,7 +201,7 @@ __setup("nohalt", nohalt_setup); #ifdef CONFIG_HOTPLUG_CPU /* We don't actually take CPU down, just spin without interrupts. */ -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { unsigned int this_cpu = smp_processor_id(); @@ -219,13 +219,13 @@ static inline void play_dead(void) BUG(); } #else -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h index d82687390b4a..416b653bccb4 100644 --- a/arch/loongarch/include/asm/smp.h +++ b/arch/loongarch/include/asm/smp.h @@ -99,7 +99,7 @@ static inline void __cpu_die(unsigned int cpu) loongson_cpu_die(cpu); } -extern void play_dead(void); +extern void __noreturn play_dead(void); #endif #endif /* __ASM_SMP_H */ diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index fa2443c7afb2..b71e17c1cc0c 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -62,7 +62,7 @@ unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; EXPORT_SYMBOL(boot_option_idle_override); #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 8c6e227cb29d..51f328169a7b 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -336,7 +336,7 @@ void play_dead(void) iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); init_fn(); - unreachable(); + BUG(); } #endif diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c index 89954f5f87fb..4212584e6efa 100644 --- a/arch/mips/cavium-octeon/smp.c +++ b/arch/mips/cavium-octeon/smp.c @@ -20,6 +20,7 @@ #include #include #include +#include #include diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 5d9ff61004ca..4eee29b7845c 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -88,7 +88,7 @@ static inline void __cpu_die(unsigned int cpu) mp_ops->cpu_die(cpu); } -extern void play_dead(void); +extern void __noreturn play_dead(void); #endif #ifdef CONFIG_KEXEC diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 093dbbd6b843..a3225912c862 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -40,7 +40,7 @@ #include #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index f5d7bfa3472a..15466d4cf4a0 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -54,6 +54,8 @@ static void bmips_set_reset_vec(int cpu, u32 val); #ifdef CONFIG_SMP +#include + /* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ unsigned long bmips_smp_boot_sp; unsigned long bmips_smp_boot_gp; @@ -413,6 +415,8 @@ void __ref play_dead(void) " wait\n" " j bmips_secondary_reentry\n" : : : "memory"); + + BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c index f968a319d87f..62f677b2306f 100644 --- a/arch/mips/kernel/smp-cps.c +++ b/arch/mips/kernel/smp-cps.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index 90c783981197..b0e8bb9fa036 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -808,6 +809,7 @@ out: state_addr = &per_cpu(cpu_state, cpu); mb(); play_dead_at_ckseg1(state_addr); + BUG(); } static int loongson3_disable_clock(unsigned int cpu) diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index c064719b49b0..97c6f875bd0e 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -159,7 +159,7 @@ EXPORT_SYMBOL(running_on_qemu); /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { #ifdef CONFIG_HOTPLUG_CPU idle_task_exit(); diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index 6c6cb53d7045..aaaa576d0e15 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -67,7 +67,7 @@ void start_secondary(void *unused); extern int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern void smp_send_debugger_break(void); -extern void start_secondary_resume(void); +extern void __noreturn start_secondary_resume(void); extern void smp_generic_give_timebase(void); extern void smp_generic_take_timebase(void); diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index b2e0d3ce4261..246201d0d879 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -480,7 +480,7 @@ void early_setup_secondary(void) #endif /* CONFIG_SMP */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { hard_irq_disable(); spin_begin(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 7db6b3faea65..bf9744034c3a 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1752,7 +1752,7 @@ void __cpu_die(unsigned int cpu) smp_ops->cpu_die(cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { /* * Disable on the down path. This will be re-enabled by diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c index 39235cf50652..a941adc7cbf2 100644 --- a/arch/riscv/kernel/cpu-hotplug.c +++ b/arch/riscv/kernel/cpu-hotplug.c @@ -72,7 +72,7 @@ void __cpu_die(unsigned int cpu) /* * Called from the idle thread for the CPU which has been shutdown. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { idle_task_exit(); diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c index 38e267c7bff7..e7239aaf428b 100644 --- a/arch/s390/kernel/idle.c +++ b/arch/s390/kernel/idle.c @@ -88,7 +88,7 @@ void arch_cpu_idle_exit(void) { } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 8ec5cdf9dadc..4259b6c50516 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -396,7 +396,7 @@ int __init arch_early_irq_init(void) return 0; } -void __init arch_call_rest_init(void) +void __init __noreturn arch_call_rest_init(void) { unsigned long stack; diff --git a/arch/sh/include/asm/smp-ops.h b/arch/sh/include/asm/smp-ops.h index e27702130eb6..97331fcb7b85 100644 --- a/arch/sh/include/asm/smp-ops.h +++ b/arch/sh/include/asm/smp-ops.h @@ -24,9 +24,10 @@ static inline void plat_smp_setup(void) mp_ops->smp_setup(); } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { mp_ops->play_dead(); + BUG(); } extern void register_smp_ops(struct plat_smp_ops *ops); @@ -42,7 +43,7 @@ static inline void register_smp_ops(struct plat_smp_ops *ops) { } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 3418c40f0099..d662503b0665 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -4,6 +4,7 @@ * * Copyright (C) 2002 - 2009 Paul Mundt */ +#include #include #include #include @@ -29,7 +30,7 @@ void default_idle(void) clear_bl_bit(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index 7a8380c63aab..799e797c5cdd 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -185,6 +185,12 @@ static inline void finish_arch_post_lock_switch(void) } } +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> adi_nbits(); +} + #include #endif /* !(__ASSEMBLY__) */ diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index e75783b6abc4..505b6700805d 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -49,7 +49,7 @@ int hard_smp_processor_id(void); void smp_fill_in_cpu_possible_map(void); void smp_fill_in_sib_core_maps(void); -void cpu_play_dead(void); +void __noreturn cpu_play_dead(void); void smp_fetch_global_regs(void); void smp_fetch_global_pmu(void); diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 94266a5c5b04..b825a5dd0210 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -8,8 +8,10 @@ #include #include +#include #include #include +#include #include #include diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 91c2b8124527..b51d8fb0ecdc 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -95,7 +95,7 @@ void arch_cpu_idle(void) } #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { sched_preempt_enable_no_resched(); cpu_play_dead(); diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f87590c3c382..53bab123a8ee 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2290,6 +2290,17 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING If unsure, leave at the default value. +config ADDRESS_MASKING + bool "Linear Address Masking support" + depends on X86_64 + help + Linear Address Masking (LAM) modifies the checking that is applied + to 64-bit linear addresses, allowing software to use of the + untranslated address bits for metadata. + + The capability can be used for efficient address sanitizers (ASAN) + implementation and for optimizations in JITs. + config HOTPLUG_CPU def_bool y depends on SMP diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index 321a5011042d..bcc956c17872 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -8,14 +8,6 @@ * Copyright (C) 2016 Kees Cook */ -/* - * Since we're dealing with identity mappings, physical and virtual - * addresses are the same, so override these defines which are ultimately - * used by the headers in misc.h. - */ -#define __pa(x) ((unsigned long)(x)) -#define __va(x) ((void *)((unsigned long)(x))) - /* No PAGE_TABLE_ISOLATION support needed either: */ #undef CONFIG_PAGE_TABLE_ISOLATION diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 20118fb7c53b..2f155a0e3041 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -19,6 +19,15 @@ /* cpu_feature_enabled() cannot be used this early */ #define USE_EARLY_PGTABLE_L5 +/* + * Boot stub deals with identity mappings, physical and virtual addresses are + * the same, so override these defines. + * + * will not define them if they are already defined. + */ +#define __pa(x) ((unsigned long)(x)) +#define __va(x) ((void *)((unsigned long)(x))) + #include #include #include diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index d63ad8f99f83..014b89c89088 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -104,9 +104,7 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt, } #undef __init -#undef __pa #define __init -#define __pa(x) ((unsigned long)(x)) #define __BOOT_COMPRESSED diff --git a/arch/x86/boot/compressed/tdx.c b/arch/x86/boot/compressed/tdx.c index 918a7606f53c..2d81d3cc72a1 100644 --- a/arch/x86/boot/compressed/tdx.c +++ b/arch/x86/boot/compressed/tdx.c @@ -26,7 +26,7 @@ static inline unsigned int tdx_io_in(int size, u16 port) .r14 = port, }; - if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT)) + if (__tdx_hypercall_ret(&args)) return UINT_MAX; return args.r11; @@ -43,7 +43,7 @@ static inline void tdx_io_out(int size, u16 port, u32 value) .r15 = value, }; - __tdx_hypercall(&args, 0); + __tdx_hypercall(&args); } static inline u8 tdx_inb(u16 port) diff --git a/arch/x86/coco/tdx/tdcall.S b/arch/x86/coco/tdx/tdcall.S index 6a255e6809bc..b193c0a1d8db 100644 --- a/arch/x86/coco/tdx/tdcall.S +++ b/arch/x86/coco/tdx/tdcall.S @@ -85,12 +85,12 @@ SYM_FUNC_START(__tdx_module_call) SYM_FUNC_END(__tdx_module_call) /* - * __tdx_hypercall() - Make hypercalls to a TDX VMM using TDVMCALL leaf - * of TDCALL instruction + * TDX_HYPERCALL - Make hypercalls to a TDX VMM using TDVMCALL leaf of TDCALL + * instruction * * Transforms values in function call argument struct tdx_hypercall_args @args * into the TDCALL register ABI. After TDCALL operation, VMM output is saved - * back in @args. + * back in @args, if \ret is 1. * *------------------------------------------------------------------------- * TD VMCALL ABI: @@ -105,26 +105,18 @@ SYM_FUNC_END(__tdx_module_call) * specification. Non zero value indicates vendor * specific ABI. * R11 - VMCALL sub function number - * RBX, RBP, RDI, RSI - Used to pass VMCALL sub function specific arguments. + * RBX, RDX, RDI, RSI - Used to pass VMCALL sub function specific arguments. * R8-R9, R12-R15 - Same as above. * * Output Registers: * * RAX - TDCALL instruction status (Not related to hypercall * output). - * R10 - Hypercall output error code. - * R11-R15 - Hypercall sub function specific output values. + * RBX, RDX, RDI, RSI - Hypercall sub function specific output values. + * R8-R15 - Same as above. * - *------------------------------------------------------------------------- - * - * __tdx_hypercall() function ABI: - * - * @args (RDI) - struct tdx_hypercall_args for input and output - * @flags (RSI) - TDX_HCALL_* flags - * - * On successful completion, return the hypercall error code. */ -SYM_FUNC_START(__tdx_hypercall) +.macro TDX_HYPERCALL ret:req FRAME_BEGIN /* Save callee-saved GPRs as mandated by the x86_64 ABI */ @@ -134,9 +126,8 @@ SYM_FUNC_START(__tdx_hypercall) push %r12 push %rbx - /* Free RDI and RSI to be used as TDVMCALL arguments */ + /* Free RDI to be used as TDVMCALL arguments */ movq %rdi, %rax - push %rsi /* Copy hypercall registers from arg struct: */ movq TDX_HYPERCALL_r8(%rax), %r8 @@ -171,14 +162,11 @@ SYM_FUNC_START(__tdx_hypercall) * and are handled by callers. */ testq %rax, %rax - jne .Lpanic + jne .Lpanic\@ pop %rax - /* Copy hypercall result registers to arg struct if needed */ - testq $TDX_HCALL_HAS_OUTPUT, (%rsp) - jz .Lout - + .if \ret movq %r8, TDX_HYPERCALL_r8(%rax) movq %r9, TDX_HYPERCALL_r9(%rax) movq %r10, TDX_HYPERCALL_r10(%rax) @@ -191,7 +179,8 @@ SYM_FUNC_START(__tdx_hypercall) movq %rsi, TDX_HYPERCALL_rsi(%rax) movq %rbx, TDX_HYPERCALL_rbx(%rax) movq %rdx, TDX_HYPERCALL_rdx(%rax) -.Lout: + .endif + /* TDVMCALL leaf return code is in R10 */ movq %r10, %rax @@ -208,9 +197,6 @@ SYM_FUNC_START(__tdx_hypercall) xor %rdi, %rdi xor %rdx, %rdx - /* Remove TDX_HCALL_* flags from the stack */ - pop %rsi - /* Restore callee-saved GPRs as mandated by the x86_64 ABI */ pop %rbx pop %r12 @@ -221,9 +207,33 @@ SYM_FUNC_START(__tdx_hypercall) FRAME_END RET -.Lpanic: +.Lpanic\@: call __tdx_hypercall_failed /* __tdx_hypercall_failed never returns */ REACHABLE - jmp .Lpanic + jmp .Lpanic\@ +.endm + +/* + * + * __tdx_hypercall() function ABI: + * + * @args (RDI) - struct tdx_hypercall_args for input + * + * On successful completion, return the hypercall error code. + */ +SYM_FUNC_START(__tdx_hypercall) + TDX_HYPERCALL ret=0 SYM_FUNC_END(__tdx_hypercall) + +/* + * + * __tdx_hypercall_ret() function ABI: + * + * @args (RDI) - struct tdx_hypercall_args for input and output + * + * On successful completion, return the hypercall error code. + */ +SYM_FUNC_START(__tdx_hypercall_ret) + TDX_HYPERCALL ret=1 +SYM_FUNC_END(__tdx_hypercall_ret) diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 055300e08fb3..e146b599260f 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -66,7 +66,7 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15) .r15 = r15, }; - return __tdx_hypercall(&args, 0); + return __tdx_hypercall(&args); } /* Called from __tdx_hypercall() for unrecoverable failure */ @@ -99,7 +99,7 @@ long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2, .r14 = p4, }; - return __tdx_hypercall(&args, 0); + return __tdx_hypercall(&args); } EXPORT_SYMBOL_GPL(tdx_kvm_hypercall); #endif @@ -179,7 +179,7 @@ static void __noreturn tdx_panic(const char *msg) * happens to return. */ while (1) - __tdx_hypercall(&args, 0); + __tdx_hypercall(&args); } static void tdx_parse_tdinfo(u64 *cc_mask) @@ -289,7 +289,7 @@ static u64 __cpuidle __halt(const bool irq_disabled) * can keep the vCPU in virtual HLT, even if an IRQ is * pending, without hanging/breaking the guest. */ - return __tdx_hypercall(&args, 0); + return __tdx_hypercall(&args); } static int handle_halt(struct ve_info *ve) @@ -326,7 +326,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve) * can be found in TDX Guest-Host-Communication Interface * (GHCI), section titled "TDG.VP.VMCALL". */ - if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT)) + if (__tdx_hypercall_ret(&args)) return -EIO; regs->ax = lower_32_bits(args.r11); @@ -348,7 +348,7 @@ static int write_msr(struct pt_regs *regs, struct ve_info *ve) * can be found in TDX Guest-Host-Communication Interface * (GHCI) section titled "TDG.VP.VMCALL". */ - if (__tdx_hypercall(&args, 0)) + if (__tdx_hypercall(&args)) return -EIO; return ve_instr_len(ve); @@ -380,7 +380,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve) * ABI can be found in TDX Guest-Host-Communication Interface * (GHCI), section titled "VP.VMCALL". */ - if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT)) + if (__tdx_hypercall_ret(&args)) return -EIO; /* @@ -407,7 +407,7 @@ static bool mmio_read(int size, unsigned long addr, unsigned long *val) .r15 = *val, }; - if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT)) + if (__tdx_hypercall_ret(&args)) return false; *val = args.r11; return true; @@ -541,7 +541,7 @@ static bool handle_in(struct pt_regs *regs, int size, int port) * in TDX Guest-Host-Communication Interface (GHCI) section titled * "TDG.VP.VMCALL". */ - success = !__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT); + success = !__tdx_hypercall_ret(&args); /* Update part of the register affected by the emulated instruction */ regs->ax &= ~mask; diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index d94d361f506f..f31e286c2977 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -205,7 +205,7 @@ syscall_return_via_sysret: */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK pushq RSP-RDI(%rdi) /* RSP */ pushq (%rdi) /* RDI */ @@ -286,7 +286,7 @@ SYM_FUNC_END(__switch_to_asm) .pushsection .text, "ax" __FUNC_ALIGN SYM_CODE_START_NOALIGN(ret_from_fork) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // copy_thread CALL_DEPTH_ACCOUNT movq %rax, %rdi @@ -303,7 +303,7 @@ SYM_CODE_START_NOALIGN(ret_from_fork) 1: /* kernel thread */ - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK movq %r12, %rdi CALL_NOSPEC rbx /* @@ -388,9 +388,9 @@ SYM_CODE_START(\asmsym) .if \vector == X86_TRAP_BP /* #BP advances %rip to the next instruction */ - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 signal=0 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0 .else - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 .endif ENDBR @@ -461,7 +461,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_mce_db vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -518,7 +518,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_vc vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR ASM_CLAC cld @@ -582,7 +582,7 @@ SYM_CODE_END(\asmsym) */ .macro idtentry_df vector asmsym cfunc SYM_CODE_START(\asmsym) - UNWIND_HINT_IRET_REGS offset=8 + UNWIND_HINT_IRET_ENTRY offset=8 ENDBR ASM_CLAC cld @@ -643,7 +643,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) */ movq %rsp, %rdi movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Copy the IRET frame to the trampoline stack. */ pushq 6*8(%rdi) /* SS */ @@ -869,7 +869,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) */ __FUNC_ALIGN SYM_CODE_START_NOALIGN(xen_failsafe_callback) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR movl %ds, %ecx cmpw %cx, 0x10(%rsp) @@ -1027,7 +1027,7 @@ SYM_CODE_START_LOCAL(paranoid_exit) * * NB to anyone to try to optimize this code: this code does * not execute at all for exceptions from user mode. Those - * exceptions go through error_exit instead. + * exceptions go through error_return instead. */ RESTORE_CR3 scratch_reg=%rax save_reg=%r14 @@ -1107,7 +1107,7 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - ANNOTATE_UNRET_END + VALIDATE_UNRET_END RET .Lbstep_iret: @@ -1153,7 +1153,7 @@ SYM_CODE_END(error_return) * when PAGE_TABLE_ISOLATION is in use. Do not clobber. */ SYM_CODE_START(asm_exc_nmi) - UNWIND_HINT_IRET_REGS + UNWIND_HINT_IRET_ENTRY ENDBR /* @@ -1520,7 +1520,7 @@ SYM_CODE_END(asm_exc_nmi) * MSRs to fully disable 32-bit SYSCALL. */ SYM_CODE_START(ignore_sysret) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR mov $-ENOSYS, %eax sysretl diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c index 3b300a773c7e..f3b3cacbcbb0 100644 --- a/arch/x86/entry/vdso/vdso32-setup.c +++ b/arch/x86/entry/vdso/vdso32-setup.c @@ -70,18 +70,9 @@ static struct ctl_table abi_table2[] = { {} }; -static struct ctl_table abi_root_table2[] = { - { - .procname = "abi", - .mode = 0555, - .child = abi_table2 - }, - {} -}; - static __init int ia32_binfmt_init(void) { - register_sysctl_table(abi_root_table2); + register_sysctl("abi", abi_table2); return 0; } __initcall(ia32_binfmt_init); diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index d234ca797e4a..e0ca8120aea8 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -317,7 +317,7 @@ static struct vm_area_struct gate_vma __ro_after_init = { struct vm_area_struct *get_gate_vma(struct mm_struct *mm) { #ifdef CONFIG_COMPAT - if (!mm || !(mm->context.flags & MM_CONTEXT_HAS_VSYSCALL)) + if (!mm || !test_bit(MM_CONTEXT_HAS_VSYSCALL, &mm->context.flags)) return NULL; #endif if (vsyscall_mode == NONE) diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 127d5b7b63de..cc92388b7a99 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -129,7 +129,7 @@ static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code, return ES_OK; } -void hv_ghcb_terminate(unsigned int set, unsigned int reason) +void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason) { u64 val = GHCB_MSR_TERM_REQ; diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 73c9672c123b..353b054812de 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -321,6 +321,7 @@ #define X86_FEATURE_LKGS (12*32+18) /* "" Load "kernel" (userspace) GS */ #define X86_FEATURE_AMX_FP16 (12*32+21) /* "" AMX fp16 Support */ #define X86_FEATURE_AVX_IFMA (12*32+23) /* "" Support for VPMADD52[H,L]UQ */ +#define X86_FEATURE_LAM (12*32+26) /* Linear Address Masking */ /* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */ #define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 5dfa4fb76f4b..fafe9be7a6f4 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -75,6 +75,12 @@ # define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31)) #endif +#ifdef CONFIG_ADDRESS_MASKING +# define DISABLE_LAM 0 +#else +# define DISABLE_LAM (1 << (X86_FEATURE_LAM & 31)) +#endif + #ifdef CONFIG_INTEL_IOMMU_SVM # define DISABLE_ENQCMD 0 #else @@ -115,7 +121,7 @@ #define DISABLED_MASK10 0 #define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \ DISABLE_CALL_DEPTH_TRACKING) -#define DISABLED_MASK12 0 +#define DISABLED_MASK12 (DISABLE_LAM) #define DISABLED_MASK13 0 #define DISABLED_MASK14 0 #define DISABLED_MASK15 0 diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index c201083b34f6..a3abdcd89a32 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -20,25 +20,4 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); -#ifdef CONFIG_X86_INTEL_MID - -extern void intel_scu_devices_create(void); -extern void intel_scu_devices_destroy(void); - -#else /* !CONFIG_X86_INTEL_MID */ - -static inline void intel_scu_devices_create(void) { } -static inline void intel_scu_devices_destroy(void) { } - -#endif /* !CONFIG_X86_INTEL_MID */ - -/* Bus Select SoC Fuse value */ -#define BSEL_SOC_FUSE_MASK 0x7 -/* FSB 133MHz */ -#define BSEL_SOC_FUSE_001 0x1 -/* FSB 100MHz */ -#define BSEL_SOC_FUSE_101 0x5 -/* FSB 83MHz */ -#define BSEL_SOC_FUSE_111 0x7 - #endif /* _ASM_X86_INTEL_MID_H */ diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index dd9b8118f784..0953aa32a324 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -99,7 +99,7 @@ /* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */ #define SYM_TYPED_FUNC_START(name) \ - SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \ ENDBR /* SYM_FUNC_START -- use for global functions */ diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 5d7494631ea9..0da5c227f490 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -9,9 +9,13 @@ #include /* Uprobes on this MM assume 32-bit code */ -#define MM_CONTEXT_UPROBE_IA32 BIT(0) +#define MM_CONTEXT_UPROBE_IA32 0 /* vsyscall page is accessible on this MM */ -#define MM_CONTEXT_HAS_VSYSCALL BIT(1) +#define MM_CONTEXT_HAS_VSYSCALL 1 +/* Do not allow changing LAM mode */ +#define MM_CONTEXT_LOCK_LAM 2 +/* Allow LAM and SVA coexisting */ +#define MM_CONTEXT_FORCE_TAGGED_SVA 3 /* * x86 has arch-specific MMU state beyond what lives in mm_struct. @@ -39,7 +43,15 @@ typedef struct { #endif #ifdef CONFIG_X86_64 - unsigned short flags; + unsigned long flags; +#endif + +#ifdef CONFIG_ADDRESS_MASKING + /* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */ + unsigned long lam_cr3_mask; + + /* Significant bits of the virtual address. Excludes tag bits. */ + u64 untag_mask; #endif struct mutex lock; diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index c3ad8a526378..1d29dc791f5a 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -85,6 +85,51 @@ static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next) } #endif +#ifdef CONFIG_ADDRESS_MASKING +static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm) +{ + return mm->context.lam_cr3_mask; +} + +static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) +{ + mm->context.lam_cr3_mask = oldmm->context.lam_cr3_mask; + mm->context.untag_mask = oldmm->context.untag_mask; +} + +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return mm->context.untag_mask; +} + +static inline void mm_reset_untag_mask(struct mm_struct *mm) +{ + mm->context.untag_mask = -1UL; +} + +#define arch_pgtable_dma_compat arch_pgtable_dma_compat +static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) +{ + return !mm_lam_cr3_mask(mm) || + test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags); +} +#else + +static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm) +{ + return 0; +} + +static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm) +{ +} + +static inline void mm_reset_untag_mask(struct mm_struct *mm) +{ +} +#endif + #define enter_lazy_tlb enter_lazy_tlb extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); @@ -109,6 +154,7 @@ static inline int init_new_context(struct task_struct *tsk, mm->context.execute_only_pkey = -1; } #endif + mm_reset_untag_mask(mm); init_new_context_ldt(mm); return 0; } @@ -162,6 +208,7 @@ static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { arch_dup_pkeys(oldmm, mm); paravirt_enter_mmap(mm); + dup_lam(oldmm, mm); return ldt_dup_context(oldmm, mm); } @@ -175,7 +222,7 @@ static inline void arch_exit_mmap(struct mm_struct *mm) static inline bool is_64bit_mm(struct mm_struct *mm) { return !IS_ENABLED(CONFIG_IA32_EMULATION) || - !(mm->context.flags & MM_CONTEXT_UPROBE_IA32); + !test_bit(MM_CONTEXT_UPROBE_IA32, &mm->context.flags); } #else static inline bool is_64bit_mm(struct mm_struct *mm) diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index b445e252aa83..49bb4f2bd300 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -228,7 +228,7 @@ int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry); void hv_ghcb_msr_write(u64 msr, u64 value); void hv_ghcb_msr_read(u64 msr, u64 *value); bool hv_ghcb_negotiate_protocol(void); -void hv_ghcb_terminate(unsigned int set, unsigned int reason); +void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason); void hv_vtom_init(void); #else static inline void hv_ghcb_msr_write(u64 msr, u64 value) {} diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 3ef70e54a858..edb2b0cb8efe 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -194,9 +194,9 @@ * builds. */ .macro ANNOTATE_RETPOLINE_SAFE - .Lannotate_\@: +.Lhere_\@: .pushsection .discard.retpoline_safe - _ASM_PTR .Lannotate_\@ + .long .Lhere_\@ - . .popsection .endm @@ -210,8 +210,8 @@ * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should * eventually turn into it's own annotation. */ -.macro ANNOTATE_UNRET_END -#ifdef CONFIG_DEBUG_ENTRY +.macro VALIDATE_UNRET_END +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) ANNOTATE_RETPOLINE_SAFE nop #endif @@ -286,7 +286,7 @@ .macro UNTRAIN_RET #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ @@ -297,7 +297,7 @@ .macro UNTRAIN_RET_FROM_CALL #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \ defined(CONFIG_CALL_DEPTH_TRACKING) - ANNOTATE_UNRET_END + VALIDATE_UNRET_END ALTERNATIVE_3 "", \ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \ @@ -318,7 +318,7 @@ #define ANNOTATE_RETPOLINE_SAFE \ "999:\n\t" \ ".pushsection .discard.retpoline_safe\n\t" \ - _ASM_PTR " 999b\n\t" \ + ".long 999b - .\n\t" \ ".popsection\n\t" typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE]; diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index 1343a62106de..46d7e06763c9 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -39,6 +39,12 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 + #ifndef __ASSEMBLY__ #include @@ -56,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h index a7f3d9100adb..d8cccadc83a6 100644 --- a/arch/x86/include/asm/processor-flags.h +++ b/arch/x86/include/asm/processor-flags.h @@ -28,6 +28,8 @@ * On systems with SME, one bit (in a variable position!) is stolen to indicate * that the top-level paging structure is encrypted. * + * On systemms with LAM, bits 61 and 62 are used to indicate LAM mode. + * * All of the remaining bits indicate the physical address of the top-level * paging structure. * diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h index bc5b4d788c08..9177b4354c3f 100644 --- a/arch/x86/include/asm/reboot.h +++ b/arch/x86/include/asm/reboot.h @@ -28,7 +28,6 @@ void __noreturn machine_real_restart(unsigned int type); void cpu_emergency_disable_virtualization(void); typedef void (*nmi_shootdown_cb)(int, struct pt_regs*); -void nmi_panic_self_stop(struct pt_regs *regs); void nmi_shootdown_cpus(nmi_shootdown_cb callback); void run_crash_ipi_callback(struct pt_regs *regs); diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index f37cbff7354c..f3495623ac99 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h @@ -125,11 +125,11 @@ void clear_bss(void); #ifdef __i386__ -asmlinkage void __init i386_start_kernel(void); +asmlinkage void __init __noreturn i386_start_kernel(void); #else -asmlinkage void __init x86_64_start_kernel(char *real_mode); -asmlinkage void __init x86_64_start_reservations(char *real_mode_data); +asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode); +asmlinkage void __init __noreturn x86_64_start_reservations(char *real_mode_data); #endif /* __i386__ */ #endif /* _SETUP */ diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h index 4a03993e0785..2631e01f6e0f 100644 --- a/arch/x86/include/asm/shared/tdx.h +++ b/arch/x86/include/asm/shared/tdx.h @@ -7,8 +7,6 @@ #define TDX_HYPERCALL_STANDARD 0 -#define TDX_HCALL_HAS_OUTPUT BIT(0) - #define TDX_CPUID_LEAF_ID 0x21 #define TDX_IDENT "IntelTDX " @@ -36,7 +34,8 @@ struct tdx_hypercall_args { }; /* Used to request services from the VMM */ -u64 __tdx_hypercall(struct tdx_hypercall_args *args, unsigned long flags); +u64 __tdx_hypercall(struct tdx_hypercall_args *args); +u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args); /* Called from __tdx_hypercall() for unrecoverable failure */ void __tdx_hypercall_failed(void); diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index bf2c51df9e0b..9fa795102b93 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -93,9 +93,10 @@ static inline void __cpu_die(unsigned int cpu) smp_ops.cpu_die(cpu); } -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { smp_ops.play_dead(); + BUG(); } static inline void smp_send_reschedule(int cpu) @@ -124,7 +125,7 @@ int native_cpu_up(unsigned int cpunum, struct task_struct *tidle); int native_cpu_disable(void); int common_cpu_die(unsigned int cpu); void native_cpu_die(unsigned int cpu); -void hlt_play_dead(void); +void __noreturn hlt_play_dead(void); void native_play_dead(void); void play_dead_common(void); void wbinvd_on_cpu(int cpu); diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index cda3118f3b27..75bfaa421030 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -2,7 +2,7 @@ #ifndef _ASM_X86_TLBFLUSH_H #define _ASM_X86_TLBFLUSH_H -#include +#include #include #include @@ -12,6 +12,7 @@ #include #include #include +#include void __flush_tlb_all(void); @@ -53,6 +54,15 @@ static inline void cr4_clear_bits(unsigned long mask) local_irq_restore(flags); } +#ifdef CONFIG_ADDRESS_MASKING +DECLARE_PER_CPU(u64, tlbstate_untag_mask); + +static inline u64 current_untag_mask(void) +{ + return this_cpu_read(tlbstate_untag_mask); +} +#endif + #ifndef MODULE /* * 6 because 6 should be plenty and struct tlb_state will fit in two cache @@ -101,6 +111,16 @@ struct tlb_state { */ bool invalidate_other; +#ifdef CONFIG_ADDRESS_MASKING + /* + * Active LAM mode. + * + * X86_CR3_LAM_U57/U48 shifted right by X86_CR3_LAM_U57_BIT or 0 if LAM + * disabled. + */ + u8 lam; +#endif + /* * Mask that contains TLB_NR_DYN_ASIDS+1 bits to indicate * the corresponding user PCID needs a flush next time we @@ -357,6 +377,32 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) } #define huge_pmd_needs_flush huge_pmd_needs_flush +#ifdef CONFIG_ADDRESS_MASKING +static inline u64 tlbstate_lam_cr3_mask(void) +{ + u64 lam = this_cpu_read(cpu_tlbstate.lam); + + return lam << X86_CR3_LAM_U57_BIT; +} + +static inline void set_tlbstate_lam_mode(struct mm_struct *mm) +{ + this_cpu_write(cpu_tlbstate.lam, + mm->context.lam_cr3_mask >> X86_CR3_LAM_U57_BIT); + this_cpu_write(tlbstate_untag_mask, mm->context.untag_mask); +} + +#else + +static inline u64 tlbstate_lam_cr3_mask(void) +{ + return 0; +} + +static inline void set_tlbstate_lam_mode(struct mm_struct *mm) +{ +} +#endif #endif /* !MODULE */ static inline void __native_tlb_flush_global(unsigned long cr4) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 1cc756eafa44..457e814712af 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -7,11 +7,14 @@ #include #include #include +#include #include +#include #include #include #include #include +#include #ifdef CONFIG_DEBUG_ATOMIC_SLEEP static inline bool pagefault_disabled(void); @@ -21,6 +24,57 @@ static inline bool pagefault_disabled(void); # define WARN_ON_IN_IRQ() #endif +#ifdef CONFIG_ADDRESS_MASKING +/* + * Mask out tag bits from the address. + * + * Magic with the 'sign' allows to untag userspace pointer without any branches + * while leaving kernel addresses intact. + */ +static inline unsigned long __untagged_addr(unsigned long addr) +{ + long sign; + + /* + * Refer tlbstate_untag_mask directly to avoid RIP-relative relocation + * in alternative instructions. The relocation gets wrong when gets + * copied to the target place. + */ + asm (ALTERNATIVE("", + "sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */ + "or %%gs:tlbstate_untag_mask, %[sign]\n\t" + "and %[sign], %[addr]\n\t", X86_FEATURE_LAM) + : [addr] "+r" (addr), [sign] "=r" (sign) + : "m" (tlbstate_untag_mask), "[sign]" (addr)); + + return addr; +} + +#define untagged_addr(addr) ({ \ + unsigned long __addr = (__force unsigned long)(addr); \ + (__force __typeof__(addr))__untagged_addr(__addr); \ +}) + +static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, + unsigned long addr) +{ + long sign = addr >> 63; + + mmap_assert_locked(mm); + addr &= (mm)->context.untag_mask | sign; + + return addr; +} + +#define untagged_addr_remote(mm, addr) ({ \ + unsigned long __addr = (__force unsigned long)(addr); \ + (__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \ +}) + +#else +#define untagged_addr(addr) (addr) +#endif + /** * access_ok - Checks if a user space pointer is valid * @addr: User space pointer to start of block to check @@ -38,10 +92,10 @@ static inline bool pagefault_disabled(void); * Return: true (nonzero) if the memory block may be valid, false (zero) * if it is definitely invalid. */ -#define access_ok(addr, size) \ +#define access_ok(addr, size) \ ({ \ WARN_ON_IN_IRQ(); \ - likely(__access_ok(addr, size)); \ + likely(__access_ok(untagged_addr(addr), size)); \ }) #include diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 0a49a8de9f3c..c972bd21aa23 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -54,8 +54,6 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size); extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); -extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset, - size_t len); static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index e7c71750b309..01cb9692b160 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -7,12 +7,17 @@ #ifdef __ASSEMBLY__ -.macro UNWIND_HINT_EMPTY - UNWIND_HINT type=UNWIND_HINT_TYPE_CALL end=1 +.macro UNWIND_HINT_END_OF_STACK + UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK +.endm + +.macro UNWIND_HINT_UNDEFINED + UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED .endm .macro UNWIND_HINT_ENTRY - UNWIND_HINT type=UNWIND_HINT_TYPE_ENTRY end=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_END_OF_STACK .endm .macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1 @@ -52,6 +57,11 @@ UNWIND_HINT_REGS base=\base offset=\offset partial=1 signal=\signal .endm +.macro UNWIND_HINT_IRET_ENTRY base=%rsp offset=0 signal=1 + VALIDATE_UNRET_BEGIN + UNWIND_HINT_IRET_REGS base=\base offset=\offset signal=\signal +.endm + .macro UNWIND_HINT_FUNC UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm @@ -67,7 +77,7 @@ #else #define UNWIND_HINT_FUNC \ - UNWIND_HINT(ORC_REG_SP, 8, UNWIND_HINT_TYPE_FUNC, 0, 0) + UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0) #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/uapi/asm/prctl.h b/arch/x86/include/uapi/asm/prctl.h index 500b96e71f18..e8d7ebbca1a4 100644 --- a/arch/x86/include/uapi/asm/prctl.h +++ b/arch/x86/include/uapi/asm/prctl.h @@ -16,8 +16,16 @@ #define ARCH_GET_XCOMP_GUEST_PERM 0x1024 #define ARCH_REQ_XCOMP_GUEST_PERM 0x1025 +#define ARCH_XCOMP_TILECFG 17 +#define ARCH_XCOMP_TILEDATA 18 + #define ARCH_MAP_VDSO_X32 0x2001 #define ARCH_MAP_VDSO_32 0x2002 #define ARCH_MAP_VDSO_64 0x2003 +#define ARCH_GET_UNTAG_MASK 0x4001 +#define ARCH_ENABLE_TAGGED_ADDR 0x4002 +#define ARCH_GET_MAX_TAG_BITS 0x4003 +#define ARCH_FORCE_TAGGED_SVA 0x4004 + #endif /* _ASM_X86_PRCTL_H */ diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h index c47cc7f2feeb..d898432947ff 100644 --- a/arch/x86/include/uapi/asm/processor-flags.h +++ b/arch/x86/include/uapi/asm/processor-flags.h @@ -82,6 +82,10 @@ #define X86_CR3_PCID_BITS 12 #define X86_CR3_PCID_MASK (_AC((1UL << X86_CR3_PCID_BITS) - 1, UL)) +#define X86_CR3_LAM_U57_BIT 61 /* Activate LAM for userspace, 62:57 bits masked */ +#define X86_CR3_LAM_U57 _BITULL(X86_CR3_LAM_U57_BIT) +#define X86_CR3_LAM_U48_BIT 62 /* Activate LAM for userspace, 62:48 bits masked */ +#define X86_CR3_LAM_U48 _BITULL(X86_CR3_LAM_U48_BIT) #define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */ #define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT) @@ -132,6 +136,8 @@ #define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT) #define X86_CR4_CET_BIT 23 /* enable Control-flow Enforcement Technology */ #define X86_CR4_CET _BITUL(X86_CR4_CET_BIT) +#define X86_CR4_LAM_SUP_BIT 28 /* LAM for supervisor pointers */ +#define X86_CR4_LAM_SUP _BITUL(X86_CR4_LAM_SUP_BIT) /* * x86-64 Task Priority Register, CR8 diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 0e7b6afe2fa6..ded1fc7cb7cb 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -383,41 +383,36 @@ void free_rmid(u32 rmid) list_add_tail(&entry->list, &rmid_free_lru); } +static struct mbm_state *get_mbm_state(struct rdt_domain *d, u32 rmid, + enum resctrl_event_id evtid) +{ + switch (evtid) { + case QOS_L3_MBM_TOTAL_EVENT_ID: + return &d->mbm_total[rmid]; + case QOS_L3_MBM_LOCAL_EVENT_ID: + return &d->mbm_local[rmid]; + default: + return NULL; + } +} + static int __mon_event_count(u32 rmid, struct rmid_read *rr) { struct mbm_state *m; u64 tval = 0; - if (rr->first) + if (rr->first) { resctrl_arch_reset_rmid(rr->r, rr->d, rmid, rr->evtid); + m = get_mbm_state(rr->d, rmid, rr->evtid); + if (m) + memset(m, 0, sizeof(struct mbm_state)); + return 0; + } rr->err = resctrl_arch_rmid_read(rr->r, rr->d, rmid, rr->evtid, &tval); if (rr->err) return rr->err; - switch (rr->evtid) { - case QOS_L3_OCCUP_EVENT_ID: - rr->val += tval; - return 0; - case QOS_L3_MBM_TOTAL_EVENT_ID: - m = &rr->d->mbm_total[rmid]; - break; - case QOS_L3_MBM_LOCAL_EVENT_ID: - m = &rr->d->mbm_local[rmid]; - break; - default: - /* - * Code would never reach here because an invalid - * event id would fail in resctrl_arch_rmid_read(). - */ - return -EINVAL; - } - - if (rr->first) { - memset(m, 0, sizeof(struct mbm_state)); - return 0; - } - rr->val += tval; return 0; diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 970d8445fdc4..b8c720b5dab2 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -346,7 +346,7 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__) #ifdef CONFIG_FUNCTION_GRAPH_TRACER SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR subq $16, %rsp diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c index ec6fefbfd3c0..10c27b4261eb 100644 --- a/arch/x86/kernel/head32.c +++ b/arch/x86/kernel/head32.c @@ -29,7 +29,7 @@ static void __init i386_default_early_setup(void) x86_init.mpparse.setup_ioapic_ids = setup_ioapic_ids_from_mpc; } -asmlinkage __visible void __init i386_start_kernel(void) +asmlinkage __visible void __init __noreturn i386_start_kernel(void) { /* Make sure IDT is set up before any exception happens */ idt_setup_early_handler(); diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 387e4b12e823..49f7629b17f7 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -471,7 +471,7 @@ static void __init copy_bootdata(char *real_mode_data) sme_unmap_bootdata(real_mode_data); } -asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) +asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode_data) { /* * Build-time sanity checks on the kernel image and module @@ -537,7 +537,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data) x86_64_start_reservations(real_mode_data); } -void __init x86_64_start_reservations(char *real_mode_data) +void __init __noreturn x86_64_start_reservations(char *real_mode_data) { /* version is always not zero if it is copied */ if (!boot_params.hdr.version) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 6a8238702eab..a5df3e994f04 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -42,7 +42,7 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map) __HEAD .code64 SYM_CODE_START_NOALIGN(startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, * and someone has loaded an identity mapped page table @@ -97,7 +97,7 @@ SYM_CODE_START_NOALIGN(startup_64) lretq .Lon_kernel_cs: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Sanitize CPU configuration */ call verify_cpu @@ -119,7 +119,7 @@ SYM_CODE_START_NOALIGN(startup_64) SYM_CODE_END(startup_64) SYM_CODE_START(secondary_startup_64) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0, @@ -148,7 +148,7 @@ SYM_CODE_START(secondary_startup_64) * verify_cpu() above to make sure NX is enabled. */ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* @@ -230,7 +230,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) ANNOTATE_RETPOLINE_SAFE jmp *%rax 1: - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // above #ifdef CONFIG_SMP @@ -383,7 +383,7 @@ SYM_CODE_END(secondary_startup_64) */ SYM_CODE_START(start_cpu0) ANNOTATE_NOENDBR - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* Find the idle task stack */ movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx @@ -406,8 +406,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS @@ -460,7 +458,6 @@ SYM_CODE_END(early_idt_handler_array) SYM_CODE_START_LOCAL(early_idt_handler_common) UNWIND_HINT_IRET_REGS offset=16 - ANNOTATE_UNRET_END /* * The stack is the hardware frame, an error code or zero, and the * vector number. @@ -510,8 +507,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb) UNWIND_HINT_IRET_REGS offset=8 ENDBR - ANNOTATE_UNRET_END - /* Build pt_regs */ PUSH_AND_CLEAR_REGS diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 9ff480e94511..670eb08b972a 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -77,15 +77,6 @@ static struct ctl_table itmt_kern_table[] = { {} }; -static struct ctl_table itmt_root_table[] = { - { - .procname = "kernel", - .mode = 0555, - .child = itmt_kern_table, - }, - {} -}; - static struct ctl_table_header *itmt_sysctl_header; /** @@ -114,7 +105,7 @@ int sched_set_itmt_support(void) return 0; } - itmt_sysctl_header = register_sysctl_table(itmt_root_table); + itmt_sysctl_header = register_sysctl("kernel", itmt_kern_table); if (!itmt_sysctl_header) { mutex_unlock(&itmt_update_mutex); return -ENOMEM; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index b650cde3f64d..dac41a0072ea 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include #include #include +#include #include "process.h" @@ -162,6 +164,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) savesegment(es, p->thread.es); savesegment(ds, p->thread.ds); + + if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) + set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags); #else p->thread.sp0 = (unsigned long) (childregs + 1); savesegment(gs, p->thread.gs); @@ -368,6 +373,8 @@ void arch_setup_new_exec(void) task_clear_spec_ssb_noexec(current); speculation_ctrl_update(read_thread_flags()); } + + mm_reset_untag_mask(current->mm); } #ifdef CONFIG_X86_IOPL_IOPERM @@ -715,7 +722,7 @@ static bool x86_idle_set(void) } #ifndef CONFIG_SMP -static inline void play_dead(void) +static inline void __noreturn play_dead(void) { BUG(); } @@ -727,7 +734,7 @@ void arch_cpu_idle_enter(void) local_touch_nmi(); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { play_dead(); } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index bb65a68b4b49..223b223f713f 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -671,7 +671,7 @@ void set_personality_64bit(void) task_pt_regs(current)->orig_ax = __NR_execve; current_thread_info()->status &= ~TS_COMPAT; if (current->mm) - current->mm->context.flags = MM_CONTEXT_HAS_VSYSCALL; + __set_bit(MM_CONTEXT_HAS_VSYSCALL, ¤t->mm->context.flags); /* TBD: overwrites user setup. Should have two bits. But 64bit processes have always behaved this way, @@ -708,7 +708,7 @@ static void __set_personality_ia32(void) * uprobes applied to this MM need to know this and * cannot use user_64bit_mode() at that time. */ - current->mm->context.flags = MM_CONTEXT_UPROBE_IA32; + __set_bit(MM_CONTEXT_UPROBE_IA32, ¤t->mm->context.flags); } current->personality |= force_personality32; @@ -743,6 +743,52 @@ static long prctl_map_vdso(const struct vdso_image *image, unsigned long addr) } #endif +#ifdef CONFIG_ADDRESS_MASKING + +#define LAM_U57_BITS 6 + +static int prctl_enable_tagged_addr(struct mm_struct *mm, unsigned long nr_bits) +{ + if (!cpu_feature_enabled(X86_FEATURE_LAM)) + return -ENODEV; + + /* PTRACE_ARCH_PRCTL */ + if (current->mm != mm) + return -EINVAL; + + if (mm_valid_pasid(mm) && + !test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags)) + return -EINVAL; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + if (test_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags)) { + mmap_write_unlock(mm); + return -EBUSY; + } + + if (!nr_bits) { + mmap_write_unlock(mm); + return -EINVAL; + } else if (nr_bits <= LAM_U57_BITS) { + mm->context.lam_cr3_mask = X86_CR3_LAM_U57; + mm->context.untag_mask = ~GENMASK(62, 57); + } else { + mmap_write_unlock(mm); + return -EINVAL; + } + + write_cr3(__read_cr3() | mm->context.lam_cr3_mask); + set_tlbstate_lam_mode(mm); + set_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags); + + mmap_write_unlock(mm); + + return 0; +} +#endif + long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) { int ret = 0; @@ -830,7 +876,23 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) case ARCH_MAP_VDSO_64: return prctl_map_vdso(&vdso_image_64, arg2); #endif - +#ifdef CONFIG_ADDRESS_MASKING + case ARCH_GET_UNTAG_MASK: + return put_user(task->mm->context.untag_mask, + (unsigned long __user *)arg2); + case ARCH_ENABLE_TAGGED_ADDR: + return prctl_enable_tagged_addr(task->mm, arg2); + case ARCH_FORCE_TAGGED_SVA: + if (current != task) + return -EINVAL; + set_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &task->mm->context.flags); + return 0; + case ARCH_GET_MAX_TAG_BITS: + if (!cpu_feature_enabled(X86_FEATURE_LAM)) + return put_user(0, (unsigned long __user *)arg2); + else + return put_user(LAM_U57_BITS, (unsigned long __user *)arg2); +#endif default: ret = -EINVAL; break; diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index d03c551defcc..3adbe97015c1 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -920,7 +920,7 @@ void run_crash_ipi_callback(struct pt_regs *regs) } /* Override the weak function in kernel/panic.c */ -void nmi_panic_self_stop(struct pt_regs *regs) +void __noreturn nmi_panic_self_stop(struct pt_regs *regs) { while (1) { /* If no CPU is preparing crash dump, we simply loop here. */ diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 4a73351f87f8..56cab1bb25f5 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -43,7 +43,7 @@ .code64 SYM_CODE_START_NOALIGN(relocate_range) SYM_CODE_START_NOALIGN(relocate_kernel) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR /* * %rdi indirection_page @@ -113,7 +113,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel) SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK /* set return address to 0 if not preserving context */ pushq $0 /* store the start address on the stack */ @@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR // RET target, above movq RSP(%r8), %rsp movq CR4(%r8), %rax @@ -256,8 +256,8 @@ SYM_CODE_END(virtual_mapped) /* Do the copies */ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) - UNWIND_HINT_EMPTY - movq %rdi, %rcx /* Put the page_list in %rcx */ + UNWIND_HINT_END_OF_STACK + movq %rdi, %rcx /* Put the page_list in %rcx */ xorl %edi, %edi xorl %esi, %esi jmp 1f diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 851477f7d728..352f0ce1ece4 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1824,7 +1824,7 @@ static inline void mwait_play_dead(void) } } -void hlt_play_dead(void) +void __noreturn hlt_play_dead(void) { if (__this_cpu_read(cpu_info.x86) >= 4) wbinvd(); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d317dc3d06a3..8b83d8fbce71 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -671,15 +671,15 @@ static bool try_fixup_enqcmd_gp(void) if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) return false; - pasid = current->mm->pasid; - /* * If the mm has not been allocated a * PASID, the #GP can not be fixed up. */ - if (!pasid_valid(pasid)) + if (!mm_valid_pasid(current->mm)) return false; + pasid = current->mm->pasid; + /* * Did this thread already have its PASID activated? * If so, the #GP must be from something else. diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 37307b40f8da..3ac50b7298d1 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -133,7 +133,7 @@ static struct orc_entry null_orc_entry = { .sp_offset = sizeof(long), .sp_reg = ORC_REG_SP, .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL + .type = ORC_TYPE_CALL }; #ifdef CONFIG_CALL_THUNKS @@ -153,12 +153,11 @@ static struct orc_entry *orc_callthunk_find(unsigned long ip) /* Fake frame pointer entry -- used as a fallback for generated code */ static struct orc_entry orc_fp_entry = { - .type = UNWIND_HINT_TYPE_CALL, + .type = ORC_TYPE_CALL, .sp_reg = ORC_REG_BP, .sp_offset = 16, .bp_reg = ORC_REG_PREV_SP, .bp_offset = -16, - .end = 0, }; static struct orc_entry *orc_find(unsigned long ip) @@ -250,13 +249,13 @@ static int orc_sort_cmp(const void *_a, const void *_b) return -1; /* - * The "weak" section terminator entries need to always be on the left + * The "weak" section terminator entries need to always be first * to ensure the lookup code skips them in favor of real entries. * These terminator entries exist to handle any gaps created by * whitelisted .o files which didn't get objtool generation. */ orc_a = cur_orc_table + (a - cur_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, @@ -474,14 +473,12 @@ bool unwind_next_frame(struct unwind_state *state) */ orc = &orc_fp_entry; state->error = true; - } - - /* End-of-stack check for kernel threads: */ - if (orc->sp_reg == ORC_REG_UNDEFINED) { - if (!orc->end) + } else { + if (orc->type == ORC_TYPE_UNDEFINED) goto err; - goto the_end; + if (orc->type == ORC_TYPE_END_OF_STACK) + goto the_end; } state->signal = orc->signal; @@ -554,7 +551,7 @@ bool unwind_next_frame(struct unwind_state *state) /* Find IP, SP and possibly regs: */ switch (orc->type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_CALL: ip_p = sp - sizeof(long); if (!deref_stack_reg(state, ip_p, &state->ip)) @@ -567,7 +564,7 @@ bool unwind_next_frame(struct unwind_state *state) state->prev_regs = NULL; break; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access registers at %pB\n", (void *)orig_ip); @@ -590,13 +587,13 @@ bool unwind_next_frame(struct unwind_state *state) state->full_regs = true; break; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { orc_warn_current("can't access iret registers at %pB\n", (void *)orig_ip); goto err; } - /* See UNWIND_HINT_TYPE_REGS case comment. */ + /* See ORC_TYPE_REGS case comment. */ state->ip = unwind_recover_rethook(state, state->ip, (unsigned long *)(state->sp - sizeof(long))); diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S index b70d98d79a9d..b64a2bd1a1ef 100644 --- a/arch/x86/lib/getuser.S +++ b/arch/x86/lib/getuser.S @@ -37,22 +37,22 @@ #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC -#ifdef CONFIG_X86_5LEVEL -#define LOAD_TASK_SIZE_MINUS_N(n) \ - ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \ - __stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57 -#else -#define LOAD_TASK_SIZE_MINUS_N(n) \ - mov $(TASK_SIZE_MAX - (n)),%_ASM_DX -#endif +.macro check_range size:req +.if IS_ENABLED(CONFIG_X86_64) + mov %rax, %rdx + sar $63, %rdx + or %rdx, %rax +.else + cmp $TASK_SIZE_MAX-\size+1, %eax + jae .Lbad_get_user + sbb %edx, %edx /* array_index_mask_nospec() */ + and %edx, %eax +.endif +.endm .text SYM_FUNC_START(__get_user_1) - LOAD_TASK_SIZE_MINUS_N(0) - cmp %_ASM_DX,%_ASM_AX - jae bad_get_user - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ - and %_ASM_DX, %_ASM_AX + check_range size=1 ASM_STAC 1: movzbl (%_ASM_AX),%edx xor %eax,%eax @@ -62,11 +62,7 @@ SYM_FUNC_END(__get_user_1) EXPORT_SYMBOL(__get_user_1) SYM_FUNC_START(__get_user_2) - LOAD_TASK_SIZE_MINUS_N(1) - cmp %_ASM_DX,%_ASM_AX - jae bad_get_user - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ - and %_ASM_DX, %_ASM_AX + check_range size=2 ASM_STAC 2: movzwl (%_ASM_AX),%edx xor %eax,%eax @@ -76,11 +72,7 @@ SYM_FUNC_END(__get_user_2) EXPORT_SYMBOL(__get_user_2) SYM_FUNC_START(__get_user_4) - LOAD_TASK_SIZE_MINUS_N(3) - cmp %_ASM_DX,%_ASM_AX - jae bad_get_user - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ - and %_ASM_DX, %_ASM_AX + check_range size=4 ASM_STAC 3: movl (%_ASM_AX),%edx xor %eax,%eax @@ -90,30 +82,17 @@ SYM_FUNC_END(__get_user_4) EXPORT_SYMBOL(__get_user_4) SYM_FUNC_START(__get_user_8) + check_range size=8 + ASM_STAC #ifdef CONFIG_X86_64 - LOAD_TASK_SIZE_MINUS_N(7) - cmp %_ASM_DX,%_ASM_AX - jae bad_get_user - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ - and %_ASM_DX, %_ASM_AX - ASM_STAC 4: movq (%_ASM_AX),%rdx - xor %eax,%eax - ASM_CLAC - RET #else - LOAD_TASK_SIZE_MINUS_N(7) - cmp %_ASM_DX,%_ASM_AX - jae bad_get_user_8 - sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */ - and %_ASM_DX, %_ASM_AX - ASM_STAC 4: movl (%_ASM_AX),%edx 5: movl 4(%_ASM_AX),%ecx +#endif xor %eax,%eax ASM_CLAC RET -#endif SYM_FUNC_END(__get_user_8) EXPORT_SYMBOL(__get_user_8) @@ -166,7 +145,7 @@ EXPORT_SYMBOL(__get_user_nocheck_8) SYM_CODE_START_LOCAL(.Lbad_get_user_clac) ASM_CLAC -bad_get_user: +.Lbad_get_user: xor %edx,%edx mov $(-EFAULT),%_ASM_AX RET @@ -184,23 +163,23 @@ SYM_CODE_END(.Lbad_get_user_8_clac) #endif /* get_user */ - _ASM_EXTABLE_UA(1b, .Lbad_get_user_clac) - _ASM_EXTABLE_UA(2b, .Lbad_get_user_clac) - _ASM_EXTABLE_UA(3b, .Lbad_get_user_clac) + _ASM_EXTABLE(1b, .Lbad_get_user_clac) + _ASM_EXTABLE(2b, .Lbad_get_user_clac) + _ASM_EXTABLE(3b, .Lbad_get_user_clac) #ifdef CONFIG_X86_64 - _ASM_EXTABLE_UA(4b, .Lbad_get_user_clac) + _ASM_EXTABLE(4b, .Lbad_get_user_clac) #else - _ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac) - _ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac) + _ASM_EXTABLE(4b, .Lbad_get_user_8_clac) + _ASM_EXTABLE(5b, .Lbad_get_user_8_clac) #endif /* __get_user */ - _ASM_EXTABLE_UA(6b, .Lbad_get_user_clac) - _ASM_EXTABLE_UA(7b, .Lbad_get_user_clac) - _ASM_EXTABLE_UA(8b, .Lbad_get_user_clac) + _ASM_EXTABLE(6b, .Lbad_get_user_clac) + _ASM_EXTABLE(7b, .Lbad_get_user_clac) + _ASM_EXTABLE(8b, .Lbad_get_user_clac) #ifdef CONFIG_X86_64 - _ASM_EXTABLE_UA(9b, .Lbad_get_user_clac) + _ASM_EXTABLE(9b, .Lbad_get_user_clac) #else - _ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac) - _ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac) + _ASM_EXTABLE(9b, .Lbad_get_user_8_clac) + _ASM_EXTABLE(10b, .Lbad_get_user_8_clac) #endif diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S index 32125224fcca..3062d09a776d 100644 --- a/arch/x86/lib/putuser.S +++ b/arch/x86/lib/putuser.S @@ -33,20 +33,20 @@ * as they get called from within inline assembly. */ -#ifdef CONFIG_X86_5LEVEL -#define LOAD_TASK_SIZE_MINUS_N(n) \ - ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rbx), \ - __stringify(mov $((1 << 56) - 4096 - (n)),%rbx), X86_FEATURE_LA57 -#else -#define LOAD_TASK_SIZE_MINUS_N(n) \ - mov $(TASK_SIZE_MAX - (n)),%_ASM_BX -#endif +.macro check_range size:req +.if IS_ENABLED(CONFIG_X86_64) + mov %rcx, %rbx + sar $63, %rbx + or %rbx, %rcx +.else + cmp $TASK_SIZE_MAX-\size+1, %ecx + jae .Lbad_put_user +.endif +.endm .text SYM_FUNC_START(__put_user_1) - LOAD_TASK_SIZE_MINUS_N(0) - cmp %_ASM_BX,%_ASM_CX - jae .Lbad_put_user + check_range size=1 ASM_STAC 1: movb %al,(%_ASM_CX) xor %ecx,%ecx @@ -66,9 +66,7 @@ SYM_FUNC_END(__put_user_nocheck_1) EXPORT_SYMBOL(__put_user_nocheck_1) SYM_FUNC_START(__put_user_2) - LOAD_TASK_SIZE_MINUS_N(1) - cmp %_ASM_BX,%_ASM_CX - jae .Lbad_put_user + check_range size=2 ASM_STAC 3: movw %ax,(%_ASM_CX) xor %ecx,%ecx @@ -88,9 +86,7 @@ SYM_FUNC_END(__put_user_nocheck_2) EXPORT_SYMBOL(__put_user_nocheck_2) SYM_FUNC_START(__put_user_4) - LOAD_TASK_SIZE_MINUS_N(3) - cmp %_ASM_BX,%_ASM_CX - jae .Lbad_put_user + check_range size=4 ASM_STAC 5: movl %eax,(%_ASM_CX) xor %ecx,%ecx @@ -110,9 +106,7 @@ SYM_FUNC_END(__put_user_nocheck_4) EXPORT_SYMBOL(__put_user_nocheck_4) SYM_FUNC_START(__put_user_8) - LOAD_TASK_SIZE_MINUS_N(7) - cmp %_ASM_BX,%_ASM_CX - jae .Lbad_put_user + check_range size=8 ASM_STAC 7: mov %_ASM_AX,(%_ASM_CX) #ifdef CONFIG_X86_32 @@ -144,15 +138,15 @@ SYM_CODE_START_LOCAL(.Lbad_put_user_clac) RET SYM_CODE_END(.Lbad_put_user_clac) - _ASM_EXTABLE_UA(1b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(2b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(3b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(4b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(5b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(6b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(7b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(9b, .Lbad_put_user_clac) + _ASM_EXTABLE(1b, .Lbad_put_user_clac) + _ASM_EXTABLE(2b, .Lbad_put_user_clac) + _ASM_EXTABLE(3b, .Lbad_put_user_clac) + _ASM_EXTABLE(4b, .Lbad_put_user_clac) + _ASM_EXTABLE(5b, .Lbad_put_user_clac) + _ASM_EXTABLE(6b, .Lbad_put_user_clac) + _ASM_EXTABLE(7b, .Lbad_put_user_clac) + _ASM_EXTABLE(9b, .Lbad_put_user_clac) #ifdef CONFIG_X86_32 - _ASM_EXTABLE_UA(8b, .Lbad_put_user_clac) - _ASM_EXTABLE_UA(10b, .Lbad_put_user_clac) + _ASM_EXTABLE(8b, .Lbad_put_user_clac) + _ASM_EXTABLE(10b, .Lbad_put_user_clac) #endif diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 5f61c65322be..27ef53fab6bd 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -33,7 +33,7 @@ .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ @@ -75,7 +75,7 @@ SYM_CODE_END(__x86_indirect_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR CALL_DEPTH_ACCOUNT @@ -103,7 +103,7 @@ SYM_CODE_END(__x86_indirect_call_thunk_array) .align RETPOLINE_THUNK_SIZE SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR POLINE \reg ANNOTATE_UNRET_SAFE diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index c3a5bbc0b41e..003d90138e20 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -140,13 +140,4 @@ void __memcpy_flushcache(void *_dst, const void *_src, size_t size) } } EXPORT_SYMBOL_GPL(__memcpy_flushcache); - -void memcpy_page_flushcache(char *to, struct page *page, size_t offset, - size_t len) -{ - char *from = kmap_atomic(page); - - memcpy_flushcache(to, from + offset, len); - kunmap_atomic(from); -} #endif diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index cbc53da4c1b4..3cdac0f0055d 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -1048,6 +1048,11 @@ __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = { .cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */ }; +#ifdef CONFIG_ADDRESS_MASKING +DEFINE_PER_CPU(u64, tlbstate_untag_mask); +EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask); +#endif + void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) { /* entry 0 MUST be WB (hardwired to speed up translations) */ diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 16c5292d227d..267acf27480a 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -154,26 +154,30 @@ static inline u16 user_pcid(u16 asid) return ret; } -static inline unsigned long build_cr3(pgd_t *pgd, u16 asid) +static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam) { + unsigned long cr3 = __sme_pa(pgd) | lam; + if (static_cpu_has(X86_FEATURE_PCID)) { - return __sme_pa(pgd) | kern_pcid(asid); + VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); + cr3 |= kern_pcid(asid); } else { VM_WARN_ON_ONCE(asid != 0); - return __sme_pa(pgd); } + + return cr3; } -static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid) +static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid, + unsigned long lam) { - VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE); /* * Use boot_cpu_has() instead of this_cpu_has() as this function * might be called during early boot. This should work even after * boot because all CPU's the have same capabilities: */ VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID)); - return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH; + return build_cr3(pgd, asid, lam) | CR3_NOFLUSH; } /* @@ -274,15 +278,16 @@ static inline void invalidate_user_asid(u16 asid) (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask)); } -static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) +static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, unsigned long lam, + bool need_flush) { unsigned long new_mm_cr3; if (need_flush) { invalidate_user_asid(new_asid); - new_mm_cr3 = build_cr3(pgdir, new_asid); + new_mm_cr3 = build_cr3(pgdir, new_asid, lam); } else { - new_mm_cr3 = build_cr3_noflush(pgdir, new_asid); + new_mm_cr3 = build_cr3_noflush(pgdir, new_asid, lam); } /* @@ -491,6 +496,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, { struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm); u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid); + unsigned long new_lam = mm_lam_cr3_mask(next); bool was_lazy = this_cpu_read(cpu_tlbstate_shared.is_lazy); unsigned cpu = smp_processor_id(); u64 next_tlb_gen; @@ -520,7 +526,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, * isn't free. */ #ifdef CONFIG_DEBUG_VM - if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) { + if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid, + tlbstate_lam_cr3_mask()))) { /* * If we were to BUG here, we'd be very likely to kill * the system so hard that we don't see the call trace. @@ -552,9 +559,15 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, * instruction. */ if (real_prev == next) { + /* Not actually switching mm's */ VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) != next->context.ctx_id); + /* + * If this races with another thread that enables lam, 'new_lam' + * might not match tlbstate_lam_cr3_mask(). + */ + /* * Even in lazy TLB mode, the CPU should stay set in the * mm_cpumask. The TLB shootdown code can figure out from @@ -622,15 +635,16 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, barrier(); } + set_tlbstate_lam_mode(next); if (need_flush) { this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id); this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen); - load_new_mm_cr3(next->pgd, new_asid, true); + load_new_mm_cr3(next->pgd, new_asid, new_lam, true); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); } else { /* The new ASID is already up to date. */ - load_new_mm_cr3(next->pgd, new_asid, false); + load_new_mm_cr3(next->pgd, new_asid, new_lam, false); trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0); } @@ -691,6 +705,10 @@ void initialize_tlbstate_and_flush(void) /* Assert that CR3 already references the right mm. */ WARN_ON((cr3 & CR3_ADDR_MASK) != __pa(mm->pgd)); + /* LAM expected to be disabled */ + WARN_ON(cr3 & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57)); + WARN_ON(mm_lam_cr3_mask(mm)); + /* * Assert that CR4.PCIDE is set if needed. (CR4.PCIDE initialization * doesn't work like other CR4 bits because it can only be set from @@ -699,8 +717,8 @@ void initialize_tlbstate_and_flush(void) WARN_ON(boot_cpu_has(X86_FEATURE_PCID) && !(cr4_read_shadow() & X86_CR4_PCIDE)); - /* Force ASID 0 and force a TLB flush. */ - write_cr3(build_cr3(mm->pgd, 0)); + /* Disable LAM, force ASID 0 and force a TLB flush. */ + write_cr3(build_cr3(mm->pgd, 0, 0)); /* Reinitialize tlbstate. */ this_cpu_write(cpu_tlbstate.last_user_mm_spec, LAST_USER_MM_INIT); @@ -708,6 +726,7 @@ void initialize_tlbstate_and_flush(void) this_cpu_write(cpu_tlbstate.next_asid, 1); this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id); this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, tlb_gen); + set_tlbstate_lam_mode(mm); for (i = 1; i < TLB_NR_DYN_ASIDS; i++) this_cpu_write(cpu_tlbstate.ctxs[i].ctx_id, 0); @@ -1071,8 +1090,10 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) */ unsigned long __get_current_cr3_fast(void) { - unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd, - this_cpu_read(cpu_tlbstate.loaded_mm_asid)); + unsigned long cr3 = + build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd, + this_cpu_read(cpu_tlbstate.loaded_mm_asid), + tlbstate_lam_cr3_mask()); /* For now, be very restrictive about when this can be called. */ VM_WARN_ON(in_nmi() || preemptible()); diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 7fe564eaf228..c4365a05ab83 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -50,7 +50,7 @@ #define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8) SYM_CODE_START_LOCAL(pvh_start_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK cld lgdt (_pa(gdt)) diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 236447ee9beb..7a4d5e911415 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -288,7 +288,7 @@ EXPORT_SYMBOL(restore_processor_state); #endif #if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU) -static void resume_play_dead(void) +static void __noreturn resume_play_dead(void) { play_dead_common(); tboot_shutdown(TB_SHUTDOWN_WFS); diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 4a184f6e4e4d..08f1ceb9eb81 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -165,7 +165,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ENDBR pop %rcx pop %r11 @@ -193,7 +193,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 * rsp->rax } */ SYM_CODE_START(xen_iret) - UNWIND_HINT_EMPTY + UNWIND_HINT_UNDEFINED ANNOTATE_NOENDBR pushq $0 jmp hypercall_iret diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index 91f7a53519a7..643d02900fbb 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -45,7 +45,7 @@ SYM_CODE_END(hypercall_page) #ifdef CONFIG_XEN_PV __INIT SYM_CODE_START(startup_xen) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR cld @@ -71,7 +71,7 @@ SYM_CODE_END(startup_xen) #ifdef CONFIG_XEN_PV_SMP .pushsection .text SYM_CODE_START(asm_cpu_bringup_and_idle) - UNWIND_HINT_EMPTY + UNWIND_HINT_END_OF_STACK ENDBR call cpu_bringup_and_idle diff --git a/arch/xtensa/include/asm/smp.h b/arch/xtensa/include/asm/smp.h index 4e43f5643891..5dc5bf8cdd77 100644 --- a/arch/xtensa/include/asm/smp.h +++ b/arch/xtensa/include/asm/smp.h @@ -33,7 +33,7 @@ void show_ipi_list(struct seq_file *p, int prec); void __cpu_die(unsigned int cpu); int __cpu_disable(void); -void cpu_die(void); +void __noreturn cpu_die(void); void cpu_restart(void); #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c index 4dc109dd6214..054bd64eab19 100644 --- a/arch/xtensa/kernel/smp.c +++ b/arch/xtensa/kernel/smp.c @@ -322,7 +322,7 @@ void __cpu_die(unsigned int cpu) pr_err("CPU%u: unable to kill\n", cpu); } -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } @@ -341,6 +341,8 @@ void __ref cpu_die(void) __asm__ __volatile__( " movi a2, cpu_restart\n" " jx a2\n"); + + BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 24bf9b2b58aa..dd76a1a09cf7 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -2,6 +2,7 @@ /* * Helpers for IOMMU drivers implementing SVA */ +#include #include #include #include @@ -32,16 +33,19 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max) min == 0 || max < min) return -EINVAL; + if (!arch_pgtable_dma_compat(mm)) + return -EBUSY; + mutex_lock(&iommu_sva_lock); /* Is a PASID already associated with this mm? */ - if (pasid_valid(mm->pasid)) { + if (mm_valid_pasid(mm)) { if (mm->pasid < min || mm->pasid >= max) ret = -EOVERFLOW; goto out; } pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm); - if (!pasid_valid(pasid)) + if (pasid == INVALID_IOASID) ret = -ENOMEM; else mm_pasid_set(mm, pasid); diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c index 9b3ba2df71c7..4f0afce8428d 100644 --- a/drivers/message/fusion/mptbase.c +++ b/drivers/message/fusion/mptbase.c @@ -6935,7 +6935,7 @@ EXPORT_SYMBOL(mpt_clear_taskmgmt_in_progress_flag); * @ioc: Pointer to MPT_ADAPTER structure * **/ -void +void __noreturn mpt_halt_firmware(MPT_ADAPTER *ioc) { u32 ioc_raw_state; diff --git a/drivers/message/fusion/mptbase.h b/drivers/message/fusion/mptbase.h index e0ba783de368..8031173c3655 100644 --- a/drivers/message/fusion/mptbase.h +++ b/drivers/message/fusion/mptbase.h @@ -944,7 +944,7 @@ extern int mpt_raid_phys_disk_get_num_paths(MPT_ADAPTER *ioc, u8 phys_disk_num); extern int mpt_set_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc); extern void mpt_clear_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc); -extern void mpt_halt_firmware(MPT_ADAPTER *ioc); +extern void __noreturn mpt_halt_firmware(MPT_ADAPTER *ioc); /* diff --git a/drivers/misc/lkdtm/stackleak.c b/drivers/misc/lkdtm/stackleak.c index 025b133297a6..f1d022160913 100644 --- a/drivers/misc/lkdtm/stackleak.c +++ b/drivers/misc/lkdtm/stackleak.c @@ -43,12 +43,14 @@ static void noinstr check_stackleak_irqoff(void) * STACK_END_MAGIC, and in either casee something is seriously wrong. */ if (current_sp < task_stack_low || current_sp >= task_stack_high) { + instrumentation_begin(); pr_err("FAIL: current_stack_pointer (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n", current_sp, task_stack_low, task_stack_high - 1); test_failed = true; goto out; } if (lowest_sp < task_stack_low || lowest_sp >= task_stack_high) { + instrumentation_begin(); pr_err("FAIL: current->lowest_stack (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n", lowest_sp, task_stack_low, task_stack_high - 1); test_failed = true; @@ -86,11 +88,14 @@ static void noinstr check_stackleak_irqoff(void) if (*(unsigned long *)poison_low == STACKLEAK_POISON) continue; + instrumentation_begin(); pr_err("FAIL: non-poison value %lu bytes below poison boundary: 0x%lx\n", poison_high - poison_low, *(unsigned long *)poison_low); test_failed = true; + goto out; } + instrumentation_begin(); pr_info("stackleak stack usage:\n" " high offset: %lu bytes\n" " current: %lu bytes\n" @@ -113,6 +118,7 @@ out: } else { pr_info("OK: the rest of the thread stack is properly erased\n"); } + instrumentation_end(); } static void lkdtm_STACKLEAK_ERASING(void) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 493c31de0edb..3d4dd9420c30 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -580,7 +580,7 @@ static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, goto done; } - vaddr = untagged_addr(vaddr); + vaddr = untagged_addr_remote(mm, vaddr); retry: vma = vma_lookup(mm, vaddr); diff --git a/fs/proc/array.c b/fs/proc/array.c index 425824ad85e1..d35bbf35a874 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include "internal.h" @@ -425,6 +426,11 @@ static inline void task_thp_status(struct seq_file *m, struct mm_struct *mm) seq_printf(m, "THP_enabled:\t%d\n", thp_enabled); } +static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) +{ + seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm)); +} + int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task) { @@ -440,6 +446,7 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, task_mem(m, mm); task_core_dumping(m, task); task_thp_status(m, mm); + task_untag_mask(m, mm); mmput(mm); } task_sig(m, task); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index cb49479acd2e..420510f6a545 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1688,8 +1688,13 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) - start_vaddr = untagged_addr(svpfn << PAGE_SHIFT); + if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + ret = mmap_read_lock_killable(mm); + if (ret) + goto out_free; + start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + mmap_read_unlock(mm); + } /* Ensure the address is inside the task */ if (start_vaddr > mm->task_size) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 3a7909ed5498..d3cbb6c16bab 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -98,7 +98,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) { } static inline int ct_state(void) { return -1; } static inline int __ct_state(void) { return -1; } static __always_inline bool context_tracking_guest_enter(void) { return false; } -static inline void context_tracking_guest_exit(void) { } +static __always_inline void context_tracking_guest_exit(void) { } #define CT_WARN_ON(cond) do { } while (0) #endif /* !CONFIG_CONTEXT_TRACKING_USER */ diff --git a/include/linux/cpu.h b/include/linux/cpu.h index f83e4519c5f0..8582a7142623 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -182,7 +182,7 @@ void arch_cpu_idle(void); void arch_cpu_idle_prepare(void); void arch_cpu_idle_enter(void); void arch_cpu_idle_exit(void); -void arch_cpu_idle_dead(void); +void __noreturn arch_cpu_idle_dead(void); int cpu_report_state(int cpu); int cpu_check_up_prepare(int cpu); diff --git a/include/linux/ioasid.h b/include/linux/ioasid.h index af1c9d62e642..836ae09e92c2 100644 --- a/include/linux/ioasid.h +++ b/include/linux/ioasid.h @@ -40,10 +40,6 @@ void *ioasid_find(struct ioasid_set *set, ioasid_t ioasid, int ioasid_register_allocator(struct ioasid_allocator_ops *allocator); void ioasid_unregister_allocator(struct ioasid_allocator_ops *allocator); int ioasid_set_data(ioasid_t ioasid, void *data); -static inline bool pasid_valid(ioasid_t ioasid) -{ - return ioasid != INVALID_IOASID; -} #else /* !CONFIG_IOASID */ static inline ioasid_t ioasid_alloc(struct ioasid_set *set, ioasid_t min, @@ -74,10 +70,5 @@ static inline int ioasid_set_data(ioasid_t ioasid, void *data) return -ENOTSUPP; } -static inline bool pasid_valid(ioasid_t ioasid) -{ - return false; -} - #endif /* CONFIG_IOASID */ #endif /* __LINUX_IOASID_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index 3731999cd9f0..27ce77080c79 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -98,17 +98,6 @@ extern int mmap_rnd_compat_bits __read_mostly; #include #include -/* - * Architectures that support memory tagging (assigning tags to memory regions, - * embedding these tags into addresses that point to these memory regions, and - * checking that the memory and the pointer tags match on memory accesses) - * redefine this macro to strip tags from pointers. - * It's defined as noop for architectures that don't support memory tagging. - */ -#ifndef untagged_addr -#define untagged_addr(addr) (addr) -#endif - #ifndef __pa_symbol #define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0)) #endif diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index b9b970f7ab45..f2b7a3f04099 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -28,4 +28,18 @@ static inline void leave_mm(int cpu) { } # define task_cpu_possible(cpu, p) cpumask_test_cpu((cpu), task_cpu_possible_mask(p)) #endif +#ifndef mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL; +} +#endif + +#ifndef arch_pgtable_dma_compat +static inline bool arch_pgtable_dma_compat(struct mm_struct *mm) +{ + return true; +} +#endif + #endif diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 9ac3df3fccf0..03f82c2c2ebf 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -2,47 +2,7 @@ #ifndef _LINUX_OBJTOOL_H #define _LINUX_OBJTOOL_H -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 +#include #ifdef CONFIG_OBJTOOL @@ -50,7 +10,7 @@ struct unwind_hint { #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ /* struct unwind_hint */ \ @@ -59,7 +19,6 @@ struct unwind_hint { ".byte " __stringify(sp_reg) "\n\t" \ ".byte " __stringify(type) "\n\t" \ ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ ".balign 4 \n\t" \ ".popsection\n\t" @@ -89,7 +48,7 @@ struct unwind_hint { #define ANNOTATE_NOENDBR \ "986: \n\t" \ ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ + ".long 986b - .\n\t" \ ".popsection\n\t" #define ASM_REACHABLE \ @@ -107,7 +66,7 @@ struct unwind_hint { #define ANNOTATE_INTRA_FUNCTION_CALL \ 999: \ .pushsection .discard.intra_function_calls; \ - .long 999b; \ + .long 999b - .; \ .popsection; /* @@ -131,23 +90,22 @@ struct unwind_hint { * the debuginfo as necessary. It will also warn if it sees any * inconsistencies. */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 +.Lhere_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . + .long .Lhere_\@ - . .short \sp_offset .byte \sp_reg .byte \type .byte \signal - .byte \end .balign 4 .popsection .endm .macro STACK_FRAME_NON_STANDARD func:req .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func + .long \func - . .popsection .endm @@ -160,10 +118,26 @@ struct unwind_hint { .macro ANNOTATE_NOENDBR .Lhere_\@: .pushsection .discard.noendbr - .quad .Lhere_\@ + .long .Lhere_\@ - . .popsection .endm +/* + * Use objtool to validate the entry requirement that all code paths do + * VALIDATE_UNRET_END before RET. + * + * NOTE: The macro must be used at the beginning of a global symbol, otherwise + * it will be ignored. + */ +.macro VALIDATE_UNRET_BEGIN +#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY) +.Lhere_\@: + .pushsection .discard.validate_unret + .long .Lhere_\@ - . + .popsection +#endif +.endm + .macro REACHABLE .Lhere_\@: .pushsection .discard.reachable @@ -177,15 +151,14 @@ struct unwind_hint { #ifndef __ASSEMBLY__ -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" +#define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" #define STACK_FRAME_NON_STANDARD(func) #define STACK_FRAME_NON_STANDARD_FP(func) #define ANNOTATE_NOENDBR #define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h new file mode 100644 index 000000000000..453a4f4ef39d --- /dev/null +++ b/include/linux/objtool_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. + */ +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 +/* The below hint types don't have corresponding ORC types */ +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 689dbe812563..af12fcb11005 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -485,6 +485,11 @@ static inline void mm_pasid_init(struct mm_struct *mm) mm->pasid = INVALID_IOASID; } +static inline bool mm_valid_pasid(struct mm_struct *mm) +{ + return mm->pasid != INVALID_IOASID; +} + /* Associate a PASID with an mm_struct: */ static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) { @@ -493,13 +498,14 @@ static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) static inline void mm_pasid_drop(struct mm_struct *mm) { - if (pasid_valid(mm->pasid)) { + if (mm_valid_pasid(mm)) { ioasid_free(mm->pasid); mm->pasid = INVALID_IOASID; } } #else static inline void mm_pasid_init(struct mm_struct *mm) {} +static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; } static inline void mm_pasid_set(struct mm_struct *mm, u32 pasid) {} static inline void mm_pasid_drop(struct mm_struct *mm) {} #endif diff --git a/include/linux/sched/task_stack.h b/include/linux/sched/task_stack.h index 5e799a47431e..f158b025c175 100644 --- a/include/linux/sched/task_stack.h +++ b/include/linux/sched/task_stack.h @@ -23,7 +23,7 @@ static __always_inline void *task_stack_page(const struct task_struct *task) #define setup_thread_stack(new,old) do { } while(0) -static inline unsigned long *end_of_stack(const struct task_struct *task) +static __always_inline unsigned long *end_of_stack(const struct task_struct *task) { #ifdef CONFIG_STACK_GROWSUP return (unsigned long *)((unsigned long)task->stack + THREAD_SIZE) - 1; diff --git a/include/linux/smp.h b/include/linux/smp.h index a80ab58ae3f1..7b93504eed26 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -59,8 +59,8 @@ int smp_call_function_single_async(int cpu, struct __call_single_data *csd); * Cpus stopping functions in panic. All have default weak definitions. * Architecture-dependent code may override them. */ -void panic_smp_self_stop(void); -void nmi_panic_self_stop(struct pt_regs *regs); +void __noreturn panic_smp_self_stop(void); +void __noreturn nmi_panic_self_stop(struct pt_regs *regs); void crash_smp_send_stop(void); /* diff --git a/include/linux/start_kernel.h b/include/linux/start_kernel.h index 8b369a41c03c..a9806a44a605 100644 --- a/include/linux/start_kernel.h +++ b/include/linux/start_kernel.h @@ -8,8 +8,8 @@ /* Define the prototype for start_kernel here, rather than cluttering up something else. */ -extern asmlinkage void __init start_kernel(void); -extern void __init arch_call_rest_init(void); -extern void __ref rest_init(void); +extern asmlinkage void __init __noreturn start_kernel(void); +extern void __init __noreturn arch_call_rest_init(void); +extern void __ref __noreturn rest_init(void); #endif /* _LINUX_START_KERNEL_H */ diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index ab9728138ad6..3064314f4832 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -10,6 +10,28 @@ #include +/* + * Architectures that support memory tagging (assigning tags to memory regions, + * embedding these tags into addresses that point to these memory regions, and + * checking that the memory and the pointer tags match on memory accesses) + * redefine this macro to strip tags from pointers. + * + * Passing down mm_struct allows to define untagging rules on per-process + * basis. + * + * It's defined as noop for architectures that don't support memory tagging. + */ +#ifndef untagged_addr +#define untagged_addr(addr) (addr) +#endif + +#ifndef untagged_addr_remote +#define untagged_addr_remote(mm, addr) ({ \ + mmap_assert_locked(mm); \ + untagged_addr(addr); \ +}) +#endif + /* * Architectures should provide two primitives (raw_copy_{to,from}_user()) * and get rid of their private instances of copy_{to,from}_user() and diff --git a/init/main.c b/init/main.c index ca986a391c9b..de6dbb494cec 100644 --- a/init/main.c +++ b/init/main.c @@ -684,7 +684,7 @@ static void __init setup_command_line(char *command_line) static __initdata DECLARE_COMPLETION(kthreadd_done); -noinline void __ref rest_init(void) +noinline void __ref __noreturn rest_init(void) { struct task_struct *tsk; int pid; @@ -827,7 +827,7 @@ static int __init early_randomize_kstack_offset(char *buf) early_param("randomize_kstack_offset", early_randomize_kstack_offset); #endif -void __init __weak arch_call_rest_init(void) +void __init __weak __noreturn arch_call_rest_init(void) { rest_init(); } @@ -875,7 +875,7 @@ static void __init print_unknown_bootoptions(void) memblock_free(unknown_options, len); } -asmlinkage __visible void __init __no_sanitize_address start_kernel(void) +asmlinkage __visible void __init __no_sanitize_address __noreturn start_kernel(void) { char *command_line; char *after_dashes; diff --git a/kernel/panic.c b/kernel/panic.c index 5cfea8302d23..886d2ebd0a0d 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -141,7 +141,7 @@ EXPORT_SYMBOL(panic_blink); /* * Stop ourself in panic -- architecture code may override this */ -void __weak panic_smp_self_stop(void) +void __weak __noreturn panic_smp_self_stop(void) { while (1) cpu_relax(); @@ -151,7 +151,7 @@ void __weak panic_smp_self_stop(void) * Stop ourselves in NMI context if another CPU has already panicked. Arch code * may override this to prepare for crash dumping, e.g. save regs info. */ -void __weak nmi_panic_self_stop(struct pt_regs *regs) +void __weak __noreturn nmi_panic_self_stop(struct pt_regs *regs) { panic_smp_self_stop(); } diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index e9ef66be2870..342f58a329f5 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -75,7 +75,7 @@ static noinline int __cpuidle cpu_idle_poll(void) void __weak arch_cpu_idle_prepare(void) { } void __weak arch_cpu_idle_enter(void) { } void __weak arch_cpu_idle_exit(void) { } -void __weak arch_cpu_idle_dead(void) { } +void __weak __noreturn arch_cpu_idle_dead(void) { while (1); } void __weak arch_cpu_idle(void) { cpu_idle_force_poll = 1; diff --git a/mm/gup.c b/mm/gup.c index 1f72a717232b..ff689c88a357 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1085,7 +1085,7 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; - start = untagged_addr(start); + start = untagged_addr_remote(mm, start); VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1259,7 +1259,7 @@ int fixup_user_fault(struct mm_struct *mm, struct vm_area_struct *vma; vm_fault_t ret; - address = untagged_addr(address); + address = untagged_addr_remote(mm, address); if (unlocked) fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; diff --git a/mm/madvise.c b/mm/madvise.c index 24c5cffe3e6c..b5ffbaf616f5 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1390,8 +1390,6 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh size_t len; struct blk_plug plug; - start = untagged_addr(start); - if (!madvise_behavior_valid(behavior)) return -EINVAL; @@ -1423,6 +1421,9 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh mmap_read_lock(mm); } + start = untagged_addr_remote(mm, start); + end = start + len; + blk_start_plug(&plug); error = madvise_walk_vmas(mm, start, end, behavior, madvise_vma_behavior); diff --git a/mm/migrate.c b/mm/migrate.c index 02cace7955d4..01cac26a3127 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2099,15 +2099,18 @@ static int do_move_pages_to_node(struct mm_struct *mm, * target node * 1 - when it has been queued */ -static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, +static int add_page_for_migration(struct mm_struct *mm, const void __user *p, int node, struct list_head *pagelist, bool migrate_all) { struct vm_area_struct *vma; + unsigned long addr; struct page *page; int err; bool isolated; mmap_read_lock(mm); + addr = (unsigned long)untagged_addr_remote(mm, p); + err = -EFAULT; vma = vma_lookup(mm, addr); if (!vma || !vma_migratable(vma)) @@ -2213,7 +2216,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, for (i = start = 0; i < nr_pages; i++) { const void __user *p; - unsigned long addr; int node; err = -EFAULT; @@ -2221,7 +2223,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, goto out_flush; if (get_user(node, nodes + i)) goto out_flush; - addr = (unsigned long)untagged_addr(p); err = -ENODEV; if (node < 0 || node >= MAX_NUMNODES) @@ -2249,8 +2250,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, * Errors in the page lookup or isolation are not fatal and we simply * report them via status */ - err = add_page_for_migration(mm, addr, current_node, - &pagelist, flags & MPOL_MF_MOVE_ALL); + err = add_page_for_migration(mm, p, current_node, &pagelist, + flags & MPOL_MF_MOVE_ALL); if (err > 0) { /* The page is successfully queued for migration */ diff --git a/scripts/objdump-func b/scripts/objdump-func index 4eb463dd9f52..7b15b873d0e2 100755 --- a/scripts/objdump-func +++ b/scripts/objdump-func @@ -3,7 +3,7 @@ # # Disassemble a single function. # -# usage: objdump-func +# usage: objdump-func [ ...] set -o errexit set -o nounset @@ -13,17 +13,33 @@ OBJDUMP="${CROSS_COMPILE:-}objdump" command -v gawk >/dev/null 2>&1 || die "gawk isn't installed" usage() { - echo "usage: objdump-func " >&2 + echo "usage: objdump-func [ ...]" >&2 exit 1 } [[ $# -lt 2 ]] && usage OBJ=$1; shift -FUNC=$1; shift +FUNCS=("$@") -# Secret feature to allow adding extra objdump args at the end -EXTRA_ARGS=$@ - -# Note this also matches compiler-added suffixes like ".cold", etc -${OBJDUMP} -wdr $EXTRA_ARGS $OBJ | gawk -M -v f=$FUNC '/^$/ { P=0; } $0 ~ "<" f "(\\..*)?>:" { P=1; O=strtonum("0x" $1); } { if (P) { o=strtonum("0x" $1); printf("%04x ", o-O); print $0; } }' +${OBJDUMP} -wdr $OBJ | gawk -M -v _funcs="${FUNCS[*]}" ' + BEGIN { split(_funcs, funcs); } + /^$/ { func_match=0; } + /<.*>:/ { + f = gensub(/.*<(.*)>:/, "\\1", 1); + for (i in funcs) { + # match compiler-added suffixes like ".cold", etc + if (f ~ "^" funcs[i] "(\\..*)?") { + func_match = 1; + base = strtonum("0x" $1); + break; + } + } + } + { + if (func_match) { + addr = strtonum("0x" $1); + printf("%04x ", addr - base); + print; + } + }' diff --git a/scripts/sorttable.h b/scripts/sorttable.h index deb7c1d3e979..7bd0184380d3 100644 --- a/scripts/sorttable.h +++ b/scripts/sorttable.h @@ -128,7 +128,7 @@ static int orc_sort_cmp(const void *_a, const void *_b) * whitelisted .o files which didn't get objtool generation. */ orc_a = g_orc_table + (a - g_orc_ip_table); - return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; + return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; } static void *sort_orctable(void *arg) diff --git a/tools/arch/x86/include/asm/orc_types.h b/tools/arch/x86/include/asm/orc_types.h index 1343a62106de..46d7e06763c9 100644 --- a/tools/arch/x86/include/asm/orc_types.h +++ b/tools/arch/x86/include/asm/orc_types.h @@ -39,6 +39,12 @@ #define ORC_REG_SP_INDIRECT 9 #define ORC_REG_MAX 15 +#define ORC_TYPE_UNDEFINED 0 +#define ORC_TYPE_END_OF_STACK 1 +#define ORC_TYPE_CALL 2 +#define ORC_TYPE_REGS 3 +#define ORC_TYPE_REGS_PARTIAL 4 + #ifndef __ASSEMBLY__ #include @@ -56,16 +62,14 @@ struct orc_entry { #if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; - unsigned type:2; + unsigned type:3; unsigned signal:1; - unsigned end:1; #elif defined(__BIG_ENDIAN_BITFIELD) unsigned bp_reg:4; unsigned sp_reg:4; unsigned unused:4; - unsigned end:1; unsigned signal:1; - unsigned type:2; + unsigned type:3; #endif } __packed; diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h deleted file mode 100644 index 9ac3df3fccf0..000000000000 --- a/tools/include/linux/objtool.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_OBJTOOL_H -#define _LINUX_OBJTOOL_H - -#ifndef __ASSEMBLY__ - -#include - -/* - * This struct is used by asm and inline asm code to manually annotate the - * location of registers on the stack. - */ -struct unwind_hint { - u32 ip; - s16 sp_offset; - u8 sp_reg; - u8 type; - u8 signal; - u8 end; -}; -#endif - -/* - * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP - * (the caller's SP right before it made the call). Used for all callable - * functions, i.e. all C code and all callable asm functions. - * - * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset - * points to a fully populated pt_regs from a syscall, interrupt, or exception. - * - * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that - * sp_reg+sp_offset points to the iret return frame. - * - * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. - * Useful for code which doesn't have an ELF function annotation. - * - * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. - */ -#define UNWIND_HINT_TYPE_CALL 0 -#define UNWIND_HINT_TYPE_REGS 1 -#define UNWIND_HINT_TYPE_REGS_PARTIAL 2 -#define UNWIND_HINT_TYPE_FUNC 3 -#define UNWIND_HINT_TYPE_ENTRY 4 -#define UNWIND_HINT_TYPE_SAVE 5 -#define UNWIND_HINT_TYPE_RESTORE 6 - -#ifdef CONFIG_OBJTOOL - -#include - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "987: \n\t" \ - ".pushsection .discard.unwind_hints\n\t" \ - /* struct unwind_hint */ \ - ".long 987b - .\n\t" \ - ".short " __stringify(sp_offset) "\n\t" \ - ".byte " __stringify(sp_reg) "\n\t" \ - ".byte " __stringify(type) "\n\t" \ - ".byte " __stringify(signal) "\n\t" \ - ".byte " __stringify(end) "\n\t" \ - ".balign 4 \n\t" \ - ".popsection\n\t" - -/* - * This macro marks the given function's stack frame as "non-standard", which - * tells objtool to ignore the function when doing stack metadata validation. - * It should only be used in special cases where you're 100% sure it won't - * affect the reliability of frame pointers and kernel stack traces. - * - * For more information, see tools/objtool/Documentation/objtool.txt. - */ -#define STACK_FRAME_NON_STANDARD(func) \ - static void __used __section(".discard.func_stack_frame_non_standard") \ - *__func_stack_frame_non_standard_##func = func - -/* - * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore - * for the case where a function is intentionally missing frame pointer setup, - * but otherwise needs objtool/ORC coverage when frame pointers are disabled. - */ -#ifdef CONFIG_FRAME_POINTER -#define STACK_FRAME_NON_STANDARD_FP(func) STACK_FRAME_NON_STANDARD(func) -#else -#define STACK_FRAME_NON_STANDARD_FP(func) -#endif - -#define ANNOTATE_NOENDBR \ - "986: \n\t" \ - ".pushsection .discard.noendbr\n\t" \ - _ASM_PTR " 986b\n\t" \ - ".popsection\n\t" - -#define ASM_REACHABLE \ - "998:\n\t" \ - ".pushsection .discard.reachable\n\t" \ - ".long 998b - .\n\t" \ - ".popsection\n\t" - -#else /* __ASSEMBLY__ */ - -/* - * This macro indicates that the following intra-function call is valid. - * Any non-annotated intra-function call will cause objtool to issue a warning. - */ -#define ANNOTATE_INTRA_FUNCTION_CALL \ - 999: \ - .pushsection .discard.intra_function_calls; \ - .long 999b; \ - .popsection; - -/* - * In asm, there are two kinds of code: normal C-type callable functions and - * the rest. The normal callable functions can be called by other code, and - * don't do anything unusual with the stack. Such normal callable functions - * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this - * category. In this case, no special debugging annotations are needed because - * objtool can automatically generate the ORC data for the ORC unwinder to read - * at runtime. - * - * Anything which doesn't fall into the above category, such as syscall and - * interrupt handlers, tends to not be called directly by other functions, and - * often does unusual non-C-function-type things with the stack pointer. Such - * code needs to be annotated such that objtool can understand it. The - * following CFI hint macros are for this type of code. - * - * These macros provide hints to objtool about the state of the stack at each - * instruction. Objtool starts from the hints and follows the code flow, - * making automatic CFI adjustments when it sees pushes and pops, filling out - * the debuginfo as necessary. It will also warn if it sees any - * inconsistencies. - */ -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.Lunwind_hint_ip_\@: - .pushsection .discard.unwind_hints - /* struct unwind_hint */ - .long .Lunwind_hint_ip_\@ - . - .short \sp_offset - .byte \sp_reg - .byte \type - .byte \signal - .byte \end - .balign 4 - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD func:req - .pushsection .discard.func_stack_frame_non_standard, "aw" - _ASM_PTR \func - .popsection -.endm - -.macro STACK_FRAME_NON_STANDARD_FP func:req -#ifdef CONFIG_FRAME_POINTER - STACK_FRAME_NON_STANDARD \func -#endif -.endm - -.macro ANNOTATE_NOENDBR -.Lhere_\@: - .pushsection .discard.noendbr - .quad .Lhere_\@ - .popsection -.endm - -.macro REACHABLE -.Lhere_\@: - .pushsection .discard.reachable - .long .Lhere_\@ - . - .popsection -.endm - -#endif /* __ASSEMBLY__ */ - -#else /* !CONFIG_OBJTOOL */ - -#ifndef __ASSEMBLY__ - -#define UNWIND_HINT(sp_reg, sp_offset, type, signal, end) \ - "\n\t" -#define STACK_FRAME_NON_STANDARD(func) -#define STACK_FRAME_NON_STANDARD_FP(func) -#define ANNOTATE_NOENDBR -#define ASM_REACHABLE -#else -#define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 signal=0 end=0 -.endm -.macro STACK_FRAME_NON_STANDARD func:req -.endm -.macro ANNOTATE_NOENDBR -.endm -.macro REACHABLE -.endm -#endif - -#endif /* CONFIG_OBJTOOL */ - -#endif /* _LINUX_OBJTOOL_H */ diff --git a/tools/include/linux/objtool_types.h b/tools/include/linux/objtool_types.h new file mode 100644 index 000000000000..453a4f4ef39d --- /dev/null +++ b/tools/include/linux/objtool_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_OBJTOOL_TYPES_H +#define _LINUX_OBJTOOL_TYPES_H + +#ifndef __ASSEMBLY__ + +#include + +/* + * This struct is used by asm and inline asm code to manually annotate the + * location of registers on the stack. + */ +struct unwind_hint { + u32 ip; + s16 sp_offset; + u8 sp_reg; + u8 type; + u8 signal; +}; + +#endif /* __ASSEMBLY__ */ + +/* + * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in + * a truncated and unreliable stack unwind. + * + * UNWIND_HINT_TYPE_END_OF_STACK: The end of the kernel stack unwind before + * hitting user entry, boot code, or fork entry (when there are no pt_regs + * available). + * + * UNWIND_HINT_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP + * (the caller's SP right before it made the call). Used for all callable + * functions, i.e. all C code and all callable asm functions. + * + * UNWIND_HINT_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset + * points to a fully populated pt_regs from a syscall, interrupt, or exception. + * + * UNWIND_HINT_TYPE_REGS_PARTIAL: Used in entry code to indicate that + * sp_reg+sp_offset points to the iret return frame. + * + * UNWIND_HINT_TYPE_FUNC: Generate the unwind metadata of a callable function. + * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_TYPE_{SAVE,RESTORE}: Save the unwind metadata at a certain + * location so that it can be restored later. + */ +#define UNWIND_HINT_TYPE_UNDEFINED 0 +#define UNWIND_HINT_TYPE_END_OF_STACK 1 +#define UNWIND_HINT_TYPE_CALL 2 +#define UNWIND_HINT_TYPE_REGS 3 +#define UNWIND_HINT_TYPE_REGS_PARTIAL 4 +/* The below hint types don't have corresponding ORC types */ +#define UNWIND_HINT_TYPE_FUNC 5 +#define UNWIND_HINT_TYPE_SAVE 6 +#define UNWIND_HINT_TYPE_RESTORE 7 + +#endif /* _LINUX_OBJTOOL_TYPES_H */ diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 931cdb7dba19..0fcf99c91400 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include #include @@ -202,6 +202,8 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "__reiserfs_panic", "__stack_chk_fail", "__ubsan_handle_builtin_unreachable", + "arch_call_rest_init", + "arch_cpu_idle_dead", "btrfs_assertfail", "cpu_bringup_and_idle", "cpu_startup_entry", @@ -210,18 +212,28 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, "do_task_dead", "ex_handler_msr_mce", "fortify_panic", + "hlt_play_dead", + "hv_ghcb_terminate", "kthread_complete_and_exit", "kthread_exit", "kunit_try_catch_throw", "lbug_with_loc", "machine_real_restart", "make_task_dead", + "mpt_halt_firmware", + "nmi_panic_self_stop", "panic", + "panic_smp_self_stop", + "rest_init", + "resume_play_dead", "rewind_stack_and_make_dead", "sev_es_terminate", "snp_abort", + "start_kernel", "stop_this_cpu", "usercopy_abort", + "x86_64_start_kernel", + "x86_64_start_reservations", "xen_cpu_bringup_again", "xen_start_kernel", }; @@ -229,14 +241,14 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, if (!func) return false; - if (func->bind == STB_WEAK) - return false; - - if (func->bind == STB_GLOBAL) + if (func->bind == STB_GLOBAL || func->bind == STB_WEAK) for (i = 0; i < ARRAY_SIZE(global_noreturns); i++) if (!strcmp(func->name, global_noreturns[i])) return true; + if (func->bind == STB_WEAK) + return false; + if (!func->len) return false; @@ -470,7 +482,7 @@ static int decode_instructions(struct objtool_file *file) // printf("%s: last chunk used: %d\n", sec->name, (int)idx); - list_for_each_entry(func, &sec->symbol_list, list) { + sec_for_each_sym(sec, func) { if (func->type != STT_NOTYPE && func->type != STT_FUNC) continue; @@ -924,7 +936,7 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) static int create_cfi_sections(struct objtool_file *file) { - struct section *sec, *s; + struct section *sec; struct symbol *sym; unsigned int *loc; int idx; @@ -937,19 +949,14 @@ static int create_cfi_sections(struct objtool_file *file) } idx = 0; - for_each_sec(file, s) { - if (!s->text) + for_each_sym(file, sym) { + if (sym->type != STT_FUNC) continue; - list_for_each_entry(sym, &s->symbol_list, list) { - if (sym->type != STT_FUNC) - continue; + if (strncmp(sym->name, "__cfi_", 6)) + continue; - if (strncmp(sym->name, "__cfi_", 6)) - continue; - - idx++; - } + idx++; } sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx); @@ -957,28 +964,23 @@ static int create_cfi_sections(struct objtool_file *file) return -1; idx = 0; - for_each_sec(file, s) { - if (!s->text) + for_each_sym(file, sym) { + if (sym->type != STT_FUNC) continue; - list_for_each_entry(sym, &s->symbol_list, list) { - if (sym->type != STT_FUNC) - continue; + if (strncmp(sym->name, "__cfi_", 6)) + continue; - if (strncmp(sym->name, "__cfi_", 6)) - continue; + loc = (unsigned int *)sec->data->d_buf + idx; + memset(loc, 0, sizeof(unsigned int)); - loc = (unsigned int *)sec->data->d_buf + idx; - memset(loc, 0, sizeof(unsigned int)); + if (elf_add_reloc_to_insn(file->elf, sec, + idx * sizeof(unsigned int), + R_X86_64_PC32, + sym->sec, sym->offset)) + return -1; - if (elf_add_reloc_to_insn(file->elf, sec, - idx * sizeof(unsigned int), - R_X86_64_PC32, - s, sym->offset)) - return -1; - - idx++; - } + idx++; } return 0; @@ -1279,6 +1281,8 @@ static const char *uaccess_safe_builtin[] = { "__ubsan_handle_type_mismatch_v1", "__ubsan_handle_shift_out_of_bounds", "__ubsan_handle_load_invalid_value", + /* STACKLEAK */ + "stackleak_track_stack", /* misc */ "csum_partial_copy_generic", "copy_mc_fragile", @@ -1444,7 +1448,7 @@ static void annotate_call_site(struct objtool_file *file, if (opts.mcount && sym->fentry) { if (sibling) - WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); + WARN_INSN(insn, "tail call to __fentry__ !?!?"); if (opts.mnop) { if (reloc) { reloc->type = R_NONE; @@ -1646,9 +1650,8 @@ static int add_jump_destinations(struct objtool_file *file) continue; } - WARN_FUNC("can't find jump dest instruction at %s+0x%lx", - insn->sec, insn->offset, dest_sec->name, - dest_off); + WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx", + dest_sec->name, dest_off); return -1; } @@ -1731,13 +1734,12 @@ static int add_call_destinations(struct objtool_file *file) continue; if (!insn_call_dest(insn)) { - WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset); + WARN_INSN(insn, "unannotated intra-function call"); return -1; } if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) { - WARN_FUNC("unsupported call to non-function", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported call to non-function"); return -1; } @@ -1745,10 +1747,8 @@ static int add_call_destinations(struct objtool_file *file) dest_off = arch_dest_reloc_offset(reloc->addend); dest = find_call_destination(reloc->sym->sec, dest_off); if (!dest) { - WARN_FUNC("can't find call dest symbol at %s+0x%lx", - insn->sec, insn->offset, - reloc->sym->sec->name, - dest_off); + WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx", + reloc->sym->sec->name, dest_off); return -1; } @@ -1808,8 +1808,7 @@ static int handle_group_alt(struct objtool_file *file, } else { if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - orig_alt_group->first_insn->offset != special_alt->orig_len) { - WARN_FUNC("weirdly overlapping alternative! %ld != %d", - orig_insn->sec, orig_insn->offset, + WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d", orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len - orig_alt_group->first_insn->offset, @@ -1878,8 +1877,7 @@ static int handle_group_alt(struct objtool_file *file, if (alt_reloc && arch_pc_relative_reloc(alt_reloc) && !arch_support_alt_relocation(special_alt, insn, alt_reloc)) { - WARN_FUNC("unsupported relocation in alternatives section", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported relocation in alternatives section"); return -1; } @@ -1893,8 +1891,7 @@ static int handle_group_alt(struct objtool_file *file, if (dest_off == special_alt->new_off + special_alt->new_len) { insn->jump_dest = next_insn_same_sec(file, orig_alt_group->last_insn); if (!insn->jump_dest) { - WARN_FUNC("can't find alternative jump destination", - insn->sec, insn->offset); + WARN_INSN(insn, "can't find alternative jump destination"); return -1; } } @@ -1928,8 +1925,7 @@ static int handle_jump_alt(struct objtool_file *file, if (orig_insn->type != INSN_JUMP_UNCONDITIONAL && orig_insn->type != INSN_NOP) { - WARN_FUNC("unsupported instruction at jump label", - orig_insn->sec, orig_insn->offset); + WARN_INSN(orig_insn, "unsupported instruction at jump label"); return -1; } @@ -2008,8 +2004,7 @@ static int add_special_section_alts(struct objtool_file *file) if (special_alt->group) { if (!special_alt->orig_len) { - WARN_FUNC("empty alternative entry", - orig_insn->sec, orig_insn->offset); + WARN_INSN(orig_insn, "empty alternative entry"); continue; } @@ -2100,8 +2095,7 @@ static int add_jump_table(struct objtool_file *file, struct instruction *insn, } if (!prev_offset) { - WARN_FUNC("can't find switch jump table", - insn->sec, insn->offset); + WARN_INSN(insn, "can't find switch jump table"); return -1; } @@ -2215,23 +2209,20 @@ static int add_func_jump_tables(struct objtool_file *file, */ static int add_jump_table_alts(struct objtool_file *file) { - struct section *sec; struct symbol *func; int ret; if (!file->rodata) return 0; - for_each_sec(file, sec) { - list_for_each_entry(func, &sec->symbol_list, list) { - if (func->type != STT_FUNC) - continue; + for_each_sym(file, func) { + if (func->type != STT_FUNC) + continue; - mark_func_jump_tables(file, func); - ret = add_func_jump_tables(file, func); - if (ret) - return ret; - } + mark_func_jump_tables(file, func); + ret = add_func_jump_tables(file, func); + if (ret) + return ret; } return 0; @@ -2243,6 +2234,7 @@ static void set_func_state(struct cfi_state *state) memcpy(&state->regs, &initial_func_cfi.regs, CFI_NUM_REGS * sizeof(struct cfi_reg)); state->stack_size = initial_func_cfi.cfa.offset; + state->type = UNWIND_HINT_TYPE_CALL; } static int read_unwind_hints(struct objtool_file *file) @@ -2304,19 +2296,11 @@ static int read_unwind_hints(struct objtool_file *file) if (sym && sym->bind == STB_GLOBAL) { if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { - WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", - insn->sec, insn->offset); + WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR"); } - - insn->entry = 1; } } - if (hint->type == UNWIND_HINT_TYPE_ENTRY) { - hint->type = UNWIND_HINT_TYPE_CALL; - insn->entry = 1; - } - if (hint->type == UNWIND_HINT_TYPE_FUNC) { insn->cfi = &func_cfi; continue; @@ -2326,15 +2310,13 @@ static int read_unwind_hints(struct objtool_file *file) cfi = *(insn->cfi); if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) { - WARN_FUNC("unsupported unwind_hint sp base reg %d", - insn->sec, insn->offset, hint->sp_reg); + WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg); return -1; } cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset); cfi.type = hint->type; cfi.signal = hint->signal; - cfi.end = hint->end; insn->cfi = cfi_hash_find_or_add(&cfi); } @@ -2391,8 +2373,7 @@ static int read_retpoline_hints(struct objtool_file *file) insn->type != INSN_CALL_DYNAMIC && insn->type != INSN_RETURN && insn->type != INSN_NOP) { - WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", - insn->sec, insn->offset); + WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop"); return -1; } @@ -2449,6 +2430,34 @@ static int read_instr_hints(struct objtool_file *file) return 0; } +static int read_validate_unret_hints(struct objtool_file *file) +{ + struct section *sec; + struct instruction *insn; + struct reloc *reloc; + + sec = find_section_by_name(file->elf, ".rela.discard.validate_unret"); + if (!sec) + return 0; + + list_for_each_entry(reloc, &sec->reloc_list, list) { + if (reloc->sym->type != STT_SECTION) { + WARN("unexpected relocation symbol type in %s", sec->name); + return -1; + } + + insn = find_insn(file, reloc->sym->sec, reloc->addend); + if (!insn) { + WARN("bad .discard.instr_end entry"); + return -1; + } + insn->unret = 1; + } + + return 0; +} + + static int read_intra_function_calls(struct objtool_file *file) { struct instruction *insn; @@ -2475,8 +2484,7 @@ static int read_intra_function_calls(struct objtool_file *file) } if (insn->type != INSN_CALL) { - WARN_FUNC("intra_function_call not a direct call", - insn->sec, insn->offset); + WARN_INSN(insn, "intra_function_call not a direct call"); return -1; } @@ -2490,8 +2498,7 @@ static int read_intra_function_calls(struct objtool_file *file) dest_off = arch_jump_destination(insn); insn->jump_dest = find_insn(file, insn->sec, dest_off); if (!insn->jump_dest) { - WARN_FUNC("can't find call dest at %s+0x%lx", - insn->sec, insn->offset, + WARN_INSN(insn, "can't find call dest at %s+0x%lx", insn->sec->name, dest_off); return -1; } @@ -2527,30 +2534,27 @@ static bool is_profiling_func(const char *name) static int classify_symbols(struct objtool_file *file) { - struct section *sec; struct symbol *func; - for_each_sec(file, sec) { - list_for_each_entry(func, &sec->symbol_list, list) { - if (func->bind != STB_GLOBAL) - continue; + for_each_sym(file, func) { + if (func->bind != STB_GLOBAL) + continue; - if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, - strlen(STATIC_CALL_TRAMP_PREFIX_STR))) - func->static_call_tramp = true; + if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR, + strlen(STATIC_CALL_TRAMP_PREFIX_STR))) + func->static_call_tramp = true; - if (arch_is_retpoline(func)) - func->retpoline_thunk = true; + if (arch_is_retpoline(func)) + func->retpoline_thunk = true; - if (arch_is_rethunk(func)) - func->return_thunk = true; + if (arch_is_rethunk(func)) + func->return_thunk = true; - if (arch_ftrace_match(func->name)) - func->fentry = true; + if (arch_ftrace_match(func->name)) + func->fentry = true; - if (is_profiling_func(func->name)) - func->profiling_func = true; - } + if (is_profiling_func(func->name)) + func->profiling_func = true; } return 0; @@ -2667,6 +2671,10 @@ static int decode_sections(struct objtool_file *file) if (ret) return ret; + ret = read_validate_unret_hints(file); + if (ret) + return ret; + return 0; } @@ -2828,7 +2836,7 @@ static int update_cfi_state(struct instruction *insn, /* stack operations don't make sense with an undefined CFA */ if (cfa->base == CFI_UNDEFINED) { if (insn_func(insn)) { - WARN_FUNC("undefined stack state", insn->sec, insn->offset); + WARN_INSN(insn, "undefined stack state"); return -1; } return 0; @@ -2977,17 +2985,6 @@ static int update_cfi_state(struct instruction *insn, break; } - if (!cfi->drap && op->src.reg == CFI_SP && - op->dest.reg == CFI_BP && cfa->base == CFI_SP && - check_reg_frame_pos(®s[CFI_BP], -cfa->offset + op->src.offset)) { - - /* lea disp(%rsp), %rbp */ - cfa->base = CFI_BP; - cfa->offset -= op->src.offset; - cfi->bp_scratch = false; - break; - } - if (op->src.reg == CFI_SP && cfa->base == CFI_SP) { /* drap: lea disp(%rsp), %drap */ @@ -3022,8 +3019,7 @@ static int update_cfi_state(struct instruction *insn, } if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) { - WARN_FUNC("unsupported stack register modification", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported stack register modification"); return -1; } @@ -3033,8 +3029,7 @@ static int update_cfi_state(struct instruction *insn, if (op->dest.reg != CFI_SP || (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) || (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) { - WARN_FUNC("unsupported stack pointer realignment", - insn->sec, insn->offset); + WARN_INSN(insn, "unsupported stack pointer realignment"); return -1; } @@ -3129,8 +3124,7 @@ static int update_cfi_state(struct instruction *insn, break; default: - WARN_FUNC("unknown stack-related instruction", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related instruction"); return -1; } @@ -3219,8 +3213,7 @@ static int update_cfi_state(struct instruction *insn, case OP_DEST_MEM: if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) { - WARN_FUNC("unknown stack-related memory operation", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related memory operation"); return -1; } @@ -3232,8 +3225,7 @@ static int update_cfi_state(struct instruction *insn, break; default: - WARN_FUNC("unknown stack-related instruction", - insn->sec, insn->offset); + WARN_INSN(insn, "unknown stack-related instruction"); return -1; } @@ -3272,8 +3264,7 @@ static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group; struct instruction *orig = orig_group->first_insn; char *where = offstr(insn->sec, insn->offset); - WARN_FUNC("stack layout conflict in alternatives: %s", - orig->sec, orig->offset, where); + WARN_INSN(orig, "stack layout conflict in alternatives: %s", where); free(where); return -1; } @@ -3300,8 +3291,7 @@ static int handle_insn_ops(struct instruction *insn, if (!state->uaccess_stack) { state->uaccess_stack = 1; } else if (state->uaccess_stack >> 31) { - WARN_FUNC("PUSHF stack exhausted", - insn->sec, insn->offset); + WARN_INSN(insn, "PUSHF stack exhausted"); return 1; } state->uaccess_stack <<= 1; @@ -3333,8 +3323,7 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) { - WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d", cfi1->cfa.base, cfi1->cfa.offset, cfi2->cfa.base, cfi2->cfa.offset); @@ -3344,8 +3333,7 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) sizeof(struct cfi_reg))) continue; - WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d", i, cfi1->regs[i].base, cfi1->regs[i].offset, i, cfi2->regs[i].base, cfi2->regs[i].offset); break; @@ -3353,15 +3341,14 @@ static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) } else if (cfi1->type != cfi2->type) { - WARN_FUNC("stack state mismatch: type1=%d type2=%d", - insn->sec, insn->offset, cfi1->type, cfi2->type); + WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d", + cfi1->type, cfi2->type); } else if (cfi1->drap != cfi2->drap || (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) || (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) { - WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", - insn->sec, insn->offset, + WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)", cfi1->drap, cfi1->drap_reg, cfi1->drap_offset, cfi2->drap, cfi2->drap_reg, cfi2->drap_offset); @@ -3469,20 +3456,17 @@ static int validate_call(struct objtool_file *file, { if (state->noinstr && state->instr <= 0 && !noinstr_call_dest(file, insn, insn_call_dest(insn))) { - WARN_FUNC("call to %s() leaves .noinstr.text section", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); return 1; } if (state->uaccess && !func_uaccess_safe(insn_call_dest(insn))) { - WARN_FUNC("call to %s() with UACCESS enabled", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn)); return 1; } if (state->df) { - WARN_FUNC("call to %s() with DF set", - insn->sec, insn->offset, call_dest_name(insn)); + WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn)); return 1; } @@ -3494,8 +3478,7 @@ static int validate_sibling_call(struct objtool_file *file, struct insn_state *state) { if (insn_func(insn) && has_modified_stack_frame(insn, state)) { - WARN_FUNC("sibling call from callable instruction with modified stack frame", - insn->sec, insn->offset); + WARN_INSN(insn, "sibling call from callable instruction with modified stack frame"); return 1; } @@ -3505,38 +3488,32 @@ static int validate_sibling_call(struct objtool_file *file, static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state) { if (state->noinstr && state->instr > 0) { - WARN_FUNC("return with instrumentation enabled", - insn->sec, insn->offset); + WARN_INSN(insn, "return with instrumentation enabled"); return 1; } if (state->uaccess && !func_uaccess_safe(func)) { - WARN_FUNC("return with UACCESS enabled", - insn->sec, insn->offset); + WARN_INSN(insn, "return with UACCESS enabled"); return 1; } if (!state->uaccess && func_uaccess_safe(func)) { - WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function", - insn->sec, insn->offset); + WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function"); return 1; } if (state->df) { - WARN_FUNC("return with DF set", - insn->sec, insn->offset); + WARN_INSN(insn, "return with DF set"); return 1; } if (func && has_modified_stack_frame(insn, state)) { - WARN_FUNC("return with modified stack frame", - insn->sec, insn->offset); + WARN_INSN(insn, "return with modified stack frame"); return 1; } if (state->cfi.bp_scratch) { - WARN_FUNC("BP used as a scratch register", - insn->sec, insn->offset); + WARN_INSN(insn, "BP used as a scratch register"); return 1; } @@ -3608,8 +3585,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, } if (func && insn->ignore) { - WARN_FUNC("BUG: why am I validating an ignored function?", - sec, insn->offset); + WARN_INSN(insn, "BUG: why am I validating an ignored function?"); return 1; } @@ -3642,14 +3618,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, } if (!save_insn) { - WARN_FUNC("no corresponding CFI save for CFI restore", - sec, insn->offset); + WARN_INSN(insn, "no corresponding CFI save for CFI restore"); return 1; } if (!save_insn->visited) { - WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", - sec, insn->offset); + WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo"); return 1; } @@ -3709,8 +3683,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, if (opts.stackval && func && !is_fentry_call(insn) && !has_valid_stack_frame(&state)) { - WARN_FUNC("call without frame pointer save/setup", - sec, insn->offset); + WARN_INSN(insn, "call without frame pointer save/setup"); return 1; } @@ -3756,15 +3729,14 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CONTEXT_SWITCH: if (func && (!next_insn || !next_insn->hint)) { - WARN_FUNC("unsupported instruction in callable function", - sec, insn->offset); + WARN_INSN(insn, "unsupported instruction in callable function"); return 1; } return 0; case INSN_STAC: if (state.uaccess) { - WARN_FUNC("recursive UACCESS enable", sec, insn->offset); + WARN_INSN(insn, "recursive UACCESS enable"); return 1; } @@ -3773,12 +3745,12 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CLAC: if (!state.uaccess && func) { - WARN_FUNC("redundant UACCESS disable", sec, insn->offset); + WARN_INSN(insn, "redundant UACCESS disable"); return 1; } if (func_uaccess_safe(func) && !state.uaccess_stack) { - WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset); + WARN_INSN(insn, "UACCESS-safe disables UACCESS"); return 1; } @@ -3787,7 +3759,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_STD: if (state.df) { - WARN_FUNC("recursive STD", sec, insn->offset); + WARN_INSN(insn, "recursive STD"); return 1; } @@ -3796,7 +3768,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, case INSN_CLD: if (!state.df && func) { - WARN_FUNC("redundant CLD", sec, insn->offset); + WARN_INSN(insn, "redundant CLD"); return 1; } @@ -3863,10 +3835,10 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) /* * Validate rethunk entry constraint: must untrain RET before the first RET. * - * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes + * Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes * before an actual RET instruction. */ -static int validate_entry(struct objtool_file *file, struct instruction *insn) +static int validate_unret(struct objtool_file *file, struct instruction *insn) { struct instruction *next, *dest; int ret, warnings = 0; @@ -3874,10 +3846,10 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) for (;;) { next = next_insn_to_validate(file, insn); - if (insn->visited & VISITED_ENTRY) + if (insn->visited & VISITED_UNRET) return 0; - insn->visited |= VISITED_ENTRY; + insn->visited |= VISITED_UNRET; if (!insn->ignore_alts && insn->alts) { struct alternative *alt; @@ -3887,7 +3859,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) if (alt->skip_orig) skip_orig = true; - ret = validate_entry(file, alt->insn); + ret = validate_unret(file, alt->insn); if (ret) { if (opts.backtrace) BT_FUNC("(alt)", insn); @@ -3904,18 +3876,17 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) case INSN_CALL_DYNAMIC: case INSN_JUMP_DYNAMIC: case INSN_JUMP_DYNAMIC_CONDITIONAL: - WARN_FUNC("early indirect call", insn->sec, insn->offset); + WARN_INSN(insn, "early indirect call"); return 1; case INSN_JUMP_UNCONDITIONAL: case INSN_JUMP_CONDITIONAL: if (!is_sibling_call(insn)) { if (!insn->jump_dest) { - WARN_FUNC("unresolved jump target after linking?!?", - insn->sec, insn->offset); + WARN_INSN(insn, "unresolved jump target after linking?!?"); return -1; } - ret = validate_entry(file, insn->jump_dest); + ret = validate_unret(file, insn->jump_dest); if (ret) { if (opts.backtrace) { BT_FUNC("(branch%s)", insn, @@ -3940,7 +3911,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) return -1; } - ret = validate_entry(file, dest); + ret = validate_unret(file, dest); if (ret) { if (opts.backtrace) BT_FUNC("(call)", insn); @@ -3953,7 +3924,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) return 0; case INSN_RETURN: - WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); + WARN_INSN(insn, "RET before UNTRAIN"); return 1; case INSN_NOP: @@ -3966,7 +3937,7 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) } if (!next) { - WARN_FUNC("teh end!", insn->sec, insn->offset); + WARN_INSN(insn, "teh end!"); return -1; } insn = next; @@ -3976,21 +3947,21 @@ static int validate_entry(struct objtool_file *file, struct instruction *insn) } /* - * Validate that all branches starting at 'insn->entry' encounter UNRET_END - * before RET. + * Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter + * VALIDATE_UNRET_END before RET. */ -static int validate_unret(struct objtool_file *file) +static int validate_unrets(struct objtool_file *file) { struct instruction *insn; int ret, warnings = 0; for_each_insn(file, insn) { - if (!insn->entry) + if (!insn->unret) continue; - ret = validate_entry(file, insn); + ret = validate_unret(file, insn); if (ret < 0) { - WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); + WARN_INSN(insn, "Failed UNRET validation"); return ret; } warnings += ret; @@ -4018,13 +3989,11 @@ static int validate_retpoline(struct objtool_file *file) if (insn->type == INSN_RETURN) { if (opts.rethunk) { - WARN_FUNC("'naked' return found in RETHUNK build", - insn->sec, insn->offset); + WARN_INSN(insn, "'naked' return found in RETHUNK build"); } else continue; } else { - WARN_FUNC("indirect %s found in RETPOLINE build", - insn->sec, insn->offset, + WARN_INSN(insn, "indirect %s found in RETPOLINE build", insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); } @@ -4121,8 +4090,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio * It may also insert a UD2 after calling a __noreturn function. */ prev_insn = prev_insn_same_sec(file, insn); - if ((prev_insn->dead_end || - dead_end_function(file, insn_call_dest(prev_insn))) && + if (prev_insn->dead_end && (insn->type == INSN_BUG || (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && insn->jump_dest->type == INSN_BUG))) @@ -4158,54 +4126,75 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio return false; } -static int add_prefix_symbol(struct objtool_file *file, struct symbol *func, - struct instruction *insn) +static int add_prefix_symbol(struct objtool_file *file, struct symbol *func) { - if (!opts.prefix) - return 0; + struct instruction *insn, *prev; + struct cfi_state *cfi; - for (;;) { - struct instruction *prev = prev_insn_same_sec(file, insn); + insn = find_insn(file, func->sec, func->offset); + if (!insn) + return -1; + + for (prev = prev_insn_same_sec(file, insn); + prev; + prev = prev_insn_same_sec(file, prev)) { u64 offset; - if (!prev) - break; - if (prev->type != INSN_NOP) - break; + return -1; offset = func->offset - prev->offset; - if (offset >= opts.prefix) { - if (offset == opts.prefix) { - /* - * Since the sec->symbol_list is ordered by - * offset (see elf_add_symbol()) the added - * symbol will not be seen by the iteration in - * validate_section(). - * - * Hence the lack of list_for_each_entry_safe() - * there. - * - * The direct concequence is that prefix symbols - * don't get visited (because pointless), except - * for the logic in ignore_unreachable_insn() - * that needs the terminating insn to be visited - * otherwise it will report the hole. - * - * Hence mark the first instruction of the - * prefix symbol as visisted. - */ - prev->visited |= VISITED_BRANCH; - elf_create_prefix_symbol(file->elf, func, opts.prefix); - } - break; - } - insn = prev; + + if (offset > opts.prefix) + return -1; + + if (offset < opts.prefix) + continue; + + elf_create_prefix_symbol(file->elf, func, opts.prefix); + break; } + if (!prev) + return -1; + + if (!insn->cfi) { + /* + * This can happen if stack validation isn't enabled or the + * function is annotated with STACK_FRAME_NON_STANDARD. + */ + return 0; + } + + /* Propagate insn->cfi to the prefix code */ + cfi = cfi_hash_find_or_add(insn->cfi); + for (; prev != insn; prev = next_insn_same_sec(file, prev)) + prev->cfi = cfi; + return 0; } +static int add_prefix_symbols(struct objtool_file *file) +{ + struct section *sec; + struct symbol *func; + int warnings = 0; + + for_each_sec(file, sec) { + if (!(sec->sh.sh_flags & SHF_EXECINSTR)) + continue; + + sec_for_each_sym(sec, func) { + if (func->type != STT_FUNC) + continue; + + add_prefix_symbol(file, func); + } + } + + return warnings; +} + static int validate_symbol(struct objtool_file *file, struct section *sec, struct symbol *sym, struct insn_state *state) { @@ -4224,8 +4213,6 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, if (!insn || insn->ignore || insn->visited) return 0; - add_prefix_symbol(file, sym, insn); - state->uaccess = sym->uaccess_safe; ret = validate_branch(file, insn_func(insn), insn, *state); @@ -4240,7 +4227,7 @@ static int validate_section(struct objtool_file *file, struct section *sec) struct symbol *func; int warnings = 0; - list_for_each_entry(func, &sec->symbol_list, list) { + sec_for_each_sym(sec, func) { if (func->type != STT_FUNC) continue; @@ -4403,9 +4390,7 @@ static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn if (noendbr_range(file, dest)) continue; - WARN_FUNC("relocation to !ENDBR: %s", - insn->sec, insn->offset, - offstr(dest->sec, dest->offset)); + WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset)); warnings++; } @@ -4507,16 +4492,14 @@ static int validate_sls(struct objtool_file *file) switch (insn->type) { case INSN_RETURN: if (!next_insn || next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after ret", - insn->sec, insn->offset); + WARN_INSN(insn, "missing int3 after ret"); warnings++; } break; case INSN_JUMP_DYNAMIC: if (!next_insn || next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after indirect jump", - insn->sec, insn->offset); + WARN_INSN(insn, "missing int3 after indirect jump"); warnings++; } break; @@ -4539,7 +4522,7 @@ static int validate_reachable_instructions(struct objtool_file *file) if (insn->visited || ignore_unreachable_insn(file, insn)) continue; - WARN_FUNC("unreachable instruction", insn->sec, insn->offset); + WARN_INSN(insn, "unreachable instruction"); return 1; } @@ -4607,7 +4590,7 @@ int check(struct objtool_file *file) * Must be after validate_branch() and friends, it plays * further games with insn->visited. */ - ret = validate_unret(file); + ret = validate_unrets(file); if (ret < 0) return ret; warnings += ret; @@ -4669,6 +4652,13 @@ int check(struct objtool_file *file) warnings += ret; } + if (opts.prefix) { + ret = add_prefix_symbols(file); + if (ret < 0) + return ret; + warnings += ret; + } + if (opts.ibt) { ret = create_ibt_endbr_seal_sections(file); if (ret < 0) diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index 6806ce01d933..500e92979a31 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -474,7 +474,7 @@ static int read_symbols(struct elf *elf) /* Create parent/child links for any cold subfunctions */ list_for_each_entry(sec, &elf->sections, list) { - list_for_each_entry(sym, &sec->symbol_list, list) { + sec_for_each_sym(sec, sym) { char pname[MAX_NAME_LEN + 1]; size_t pnamelen; if (sym->type != STT_FUNC) diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index 3e7c7004f7df..daa46f1f0965 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -61,7 +61,7 @@ struct instruction { restore : 1, retpoline_safe : 1, noendbr : 1, - entry : 1, + unret : 1, visited : 4, no_reloc : 1; /* 10 bit hole */ @@ -92,7 +92,7 @@ static inline struct symbol *insn_func(struct instruction *insn) #define VISITED_BRANCH 0x01 #define VISITED_BRANCH_UACCESS 0x02 #define VISITED_BRANCH_MASK 0x03 -#define VISITED_ENTRY 0x04 +#define VISITED_UNRET 0x04 static inline bool is_static_jump(struct instruction *insn) { diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index ad0024da262b..e1ca588eb69d 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -188,4 +188,13 @@ struct symbol *find_func_containing(struct section *sec, unsigned long offset); #define for_each_sec(file, sec) \ list_for_each_entry(sec, &file->elf->sections, list) +#define sec_for_each_sym(sec, sym) \ + list_for_each_entry(sym, &sec->symbol_list, list) + +#define for_each_sym(file, sym) \ + for (struct section *__sec, *__fake = (struct section *)1; \ + __fake; __fake = NULL) \ + for_each_sec(file, __sec) \ + sec_for_each_sym(__sec, sym) + #endif /* _OBJTOOL_ELF_H */ diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h index a3e79ae75f2e..b1c920dc9516 100644 --- a/tools/objtool/include/objtool/warn.h +++ b/tools/objtool/include/objtool/warn.h @@ -53,6 +53,11 @@ static inline char *offstr(struct section *sec, unsigned long offset) free(_str); \ }) +#define WARN_INSN(insn, format, ...) \ +({ \ + WARN_FUNC(format, insn->sec, insn->offset, ##__VA_ARGS__); \ +}) + #define BT_FUNC(format, insn, ...) \ ({ \ struct instruction *_insn = (insn); \ diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c index 2d8ebdcd1db3..0e183bb1c720 100644 --- a/tools/objtool/orc_dump.c +++ b/tools/objtool/orc_dump.c @@ -4,7 +4,6 @@ */ #include -#include #include #include #include @@ -39,11 +38,15 @@ static const char *reg_name(unsigned int reg) static const char *orc_type_name(unsigned int type) { switch (type) { - case UNWIND_HINT_TYPE_CALL: + case ORC_TYPE_UNDEFINED: + return "(und)"; + case ORC_TYPE_END_OF_STACK: + return "end"; + case ORC_TYPE_CALL: return "call"; - case UNWIND_HINT_TYPE_REGS: + case ORC_TYPE_REGS: return "regs"; - case UNWIND_HINT_TYPE_REGS_PARTIAL: + case ORC_TYPE_REGS_PARTIAL: return "regs (partial)"; default: return "?"; @@ -202,6 +205,7 @@ int orc_dump(const char *_objname) printf("%llx:", (unsigned long long)(orc_ip_addr + (i * sizeof(int)) + orc_ip[i])); } + printf("type:%s", orc_type_name(orc[i].type)); printf(" sp:"); @@ -211,8 +215,7 @@ int orc_dump(const char *_objname) print_reg(orc[i].bp_reg, bswap_if_needed(&dummy_elf, orc[i].bp_offset)); - printf(" type:%s signal:%d end:%d\n", - orc_type_name(orc[i].type), orc[i].signal, orc[i].end); + printf(" signal:%d\n", orc[i].signal); } elf_end(elf); diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c index 57a4527d5988..48efd1e2f00d 100644 --- a/tools/objtool/orc_gen.c +++ b/tools/objtool/orc_gen.c @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -21,19 +21,38 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, memset(orc, 0, sizeof(*orc)); if (!cfi) { - orc->end = 0; - orc->sp_reg = ORC_REG_UNDEFINED; + /* + * This is usually either unreachable nops/traps (which don't + * trigger unreachable instruction warnings), or + * STACK_FRAME_NON_STANDARD functions. + */ + orc->type = ORC_TYPE_UNDEFINED; return 0; } - orc->end = cfi->end; + switch (cfi->type) { + case UNWIND_HINT_TYPE_UNDEFINED: + orc->type = ORC_TYPE_UNDEFINED; + return 0; + case UNWIND_HINT_TYPE_END_OF_STACK: + orc->type = ORC_TYPE_END_OF_STACK; + return 0; + case UNWIND_HINT_TYPE_CALL: + orc->type = ORC_TYPE_CALL; + break; + case UNWIND_HINT_TYPE_REGS: + orc->type = ORC_TYPE_REGS; + break; + case UNWIND_HINT_TYPE_REGS_PARTIAL: + orc->type = ORC_TYPE_REGS_PARTIAL; + break; + default: + WARN_INSN(insn, "unknown unwind hint type %d", cfi->type); + return -1; + } + orc->signal = cfi->signal; - if (cfi->cfa.base == CFI_UNDEFINED) { - orc->sp_reg = ORC_REG_UNDEFINED; - return 0; - } - switch (cfi->cfa.base) { case CFI_SP: orc->sp_reg = ORC_REG_SP; @@ -60,8 +79,7 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->sp_reg = ORC_REG_DX; break; default: - WARN_FUNC("unknown CFA base reg %d", - insn->sec, insn->offset, cfi->cfa.base); + WARN_INSN(insn, "unknown CFA base reg %d", cfi->cfa.base); return -1; } @@ -76,14 +94,12 @@ static int init_orc_entry(struct orc_entry *orc, struct cfi_state *cfi, orc->bp_reg = ORC_REG_BP; break; default: - WARN_FUNC("unknown BP base reg %d", - insn->sec, insn->offset, bp->base); + WARN_INSN(insn, "unknown BP base reg %d", bp->base); return -1; } orc->sp_offset = cfi->cfa.offset; orc->bp_offset = bp->offset; - orc->type = cfi->type; return 0; } @@ -148,11 +164,7 @@ int orc_create(struct objtool_file *file) struct orc_list_entry *entry; struct list_head orc_list; - struct orc_entry null = { - .sp_reg = ORC_REG_UNDEFINED, - .bp_reg = ORC_REG_UNDEFINED, - .type = UNWIND_HINT_TYPE_CALL, - }; + struct orc_entry null = { .type = ORC_TYPE_UNDEFINED }; /* Build a deduplicated list of ORC entries: */ INIT_LIST_HEAD(&orc_list); diff --git a/tools/objtool/sync-check.sh b/tools/objtool/sync-check.sh index 105a291ff8e7..81d120d05442 100755 --- a/tools/objtool/sync-check.sh +++ b/tools/objtool/sync-check.sh @@ -6,7 +6,7 @@ if [ -z "$SRCARCH" ]; then exit 1 fi -FILES="include/linux/objtool.h" +FILES="include/linux/objtool_types.h" if [ "$SRCARCH" = "x86" ]; then FILES="$FILES diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile index ca9374b56ead..598135d3162b 100644 --- a/tools/testing/selftests/x86/Makefile +++ b/tools/testing/selftests/x86/Makefile @@ -18,7 +18,7 @@ TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ test_FCMOV test_FCOMI test_FISTTP \ vdso_restorer TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \ - corrupt_xstate_header amx + corrupt_xstate_header amx lam # Some selftests require 32bit support enabled also on 64bit systems TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall diff --git a/tools/testing/selftests/x86/lam.c b/tools/testing/selftests/x86/lam.c new file mode 100644 index 000000000000..eb0e46905bf9 --- /dev/null +++ b/tools/testing/selftests/x86/lam.c @@ -0,0 +1,1241 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "../kselftest.h" + +#ifndef __x86_64__ +# error This test is 64-bit only +#endif + +/* LAM modes, these definitions were copied from kernel code */ +#define LAM_NONE 0 +#define LAM_U57_BITS 6 + +#define LAM_U57_MASK (0x3fULL << 57) +/* arch prctl for LAM */ +#define ARCH_GET_UNTAG_MASK 0x4001 +#define ARCH_ENABLE_TAGGED_ADDR 0x4002 +#define ARCH_GET_MAX_TAG_BITS 0x4003 +#define ARCH_FORCE_TAGGED_SVA 0x4004 + +/* Specified test function bits */ +#define FUNC_MALLOC 0x1 +#define FUNC_BITS 0x2 +#define FUNC_MMAP 0x4 +#define FUNC_SYSCALL 0x8 +#define FUNC_URING 0x10 +#define FUNC_INHERITE 0x20 +#define FUNC_PASID 0x40 + +#define TEST_MASK 0x7f + +#define LOW_ADDR (0x1UL << 30) +#define HIGH_ADDR (0x3UL << 48) + +#define MALLOC_LEN 32 + +#define PAGE_SIZE (4 << 10) + +#define STACK_SIZE 65536 + +#define barrier() ({ \ + __asm__ __volatile__("" : : : "memory"); \ +}) + +#define URING_QUEUE_SZ 1 +#define URING_BLOCK_SZ 2048 + +/* Pasid test define */ +#define LAM_CMD_BIT 0x1 +#define PAS_CMD_BIT 0x2 +#define SVA_CMD_BIT 0x4 + +#define PAS_CMD(cmd1, cmd2, cmd3) (((cmd3) << 8) | ((cmd2) << 4) | ((cmd1) << 0)) + +struct testcases { + unsigned int later; + int expected; /* 2: SIGSEGV Error; 1: other errors */ + unsigned long lam; + uint64_t addr; + uint64_t cmd; + int (*test_func)(struct testcases *test); + const char *msg; +}; + +/* Used by CQ of uring, source file handler and file's size */ +struct file_io { + int file_fd; + off_t file_sz; + struct iovec iovecs[]; +}; + +struct io_uring_queue { + unsigned int *head; + unsigned int *tail; + unsigned int *ring_mask; + unsigned int *ring_entries; + unsigned int *flags; + unsigned int *array; + union { + struct io_uring_cqe *cqes; + struct io_uring_sqe *sqes; + } queue; + size_t ring_sz; +}; + +struct io_ring { + int ring_fd; + struct io_uring_queue sq_ring; + struct io_uring_queue cq_ring; +}; + +int tests_cnt; +jmp_buf segv_env; + +static void segv_handler(int sig) +{ + ksft_print_msg("Get segmentation fault(%d).", sig); + + siglongjmp(segv_env, 1); +} + +static inline int cpu_has_lam(void) +{ + unsigned int cpuinfo[4]; + + __cpuid_count(0x7, 1, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]); + + return (cpuinfo[0] & (1 << 26)); +} + +/* Check 5-level page table feature in CPUID.(EAX=07H, ECX=00H):ECX.[bit 16] */ +static inline int cpu_has_la57(void) +{ + unsigned int cpuinfo[4]; + + __cpuid_count(0x7, 0, cpuinfo[0], cpuinfo[1], cpuinfo[2], cpuinfo[3]); + + return (cpuinfo[2] & (1 << 16)); +} + +/* + * Set tagged address and read back untag mask. + * check if the untagged mask is expected. + * + * @return: + * 0: Set LAM mode successfully + * others: failed to set LAM + */ +static int set_lam(unsigned long lam) +{ + int ret = 0; + uint64_t ptr = 0; + + if (lam != LAM_U57_BITS && lam != LAM_NONE) + return -1; + + /* Skip check return */ + syscall(SYS_arch_prctl, ARCH_ENABLE_TAGGED_ADDR, lam); + + /* Get untagged mask */ + syscall(SYS_arch_prctl, ARCH_GET_UNTAG_MASK, &ptr); + + /* Check mask returned is expected */ + if (lam == LAM_U57_BITS) + ret = (ptr != ~(LAM_U57_MASK)); + else if (lam == LAM_NONE) + ret = (ptr != -1ULL); + + return ret; +} + +static unsigned long get_default_tag_bits(void) +{ + pid_t pid; + int lam = LAM_NONE; + int ret = 0; + + pid = fork(); + if (pid < 0) { + perror("Fork failed."); + } else if (pid == 0) { + /* Set LAM mode in child process */ + if (set_lam(LAM_U57_BITS) == 0) + lam = LAM_U57_BITS; + else + lam = LAM_NONE; + exit(lam); + } else { + wait(&ret); + lam = WEXITSTATUS(ret); + } + + return lam; +} + +/* + * Set tagged address and read back untag mask. + * check if the untag mask is expected. + */ +static int get_lam(void) +{ + uint64_t ptr = 0; + int ret = -1; + /* Get untagged mask */ + if (syscall(SYS_arch_prctl, ARCH_GET_UNTAG_MASK, &ptr) == -1) + return -1; + + /* Check mask returned is expected */ + if (ptr == ~(LAM_U57_MASK)) + ret = LAM_U57_BITS; + else if (ptr == -1ULL) + ret = LAM_NONE; + + + return ret; +} + +/* According to LAM mode, set metadata in high bits */ +static uint64_t set_metadata(uint64_t src, unsigned long lam) +{ + uint64_t metadata; + + srand(time(NULL)); + + switch (lam) { + case LAM_U57_BITS: /* Set metadata in bits 62:57 */ + /* Get a random non-zero value as metadata */ + metadata = (rand() % ((1UL << LAM_U57_BITS) - 1) + 1) << 57; + metadata |= (src & ~(LAM_U57_MASK)); + break; + default: + metadata = src; + break; + } + + return metadata; +} + +/* + * Set metadata in user pointer, compare new pointer with original pointer. + * both pointers should point to the same address. + * + * @return: + * 0: value on the pointer with metadate and value on original are same + * 1: not same. + */ +static int handle_lam_test(void *src, unsigned int lam) +{ + char *ptr; + + strcpy((char *)src, "USER POINTER"); + + ptr = (char *)set_metadata((uint64_t)src, lam); + if (src == ptr) + return 0; + + /* Copy a string into the pointer with metadata */ + strcpy((char *)ptr, "METADATA POINTER"); + + return (!!strcmp((char *)src, (char *)ptr)); +} + + +int handle_max_bits(struct testcases *test) +{ + unsigned long exp_bits = get_default_tag_bits(); + unsigned long bits = 0; + + if (exp_bits != LAM_NONE) + exp_bits = LAM_U57_BITS; + + /* Get LAM max tag bits */ + if (syscall(SYS_arch_prctl, ARCH_GET_MAX_TAG_BITS, &bits) == -1) + return 1; + + return (exp_bits != bits); +} + +/* + * Test lam feature through dereference pointer get from malloc. + * @return 0: Pass test. 1: Get failure during test 2: Get SIGSEGV + */ +static int handle_malloc(struct testcases *test) +{ + char *ptr = NULL; + int ret = 0; + + if (test->later == 0 && test->lam != 0) + if (set_lam(test->lam) == -1) + return 1; + + ptr = (char *)malloc(MALLOC_LEN); + if (ptr == NULL) { + perror("malloc() failure\n"); + return 1; + } + + /* Set signal handler */ + if (sigsetjmp(segv_env, 1) == 0) { + signal(SIGSEGV, segv_handler); + ret = handle_lam_test(ptr, test->lam); + } else { + ret = 2; + } + + if (test->later != 0 && test->lam != 0) + if (set_lam(test->lam) == -1 && ret == 0) + ret = 1; + + free(ptr); + + return ret; +} + +static int handle_mmap(struct testcases *test) +{ + void *ptr; + unsigned int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED; + int ret = 0; + + if (test->later == 0 && test->lam != 0) + if (set_lam(test->lam) != 0) + return 1; + + ptr = mmap((void *)test->addr, PAGE_SIZE, PROT_READ | PROT_WRITE, + flags, -1, 0); + if (ptr == MAP_FAILED) { + if (test->addr == HIGH_ADDR) + if (!cpu_has_la57()) + return 3; /* unsupport LA57 */ + return 1; + } + + if (test->later != 0 && test->lam != 0) + if (set_lam(test->lam) != 0) + ret = 1; + + if (ret == 0) { + if (sigsetjmp(segv_env, 1) == 0) { + signal(SIGSEGV, segv_handler); + ret = handle_lam_test(ptr, test->lam); + } else { + ret = 2; + } + } + + munmap(ptr, PAGE_SIZE); + return ret; +} + +static int handle_syscall(struct testcases *test) +{ + struct utsname unme, *pu; + int ret = 0; + + if (test->later == 0 && test->lam != 0) + if (set_lam(test->lam) != 0) + return 1; + + if (sigsetjmp(segv_env, 1) == 0) { + signal(SIGSEGV, segv_handler); + pu = (struct utsname *)set_metadata((uint64_t)&unme, test->lam); + ret = uname(pu); + if (ret < 0) + ret = 1; + } else { + ret = 2; + } + + if (test->later != 0 && test->lam != 0) + if (set_lam(test->lam) != -1 && ret == 0) + ret = 1; + + return ret; +} + +int sys_uring_setup(unsigned int entries, struct io_uring_params *p) +{ + return (int)syscall(__NR_io_uring_setup, entries, p); +} + +int sys_uring_enter(int fd, unsigned int to, unsigned int min, unsigned int flags) +{ + return (int)syscall(__NR_io_uring_enter, fd, to, min, flags, NULL, 0); +} + +/* Init submission queue and completion queue */ +int mmap_io_uring(struct io_uring_params p, struct io_ring *s) +{ + struct io_uring_queue *sring = &s->sq_ring; + struct io_uring_queue *cring = &s->cq_ring; + + sring->ring_sz = p.sq_off.array + p.sq_entries * sizeof(unsigned int); + cring->ring_sz = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe); + + if (p.features & IORING_FEAT_SINGLE_MMAP) { + if (cring->ring_sz > sring->ring_sz) + sring->ring_sz = cring->ring_sz; + + cring->ring_sz = sring->ring_sz; + } + + void *sq_ptr = mmap(0, sring->ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, s->ring_fd, + IORING_OFF_SQ_RING); + + if (sq_ptr == MAP_FAILED) { + perror("sub-queue!"); + return 1; + } + + void *cq_ptr = sq_ptr; + + if (!(p.features & IORING_FEAT_SINGLE_MMAP)) { + cq_ptr = mmap(0, cring->ring_sz, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, s->ring_fd, + IORING_OFF_CQ_RING); + if (cq_ptr == MAP_FAILED) { + perror("cpl-queue!"); + munmap(sq_ptr, sring->ring_sz); + return 1; + } + } + + sring->head = sq_ptr + p.sq_off.head; + sring->tail = sq_ptr + p.sq_off.tail; + sring->ring_mask = sq_ptr + p.sq_off.ring_mask; + sring->ring_entries = sq_ptr + p.sq_off.ring_entries; + sring->flags = sq_ptr + p.sq_off.flags; + sring->array = sq_ptr + p.sq_off.array; + + /* Map a queue as mem map */ + s->sq_ring.queue.sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + s->ring_fd, IORING_OFF_SQES); + if (s->sq_ring.queue.sqes == MAP_FAILED) { + munmap(sq_ptr, sring->ring_sz); + if (sq_ptr != cq_ptr) { + ksft_print_msg("failed to mmap uring queue!"); + munmap(cq_ptr, cring->ring_sz); + return 1; + } + } + + cring->head = cq_ptr + p.cq_off.head; + cring->tail = cq_ptr + p.cq_off.tail; + cring->ring_mask = cq_ptr + p.cq_off.ring_mask; + cring->ring_entries = cq_ptr + p.cq_off.ring_entries; + cring->queue.cqes = cq_ptr + p.cq_off.cqes; + + return 0; +} + +/* Init io_uring queues */ +int setup_io_uring(struct io_ring *s) +{ + struct io_uring_params para; + + memset(¶, 0, sizeof(para)); + s->ring_fd = sys_uring_setup(URING_QUEUE_SZ, ¶); + if (s->ring_fd < 0) + return 1; + + return mmap_io_uring(para, s); +} + +/* + * Get data from completion queue. the data buffer saved the file data + * return 0: success; others: error; + */ +int handle_uring_cq(struct io_ring *s) +{ + struct file_io *fi = NULL; + struct io_uring_queue *cring = &s->cq_ring; + struct io_uring_cqe *cqe; + unsigned int head; + off_t len = 0; + + head = *cring->head; + + do { + barrier(); + if (head == *cring->tail) + break; + /* Get the entry */ + cqe = &cring->queue.cqes[head & *s->cq_ring.ring_mask]; + fi = (struct file_io *)cqe->user_data; + if (cqe->res < 0) + break; + + int blocks = (int)(fi->file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; + + for (int i = 0; i < blocks; i++) + len += fi->iovecs[i].iov_len; + + head++; + } while (1); + + *cring->head = head; + barrier(); + + return (len != fi->file_sz); +} + +/* + * Submit squeue. specify via IORING_OP_READV. + * the buffer need to be set metadata according to LAM mode + */ +int handle_uring_sq(struct io_ring *ring, struct file_io *fi, unsigned long lam) +{ + int file_fd = fi->file_fd; + struct io_uring_queue *sring = &ring->sq_ring; + unsigned int index = 0, cur_block = 0, tail = 0, next_tail = 0; + struct io_uring_sqe *sqe; + + off_t remain = fi->file_sz; + int blocks = (int)(remain + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; + + while (remain) { + off_t bytes = remain; + void *buf; + + if (bytes > URING_BLOCK_SZ) + bytes = URING_BLOCK_SZ; + + fi->iovecs[cur_block].iov_len = bytes; + + if (posix_memalign(&buf, URING_BLOCK_SZ, URING_BLOCK_SZ)) + return 1; + + fi->iovecs[cur_block].iov_base = (void *)set_metadata((uint64_t)buf, lam); + remain -= bytes; + cur_block++; + } + + next_tail = *sring->tail; + tail = next_tail; + next_tail++; + + barrier(); + + index = tail & *ring->sq_ring.ring_mask; + + sqe = &ring->sq_ring.queue.sqes[index]; + sqe->fd = file_fd; + sqe->flags = 0; + sqe->opcode = IORING_OP_READV; + sqe->addr = (unsigned long)fi->iovecs; + sqe->len = blocks; + sqe->off = 0; + sqe->user_data = (uint64_t)fi; + + sring->array[index] = index; + tail = next_tail; + + if (*sring->tail != tail) { + *sring->tail = tail; + barrier(); + } + + if (sys_uring_enter(ring->ring_fd, 1, 1, IORING_ENTER_GETEVENTS) < 0) + return 1; + + return 0; +} + +/* + * Test LAM in async I/O and io_uring, read current binery through io_uring + * Set metadata in pointers to iovecs buffer. + */ +int do_uring(unsigned long lam) +{ + struct io_ring *ring; + struct file_io *fi; + struct stat st; + int ret = 1; + char path[PATH_MAX] = {0}; + + /* get current process path */ + if (readlink("/proc/self/exe", path, PATH_MAX) <= 0) + return 1; + + int file_fd = open(path, O_RDONLY); + + if (file_fd < 0) + return 1; + + if (fstat(file_fd, &st) < 0) + return 1; + + off_t file_sz = st.st_size; + + int blocks = (int)(file_sz + URING_BLOCK_SZ - 1) / URING_BLOCK_SZ; + + fi = malloc(sizeof(*fi) + sizeof(struct iovec) * blocks); + if (!fi) + return 1; + + fi->file_sz = file_sz; + fi->file_fd = file_fd; + + ring = malloc(sizeof(*ring)); + if (!ring) + return 1; + + memset(ring, 0, sizeof(struct io_ring)); + + if (setup_io_uring(ring)) + goto out; + + if (handle_uring_sq(ring, fi, lam)) + goto out; + + ret = handle_uring_cq(ring); + +out: + free(ring); + + for (int i = 0; i < blocks; i++) { + if (fi->iovecs[i].iov_base) { + uint64_t addr = ((uint64_t)fi->iovecs[i].iov_base); + + switch (lam) { + case LAM_U57_BITS: /* Clear bits 62:57 */ + addr = (addr & ~(LAM_U57_MASK)); + break; + } + free((void *)addr); + fi->iovecs[i].iov_base = NULL; + } + } + + free(fi); + + return ret; +} + +int handle_uring(struct testcases *test) +{ + int ret = 0; + + if (test->later == 0 && test->lam != 0) + if (set_lam(test->lam) != 0) + return 1; + + if (sigsetjmp(segv_env, 1) == 0) { + signal(SIGSEGV, segv_handler); + ret = do_uring(test->lam); + } else { + ret = 2; + } + + return ret; +} + +static int fork_test(struct testcases *test) +{ + int ret, child_ret; + pid_t pid; + + pid = fork(); + if (pid < 0) { + perror("Fork failed."); + ret = 1; + } else if (pid == 0) { + ret = test->test_func(test); + exit(ret); + } else { + wait(&child_ret); + ret = WEXITSTATUS(child_ret); + } + + return ret; +} + +static int handle_execve(struct testcases *test) +{ + int ret, child_ret; + int lam = test->lam; + pid_t pid; + + pid = fork(); + if (pid < 0) { + perror("Fork failed."); + ret = 1; + } else if (pid == 0) { + char path[PATH_MAX]; + + /* Set LAM mode in parent process */ + if (set_lam(lam) != 0) + return 1; + + /* Get current binary's path and the binary was run by execve */ + if (readlink("/proc/self/exe", path, PATH_MAX) <= 0) + exit(-1); + + /* run binary to get LAM mode and return to parent process */ + if (execlp(path, path, "-t 0x0", NULL) < 0) { + perror("error on exec"); + exit(-1); + } + } else { + wait(&child_ret); + ret = WEXITSTATUS(child_ret); + if (ret != LAM_NONE) + return 1; + } + + return 0; +} + +static int handle_inheritance(struct testcases *test) +{ + int ret, child_ret; + int lam = test->lam; + pid_t pid; + + /* Set LAM mode in parent process */ + if (set_lam(lam) != 0) + return 1; + + pid = fork(); + if (pid < 0) { + perror("Fork failed."); + return 1; + } else if (pid == 0) { + /* Set LAM mode in parent process */ + int child_lam = get_lam(); + + exit(child_lam); + } else { + wait(&child_ret); + ret = WEXITSTATUS(child_ret); + + if (lam != ret) + return 1; + } + + return 0; +} + +static int thread_fn_get_lam(void *arg) +{ + return get_lam(); +} + +static int thread_fn_set_lam(void *arg) +{ + struct testcases *test = arg; + + return set_lam(test->lam); +} + +static int handle_thread(struct testcases *test) +{ + char stack[STACK_SIZE]; + int ret, child_ret; + int lam = 0; + pid_t pid; + + /* Set LAM mode in parent process */ + if (!test->later) { + lam = test->lam; + if (set_lam(lam) != 0) + return 1; + } + + pid = clone(thread_fn_get_lam, stack + STACK_SIZE, + SIGCHLD | CLONE_FILES | CLONE_FS | CLONE_VM, NULL); + if (pid < 0) { + perror("Clone failed."); + return 1; + } + + waitpid(pid, &child_ret, 0); + ret = WEXITSTATUS(child_ret); + + if (lam != ret) + return 1; + + if (test->later) { + if (set_lam(test->lam) != 0) + return 1; + } + + return 0; +} + +static int handle_thread_enable(struct testcases *test) +{ + char stack[STACK_SIZE]; + int ret, child_ret; + int lam = test->lam; + pid_t pid; + + pid = clone(thread_fn_set_lam, stack + STACK_SIZE, + SIGCHLD | CLONE_FILES | CLONE_FS | CLONE_VM, test); + if (pid < 0) { + perror("Clone failed."); + return 1; + } + + waitpid(pid, &child_ret, 0); + ret = WEXITSTATUS(child_ret); + + if (lam != ret) + return 1; + + return 0; +} +static void run_test(struct testcases *test, int count) +{ + int i, ret = 0; + + for (i = 0; i < count; i++) { + struct testcases *t = test + i; + + /* fork a process to run test case */ + tests_cnt++; + ret = fork_test(t); + + /* return 3 is not support LA57, the case should be skipped */ + if (ret == 3) { + ksft_test_result_skip(t->msg); + continue; + } + + if (ret != 0) + ret = (t->expected == ret); + else + ret = !(t->expected); + + ksft_test_result(ret, t->msg); + } +} + +static struct testcases uring_cases[] = { + { + .later = 0, + .lam = LAM_U57_BITS, + .test_func = handle_uring, + .msg = "URING: LAM_U57. Dereferencing pointer with metadata\n", + }, + { + .later = 1, + .expected = 1, + .lam = LAM_U57_BITS, + .test_func = handle_uring, + .msg = "URING:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", + }, +}; + +static struct testcases malloc_cases[] = { + { + .later = 0, + .lam = LAM_U57_BITS, + .test_func = handle_malloc, + .msg = "MALLOC: LAM_U57. Dereferencing pointer with metadata\n", + }, + { + .later = 1, + .expected = 2, + .lam = LAM_U57_BITS, + .test_func = handle_malloc, + .msg = "MALLOC:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", + }, +}; + +static struct testcases bits_cases[] = { + { + .test_func = handle_max_bits, + .msg = "BITS: Check default tag bits\n", + }, +}; + +static struct testcases syscall_cases[] = { + { + .later = 0, + .lam = LAM_U57_BITS, + .test_func = handle_syscall, + .msg = "SYSCALL: LAM_U57. syscall with metadata\n", + }, + { + .later = 1, + .expected = 1, + .lam = LAM_U57_BITS, + .test_func = handle_syscall, + .msg = "SYSCALL:[Negative] Disable LAM. Dereferencing pointer with metadata.\n", + }, +}; + +static struct testcases mmap_cases[] = { + { + .later = 1, + .expected = 0, + .lam = LAM_U57_BITS, + .addr = HIGH_ADDR, + .test_func = handle_mmap, + .msg = "MMAP: First mmap high address, then set LAM_U57.\n", + }, + { + .later = 0, + .expected = 0, + .lam = LAM_U57_BITS, + .addr = HIGH_ADDR, + .test_func = handle_mmap, + .msg = "MMAP: First LAM_U57, then High address.\n", + }, + { + .later = 0, + .expected = 0, + .lam = LAM_U57_BITS, + .addr = LOW_ADDR, + .test_func = handle_mmap, + .msg = "MMAP: First LAM_U57, then Low address.\n", + }, +}; + +static struct testcases inheritance_cases[] = { + { + .expected = 0, + .lam = LAM_U57_BITS, + .test_func = handle_inheritance, + .msg = "FORK: LAM_U57, child process should get LAM mode same as parent\n", + }, + { + .expected = 0, + .lam = LAM_U57_BITS, + .test_func = handle_thread, + .msg = "THREAD: LAM_U57, child thread should get LAM mode same as parent\n", + }, + { + .expected = 1, + .lam = LAM_U57_BITS, + .test_func = handle_thread_enable, + .msg = "THREAD: [NEGATIVE] Enable LAM in child.\n", + }, + { + .expected = 1, + .later = 1, + .lam = LAM_U57_BITS, + .test_func = handle_thread, + .msg = "THREAD: [NEGATIVE] Enable LAM in parent after thread created.\n", + }, + { + .expected = 0, + .lam = LAM_U57_BITS, + .test_func = handle_execve, + .msg = "EXECVE: LAM_U57, child process should get disabled LAM mode\n", + }, +}; + +static void cmd_help(void) +{ + printf("usage: lam [-h] [-t test list]\n"); + printf("\t-t test list: run tests specified in the test list, default:0x%x\n", TEST_MASK); + printf("\t\t0x1:malloc; 0x2:max_bits; 0x4:mmap; 0x8:syscall; 0x10:io_uring; 0x20:inherit;\n"); + printf("\t-h: help\n"); +} + +/* Check for file existence */ +uint8_t file_Exists(const char *fileName) +{ + struct stat buffer; + + uint8_t ret = (stat(fileName, &buffer) == 0); + + return ret; +} + +/* Sysfs idxd files */ +const char *dsa_configs[] = { + "echo 1 > /sys/bus/dsa/devices/dsa0/wq0.1/group_id", + "echo shared > /sys/bus/dsa/devices/dsa0/wq0.1/mode", + "echo 10 > /sys/bus/dsa/devices/dsa0/wq0.1/priority", + "echo 16 > /sys/bus/dsa/devices/dsa0/wq0.1/size", + "echo 15 > /sys/bus/dsa/devices/dsa0/wq0.1/threshold", + "echo user > /sys/bus/dsa/devices/dsa0/wq0.1/type", + "echo MyApp1 > /sys/bus/dsa/devices/dsa0/wq0.1/name", + "echo 1 > /sys/bus/dsa/devices/dsa0/engine0.1/group_id", + "echo dsa0 > /sys/bus/dsa/drivers/idxd/bind", + /* bind files and devices, generated a device file in /dev */ + "echo wq0.1 > /sys/bus/dsa/drivers/user/bind", +}; + +/* DSA device file */ +const char *dsaDeviceFile = "/dev/dsa/wq0.1"; +/* file for io*/ +const char *dsaPasidEnable = "/sys/bus/dsa/devices/dsa0/pasid_enabled"; + +/* + * DSA depends on kernel cmdline "intel_iommu=on,sm_on" + * return pasid_enabled (0: disable 1:enable) + */ +int Check_DSA_Kernel_Setting(void) +{ + char command[256] = ""; + char buf[256] = ""; + char *ptr; + int rv = -1; + + snprintf(command, sizeof(command) - 1, "cat %s", dsaPasidEnable); + + FILE *cmd = popen(command, "r"); + + if (cmd) { + while (fgets(buf, sizeof(buf) - 1, cmd) != NULL); + + pclose(cmd); + rv = strtol(buf, &ptr, 16); + } + + return rv; +} + +/* + * Config DSA's sysfs files as shared DSA's WQ. + * Generated a device file /dev/dsa/wq0.1 + * Return: 0 OK; 1 Failed; 3 Skip(SVA disabled). + */ +int Dsa_Init_Sysfs(void) +{ + uint len = ARRAY_SIZE(dsa_configs); + const char **p = dsa_configs; + + if (file_Exists(dsaDeviceFile) == 1) + return 0; + + /* check the idxd driver */ + if (file_Exists(dsaPasidEnable) != 1) { + printf("Please make sure idxd driver was loaded\n"); + return 3; + } + + /* Check SVA feature */ + if (Check_DSA_Kernel_Setting() != 1) { + printf("Please enable SVA.(Add intel_iommu=on,sm_on in kernel cmdline)\n"); + return 3; + } + + /* Check the idxd device file on /dev/dsa/ */ + for (int i = 0; i < len; i++) { + if (system(p[i])) + return 1; + } + + /* After config, /dev/dsa/wq0.1 should be generated */ + return (file_Exists(dsaDeviceFile) != 1); +} + +/* + * Open DSA device file, triger API: iommu_sva_alloc_pasid + */ +void *allocate_dsa_pasid(void) +{ + int fd; + void *wq; + + fd = open(dsaDeviceFile, O_RDWR); + if (fd < 0) { + perror("open"); + return MAP_FAILED; + } + + wq = mmap(NULL, 0x1000, PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd, 0); + if (wq == MAP_FAILED) + perror("mmap"); + + return wq; +} + +int set_force_svm(void) +{ + int ret = 0; + + ret = syscall(SYS_arch_prctl, ARCH_FORCE_TAGGED_SVA); + + return ret; +} + +int handle_pasid(struct testcases *test) +{ + uint tmp = test->cmd; + uint runed = 0x0; + int ret = 0; + void *wq = NULL; + + ret = Dsa_Init_Sysfs(); + if (ret != 0) + return ret; + + for (int i = 0; i < 3; i++) { + int err = 0; + + if (tmp & 0x1) { + /* run set lam mode*/ + if ((runed & 0x1) == 0) { + err = set_lam(LAM_U57_BITS); + runed = runed | 0x1; + } else + err = 1; + } else if (tmp & 0x4) { + /* run force svm */ + if ((runed & 0x4) == 0) { + err = set_force_svm(); + runed = runed | 0x4; + } else + err = 1; + } else if (tmp & 0x2) { + /* run allocate pasid */ + if ((runed & 0x2) == 0) { + runed = runed | 0x2; + wq = allocate_dsa_pasid(); + if (wq == MAP_FAILED) + err = 1; + } else + err = 1; + } + + ret = ret + err; + if (ret > 0) + break; + + tmp = tmp >> 4; + } + + if (wq != MAP_FAILED && wq != NULL) + if (munmap(wq, 0x1000)) + printf("munmap failed %d\n", errno); + + if (runed != 0x7) + ret = 1; + + return (ret != 0); +} + +/* + * Pasid test depends on idxd and SVA, kernel should enable iommu and sm. + * command line(intel_iommu=on,sm_on) + */ +static struct testcases pasid_cases[] = { + { + .expected = 1, + .cmd = PAS_CMD(LAM_CMD_BIT, PAS_CMD_BIT, SVA_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: [Negative] Execute LAM, PASID, SVA in sequence\n", + }, + { + .expected = 0, + .cmd = PAS_CMD(LAM_CMD_BIT, SVA_CMD_BIT, PAS_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: Execute LAM, SVA, PASID in sequence\n", + }, + { + .expected = 1, + .cmd = PAS_CMD(PAS_CMD_BIT, LAM_CMD_BIT, SVA_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: [Negative] Execute PASID, LAM, SVA in sequence\n", + }, + { + .expected = 0, + .cmd = PAS_CMD(PAS_CMD_BIT, SVA_CMD_BIT, LAM_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: Execute PASID, SVA, LAM in sequence\n", + }, + { + .expected = 0, + .cmd = PAS_CMD(SVA_CMD_BIT, LAM_CMD_BIT, PAS_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: Execute SVA, LAM, PASID in sequence\n", + }, + { + .expected = 0, + .cmd = PAS_CMD(SVA_CMD_BIT, PAS_CMD_BIT, LAM_CMD_BIT), + .test_func = handle_pasid, + .msg = "PASID: Execute SVA, PASID, LAM in sequence\n", + }, +}; + +int main(int argc, char **argv) +{ + int c = 0; + unsigned int tests = TEST_MASK; + + tests_cnt = 0; + + if (!cpu_has_lam()) { + ksft_print_msg("Unsupported LAM feature!\n"); + return -1; + } + + while ((c = getopt(argc, argv, "ht:")) != -1) { + switch (c) { + case 't': + tests = strtoul(optarg, NULL, 16); + if (tests && !(tests & TEST_MASK)) { + ksft_print_msg("Invalid argument!\n"); + return -1; + } + break; + case 'h': + cmd_help(); + return 0; + default: + ksft_print_msg("Invalid argument\n"); + return -1; + } + } + + /* + * When tests is 0, it is not a real test case; + * the option used by test case(execve) to check the lam mode in + * process generated by execve, the process read back lam mode and + * check with lam mode in parent process. + */ + if (!tests) + return (get_lam()); + + /* Run test cases */ + if (tests & FUNC_MALLOC) + run_test(malloc_cases, ARRAY_SIZE(malloc_cases)); + + if (tests & FUNC_BITS) + run_test(bits_cases, ARRAY_SIZE(bits_cases)); + + if (tests & FUNC_MMAP) + run_test(mmap_cases, ARRAY_SIZE(mmap_cases)); + + if (tests & FUNC_SYSCALL) + run_test(syscall_cases, ARRAY_SIZE(syscall_cases)); + + if (tests & FUNC_URING) + run_test(uring_cases, ARRAY_SIZE(uring_cases)); + + if (tests & FUNC_INHERITE) + run_test(inheritance_cases, ARRAY_SIZE(inheritance_cases)); + + if (tests & FUNC_PASID) + run_test(pasid_cases, ARRAY_SIZE(pasid_cases)); + + ksft_set_plan(tests_cnt); + + return ksft_exit_pass(); +}