tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 2aff7c706c ("Merge tag 'objtool-core-2023-04-27' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip") into android-mainline

Browse Source 
Steps on the way to 6.4-rc1

Change-Id: I6180cd865e58218c704f11669600397ef6299f2a
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
This commit is contained in:
Greg Kroah-Hartman
2023-05-29 19:19:27 +00:00
parent a873a66576 2aff7c706c
commit 968ce63988
132 changed files with 2527 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/arch/x86/xstate.rst
+100
View File
@@ -11,6 +11,22 @@ are enabled by XCR0 as well, but the first use of related instruction is
trapped by the kernel because by default the required large XSTATE buffers
are not allocated automatically.
The purpose for dynamic features
--------------------------------
Legacy userspace libraries often have hard-coded, static sizes for
alternate signal stacks, often using MINSIGSTKSZ which is typically 2KB.
That stack must be able to store at *least* the signal frame that the
kernel sets up before jumping into the signal handler. That signal frame
must include an XSAVE buffer defined by the CPU.
However, that means that the size of signal stacks is dynamic, not static,
because different CPUs have differently-sized XSAVE buffers. A compiled-in
size of 2KB with existing applications is too small for new CPU features
like AMX. Instead of universally requiring larger stack, with the dynamic
enabling, the kernel can enforce userspace applications to have
properly-sized altstacks.
Using dynamically enabled XSTATE features in user space applications
--------------------------------------------------------------------
@@ -64,6 +80,61 @@ the handler allocates a larger xstate buffer for the task so the large
state can be context switched. In the unlikely cases that the allocation
fails, the kernel sends SIGSEGV.
AMX TILE_DATA enabling example
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Below is the example of how userspace applications enable
TILE_DATA dynamically:
1. The application first needs to query the kernel for AMX
support::
#include <asm/prctl.h>
#include <sys/syscall.h>
#include <stdio.h>
#include <unistd.h>
#ifndef ARCH_GET_XCOMP_SUPP
#define ARCH_GET_XCOMP_SUPP 0x1021
#endif
#ifndef ARCH_XCOMP_TILECFG
#define ARCH_XCOMP_TILECFG 17
#endif
#ifndef ARCH_XCOMP_TILEDATA
#define ARCH_XCOMP_TILEDATA 18
#endif
#define MASK_XCOMP_TILE ((1 << ARCH_XCOMP_TILECFG) | \
(1 << ARCH_XCOMP_TILEDATA))
unsigned long features;
long rc;
...
rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_SUPP, &features);
if (!rc && (features & MASK_XCOMP_TILE) == MASK_XCOMP_TILE)
printf("AMX is available.\n");
2. After that, determining support for AMX, an application must
explicitly ask permission to use it::
#ifndef ARCH_REQ_XCOMP_PERM
#define ARCH_REQ_XCOMP_PERM 0x1023
#endif
...
rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, ARCH_XCOMP_TILEDATA);
if (!rc)
printf("AMX is ready for use.\n");
Note this example does not include the sigaltstack preparation.
Dynamic features in signal frames
---------------------------------
@@ -72,3 +143,32 @@ entry if the feature is in its initial configuration. This differs from
non-dynamic features which are always written regardless of their
configuration. Signal handlers can examine the XSAVE buffer's XSTATE_BV
field to determine if a features was written.
Dynamic features for virtual machines
-------------------------------------
The permission for the guest state component needs to be managed separately
from the host, as they are exclusive to each other. A coupled of options
are extended to control the guest permission:
-ARCH_GET_XCOMP_GUEST_PERM
arch_prctl(ARCH_GET_XCOMP_GUEST_PERM, &features);
ARCH_GET_XCOMP_GUEST_PERM is a variant of ARCH_GET_XCOMP_PERM. So it
provides the same semantics and functionality but for the guest
components.
-ARCH_REQ_XCOMP_GUEST_PERM
arch_prctl(ARCH_REQ_XCOMP_GUEST_PERM, feature_nr);
ARCH_REQ_XCOMP_GUEST_PERM is a variant of ARCH_REQ_XCOMP_PERM. It has the
same semantics for the guest permission. While providing a similar
functionality, this comes with a constraint. Permission is frozen when the
first VCPU is created. Any attempt to change permission after that point
is going to be rejected. So, the permission has to be requested before the
first VCPU creation.
Note that some VMMs may have already established a set of supported state
components. These options are not presumed to support any particular VMM.
Documentation/livepatch/reliable-stacktrace.rst
+1 -1
View File
@@ -183,7 +183,7 @@ trampoline or return trampoline. For example, considering the x86_64
.. code-block:: none
SYM_CODE_START(return_to_handler)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
subq $24, %rsp
/* Save the return values */
MAINTAINERS
+1 -1
View File
@@ -15187,8 +15187,8 @@ OBJTOOL
M: Josh Poimboeuf <jpoimboe@kernel.org>
M: Peter Zijlstra <peterz@infradead.org>
S: Supported
F: include/linux/objtool*.h
F: tools/objtool/
F: include/linux/objtool.h
OCELOT ETHERNET SWITCH DRIVER
M: Vladimir Oltean <vladimir.oltean@nxp.com>
arch/alpha/kernel/process.c
+3 -1
View File
@@ -9,6 +9,7 @@
* This file handles the architecture-dependent parts of process handling.
*/
#include <linux/cpu.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/sched.h>
@@ -59,9 +60,10 @@ void arch_cpu_idle(void)
wtint(0);
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
wtint(INT_MAX);
BUG();
}
#endif /* ALPHA_WTINT */
arch/arm/kernel/smp.c
+4 -2
View File
@@ -324,7 +324,7 @@ void __cpu_die(unsigned int cpu)
* of the other hotplug-cpu capable cores, so presumably coming
* out of idle fixes this.
*/
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
unsigned int cpu = smp_processor_id();
@@ -386,6 +386,8 @@ void arch_cpu_idle_dead(void)
: "r" (task_stack_page(current) + THREAD_SIZE - 8),
"r" (current)
: "r0");
unreachable();
}
#endif /* CONFIG_HOTPLUG_CPU */
@@ -781,7 +783,7 @@ void smp_send_stop(void)
* kdump fails. So split out the panic_smp_self_stop() and add
* set_cpu_online(smp_processor_id(), false).
*/
void panic_smp_self_stop(void)
void __noreturn panic_smp_self_stop(void)
{
pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n",
smp_processor_id());
arch/arm64/include/asm/exception.h
+2 -2
View File
@@ -31,7 +31,7 @@ static inline unsigned long disr_to_esr(u64 disr)
return esr;
}
asmlinkage void handle_bad_stack(struct pt_regs *regs);
asmlinkage void __noreturn handle_bad_stack(struct pt_regs *regs);
asmlinkage void el1t_64_sync_handler(struct pt_regs *regs);
asmlinkage void el1t_64_irq_handler(struct pt_regs *regs);
@@ -80,5 +80,5 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr);
void do_serror(struct pt_regs *regs, unsigned long esr);
void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags);
void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far);
void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far);
#endif /* __ASM_EXCEPTION_H */
arch/arm64/include/asm/mmu_context.h
+6
View File
@@ -288,6 +288,12 @@ void post_ttbr_update_workaround(void);
unsigned long arm64_mm_context_get(struct mm_struct *mm);
void arm64_mm_context_put(struct mm_struct *mm);
#define mm_untag_mask mm_untag_mask
static inline unsigned long mm_untag_mask(struct mm_struct *mm)
{
return -1UL >> 8;
}
#include <asm-generic/mmu_context.h>
#endif /* !__ASSEMBLY__ */
arch/arm64/include/asm/smp.h
+4 -5
View File
@@ -102,10 +102,10 @@ static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
extern int __cpu_disable(void);
extern void __cpu_die(unsigned int cpu);
extern void cpu_die(void);
extern void cpu_die_early(void);
extern void __noreturn cpu_die(void);
extern void __noreturn cpu_die_early(void);
static inline void cpu_park_loop(void)
static inline void __noreturn cpu_park_loop(void)
{
for (;;) {
wfe();
@@ -125,7 +125,7 @@ static inline void update_cpu_boot_status(int val)
* which calls for a kernel panic. Update the boot status and park the calling
* CPU.
*/
static inline void cpu_panic_kernel(void)
static inline void __noreturn cpu_panic_kernel(void)
{
update_cpu_boot_status(CPU_PANIC_KERNEL);
cpu_park_loop();
@@ -145,7 +145,6 @@ bool cpus_are_stuck_in_kernel(void);
extern void crash_smp_send_stop(void);
extern bool smp_crash_stop_failed(void);
extern void panic_smp_self_stop(void);
#endif /* ifndef __ASSEMBLY__ */
arch/arm64/kernel/entry-common.c
+1 -1
View File
@@ -840,7 +840,7 @@ UNHANDLED(el0t, 32, error)
#endif /* CONFIG_COMPAT */
#ifdef CONFIG_VMAP_STACK
asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
unsigned long far = read_sysreg(far_el1);
arch/arm64/kernel/process.c
+1 -1
View File
@@ -70,7 +70,7 @@ void (*pm_power_off)(void);
EXPORT_SYMBOL_GPL(pm_power_off);
#ifdef CONFIG_HOTPLUG_CPU
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
cpu_die();
}
arch/arm64/kernel/smp.c
+7 -5
View File
@@ -366,7 +366,7 @@ void __cpu_die(unsigned int cpu)
* Called from the idle thread for the CPU which has been shutdown.
*
*/
void cpu_die(void)
void __noreturn cpu_die(void)
{
unsigned int cpu = smp_processor_id();
const struct cpu_operations *ops = get_cpu_ops(cpu);
@@ -403,7 +403,7 @@ static void __cpu_try_die(int cpu)
* Kill the calling secondary CPU, early in bringup before it is turned
* online.
*/
void cpu_die_early(void)
void __noreturn cpu_die_early(void)
{
int cpu = smp_processor_id();
@@ -821,7 +821,7 @@ void arch_irq_work_raise(void)
}
#endif
static void local_cpu_stop(void)
static void __noreturn local_cpu_stop(void)
{
set_cpu_online(smp_processor_id(), false);
@@ -835,7 +835,7 @@ static void local_cpu_stop(void)
* that cpu_online_mask gets correctly updated and smp_send_stop() can skip
* CPUs that have already stopped themselves.
*/
void panic_smp_self_stop(void)
void __noreturn panic_smp_self_stop(void)
{
local_cpu_stop();
}
@@ -844,7 +844,7 @@ void panic_smp_self_stop(void)
static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0);
#endif
static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
{
#ifdef CONFIG_KEXEC_CORE
crash_save_cpu(regs, cpu);
@@ -859,6 +859,8 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
/* just in case */
cpu_park_loop();
#else
BUG();
#endif
}
arch/arm64/kernel/traps.c
+1 -2
View File
@@ -863,7 +863,7 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr)
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
__aligned(16);
void panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far)
void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far)
{
unsigned long tsk_stk = (unsigned long)current->stack;
unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
@@ -905,7 +905,6 @@ void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr)
nmi_panic(regs, "Asynchronous SError Interrupt");
cpu_park_loop();
unreachable();
}
bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr)
arch/csky/kernel/smp.c
+3 -1
View File
@@ -300,7 +300,7 @@ void __cpu_die(unsigned int cpu)
pr_notice("CPU%u: shutdown\n", cpu);
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
idle_task_exit();
@@ -317,5 +317,7 @@ void arch_cpu_idle_dead(void)
"jmpi csky_start_secondary"
:
: "r" (secondary_stack));
BUG();
}
#endif
arch/ia64/kernel/process.c
+3 -3
View File
@@ -201,7 +201,7 @@ __setup("nohalt", nohalt_setup);
#ifdef CONFIG_HOTPLUG_CPU
/* We don't actually take CPU down, just spin without interrupts. */
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
unsigned int this_cpu = smp_processor_id();
@@ -219,13 +219,13 @@ static inline void play_dead(void)
BUG();
}
#else
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
BUG();
}
#endif /* CONFIG_HOTPLUG_CPU */
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
play_dead();
}
arch/loongarch/include/asm/smp.h
+1 -1
View File
@@ -99,7 +99,7 @@ static inline void __cpu_die(unsigned int cpu)
loongson_cpu_die(cpu);
}
extern void play_dead(void);
extern void __noreturn play_dead(void);
#endif
#endif /* __ASM_SMP_H */
arch/loongarch/kernel/process.c
+1 -1
View File
@@ -62,7 +62,7 @@ unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
EXPORT_SYMBOL(boot_option_idle_override);
#ifdef CONFIG_HOTPLUG_CPU
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
play_dead();
}
arch/loongarch/kernel/smp.c
+1 -1
View File
@@ -336,7 +336,7 @@ void play_dead(void)
iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR);
init_fn();
unreachable();
BUG();
}
#endif
arch/mips/cavium-octeon/smp.c
+1
View File
@@ -20,6 +20,7 @@
#include <asm/mmu_context.h>
#include <asm/time.h>
#include <asm/setup.h>
#include <asm/smp.h>
#include <asm/octeon/octeon.h>
arch/mips/include/asm/smp.h
+1 -1
View File
@@ -88,7 +88,7 @@ static inline void __cpu_die(unsigned int cpu)
mp_ops->cpu_die(cpu);
}
extern void play_dead(void);
extern void __noreturn play_dead(void);
#endif
#ifdef CONFIG_KEXEC
arch/mips/kernel/process.c
+1 -1
View File
@@ -40,7 +40,7 @@
#include <asm/stacktrace.h>
#ifdef CONFIG_HOTPLUG_CPU
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
play_dead();
}
arch/mips/kernel/smp-bmips.c
+4
View File
@@ -54,6 +54,8 @@ static void bmips_set_reset_vec(int cpu, u32 val);
#ifdef CONFIG_SMP
#include <asm/smp.h>
/* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */
unsigned long bmips_smp_boot_sp;
unsigned long bmips_smp_boot_gp;
@@ -413,6 +415,8 @@ void __ref play_dead(void)
" wait\n"
" j bmips_secondary_reentry\n"
: : : "memory");
BUG();
}
#endif /* CONFIG_HOTPLUG_CPU */
arch/mips/kernel/smp-cps.c
+1
View File
@@ -20,6 +20,7 @@
#include <asm/mipsregs.h>
#include <asm/pm-cps.h>
#include <asm/r4kcache.h>
#include <asm/smp.h>
#include <asm/smp-cps.h>
#include <asm/time.h>
#include <asm/uasm.h>
arch/mips/loongson64/smp.c
+2
View File
@@ -14,6 +14,7 @@
#include <linux/cpufreq.h>
#include <linux/kexec.h>
#include <asm/processor.h>
#include <asm/smp.h>
#include <asm/time.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
@@ -808,6 +809,7 @@ out:
state_addr = &per_cpu(cpu_state, cpu);
mb();
play_dead_at_ckseg1(state_addr);
BUG();
}
static int loongson3_disable_clock(unsigned int cpu)
arch/parisc/kernel/process.c
+1 -1
View File
@@ -159,7 +159,7 @@ EXPORT_SYMBOL(running_on_qemu);
/*
* Called from the idle thread for the CPU which has been shutdown.
*/
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
#ifdef CONFIG_HOTPLUG_CPU
idle_task_exit();
arch/powerpc/include/asm/smp.h
+1 -1
View File
@@ -67,7 +67,7 @@ void start_secondary(void *unused);
extern int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us);
extern int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us);
extern void smp_send_debugger_break(void);
extern void start_secondary_resume(void);
extern void __noreturn start_secondary_resume(void);
extern void smp_generic_give_timebase(void);
extern void smp_generic_take_timebase(void);
arch/powerpc/kernel/setup_64.c
+1 -1
View File
@@ -480,7 +480,7 @@ void early_setup_secondary(void)
#endif /* CONFIG_SMP */
void panic_smp_self_stop(void)
void __noreturn panic_smp_self_stop(void)
{
hard_irq_disable();
spin_begin();
arch/powerpc/kernel/smp.c
+1 -1
View File
@@ -1752,7 +1752,7 @@ void __cpu_die(unsigned int cpu)
smp_ops->cpu_die(cpu);
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
/*
* Disable on the down path. This will be re-enabled by
arch/riscv/kernel/cpu-hotplug.c
+1 -1
View File
@@ -72,7 +72,7 @@ void __cpu_die(unsigned int cpu)
/*
* Called from the idle thread for the CPU which has been shutdown.
*/
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
idle_task_exit();
arch/s390/kernel/idle.c
+1 -1
View File
@@ -88,7 +88,7 @@ void arch_cpu_idle_exit(void)
{
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
cpu_die();
}
arch/s390/kernel/setup.c
+1 -1
View File
@@ -396,7 +396,7 @@ int __init arch_early_irq_init(void)
return 0;
}
void __init arch_call_rest_init(void)
void __init __noreturn arch_call_rest_init(void)
{
unsigned long stack;
arch/sh/include/asm/smp-ops.h
+3 -2
View File
@@ -24,9 +24,10 @@ static inline void plat_smp_setup(void)
mp_ops->smp_setup();
}
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
mp_ops->play_dead();
BUG();
}
extern void register_smp_ops(struct plat_smp_ops *ops);
@@ -42,7 +43,7 @@ static inline void register_smp_ops(struct plat_smp_ops *ops)
{
}
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
BUG();
}
arch/sh/kernel/idle.c
+2 -1
View File
@@ -4,6 +4,7 @@
*
* Copyright (C) 2002 - 2009 Paul Mundt
*/
#include <linux/cpu.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
@@ -29,7 +30,7 @@ void default_idle(void)
clear_bl_bit();
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
play_dead();
}
arch/sparc/include/asm/mmu_context_64.h
+6
View File
@@ -185,6 +185,12 @@ static inline void finish_arch_post_lock_switch(void)
}
}
#define mm_untag_mask mm_untag_mask
static inline unsigned long mm_untag_mask(struct mm_struct *mm)
{
return -1UL >> adi_nbits();
}
#include <asm-generic/mmu_context.h>
#endif /* !(__ASSEMBLY__) */
arch/sparc/include/asm/smp_64.h
+1 -1
View File
@@ -49,7 +49,7 @@ int hard_smp_processor_id(void);
void smp_fill_in_cpu_possible_map(void);
void smp_fill_in_sib_core_maps(void);
void cpu_play_dead(void);
void __noreturn cpu_play_dead(void);
void smp_fetch_global_regs(void);
void smp_fetch_global_pmu(void);
arch/sparc/include/asm/uaccess_64.h
+2
View File
@@ -8,8 +8,10 @@
#include <linux/compiler.h>
#include <linux/string.h>
#include <linux/mm_types.h>
#include <asm/asi.h>
#include <asm/spitfire.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm-generic/access_ok.h>
arch/sparc/kernel/process_64.c
+1 -1
View File
@@ -95,7 +95,7 @@ void arch_cpu_idle(void)
}
#ifdef CONFIG_HOTPLUG_CPU
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
sched_preempt_enable_no_resched();
cpu_play_dead();
arch/x86/Kconfig
+11
View File
@@ -2290,6 +2290,17 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
config ADDRESS_MASKING
bool "Linear Address Masking support"
depends on X86_64
help
Linear Address Masking (LAM) modifies the checking that is applied
to 64-bit linear addresses, allowing software to use of the
untranslated address bits for metadata.
The capability can be used for efficient address sanitizers (ASAN)
implementation and for optimizations in JITs.
config HOTPLUG_CPU
def_bool y
depends on SMP
arch/x86/boot/compressed/ident_map_64.c
-8
View File
@@ -8,14 +8,6 @@
* Copyright (C) 2016 Kees Cook
*/
/*
* Since we're dealing with identity mappings, physical and virtual
* addresses are the same, so override these defines which are ultimately
* used by the headers in misc.h.
*/
#define __pa(x) ((unsigned long)(x))
#define __va(x) ((void *)((unsigned long)(x)))
/* No PAGE_TABLE_ISOLATION support needed either: */
#undef CONFIG_PAGE_TABLE_ISOLATION
arch/x86/boot/compressed/misc.h
+9
View File
@@ -19,6 +19,15 @@
/* cpu_feature_enabled() cannot be used this early */
#define USE_EARLY_PGTABLE_L5
/*
* Boot stub deals with identity mappings, physical and virtual addresses are
* the same, so override these defines.
*
* <asm/page.h> will not define them if they are already defined.
*/
#define __pa(x) ((unsigned long)(x))
#define __va(x) ((void *)((unsigned long)(x)))
#include <linux/linkage.h>
#include <linux/screen_info.h>
#include <linux/elf.h>
arch/x86/boot/compressed/sev.c
-2
View File
@@ -104,9 +104,7 @@ static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
}
#undef __init
#undef __pa
#define __init
#define __pa(x) ((unsigned long)(x))
#define __BOOT_COMPRESSED
arch/x86/boot/compressed/tdx.c
+2 -2
View File
@@ -26,7 +26,7 @@ static inline unsigned int tdx_io_in(int size, u16 port)
.r14 = port,
};
if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT))
if (__tdx_hypercall_ret(&args))
return UINT_MAX;
return args.r11;
@@ -43,7 +43,7 @@ static inline void tdx_io_out(int size, u16 port, u32 value)
.r15 = value,
};
__tdx_hypercall(&args, 0);
__tdx_hypercall(&args);
}
static inline u8 tdx_inb(u16 port)
arch/x86/coco/tdx/tdcall.S
+38 -28
View File
@@ -85,12 +85,12 @@ SYM_FUNC_START(__tdx_module_call)
SYM_FUNC_END(__tdx_module_call)
/*
* __tdx_hypercall() - Make hypercalls to a TDX VMM using TDVMCALL leaf
* of TDCALL instruction
* TDX_HYPERCALL - Make hypercalls to a TDX VMM using TDVMCALL leaf of TDCALL
* instruction
*
* Transforms values in function call argument struct tdx_hypercall_args @args
* into the TDCALL register ABI. After TDCALL operation, VMM output is saved
* back in @args.
* back in @args, if \ret is 1.
*
*-------------------------------------------------------------------------
* TD VMCALL ABI:
@@ -105,26 +105,18 @@ SYM_FUNC_END(__tdx_module_call)
* specification. Non zero value indicates vendor
* specific ABI.
* R11 - VMCALL sub function number
* RBX, RBP, RDI, RSI - Used to pass VMCALL sub function specific arguments.
* RBX, RDX, RDI, RSI - Used to pass VMCALL sub function specific arguments.
* R8-R9, R12-R15 - Same as above.
*
* Output Registers:
*
* RAX - TDCALL instruction status (Not related to hypercall
* output).
* R10 - Hypercall output error code.
* R11-R15 - Hypercall sub function specific output values.
* RBX, RDX, RDI, RSI - Hypercall sub function specific output values.
* R8-R15 - Same as above.
*
*-------------------------------------------------------------------------
*
* __tdx_hypercall() function ABI:
*
* @args (RDI) - struct tdx_hypercall_args for input and output
* @flags (RSI) - TDX_HCALL_* flags
*
* On successful completion, return the hypercall error code.
*/
SYM_FUNC_START(__tdx_hypercall)
.macro TDX_HYPERCALL ret:req
FRAME_BEGIN
/* Save callee-saved GPRs as mandated by the x86_64 ABI */
@@ -134,9 +126,8 @@ SYM_FUNC_START(__tdx_hypercall)
push %r12
push %rbx
/* Free RDI and RSI to be used as TDVMCALL arguments */
/* Free RDI to be used as TDVMCALL arguments */
movq %rdi, %rax
push %rsi
/* Copy hypercall registers from arg struct: */
movq TDX_HYPERCALL_r8(%rax), %r8
@@ -171,14 +162,11 @@ SYM_FUNC_START(__tdx_hypercall)
* and are handled by callers.
*/
testq %rax, %rax
jne .Lpanic
jne .Lpanic\@
pop %rax
/* Copy hypercall result registers to arg struct if needed */
testq $TDX_HCALL_HAS_OUTPUT, (%rsp)
jz .Lout
.if \ret
movq %r8, TDX_HYPERCALL_r8(%rax)
movq %r9, TDX_HYPERCALL_r9(%rax)
movq %r10, TDX_HYPERCALL_r10(%rax)
@@ -191,7 +179,8 @@ SYM_FUNC_START(__tdx_hypercall)
movq %rsi, TDX_HYPERCALL_rsi(%rax)
movq %rbx, TDX_HYPERCALL_rbx(%rax)
movq %rdx, TDX_HYPERCALL_rdx(%rax)
.Lout:
.endif
/* TDVMCALL leaf return code is in R10 */
movq %r10, %rax
@@ -208,9 +197,6 @@ SYM_FUNC_START(__tdx_hypercall)
xor %rdi, %rdi
xor %rdx, %rdx
/* Remove TDX_HCALL_* flags from the stack */
pop %rsi
/* Restore callee-saved GPRs as mandated by the x86_64 ABI */
pop %rbx
pop %r12
@@ -221,9 +207,33 @@ SYM_FUNC_START(__tdx_hypercall)
FRAME_END
RET
.Lpanic:
.Lpanic\@:
call __tdx_hypercall_failed
/* __tdx_hypercall_failed never returns */
REACHABLE
jmp .Lpanic
jmp .Lpanic\@
.endm
/*
*
* __tdx_hypercall() function ABI:
*
* @args (RDI) - struct tdx_hypercall_args for input
*
* On successful completion, return the hypercall error code.
*/
SYM_FUNC_START(__tdx_hypercall)
TDX_HYPERCALL ret=0
SYM_FUNC_END(__tdx_hypercall)
/*
*
* __tdx_hypercall_ret() function ABI:
*
* @args (RDI) - struct tdx_hypercall_args for input and output
*
* On successful completion, return the hypercall error code.
*/
SYM_FUNC_START(__tdx_hypercall_ret)
TDX_HYPERCALL ret=1
SYM_FUNC_END(__tdx_hypercall_ret)
arch/x86/coco/tdx/tdx.c
+9 -9
View File
@@ -66,7 +66,7 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
.r15 = r15,
};
return __tdx_hypercall(&args, 0);
return __tdx_hypercall(&args);
}
/* Called from __tdx_hypercall() for unrecoverable failure */
@@ -99,7 +99,7 @@ long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, unsigned long p2,
.r14 = p4,
};
return __tdx_hypercall(&args, 0);
return __tdx_hypercall(&args);
}
EXPORT_SYMBOL_GPL(tdx_kvm_hypercall);
#endif
@@ -179,7 +179,7 @@ static void __noreturn tdx_panic(const char *msg)
* happens to return.
*/
while (1)
__tdx_hypercall(&args, 0);
__tdx_hypercall(&args);
}
static void tdx_parse_tdinfo(u64 *cc_mask)
@@ -289,7 +289,7 @@ static u64 __cpuidle __halt(const bool irq_disabled)
* can keep the vCPU in virtual HLT, even if an IRQ is
* pending, without hanging/breaking the guest.
*/
return __tdx_hypercall(&args, 0);
return __tdx_hypercall(&args);
}
static int handle_halt(struct ve_info *ve)
@@ -326,7 +326,7 @@ static int read_msr(struct pt_regs *regs, struct ve_info *ve)
* can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "TDG.VP.VMCALL<Instruction.RDMSR>".
*/
if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT))
if (__tdx_hypercall_ret(&args))
return -EIO;
regs->ax = lower_32_bits(args.r11);
@@ -348,7 +348,7 @@ static int write_msr(struct pt_regs *regs, struct ve_info *ve)
* can be found in TDX Guest-Host-Communication Interface
* (GHCI) section titled "TDG.VP.VMCALL<Instruction.WRMSR>".
*/
if (__tdx_hypercall(&args, 0))
if (__tdx_hypercall(&args))
return -EIO;
return ve_instr_len(ve);
@@ -380,7 +380,7 @@ static int handle_cpuid(struct pt_regs *regs, struct ve_info *ve)
* ABI can be found in TDX Guest-Host-Communication Interface
* (GHCI), section titled "VP.VMCALL<Instruction.CPUID>".
*/
if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT))
if (__tdx_hypercall_ret(&args))
return -EIO;
/*
@@ -407,7 +407,7 @@ static bool mmio_read(int size, unsigned long addr, unsigned long *val)
.r15 = *val,
};
if (__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT))
if (__tdx_hypercall_ret(&args))
return false;
*val = args.r11;
return true;
@@ -541,7 +541,7 @@ static bool handle_in(struct pt_regs *regs, int size, int port)
* in TDX Guest-Host-Communication Interface (GHCI) section titled
* "TDG.VP.VMCALL<Instruction.IO>".
*/
success = !__tdx_hypercall(&args, TDX_HCALL_HAS_OUTPUT);
success = !__tdx_hypercall_ret(&args);
/* Update part of the register affected by the emulated instruction */
regs->ax &= ~mask;
arch/x86/entry/entry_64.S
+14 -14
View File
@@ -205,7 +205,7 @@ syscall_return_via_sysret:
*/
movq %rsp, %rdi
movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
pushq RSP-RDI(%rdi) /* RSP */
pushq (%rdi) /* RDI */
@@ -286,7 +286,7 @@ SYM_FUNC_END(__switch_to_asm)
.pushsection .text, "ax"
__FUNC_ALIGN
SYM_CODE_START_NOALIGN(ret_from_fork)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // copy_thread
CALL_DEPTH_ACCOUNT
movq %rax, %rdi
@@ -303,7 +303,7 @@ SYM_CODE_START_NOALIGN(ret_from_fork)
1:
/* kernel thread */
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
movq %r12, %rdi
CALL_NOSPEC rbx
/*
@@ -388,9 +388,9 @@ SYM_CODE_START(\asmsym)
.if \vector == X86_TRAP_BP
/* #BP advances %rip to the next instruction */
UNWIND_HINT_IRET_REGS offset=\has_error_code*8 signal=0
UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8 signal=0
.else
UNWIND_HINT_IRET_REGS offset=\has_error_code*8
UNWIND_HINT_IRET_ENTRY offset=\has_error_code*8
.endif
ENDBR
@@ -461,7 +461,7 @@ SYM_CODE_END(\asmsym)
*/
.macro idtentry_mce_db vector asmsym cfunc
SYM_CODE_START(\asmsym)
UNWIND_HINT_IRET_REGS
UNWIND_HINT_IRET_ENTRY
ENDBR
ASM_CLAC
cld
@@ -518,7 +518,7 @@ SYM_CODE_END(\asmsym)
*/
.macro idtentry_vc vector asmsym cfunc
SYM_CODE_START(\asmsym)
UNWIND_HINT_IRET_REGS
UNWIND_HINT_IRET_ENTRY
ENDBR
ASM_CLAC
cld
@@ -582,7 +582,7 @@ SYM_CODE_END(\asmsym)
*/
.macro idtentry_df vector asmsym cfunc
SYM_CODE_START(\asmsym)
UNWIND_HINT_IRET_REGS offset=8
UNWIND_HINT_IRET_ENTRY offset=8
ENDBR
ASM_CLAC
cld
@@ -643,7 +643,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
*/
movq %rsp, %rdi
movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
/* Copy the IRET frame to the trampoline stack. */
pushq 6*8(%rdi) /* SS */
@@ -869,7 +869,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback)
*/
__FUNC_ALIGN
SYM_CODE_START_NOALIGN(xen_failsafe_callback)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ENDBR
movl %ds, %ecx
cmpw %cx, 0x10(%rsp)
@@ -1027,7 +1027,7 @@ SYM_CODE_START_LOCAL(paranoid_exit)
*
* NB to anyone to try to optimize this code: this code does
* not execute at all for exceptions from user mode. Those
* exceptions go through error_exit instead.
* exceptions go through error_return instead.
*/
RESTORE_CR3 scratch_reg=%rax save_reg=%r14
@@ -1107,7 +1107,7 @@ SYM_CODE_START(error_entry)
FENCE_SWAPGS_KERNEL_ENTRY
CALL_DEPTH_ACCOUNT
leaq 8(%rsp), %rax /* return pt_regs pointer */
ANNOTATE_UNRET_END
VALIDATE_UNRET_END
RET
.Lbstep_iret:
@@ -1153,7 +1153,7 @@ SYM_CODE_END(error_return)
* when PAGE_TABLE_ISOLATION is in use. Do not clobber.
*/
SYM_CODE_START(asm_exc_nmi)
UNWIND_HINT_IRET_REGS
UNWIND_HINT_IRET_ENTRY
ENDBR
/*
@@ -1520,7 +1520,7 @@ SYM_CODE_END(asm_exc_nmi)
* MSRs to fully disable 32-bit SYSCALL.
*/
SYM_CODE_START(ignore_sysret)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ENDBR
mov $-ENOSYS, %eax
sysretl
arch/x86/entry/vdso/vdso32-setup.c
+1 -10
View File
@@ -70,18 +70,9 @@ static struct ctl_table abi_table2[] = {
{}
};
static struct ctl_table abi_root_table2[] = {
{
.procname = "abi",
.mode = 0555,
.child = abi_table2
},
{}
};
static __init int ia32_binfmt_init(void)
{
register_sysctl_table(abi_root_table2);
register_sysctl("abi", abi_table2);
return 0;
}
__initcall(ia32_binfmt_init);
arch/x86/entry/vsyscall/vsyscall_64.c
+1 -1
View File
@@ -317,7 +317,7 @@ static struct vm_area_struct gate_vma __ro_after_init = {
struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
#ifdef CONFIG_COMPAT
if (!mm || !(mm->context.flags & MM_CONTEXT_HAS_VSYSCALL))
if (!mm || !test_bit(MM_CONTEXT_HAS_VSYSCALL, &mm->context.flags))
return NULL;
#endif
if (vsyscall_mode == NONE)
arch/x86/hyperv/ivm.c
+1 -1
View File
@@ -129,7 +129,7 @@ static enum es_result hv_ghcb_hv_call(struct ghcb *ghcb, u64 exit_code,
return ES_OK;
}
void hv_ghcb_terminate(unsigned int set, unsigned int reason)
void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason)
{
u64 val = GHCB_MSR_TERM_REQ;
arch/x86/include/asm/cpufeatures.h
+1
View File
@@ -321,6 +321,7 @@
#define X86_FEATURE_LKGS (12*32+18) /* "" Load "kernel" (userspace) GS */
#define X86_FEATURE_AMX_FP16 (12*32+21) /* "" AMX fp16 Support */
#define X86_FEATURE_AVX_IFMA (12*32+23) /* "" Support for VPMADD52[H,L]UQ */
#define X86_FEATURE_LAM (12*32+26) /* Linear Address Masking */
/* AMD-defined CPU features, CPUID level 0x80000008 (EBX), word 13 */
#define X86_FEATURE_CLZERO (13*32+ 0) /* CLZERO instruction */
arch/x86/include/asm/disabled-features.h
+7 -1
View File
@@ -75,6 +75,12 @@
# define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31))
#endif
#ifdef CONFIG_ADDRESS_MASKING
# define DISABLE_LAM 0
#else
# define DISABLE_LAM (1 << (X86_FEATURE_LAM & 31))
#endif
#ifdef CONFIG_INTEL_IOMMU_SVM
# define DISABLE_ENQCMD 0
#else
@@ -115,7 +121,7 @@
#define DISABLED_MASK10 0
#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
DISABLE_CALL_DEPTH_TRACKING)
#define DISABLED_MASK12 0
#define DISABLED_MASK12 (DISABLE_LAM)
#define DISABLED_MASK13 0
#define DISABLED_MASK14 0
#define DISABLED_MASK15 0
arch/x86/include/asm/intel-mid.h
-21
View File
@@ -20,25 +20,4 @@ extern void intel_mid_pwr_power_off(void);
extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev);
#ifdef CONFIG_X86_INTEL_MID
extern void intel_scu_devices_create(void);
extern void intel_scu_devices_destroy(void);
#else /* !CONFIG_X86_INTEL_MID */
static inline void intel_scu_devices_create(void) { }
static inline void intel_scu_devices_destroy(void) { }
#endif /* !CONFIG_X86_INTEL_MID */
/* Bus Select SoC Fuse value */
#define BSEL_SOC_FUSE_MASK 0x7
/* FSB 133MHz */
#define BSEL_SOC_FUSE_001 0x1
/* FSB 100MHz */
#define BSEL_SOC_FUSE_101 0x5
/* FSB 83MHz */
#define BSEL_SOC_FUSE_111 0x7
#endif /* _ASM_X86_INTEL_MID_H */
arch/x86/include/asm/linkage.h
+1 -1
View File
@@ -99,7 +99,7 @@
/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
#define SYM_TYPED_FUNC_START(name) \
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \
ENDBR
/* SYM_FUNC_START -- use for global functions */
arch/x86/include/asm/mmu.h
+15 -3
View File
@@ -9,9 +9,13 @@
#include <linux/bits.h>
/* Uprobes on this MM assume 32-bit code */
#define MM_CONTEXT_UPROBE_IA32 BIT(0)
#define MM_CONTEXT_UPROBE_IA32 0
/* vsyscall page is accessible on this MM */
#define MM_CONTEXT_HAS_VSYSCALL BIT(1)
#define MM_CONTEXT_HAS_VSYSCALL 1
/* Do not allow changing LAM mode */
#define MM_CONTEXT_LOCK_LAM 2
/* Allow LAM and SVA coexisting */
#define MM_CONTEXT_FORCE_TAGGED_SVA 3
/*
* x86 has arch-specific MMU state beyond what lives in mm_struct.
@@ -39,7 +43,15 @@ typedef struct {
#endif
#ifdef CONFIG_X86_64
unsigned short flags;
unsigned long flags;
#endif
#ifdef CONFIG_ADDRESS_MASKING
/* Active LAM mode: X86_CR3_LAM_U48 or X86_CR3_LAM_U57 or 0 (disabled) */
unsigned long lam_cr3_mask;
/* Significant bits of the virtual address. Excludes tag bits. */
u64 untag_mask;
#endif
struct mutex lock;
arch/x86/include/asm/mmu_context.h
+48 -1
View File
@@ -85,6 +85,51 @@ static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
}
#endif
#ifdef CONFIG_ADDRESS_MASKING
static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm)
{
return mm->context.lam_cr3_mask;
}
static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm)
{
mm->context.lam_cr3_mask = oldmm->context.lam_cr3_mask;
mm->context.untag_mask = oldmm->context.untag_mask;
}
#define mm_untag_mask mm_untag_mask
static inline unsigned long mm_untag_mask(struct mm_struct *mm)
{
return mm->context.untag_mask;
}
static inline void mm_reset_untag_mask(struct mm_struct *mm)
{
mm->context.untag_mask = -1UL;
}
#define arch_pgtable_dma_compat arch_pgtable_dma_compat
static inline bool arch_pgtable_dma_compat(struct mm_struct *mm)
{
return !mm_lam_cr3_mask(mm) ||
test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags);
}
#else
static inline unsigned long mm_lam_cr3_mask(struct mm_struct *mm)
{
return 0;
}
static inline void dup_lam(struct mm_struct *oldmm, struct mm_struct *mm)
{
}
static inline void mm_reset_untag_mask(struct mm_struct *mm)
{
}
#endif
#define enter_lazy_tlb enter_lazy_tlb
extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);
@@ -109,6 +154,7 @@ static inline int init_new_context(struct task_struct *tsk,
mm->context.execute_only_pkey = -1;
}
#endif
mm_reset_untag_mask(mm);
init_new_context_ldt(mm);
return 0;
}
@@ -162,6 +208,7 @@ static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
{
arch_dup_pkeys(oldmm, mm);
paravirt_enter_mmap(mm);
dup_lam(oldmm, mm);
return ldt_dup_context(oldmm, mm);
}
@@ -175,7 +222,7 @@ static inline void arch_exit_mmap(struct mm_struct *mm)
static inline bool is_64bit_mm(struct mm_struct *mm)
{
return !IS_ENABLED(CONFIG_IA32_EMULATION) ||
!(mm->context.flags & MM_CONTEXT_UPROBE_IA32);
!test_bit(MM_CONTEXT_UPROBE_IA32, &mm->context.flags);
}
#else
static inline bool is_64bit_mm(struct mm_struct *mm)
arch/x86/include/asm/mshyperv.h
+1 -1
View File
@@ -228,7 +228,7 @@ int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry);
void hv_ghcb_msr_write(u64 msr, u64 value);
void hv_ghcb_msr_read(u64 msr, u64 *value);
bool hv_ghcb_negotiate_protocol(void);
void hv_ghcb_terminate(unsigned int set, unsigned int reason);
void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason);
void hv_vtom_init(void);
#else
static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
arch/x86/include/asm/nospec-branch.h
+7 -7
View File
@@ -194,9 +194,9 @@
* builds.
*/
.macro ANNOTATE_RETPOLINE_SAFE
.Lannotate_\@:
.Lhere_\@:
.pushsection .discard.retpoline_safe
_ASM_PTR .Lannotate_\@
.long .Lhere_\@ - .
.popsection
.endm
@@ -210,8 +210,8 @@
* Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should
* eventually turn into it's own annotation.
*/
.macro ANNOTATE_UNRET_END
#ifdef CONFIG_DEBUG_ENTRY
.macro VALIDATE_UNRET_END
#if defined(CONFIG_NOINSTR_VALIDATION) && defined(CONFIG_CPU_UNRET_ENTRY)
ANNOTATE_RETPOLINE_SAFE
nop
#endif
@@ -286,7 +286,7 @@
.macro UNTRAIN_RET
#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
defined(CONFIG_CALL_DEPTH_TRACKING)
ANNOTATE_UNRET_END
VALIDATE_UNRET_END
ALTERNATIVE_3 "", \
CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
"call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
@@ -297,7 +297,7 @@
.macro UNTRAIN_RET_FROM_CALL
#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \
defined(CONFIG_CALL_DEPTH_TRACKING)
ANNOTATE_UNRET_END
VALIDATE_UNRET_END
ALTERNATIVE_3 "", \
CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
"call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
@@ -318,7 +318,7 @@
#define ANNOTATE_RETPOLINE_SAFE \
"999:\n\t" \
".pushsection .discard.retpoline_safe\n\t" \
_ASM_PTR " 999b\n\t" \
".long 999b - .\n\t" \
".popsection\n\t"
typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
arch/x86/include/asm/orc_types.h
+8 -4
View File
@@ -39,6 +39,12 @@
#define ORC_REG_SP_INDIRECT 9
#define ORC_REG_MAX 15
#define ORC_TYPE_UNDEFINED 0
#define ORC_TYPE_END_OF_STACK 1
#define ORC_TYPE_CALL 2
#define ORC_TYPE_REGS 3
#define ORC_TYPE_REGS_PARTIAL 4
#ifndef __ASSEMBLY__
#include <asm/byteorder.h>
@@ -56,16 +62,14 @@ struct orc_entry {
#if defined(__LITTLE_ENDIAN_BITFIELD)
unsigned sp_reg:4;
unsigned bp_reg:4;
unsigned type:2;
unsigned type:3;
unsigned signal:1;
unsigned end:1;
#elif defined(__BIG_ENDIAN_BITFIELD)
unsigned bp_reg:4;
unsigned sp_reg:4;
unsigned unused:4;
unsigned end:1;
unsigned signal:1;
unsigned type:2;
unsigned type:3;
#endif
} __packed;
arch/x86/include/asm/processor-flags.h
+2
View File
@@ -28,6 +28,8 @@
* On systems with SME, one bit (in a variable position!) is stolen to indicate
* that the top-level paging structure is encrypted.
*
* On systemms with LAM, bits 61 and 62 are used to indicate LAM mode.
*
* All of the remaining bits indicate the physical address of the top-level
* paging structure.
*
arch/x86/include/asm/reboot.h
-1
View File
@@ -28,7 +28,6 @@ void __noreturn machine_real_restart(unsigned int type);
void cpu_emergency_disable_virtualization(void);
typedef void (*nmi_shootdown_cb)(int, struct pt_regs*);
void nmi_panic_self_stop(struct pt_regs *regs);
void nmi_shootdown_cpus(nmi_shootdown_cb callback);
void run_crash_ipi_callback(struct pt_regs *regs);
arch/x86/include/asm/setup.h
+3 -3
View File
@@ -125,11 +125,11 @@ void clear_bss(void);
#ifdef __i386__
asmlinkage void __init i386_start_kernel(void);
asmlinkage void __init __noreturn i386_start_kernel(void);
#else
asmlinkage void __init x86_64_start_kernel(char *real_mode);
asmlinkage void __init x86_64_start_reservations(char *real_mode_data);
asmlinkage void __init __noreturn x86_64_start_kernel(char *real_mode);
asmlinkage void __init __noreturn x86_64_start_reservations(char *real_mode_data);
#endif /* __i386__ */
#endif /* _SETUP */
arch/x86/include/asm/shared/tdx.h
+2 -3
View File
@@ -7,8 +7,6 @@
#define TDX_HYPERCALL_STANDARD 0
#define TDX_HCALL_HAS_OUTPUT BIT(0)
#define TDX_CPUID_LEAF_ID 0x21
#define TDX_IDENT "IntelTDX "
@@ -36,7 +34,8 @@ struct tdx_hypercall_args {
};
/* Used to request services from the VMM */
u64 __tdx_hypercall(struct tdx_hypercall_args *args, unsigned long flags);
u64 __tdx_hypercall(struct tdx_hypercall_args *args);
u64 __tdx_hypercall_ret(struct tdx_hypercall_args *args);
/* Called from __tdx_hypercall() for unrecoverable failure */
void __tdx_hypercall_failed(void);
arch/x86/include/asm/smp.h
+3 -2
View File
@@ -93,9 +93,10 @@ static inline void __cpu_die(unsigned int cpu)
smp_ops.cpu_die(cpu);
}
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
smp_ops.play_dead();
BUG();
}
static inline void smp_send_reschedule(int cpu)
@@ -124,7 +125,7 @@ int native_cpu_up(unsigned int cpunum, struct task_struct *tidle);
int native_cpu_disable(void);
int common_cpu_die(unsigned int cpu);
void native_cpu_die(unsigned int cpu);
void hlt_play_dead(void);
void __noreturn hlt_play_dead(void);
void native_play_dead(void);
void play_dead_common(void);
void wbinvd_on_cpu(int cpu);
arch/x86/include/asm/tlbflush.h
+47 -1
View File
@@ -2,7 +2,7 @@
#ifndef _ASM_X86_TLBFLUSH_H
#define _ASM_X86_TLBFLUSH_H
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/sched.h>
#include <asm/processor.h>
@@ -12,6 +12,7 @@
#include <asm/invpcid.h>
#include <asm/pti.h>
#include <asm/processor-flags.h>
#include <asm/pgtable.h>
void __flush_tlb_all(void);
@@ -53,6 +54,15 @@ static inline void cr4_clear_bits(unsigned long mask)
local_irq_restore(flags);
}
#ifdef CONFIG_ADDRESS_MASKING
DECLARE_PER_CPU(u64, tlbstate_untag_mask);
static inline u64 current_untag_mask(void)
{
return this_cpu_read(tlbstate_untag_mask);
}
#endif
#ifndef MODULE
/*
* 6 because 6 should be plenty and struct tlb_state will fit in two cache
@@ -101,6 +111,16 @@ struct tlb_state {
*/
bool invalidate_other;
#ifdef CONFIG_ADDRESS_MASKING
/*
* Active LAM mode.
*
* X86_CR3_LAM_U57/U48 shifted right by X86_CR3_LAM_U57_BIT or 0 if LAM
* disabled.
*/
u8 lam;
#endif
/*
* Mask that contains TLB_NR_DYN_ASIDS+1 bits to indicate
* the corresponding user PCID needs a flush next time we
@@ -357,6 +377,32 @@ static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
}
#define huge_pmd_needs_flush huge_pmd_needs_flush
#ifdef CONFIG_ADDRESS_MASKING
static inline u64 tlbstate_lam_cr3_mask(void)
{
u64 lam = this_cpu_read(cpu_tlbstate.lam);
return lam << X86_CR3_LAM_U57_BIT;
}
static inline void set_tlbstate_lam_mode(struct mm_struct *mm)
{
this_cpu_write(cpu_tlbstate.lam,
mm->context.lam_cr3_mask >> X86_CR3_LAM_U57_BIT);
this_cpu_write(tlbstate_untag_mask, mm->context.untag_mask);
}
#else
static inline u64 tlbstate_lam_cr3_mask(void)
{
return 0;
}
static inline void set_tlbstate_lam_mode(struct mm_struct *mm)
{
}
#endif
#endif /* !MODULE */
static inline void __native_tlb_flush_global(unsigned long cr4)
arch/x86/include/asm/uaccess.h
+56 -2
View File
@@ -7,11 +7,14 @@
#include <linux/compiler.h>
#include <linux/instrumented.h>
#include <linux/kasan-checks.h>
#include <linux/mm_types.h>
#include <linux/string.h>
#include <linux/mmap_lock.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
#include <asm/extable.h>
#include <asm/tlbflush.h>
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
static inline bool pagefault_disabled(void);
@@ -21,6 +24,57 @@ static inline bool pagefault_disabled(void);
# define WARN_ON_IN_IRQ()
#endif
#ifdef CONFIG_ADDRESS_MASKING
/*
* Mask out tag bits from the address.
*
* Magic with the 'sign' allows to untag userspace pointer without any branches
* while leaving kernel addresses intact.
*/
static inline unsigned long __untagged_addr(unsigned long addr)
{
long sign;
/*
* Refer tlbstate_untag_mask directly to avoid RIP-relative relocation
* in alternative instructions. The relocation gets wrong when gets
* copied to the target place.
*/
asm (ALTERNATIVE("",
"sar $63, %[sign]\n\t" /* user_ptr ? 0 : -1UL */
"or %%gs:tlbstate_untag_mask, %[sign]\n\t"
"and %[sign], %[addr]\n\t", X86_FEATURE_LAM)
: [addr] "+r" (addr), [sign] "=r" (sign)
: "m" (tlbstate_untag_mask), "[sign]" (addr));
return addr;
}
#define untagged_addr(addr) ({ \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr(__addr); \
})
static inline unsigned long __untagged_addr_remote(struct mm_struct *mm,
unsigned long addr)
{
long sign = addr >> 63;
mmap_assert_locked(mm);
addr &= (mm)->context.untag_mask | sign;
return addr;
}
#define untagged_addr_remote(mm, addr) ({ \
unsigned long __addr = (__force unsigned long)(addr); \
(__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
})
#else
#define untagged_addr(addr) (addr)
#endif
/**
* access_ok - Checks if a user space pointer is valid
* @addr: User space pointer to start of block to check
@@ -38,10 +92,10 @@ static inline bool pagefault_disabled(void);
* Return: true (nonzero) if the memory block may be valid, false (zero)
* if it is definitely invalid.
*/
#define access_ok(addr, size) \
#define access_ok(addr, size) \
({ \
WARN_ON_IN_IRQ(); \
likely(__access_ok(addr, size)); \
likely(__access_ok(untagged_addr(addr), size)); \
})
#include <asm-generic/access_ok.h>
arch/x86/include/asm/uaccess_64.h
-2
View File
@@ -54,8 +54,6 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size);
extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
size_t len);
static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
arch/x86/include/asm/unwind_hints.h
+14 -4
View File
@@ -7,12 +7,17 @@
#ifdef __ASSEMBLY__
.macro UNWIND_HINT_EMPTY
UNWIND_HINT type=UNWIND_HINT_TYPE_CALL end=1
.macro UNWIND_HINT_END_OF_STACK
UNWIND_HINT type=UNWIND_HINT_TYPE_END_OF_STACK
.endm
.macro UNWIND_HINT_UNDEFINED
UNWIND_HINT type=UNWIND_HINT_TYPE_UNDEFINED
.endm
.macro UNWIND_HINT_ENTRY
UNWIND_HINT type=UNWIND_HINT_TYPE_ENTRY end=1
VALIDATE_UNRET_BEGIN
UNWIND_HINT_END_OF_STACK
.endm
.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 partial=0 signal=1
@@ -52,6 +57,11 @@
UNWIND_HINT_REGS base=\base offset=\offset partial=1 signal=\signal
.endm
.macro UNWIND_HINT_IRET_ENTRY base=%rsp offset=0 signal=1
VALIDATE_UNRET_BEGIN
UNWIND_HINT_IRET_REGS base=\base offset=\offset signal=\signal
.endm
.macro UNWIND_HINT_FUNC
UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC
.endm
@@ -67,7 +77,7 @@
#else
#define UNWIND_HINT_FUNC \
UNWIND_HINT(ORC_REG_SP, 8, UNWIND_HINT_TYPE_FUNC, 0, 0)
UNWIND_HINT(UNWIND_HINT_TYPE_FUNC, ORC_REG_SP, 8, 0)
#endif /* __ASSEMBLY__ */
arch/x86/include/uapi/asm/prctl.h
+8
View File
@@ -16,8 +16,16 @@
#define ARCH_GET_XCOMP_GUEST_PERM 0x1024
#define ARCH_REQ_XCOMP_GUEST_PERM 0x1025
#define ARCH_XCOMP_TILECFG 17
#define ARCH_XCOMP_TILEDATA 18
#define ARCH_MAP_VDSO_X32 0x2001
#define ARCH_MAP_VDSO_32 0x2002
#define ARCH_MAP_VDSO_64 0x2003
#define ARCH_GET_UNTAG_MASK 0x4001
#define ARCH_ENABLE_TAGGED_ADDR 0x4002
#define ARCH_GET_MAX_TAG_BITS 0x4003
#define ARCH_FORCE_TAGGED_SVA 0x4004
#endif /* _ASM_X86_PRCTL_H */
arch/x86/include/uapi/asm/processor-flags.h
+6
View File
@@ -82,6 +82,10 @@
#define X86_CR3_PCID_BITS 12
#define X86_CR3_PCID_MASK (_AC((1UL << X86_CR3_PCID_BITS) - 1, UL))
#define X86_CR3_LAM_U57_BIT 61 /* Activate LAM for userspace, 62:57 bits masked */
#define X86_CR3_LAM_U57 _BITULL(X86_CR3_LAM_U57_BIT)
#define X86_CR3_LAM_U48_BIT 62 /* Activate LAM for userspace, 62:48 bits masked */
#define X86_CR3_LAM_U48 _BITULL(X86_CR3_LAM_U48_BIT)
#define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */
#define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT)
@@ -132,6 +136,8 @@
#define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT)
#define X86_CR4_CET_BIT 23 /* enable Control-flow Enforcement Technology */
#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)
#define X86_CR4_LAM_SUP_BIT 28 /* LAM for supervisor pointers */
#define X86_CR4_LAM_SUP _BITUL(X86_CR4_LAM_SUP_BIT)
/*
* x86-64 Task Priority Register, CR8
arch/x86/kernel/cpu/resctrl/monitor.c
+19 -24
View File
@@ -383,41 +383,36 @@ void free_rmid(u32 rmid)
list_add_tail(&entry->list, &rmid_free_lru);
}
static struct mbm_state *get_mbm_state(struct rdt_domain *d, u32 rmid,
enum resctrl_event_id evtid)
{
switch (evtid) {
case QOS_L3_MBM_TOTAL_EVENT_ID:
return &d->mbm_total[rmid];
case QOS_L3_MBM_LOCAL_EVENT_ID:
return &d->mbm_local[rmid];
default:
return NULL;
}
}
static int __mon_event_count(u32 rmid, struct rmid_read *rr)
{
struct mbm_state *m;
u64 tval = 0;
if (rr->first)
if (rr->first) {
resctrl_arch_reset_rmid(rr->r, rr->d, rmid, rr->evtid);
m = get_mbm_state(rr->d, rmid, rr->evtid);
if (m)
memset(m, 0, sizeof(struct mbm_state));
return 0;
}
rr->err = resctrl_arch_rmid_read(rr->r, rr->d, rmid, rr->evtid, &tval);
if (rr->err)
return rr->err;
switch (rr->evtid) {
case QOS_L3_OCCUP_EVENT_ID:
rr->val += tval;
return 0;
case QOS_L3_MBM_TOTAL_EVENT_ID:
m = &rr->d->mbm_total[rmid];
break;
case QOS_L3_MBM_LOCAL_EVENT_ID:
m = &rr->d->mbm_local[rmid];
break;
default:
/*
* Code would never reach here because an invalid
* event id would fail in resctrl_arch_rmid_read().
*/
return -EINVAL;
}
if (rr->first) {
memset(m, 0, sizeof(struct mbm_state));
return 0;
}
rr->val += tval;
return 0;
arch/x86/kernel/ftrace_64.S
+1 -1
View File
@@ -346,7 +346,7 @@ STACK_FRAME_NON_STANDARD_FP(__fentry__)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
SYM_CODE_START(return_to_handler)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
subq $16, %rsp
arch/x86/kernel/head32.c
+1 -1
View File
@@ -29,7 +29,7 @@ static void __init i386_default_early_setup(void)
x86_init.mpparse.setup_ioapic_ids = setup_ioapic_ids_from_mpc;
}
asmlinkage __visible void __init i386_start_kernel(void)
asmlinkage __visible void __init __noreturn i386_start_kernel(void)
{
/* Make sure IDT is set up before any exception happens */
idt_setup_early_handler();
arch/x86/kernel/head64.c
+2 -2
View File
@@ -471,7 +471,7 @@ static void __init copy_bootdata(char *real_mode_data)
sme_unmap_bootdata(real_mode_data);
}
asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
asmlinkage __visible void __init __noreturn x86_64_start_kernel(char * real_mode_data)
{
/*
* Build-time sanity checks on the kernel image and module
@@ -537,7 +537,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
x86_64_start_reservations(real_mode_data);
}
void __init x86_64_start_reservations(char *real_mode_data)
void __init __noreturn x86_64_start_reservations(char *real_mode_data)
{
/* version is always not zero if it is copied */
if (!boot_params.hdr.version)
arch/x86/kernel/head_64.S
+6 -11
View File
@@ -42,7 +42,7 @@ L3_START_KERNEL = pud_index(__START_KERNEL_map)
__HEAD
.code64
SYM_CODE_START_NOALIGN(startup_64)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
/*
* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
* and someone has loaded an identity mapped page table
@@ -97,7 +97,7 @@ SYM_CODE_START_NOALIGN(startup_64)
lretq
.Lon_kernel_cs:
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
/* Sanitize CPU configuration */
call verify_cpu
@@ -119,7 +119,7 @@ SYM_CODE_START_NOALIGN(startup_64)
SYM_CODE_END(startup_64)
SYM_CODE_START(secondary_startup_64)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
/*
* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
@@ -148,7 +148,7 @@ SYM_CODE_START(secondary_startup_64)
* verify_cpu() above to make sure NX is enabled.
*/
SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
/*
@@ -230,7 +230,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
ANNOTATE_RETPOLINE_SAFE
jmp *%rax
1:
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // above
#ifdef CONFIG_SMP
@@ -383,7 +383,7 @@ SYM_CODE_END(secondary_startup_64)
*/
SYM_CODE_START(start_cpu0)
ANNOTATE_NOENDBR
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
/* Find the idle task stack */
movq PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
@@ -406,8 +406,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb)
UNWIND_HINT_IRET_REGS offset=8
ENDBR
ANNOTATE_UNRET_END
/* Build pt_regs */
PUSH_AND_CLEAR_REGS
@@ -460,7 +458,6 @@ SYM_CODE_END(early_idt_handler_array)
SYM_CODE_START_LOCAL(early_idt_handler_common)
UNWIND_HINT_IRET_REGS offset=16
ANNOTATE_UNRET_END
/*
* The stack is the hardware frame, an error code or zero, and the
* vector number.
@@ -510,8 +507,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb)
UNWIND_HINT_IRET_REGS offset=8
ENDBR
ANNOTATE_UNRET_END
/* Build pt_regs */
PUSH_AND_CLEAR_REGS
arch/x86/kernel/itmt.c
+1 -10
View File
@@ -77,15 +77,6 @@ static struct ctl_table itmt_kern_table[] = {
{}
};
static struct ctl_table itmt_root_table[] = {
{
.procname = "kernel",
.mode = 0555,
.child = itmt_kern_table,
},
{}
};
static struct ctl_table_header *itmt_sysctl_header;
/**
@@ -114,7 +105,7 @@ int sched_set_itmt_support(void)
return 0;
}
itmt_sysctl_header = register_sysctl_table(itmt_root_table);
itmt_sysctl_header = register_sysctl("kernel", itmt_kern_table);
if (!itmt_sysctl_header) {
mutex_unlock(&itmt_update_mutex);
return -ENOMEM;
arch/x86/kernel/process.c
+9 -2
View File
@@ -5,6 +5,7 @@
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/prctl.h>
#include <linux/slab.h>
#include <linux/sched.h>
@@ -48,6 +49,7 @@
#include <asm/frame.h>
#include <asm/unwind.h>
#include <asm/tdx.h>
#include <asm/mmu_context.h>
#include "process.h"
@@ -162,6 +164,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
savesegment(es, p->thread.es);
savesegment(ds, p->thread.ds);
if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM)
set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags);
#else
p->thread.sp0 = (unsigned long) (childregs + 1);
savesegment(gs, p->thread.gs);
@@ -368,6 +373,8 @@ void arch_setup_new_exec(void)
task_clear_spec_ssb_noexec(current);
speculation_ctrl_update(read_thread_flags());
}
mm_reset_untag_mask(current->mm);
}
#ifdef CONFIG_X86_IOPL_IOPERM
@@ -715,7 +722,7 @@ static bool x86_idle_set(void)
}
#ifndef CONFIG_SMP
static inline void play_dead(void)
static inline void __noreturn play_dead(void)
{
BUG();
}
@@ -727,7 +734,7 @@ void arch_cpu_idle_enter(void)
local_touch_nmi();
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
play_dead();
}
arch/x86/kernel/process_64.c
+65 -3
View File
@@ -671,7 +671,7 @@ void set_personality_64bit(void)
task_pt_regs(current)->orig_ax = __NR_execve;
current_thread_info()->status &= ~TS_COMPAT;
if (current->mm)
current->mm->context.flags = MM_CONTEXT_HAS_VSYSCALL;
__set_bit(MM_CONTEXT_HAS_VSYSCALL, &current->mm->context.flags);
/* TBD: overwrites user setup. Should have two bits.
But 64bit processes have always behaved this way,
@@ -708,7 +708,7 @@ static void __set_personality_ia32(void)
* uprobes applied to this MM need to know this and
* cannot use user_64bit_mode() at that time.
*/
current->mm->context.flags = MM_CONTEXT_UPROBE_IA32;
__set_bit(MM_CONTEXT_UPROBE_IA32, &current->mm->context.flags);
}
current->personality |= force_personality32;
@@ -743,6 +743,52 @@ static long prctl_map_vdso(const struct vdso_image *image, unsigned long addr)
}
#endif
#ifdef CONFIG_ADDRESS_MASKING
#define LAM_U57_BITS 6
static int prctl_enable_tagged_addr(struct mm_struct *mm, unsigned long nr_bits)
{
if (!cpu_feature_enabled(X86_FEATURE_LAM))
return -ENODEV;
/* PTRACE_ARCH_PRCTL */
if (current->mm != mm)
return -EINVAL;
if (mm_valid_pasid(mm) &&
!test_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &mm->context.flags))
return -EINVAL;
if (mmap_write_lock_killable(mm))
return -EINTR;
if (test_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags)) {
mmap_write_unlock(mm);
return -EBUSY;
}
if (!nr_bits) {
mmap_write_unlock(mm);
return -EINVAL;
} else if (nr_bits <= LAM_U57_BITS) {
mm->context.lam_cr3_mask = X86_CR3_LAM_U57;
mm->context.untag_mask = ~GENMASK(62, 57);
} else {
mmap_write_unlock(mm);
return -EINVAL;
}
write_cr3(__read_cr3() | mm->context.lam_cr3_mask);
set_tlbstate_lam_mode(mm);
set_bit(MM_CONTEXT_LOCK_LAM, &mm->context.flags);
mmap_write_unlock(mm);
return 0;
}
#endif
long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
{
int ret = 0;
@@ -830,7 +876,23 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2)
case ARCH_MAP_VDSO_64:
return prctl_map_vdso(&vdso_image_64, arg2);
#endif
#ifdef CONFIG_ADDRESS_MASKING
case ARCH_GET_UNTAG_MASK:
return put_user(task->mm->context.untag_mask,
(unsigned long __user *)arg2);
case ARCH_ENABLE_TAGGED_ADDR:
return prctl_enable_tagged_addr(task->mm, arg2);
case ARCH_FORCE_TAGGED_SVA:
if (current != task)
return -EINVAL;
set_bit(MM_CONTEXT_FORCE_TAGGED_SVA, &task->mm->context.flags);
return 0;
case ARCH_GET_MAX_TAG_BITS:
if (!cpu_feature_enabled(X86_FEATURE_LAM))
return put_user(0, (unsigned long __user *)arg2);
else
return put_user(LAM_U57_BITS, (unsigned long __user *)arg2);
#endif
default:
ret = -EINVAL;
break;
arch/x86/kernel/reboot.c
+1 -1
View File
@@ -920,7 +920,7 @@ void run_crash_ipi_callback(struct pt_regs *regs)
}
/* Override the weak function in kernel/panic.c */
void nmi_panic_self_stop(struct pt_regs *regs)
void __noreturn nmi_panic_self_stop(struct pt_regs *regs)
{
while (1) {
/* If no CPU is preparing crash dump, we simply loop here. */
arch/x86/kernel/relocate_kernel_64.S
+5 -5
View File
@@ -43,7 +43,7 @@
.code64
SYM_CODE_START_NOALIGN(relocate_range)
SYM_CODE_START_NOALIGN(relocate_kernel)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
/*
* %rdi indirection_page
@@ -113,7 +113,7 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
SYM_CODE_END(relocate_kernel)
SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
/* set return address to 0 if not preserving context */
pushq $0
/* store the start address on the stack */
@@ -231,7 +231,7 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
SYM_CODE_END(identity_mapped)
SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR // RET target, above
movq RSP(%r8), %rsp
movq CR4(%r8), %rax
@@ -256,8 +256,8 @@ SYM_CODE_END(virtual_mapped)
/* Do the copies */
SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
UNWIND_HINT_EMPTY
movq %rdi, %rcx /* Put the page_list in %rcx */
UNWIND_HINT_END_OF_STACK
movq %rdi, %rcx /* Put the page_list in %rcx */
xorl %edi, %edi
xorl %esi, %esi
jmp 1f
arch/x86/kernel/smpboot.c
+1 -1
View File
@@ -1824,7 +1824,7 @@ static inline void mwait_play_dead(void)
}
}
void hlt_play_dead(void)
void __noreturn hlt_play_dead(void)
{
if (__this_cpu_read(cpu_info.x86) >= 4)
wbinvd();
arch/x86/kernel/traps.c
+3 -3
View File
@@ -671,15 +671,15 @@ static bool try_fixup_enqcmd_gp(void)
if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
return false;
pasid = current->mm->pasid;
/*
* If the mm has not been allocated a
* PASID, the #GP can not be fixed up.
*/
if (!pasid_valid(pasid))
if (!mm_valid_pasid(current->mm))
return false;
pasid = current->mm->pasid;
/*
* Did this thread already have its PASID activated?
* If so, the #GP must be from something else.
arch/x86/kernel/unwind_orc.c
+12 -15
View File
@@ -133,7 +133,7 @@ static struct orc_entry null_orc_entry = {
.sp_offset = sizeof(long),
.sp_reg = ORC_REG_SP,
.bp_reg = ORC_REG_UNDEFINED,
.type = UNWIND_HINT_TYPE_CALL
.type = ORC_TYPE_CALL
};
#ifdef CONFIG_CALL_THUNKS
@@ -153,12 +153,11 @@ static struct orc_entry *orc_callthunk_find(unsigned long ip)
/* Fake frame pointer entry -- used as a fallback for generated code */
static struct orc_entry orc_fp_entry = {
.type = UNWIND_HINT_TYPE_CALL,
.type = ORC_TYPE_CALL,
.sp_reg = ORC_REG_BP,
.sp_offset = 16,
.bp_reg = ORC_REG_PREV_SP,
.bp_offset = -16,
.end = 0,
};
static struct orc_entry *orc_find(unsigned long ip)
@@ -250,13 +249,13 @@ static int orc_sort_cmp(const void *_a, const void *_b)
return -1;
/*
* The "weak" section terminator entries need to always be on the left
* The "weak" section terminator entries need to always be first
* to ensure the lookup code skips them in favor of real entries.
* These terminator entries exist to handle any gaps created by
* whitelisted .o files which didn't get objtool generation.
*/
orc_a = cur_orc_table + (a - cur_orc_ip_table);
return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1;
}
void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
@@ -474,14 +473,12 @@ bool unwind_next_frame(struct unwind_state *state)
*/
orc = &orc_fp_entry;
state->error = true;
}
/* End-of-stack check for kernel threads: */
if (orc->sp_reg == ORC_REG_UNDEFINED) {
if (!orc->end)
} else {
if (orc->type == ORC_TYPE_UNDEFINED)
goto err;
goto the_end;
if (orc->type == ORC_TYPE_END_OF_STACK)
goto the_end;
}
state->signal = orc->signal;
@@ -554,7 +551,7 @@ bool unwind_next_frame(struct unwind_state *state)
/* Find IP, SP and possibly regs: */
switch (orc->type) {
case UNWIND_HINT_TYPE_CALL:
case ORC_TYPE_CALL:
ip_p = sp - sizeof(long);
if (!deref_stack_reg(state, ip_p, &state->ip))
@@ -567,7 +564,7 @@ bool unwind_next_frame(struct unwind_state *state)
state->prev_regs = NULL;
break;
case UNWIND_HINT_TYPE_REGS:
case ORC_TYPE_REGS:
if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
orc_warn_current("can't access registers at %pB\n",
(void *)orig_ip);
@@ -590,13 +587,13 @@ bool unwind_next_frame(struct unwind_state *state)
state->full_regs = true;
break;
case UNWIND_HINT_TYPE_REGS_PARTIAL:
case ORC_TYPE_REGS_PARTIAL:
if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
orc_warn_current("can't access iret registers at %pB\n",
(void *)orig_ip);
goto err;
}
/* See UNWIND_HINT_TYPE_REGS case comment. */
/* See ORC_TYPE_REGS case comment. */
state->ip = unwind_recover_rethook(state, state->ip,
(unsigned long *)(state->sp - sizeof(long)));
arch/x86/lib/getuser.S
+31 -52
View File
@@ -37,22 +37,22 @@
#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
#ifdef CONFIG_X86_5LEVEL
#define LOAD_TASK_SIZE_MINUS_N(n) \
ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \
__stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57
#else
#define LOAD_TASK_SIZE_MINUS_N(n) \
mov $(TASK_SIZE_MAX - (n)),%_ASM_DX
#endif
.macro check_range size:req
.if IS_ENABLED(CONFIG_X86_64)
mov %rax, %rdx
sar $63, %rdx
or %rdx, %rax
.else
cmp $TASK_SIZE_MAX-\size+1, %eax
jae .Lbad_get_user
sbb %edx, %edx /* array_index_mask_nospec() */
and %edx, %eax
.endif
.endm
.text
SYM_FUNC_START(__get_user_1)
LOAD_TASK_SIZE_MINUS_N(0)
cmp %_ASM_DX,%_ASM_AX
jae bad_get_user
sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
and %_ASM_DX, %_ASM_AX
check_range size=1
ASM_STAC
1: movzbl (%_ASM_AX),%edx
xor %eax,%eax
@@ -62,11 +62,7 @@ SYM_FUNC_END(__get_user_1)
EXPORT_SYMBOL(__get_user_1)
SYM_FUNC_START(__get_user_2)
LOAD_TASK_SIZE_MINUS_N(1)
cmp %_ASM_DX,%_ASM_AX
jae bad_get_user
sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
and %_ASM_DX, %_ASM_AX
check_range size=2
ASM_STAC
2: movzwl (%_ASM_AX),%edx
xor %eax,%eax
@@ -76,11 +72,7 @@ SYM_FUNC_END(__get_user_2)
EXPORT_SYMBOL(__get_user_2)
SYM_FUNC_START(__get_user_4)
LOAD_TASK_SIZE_MINUS_N(3)
cmp %_ASM_DX,%_ASM_AX
jae bad_get_user
sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
and %_ASM_DX, %_ASM_AX
check_range size=4
ASM_STAC
3: movl (%_ASM_AX),%edx
xor %eax,%eax
@@ -90,30 +82,17 @@ SYM_FUNC_END(__get_user_4)
EXPORT_SYMBOL(__get_user_4)
SYM_FUNC_START(__get_user_8)
check_range size=8
ASM_STAC
#ifdef CONFIG_X86_64
LOAD_TASK_SIZE_MINUS_N(7)
cmp %_ASM_DX,%_ASM_AX
jae bad_get_user
sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
and %_ASM_DX, %_ASM_AX
ASM_STAC
4: movq (%_ASM_AX),%rdx
xor %eax,%eax
ASM_CLAC
RET
#else
LOAD_TASK_SIZE_MINUS_N(7)
cmp %_ASM_DX,%_ASM_AX
jae bad_get_user_8
sbb %_ASM_DX, %_ASM_DX /* array_index_mask_nospec() */
and %_ASM_DX, %_ASM_AX
ASM_STAC
4: movl (%_ASM_AX),%edx
5: movl 4(%_ASM_AX),%ecx
#endif
xor %eax,%eax
ASM_CLAC
RET
#endif
SYM_FUNC_END(__get_user_8)
EXPORT_SYMBOL(__get_user_8)
@@ -166,7 +145,7 @@ EXPORT_SYMBOL(__get_user_nocheck_8)
SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
ASM_CLAC
bad_get_user:
.Lbad_get_user:
xor %edx,%edx
mov $(-EFAULT),%_ASM_AX
RET
@@ -184,23 +163,23 @@ SYM_CODE_END(.Lbad_get_user_8_clac)
#endif
/* get_user */
_ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
_ASM_EXTABLE(1b, .Lbad_get_user_clac)
_ASM_EXTABLE(2b, .Lbad_get_user_clac)
_ASM_EXTABLE(3b, .Lbad_get_user_clac)
#ifdef CONFIG_X86_64
_ASM_EXTABLE_UA(4b, .Lbad_get_user_clac)
_ASM_EXTABLE(4b, .Lbad_get_user_clac)
#else
_ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
_ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
_ASM_EXTABLE(4b, .Lbad_get_user_8_clac)
_ASM_EXTABLE(5b, .Lbad_get_user_8_clac)
#endif
/* __get_user */
_ASM_EXTABLE_UA(6b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(7b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(8b, .Lbad_get_user_clac)
_ASM_EXTABLE(6b, .Lbad_get_user_clac)
_ASM_EXTABLE(7b, .Lbad_get_user_clac)
_ASM_EXTABLE(8b, .Lbad_get_user_clac)
#ifdef CONFIG_X86_64
_ASM_EXTABLE_UA(9b, .Lbad_get_user_clac)
_ASM_EXTABLE(9b, .Lbad_get_user_clac)
#else
_ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac)
_ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac)
_ASM_EXTABLE(9b, .Lbad_get_user_8_clac)
_ASM_EXTABLE(10b, .Lbad_get_user_8_clac)
#endif
arch/x86/lib/putuser.S
+24 -30
View File
@@ -33,20 +33,20 @@
* as they get called from within inline assembly.
*/
#ifdef CONFIG_X86_5LEVEL
#define LOAD_TASK_SIZE_MINUS_N(n) \
ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rbx), \
__stringify(mov $((1 << 56) - 4096 - (n)),%rbx), X86_FEATURE_LA57
#else
#define LOAD_TASK_SIZE_MINUS_N(n) \
mov $(TASK_SIZE_MAX - (n)),%_ASM_BX
#endif
.macro check_range size:req
.if IS_ENABLED(CONFIG_X86_64)
mov %rcx, %rbx
sar $63, %rbx
or %rbx, %rcx
.else
cmp $TASK_SIZE_MAX-\size+1, %ecx
jae .Lbad_put_user
.endif
.endm
.text
SYM_FUNC_START(__put_user_1)
LOAD_TASK_SIZE_MINUS_N(0)
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
check_range size=1
ASM_STAC
1: movb %al,(%_ASM_CX)
xor %ecx,%ecx
@@ -66,9 +66,7 @@ SYM_FUNC_END(__put_user_nocheck_1)
EXPORT_SYMBOL(__put_user_nocheck_1)
SYM_FUNC_START(__put_user_2)
LOAD_TASK_SIZE_MINUS_N(1)
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
check_range size=2
ASM_STAC
3: movw %ax,(%_ASM_CX)
xor %ecx,%ecx
@@ -88,9 +86,7 @@ SYM_FUNC_END(__put_user_nocheck_2)
EXPORT_SYMBOL(__put_user_nocheck_2)
SYM_FUNC_START(__put_user_4)
LOAD_TASK_SIZE_MINUS_N(3)
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
check_range size=4
ASM_STAC
5: movl %eax,(%_ASM_CX)
xor %ecx,%ecx
@@ -110,9 +106,7 @@ SYM_FUNC_END(__put_user_nocheck_4)
EXPORT_SYMBOL(__put_user_nocheck_4)
SYM_FUNC_START(__put_user_8)
LOAD_TASK_SIZE_MINUS_N(7)
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
check_range size=8
ASM_STAC
7: mov %_ASM_AX,(%_ASM_CX)
#ifdef CONFIG_X86_32
@@ -144,15 +138,15 @@ SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
RET
SYM_CODE_END(.Lbad_put_user_clac)
_ASM_EXTABLE_UA(1b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(2b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(3b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(4b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(5b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(6b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(7b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(9b, .Lbad_put_user_clac)
_ASM_EXTABLE(1b, .Lbad_put_user_clac)
_ASM_EXTABLE(2b, .Lbad_put_user_clac)
_ASM_EXTABLE(3b, .Lbad_put_user_clac)
_ASM_EXTABLE(4b, .Lbad_put_user_clac)
_ASM_EXTABLE(5b, .Lbad_put_user_clac)
_ASM_EXTABLE(6b, .Lbad_put_user_clac)
_ASM_EXTABLE(7b, .Lbad_put_user_clac)
_ASM_EXTABLE(9b, .Lbad_put_user_clac)
#ifdef CONFIG_X86_32
_ASM_EXTABLE_UA(8b, .Lbad_put_user_clac)
_ASM_EXTABLE_UA(10b, .Lbad_put_user_clac)
_ASM_EXTABLE(8b, .Lbad_put_user_clac)
_ASM_EXTABLE(10b, .Lbad_put_user_clac)
#endif
arch/x86/lib/retpoline.S
+3 -3
View File
@@ -33,7 +33,7 @@
.align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
ALTERNATIVE_2 __stringify(RETPOLINE \reg), \
@@ -75,7 +75,7 @@ SYM_CODE_END(__x86_indirect_thunk_array)
.align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
CALL_DEPTH_ACCOUNT
@@ -103,7 +103,7 @@ SYM_CODE_END(__x86_indirect_call_thunk_array)
.align RETPOLINE_THUNK_SIZE
SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
POLINE \reg
ANNOTATE_UNRET_SAFE
arch/x86/lib/usercopy_64.c
-9
View File
@@ -140,13 +140,4 @@ void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
}
}
EXPORT_SYMBOL_GPL(__memcpy_flushcache);
void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
size_t len)
{
char *from = kmap_atomic(page);
memcpy_flushcache(to, from + offset, len);
kunmap_atomic(from);
}
#endif
arch/x86/mm/init.c
+5
View File
@@ -1048,6 +1048,11 @@ __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = {
.cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */
};
#ifdef CONFIG_ADDRESS_MASKING
DEFINE_PER_CPU(u64, tlbstate_untag_mask);
EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask);
#endif
void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
{
/* entry 0 MUST be WB (hardwired to speed up translations) */
arch/x86/mm/tlb.c
+37 -16
View File
@@ -154,26 +154,30 @@ static inline u16 user_pcid(u16 asid)
return ret;
}
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
{
unsigned long cr3 = __sme_pa(pgd) | lam;
if (static_cpu_has(X86_FEATURE_PCID)) {
return __sme_pa(pgd) | kern_pcid(asid);
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
cr3 |= kern_pcid(asid);
} else {
VM_WARN_ON_ONCE(asid != 0);
return __sme_pa(pgd);
}
return cr3;
}
static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid,
unsigned long lam)
{
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
/*
* Use boot_cpu_has() instead of this_cpu_has() as this function
* might be called during early boot. This should work even after
* boot because all CPU's the have same capabilities:
*/
VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID));
return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;
return build_cr3(pgd, asid, lam) | CR3_NOFLUSH;
}
/*
@@ -274,15 +278,16 @@ static inline void invalidate_user_asid(u16 asid)
(unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
}
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, unsigned long lam,
bool need_flush)
{
unsigned long new_mm_cr3;
if (need_flush) {
invalidate_user_asid(new_asid);
new_mm_cr3 = build_cr3(pgdir, new_asid);
new_mm_cr3 = build_cr3(pgdir, new_asid, lam);
} else {
new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);
new_mm_cr3 = build_cr3_noflush(pgdir, new_asid, lam);
}
/*
@@ -491,6 +496,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
{
struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);
u16 prev_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
unsigned long new_lam = mm_lam_cr3_mask(next);
bool was_lazy = this_cpu_read(cpu_tlbstate_shared.is_lazy);
unsigned cpu = smp_processor_id();
u64 next_tlb_gen;
@@ -520,7 +526,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
* isn't free.
*/
#ifdef CONFIG_DEBUG_VM
if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid))) {
if (WARN_ON_ONCE(__read_cr3() != build_cr3(real_prev->pgd, prev_asid,
tlbstate_lam_cr3_mask()))) {
/*
* If we were to BUG here, we'd be very likely to kill
* the system so hard that we don't see the call trace.
@@ -552,9 +559,15 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
* instruction.
*/
if (real_prev == next) {
/* Not actually switching mm's */
VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
next->context.ctx_id);
/*
* If this races with another thread that enables lam, 'new_lam'
* might not match tlbstate_lam_cr3_mask().
*/
/*
* Even in lazy TLB mode, the CPU should stay set in the
* mm_cpumask. The TLB shootdown code can figure out from
@@ -622,15 +635,16 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
barrier();
}
set_tlbstate_lam_mode(next);
if (need_flush) {
this_cpu_write(cpu_tlbstate.ctxs[new_asid].ctx_id, next->context.ctx_id);
this_cpu_write(cpu_tlbstate.ctxs[new_asid].tlb_gen, next_tlb_gen);
load_new_mm_cr3(next->pgd, new_asid, true);
load_new_mm_cr3(next->pgd, new_asid, new_lam, true);
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
} else {
/* The new ASID is already up to date. */
load_new_mm_cr3(next->pgd, new_asid, false);
load_new_mm_cr3(next->pgd, new_asid, new_lam, false);
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, 0);
}
@@ -691,6 +705,10 @@ void initialize_tlbstate_and_flush(void)
/* Assert that CR3 already references the right mm. */
WARN_ON((cr3 & CR3_ADDR_MASK) != __pa(mm->pgd));
/* LAM expected to be disabled */
WARN_ON(cr3 & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57));
WARN_ON(mm_lam_cr3_mask(mm));
/*
* Assert that CR4.PCIDE is set if needed. (CR4.PCIDE initialization
* doesn't work like other CR4 bits because it can only be set from
@@ -699,8 +717,8 @@ void initialize_tlbstate_and_flush(void)
WARN_ON(boot_cpu_has(X86_FEATURE_PCID) &&
!(cr4_read_shadow() & X86_CR4_PCIDE));
/* Force ASID 0 and force a TLB flush. */
write_cr3(build_cr3(mm->pgd, 0));
/* Disable LAM, force ASID 0 and force a TLB flush. */
write_cr3(build_cr3(mm->pgd, 0, 0));
/* Reinitialize tlbstate. */
this_cpu_write(cpu_tlbstate.last_user_mm_spec, LAST_USER_MM_INIT);
@@ -708,6 +726,7 @@ void initialize_tlbstate_and_flush(void)
this_cpu_write(cpu_tlbstate.next_asid, 1);
this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id);
this_cpu_write(cpu_tlbstate.ctxs[0].tlb_gen, tlb_gen);
set_tlbstate_lam_mode(mm);
for (i = 1; i < TLB_NR_DYN_ASIDS; i++)
this_cpu_write(cpu_tlbstate.ctxs[i].ctx_id, 0);
@@ -1071,8 +1090,10 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
*/
unsigned long __get_current_cr3_fast(void)
{
unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
this_cpu_read(cpu_tlbstate.loaded_mm_asid));
unsigned long cr3 =
build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
this_cpu_read(cpu_tlbstate.loaded_mm_asid),
tlbstate_lam_cr3_mask());
/* For now, be very restrictive about when this can be called. */
VM_WARN_ON(in_nmi() || preemptible());
arch/x86/platform/pvh/head.S
+1 -1
View File
@@ -50,7 +50,7 @@
#define PVH_DS_SEL (PVH_GDT_ENTRY_DS * 8)
SYM_CODE_START_LOCAL(pvh_start_xen)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
cld
lgdt (_pa(gdt))
arch/x86/power/cpu.c
+1 -1
View File
@@ -288,7 +288,7 @@ EXPORT_SYMBOL(restore_processor_state);
#endif
#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
static void resume_play_dead(void)
static void __noreturn resume_play_dead(void)
{
play_dead_common();
tboot_shutdown(TB_SHUTDOWN_WFS);
arch/x86/xen/xen-asm.S
+2 -2
View File
@@ -165,7 +165,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback
SYM_CODE_START(xen_early_idt_handler_array)
i = 0
.rept NUM_EXCEPTION_VECTORS
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ENDBR
pop %rcx
pop %r11
@@ -193,7 +193,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
* rsp->rax }
*/
SYM_CODE_START(xen_iret)
UNWIND_HINT_EMPTY
UNWIND_HINT_UNDEFINED
ANNOTATE_NOENDBR
pushq $0
jmp hypercall_iret
arch/x86/xen/xen-head.S
+2 -2
View File
@@ -45,7 +45,7 @@ SYM_CODE_END(hypercall_page)
#ifdef CONFIG_XEN_PV
__INIT
SYM_CODE_START(startup_xen)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ANNOTATE_NOENDBR
cld
@@ -71,7 +71,7 @@ SYM_CODE_END(startup_xen)
#ifdef CONFIG_XEN_PV_SMP
.pushsection .text
SYM_CODE_START(asm_cpu_bringup_and_idle)
UNWIND_HINT_EMPTY
UNWIND_HINT_END_OF_STACK
ENDBR
call cpu_bringup_and_idle
arch/xtensa/include/asm/smp.h
+1 -1
View File
@@ -33,7 +33,7 @@ void show_ipi_list(struct seq_file *p, int prec);
void __cpu_die(unsigned int cpu);
int __cpu_disable(void);
void cpu_die(void);
void __noreturn cpu_die(void);
void cpu_restart(void);
#endif /* CONFIG_HOTPLUG_CPU */
arch/xtensa/kernel/smp.c
+3 -1
View File
@@ -322,7 +322,7 @@ void __cpu_die(unsigned int cpu)
pr_err("CPU%u: unable to kill\n", cpu);
}
void arch_cpu_idle_dead(void)
void __noreturn arch_cpu_idle_dead(void)
{
cpu_die();
}
@@ -341,6 +341,8 @@ void __ref cpu_die(void)
__asm__ __volatile__(
" movi a2, cpu_restart\n"
" jx a2\n");
BUG();
}
#endif /* CONFIG_HOTPLUG_CPU */
drivers/iommu/iommu-sva.c
+6 -2
View File
@@ -2,6 +2,7 @@
/*
* Helpers for IOMMU drivers implementing SVA
*/
#include <linux/mmu_context.h>
#include <linux/mutex.h>
#include <linux/sched/mm.h>
#include <linux/iommu.h>
@@ -32,16 +33,19 @@ int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t max)
min == 0 || max < min)
return -EINVAL;
if (!arch_pgtable_dma_compat(mm))
return -EBUSY;
mutex_lock(&iommu_sva_lock);
/* Is a PASID already associated with this mm? */
if (pasid_valid(mm->pasid)) {
if (mm_valid_pasid(mm)) {
if (mm->pasid < min || mm->pasid >= max)
ret = -EOVERFLOW;
goto out;
}
pasid = ioasid_alloc(&iommu_sva_pasid, min, max, mm);
if (!pasid_valid(pasid))
if (pasid == INVALID_IOASID)
ret = -ENOMEM;
else
mm_pasid_set(mm, pasid);
drivers/message/fusion/mptbase.c
+1 -1
View File
@@ -6935,7 +6935,7 @@ EXPORT_SYMBOL(mpt_clear_taskmgmt_in_progress_flag);
* @ioc: Pointer to MPT_ADAPTER structure
*
**/
void
void __noreturn
mpt_halt_firmware(MPT_ADAPTER *ioc)
{
u32 ioc_raw_state;
drivers/message/fusion/mptbase.h
+1 -1
View File
@@ -944,7 +944,7 @@ extern int mpt_raid_phys_disk_get_num_paths(MPT_ADAPTER *ioc,
u8 phys_disk_num);
extern int mpt_set_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc);
extern void mpt_clear_taskmgmt_in_progress_flag(MPT_ADAPTER *ioc);
extern void mpt_halt_firmware(MPT_ADAPTER *ioc);
extern void __noreturn mpt_halt_firmware(MPT_ADAPTER *ioc);
/*
drivers/misc/lkdtm/stackleak.c
+6
View File
@@ -43,12 +43,14 @@ static void noinstr check_stackleak_irqoff(void)
* STACK_END_MAGIC, and in either casee something is seriously wrong.
*/
if (current_sp < task_stack_low || current_sp >= task_stack_high) {
instrumentation_begin();
pr_err("FAIL: current_stack_pointer (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n",
current_sp, task_stack_low, task_stack_high - 1);
test_failed = true;
goto out;
}
if (lowest_sp < task_stack_low || lowest_sp >= task_stack_high) {
instrumentation_begin();
pr_err("FAIL: current->lowest_stack (0x%lx) outside of task stack bounds [0x%lx..0x%lx]\n",
lowest_sp, task_stack_low, task_stack_high - 1);
test_failed = true;
@@ -86,11 +88,14 @@ static void noinstr check_stackleak_irqoff(void)
if (*(unsigned long *)poison_low == STACKLEAK_POISON)
continue;
instrumentation_begin();
pr_err("FAIL: non-poison value %lu bytes below poison boundary: 0x%lx\n",
poison_high - poison_low, *(unsigned long *)poison_low);
test_failed = true;
goto out;
}
instrumentation_begin();
pr_info("stackleak stack usage:\n"
" high offset: %lu bytes\n"
" current: %lu bytes\n"
@@ -113,6 +118,7 @@ out:
} else {
pr_info("OK: the rest of the thread stack is properly erased\n");
}
instrumentation_end();
}
static void lkdtm_STACKLEAK_ERASING(void)
drivers/vfio/vfio_iommu_type1.c
+1 -1
View File
@@ -580,7 +580,7 @@ static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
goto done;
}
vaddr = untagged_addr(vaddr);
vaddr = untagged_addr_remote(mm, vaddr);
retry:
vma = vma_lookup(mm, vaddr);
fs/proc/array.c
+7
View File
@@ -91,6 +91,7 @@
#include <linux/user_namespace.h>
#include <linux/fs_struct.h>
#include <linux/kthread.h>
#include <linux/mmu_context.h>
#include <asm/processor.h>
#include "internal.h"
@@ -425,6 +426,11 @@ static inline void task_thp_status(struct seq_file *m, struct mm_struct *mm)
seq_printf(m, "THP_enabled:\t%d\n", thp_enabled);
}
static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm)
{
seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm));
}
int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task)
{
@@ -440,6 +446,7 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,
task_mem(m, mm);
task_core_dumping(m, task);
task_thp_status(m, mm);
task_untag_mask(m, mm);
mmput(mm);
}
task_sig(m, task);
fs/proc/task_mmu.c
+7 -2
View File
@@ -1688,8 +1688,13 @@ static ssize_t pagemap_read(struct file *file, char __user *buf,
/* watch out for wraparound */
start_vaddr = end_vaddr;
if (svpfn <= (ULONG_MAX >> PAGE_SHIFT))
start_vaddr = untagged_addr(svpfn << PAGE_SHIFT);
if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
ret = mmap_read_lock_killable(mm);
if (ret)
goto out_free;
start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
mmap_read_unlock(mm);
}
/* Ensure the address is inside the task */
if (start_vaddr > mm->task_size)
include/linux/context_tracking.h
+1 -1
View File
@@ -98,7 +98,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) { }
static inline int ct_state(void) { return -1; }
static inline int __ct_state(void) { return -1; }
static __always_inline bool context_tracking_guest_enter(void) { return false; }
static inline void context_tracking_guest_exit(void) { }
static __always_inline void context_tracking_guest_exit(void) { }
#define CT_WARN_ON(cond) do { } while (0)
#endif /* !CONFIG_CONTEXT_TRACKING_USER */

Some files were not shown because too many files have changed in this diff Show More