diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 7ec7b5bed298..6ad3693489ba 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7439,17 +7439,18 @@ Crash from Xen panic notifier, without executing late panic() code such as dumping handler. + xen_mc_debug [X86,XEN,EARLY] + Enable multicall debugging when running as a Xen PV guest. + Enabling this feature will reduce performance a little + bit, so it should only be enabled for obtaining extended + debug data in case of multicall errors. + xen_msr_safe= [X86,XEN,EARLY] Format: Select whether to always use non-faulting (safe) MSR access functions when running as Xen PV guest. The default value is controlled by CONFIG_XEN_PV_MSR_SAFE. - xen_nopvspin [X86,XEN,EARLY] - Disables the qspinlock slowpath using Xen PV optimizations. - This parameter is obsoleted by "nopvspin" parameter, which - has equivalent effect for XEN platform. - xen_nopv [X86] Disables the PV optimizations forcing the HVM guest to run as generic HVM guest with no PV drivers. diff --git a/MAINTAINERS b/MAINTAINERS index 29e287265d89..a68ae2c3a7a5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8285,7 +8285,9 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/execve F: Documentation/userspace-api/ELF.rst F: fs/*binfmt_*.c +F: fs/Kconfig.binfmt F: fs/exec.c +F: fs/exec_test.c F: include/linux/binfmts.h F: include/linux/elf.h F: include/uapi/linux/binfmts.h diff --git a/arch/arm/xen/p2m.c b/arch/arm/xen/p2m.c index 309648c17f48..9da57a5b81c7 100644 --- a/arch/arm/xen/p2m.c +++ b/arch/arm/xen/p2m.c @@ -109,7 +109,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, * immediate unmapping. */ map_ops[i].status = GNTST_general_error; - unmap.host_addr = map_ops[i].host_addr, + unmap.host_addr = map_ops[i].host_addr; unmap.handle = map_ops[i].handle; map_ops[i].handle = INVALID_GRANT_HANDLE; if (map_ops[i].flags & GNTMAP_device_map) diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 8b045dd25196..bb0f3f368446 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -10,8 +10,6 @@ #include #include #include "xen-ops.h" -#include "pmu.h" -#include "smp.h" static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) { diff --git a/arch/x86/xen/debugfs.c b/arch/x86/xen/debugfs.c index 532410998684..b8c9f2a7d9b6 100644 --- a/arch/x86/xen/debugfs.c +++ b/arch/x86/xen/debugfs.c @@ -3,7 +3,7 @@ #include #include -#include "debugfs.h" +#include "xen-ops.h" static struct dentry *d_xen_debug; diff --git a/arch/x86/xen/debugfs.h b/arch/x86/xen/debugfs.h deleted file mode 100644 index 6b813ad1091c..000000000000 --- a/arch/x86/xen/debugfs.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _XEN_DEBUGFS_H -#define _XEN_DEBUGFS_H - -struct dentry * __init xen_init_debugfs(void); - -#endif /* _XEN_DEBUGFS_H */ diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 0305485edcd3..84e5adbd0925 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -20,8 +20,6 @@ #include #include "xen-ops.h" -#include "smp.h" -#include "pmu.h" EXPORT_SYMBOL_GPL(hypercall_page); diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index c001a2296582..24d2957a4726 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -28,8 +28,6 @@ #include #include "xen-ops.h" -#include "mmu.h" -#include "smp.h" static unsigned long shared_info_pfn; diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 9ba53814ed6a..2c12ae42dc8b 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -85,10 +85,6 @@ #endif #include "xen-ops.h" -#include "mmu.h" -#include "smp.h" -#include "multicalls.h" -#include "pmu.h" #include "../kernel/cpu/cpu.h" /* get_cpu_cap() */ diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 60e9c37fd79f..c4c479373249 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -5,8 +5,7 @@ #include #include -#include "multicalls.h" -#include "mmu.h" +#include "xen-ops.h" unsigned long arbitrary_virt_to_mfn(void *vaddr) { diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h deleted file mode 100644 index 6e4c6bd62203..000000000000 --- a/arch/x86/xen/mmu.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _XEN_MMU_H - -#include -#include - -enum pt_level { - PT_PGD, - PT_P4D, - PT_PUD, - PT_PMD, - PT_PTE -}; - - -bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); - -void set_pte_mfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags); - -pte_t xen_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep); -void xen_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep, pte_t pte); - -unsigned long xen_read_cr2_direct(void); - -extern void xen_init_mmu_ops(void); -extern void xen_hvm_init_mmu_ops(void); -#endif /* _XEN_MMU_H */ diff --git a/arch/x86/xen/mmu_hvm.c b/arch/x86/xen/mmu_hvm.c index 509bdee3ab90..337955652202 100644 --- a/arch/x86/xen/mmu_hvm.c +++ b/arch/x86/xen/mmu_hvm.c @@ -5,7 +5,7 @@ #include #include -#include "mmu.h" +#include "xen-ops.h" #ifdef CONFIG_PROC_VMCORE /* diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 54e0d311dcc9..f1ce39d6d32c 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -82,9 +82,7 @@ #include #include -#include "multicalls.h" -#include "mmu.h" -#include "debugfs.h" +#include "xen-ops.h" /* * Prototypes for functions called via PV_CALLEE_SAVE_REGS_THUNK() in order @@ -128,7 +126,7 @@ static DEFINE_SPINLOCK(xen_reservation_lock); * looking at another vcpu's cr3 value, it should use this variable. */ DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ -DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ +static DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ static phys_addr_t xen_pt_base, xen_pt_size __initdata; @@ -305,16 +303,17 @@ static void xen_set_pte(pte_t *ptep, pte_t pteval) __xen_set_pte(ptep, pteval); } -pte_t xen_ptep_modify_prot_start(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static pte_t xen_ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { /* Just return the pte as-is. We preserve the bits on commit */ trace_xen_mmu_ptep_modify_prot_start(vma->vm_mm, addr, ptep, *ptep); return *ptep; } -void xen_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep, pte_t pte) +static void xen_ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep, pte_t pte) { struct mmu_update u; diff --git a/arch/x86/xen/multicalls.c b/arch/x86/xen/multicalls.c index 07054572297f..d4cefd8a9af4 100644 --- a/arch/x86/xen/multicalls.c +++ b/arch/x86/xen/multicalls.c @@ -23,26 +23,21 @@ #include #include #include +#include +#include #include -#include "multicalls.h" -#include "debugfs.h" +#include "xen-ops.h" #define MC_BATCH 32 -#define MC_DEBUG 0 - #define MC_ARGS (MC_BATCH * 16) struct mc_buffer { unsigned mcidx, argidx, cbidx; struct multicall_entry entries[MC_BATCH]; -#if MC_DEBUG - struct multicall_entry debug[MC_BATCH]; - void *caller[MC_BATCH]; -#endif unsigned char args[MC_ARGS]; struct callback { void (*fn)(void *); @@ -50,13 +45,98 @@ struct mc_buffer { } callbacks[MC_BATCH]; }; +struct mc_debug_data { + struct multicall_entry entries[MC_BATCH]; + void *caller[MC_BATCH]; + size_t argsz[MC_BATCH]; + unsigned long *args[MC_BATCH]; +}; + static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); +static struct mc_debug_data mc_debug_data_early __initdata; +static struct mc_debug_data __percpu *mc_debug_data __refdata = + &mc_debug_data_early; DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); +static struct static_key mc_debug __ro_after_init; +static bool mc_debug_enabled __initdata; + +static int __init xen_parse_mc_debug(char *arg) +{ + mc_debug_enabled = true; + static_key_slow_inc(&mc_debug); + + return 0; +} +early_param("xen_mc_debug", xen_parse_mc_debug); + +static int __init mc_debug_enable(void) +{ + struct mc_debug_data __percpu *mcdb; + unsigned long flags; + + if (!mc_debug_enabled) + return 0; + + mcdb = alloc_percpu(struct mc_debug_data); + if (!mcdb) { + pr_err("xen_mc_debug inactive\n"); + static_key_slow_dec(&mc_debug); + return -ENOMEM; + } + + /* Be careful when switching to percpu debug data. */ + local_irq_save(flags); + xen_mc_flush(); + mc_debug_data = mcdb; + local_irq_restore(flags); + + pr_info("xen_mc_debug active\n"); + + return 0; +} +early_initcall(mc_debug_enable); + +/* Number of parameters of hypercalls used via multicalls. */ +static const uint8_t hpcpars[] = { + [__HYPERVISOR_mmu_update] = 4, + [__HYPERVISOR_stack_switch] = 2, + [__HYPERVISOR_fpu_taskswitch] = 1, + [__HYPERVISOR_update_descriptor] = 2, + [__HYPERVISOR_update_va_mapping] = 3, + [__HYPERVISOR_mmuext_op] = 4, +}; + +static void print_debug_data(struct mc_buffer *b, struct mc_debug_data *mcdb, + int idx) +{ + unsigned int arg; + unsigned int opidx = mcdb->entries[idx].op & 0xff; + unsigned int pars = 0; + + pr_err(" call %2d: op=%lu result=%ld caller=%pS ", idx + 1, + mcdb->entries[idx].op, b->entries[idx].result, + mcdb->caller[idx]); + if (opidx < ARRAY_SIZE(hpcpars)) + pars = hpcpars[opidx]; + if (pars) { + pr_cont("pars="); + for (arg = 0; arg < pars; arg++) + pr_cont("%lx ", mcdb->entries[idx].args[arg]); + } + if (mcdb->argsz[idx]) { + pr_cont("args="); + for (arg = 0; arg < mcdb->argsz[idx] / 8; arg++) + pr_cont("%lx ", mcdb->args[idx][arg]); + } + pr_cont("\n"); +} + void xen_mc_flush(void) { struct mc_buffer *b = this_cpu_ptr(&mc_buffer); struct multicall_entry *mc; + struct mc_debug_data *mcdb = NULL; int ret = 0; unsigned long flags; int i; @@ -69,10 +149,11 @@ void xen_mc_flush(void) trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx); -#if MC_DEBUG - memcpy(b->debug, b->entries, - b->mcidx * sizeof(struct multicall_entry)); -#endif + if (static_key_false(&mc_debug)) { + mcdb = this_cpu_ptr(mc_debug_data); + memcpy(mcdb->entries, b->entries, + b->mcidx * sizeof(struct multicall_entry)); + } switch (b->mcidx) { case 0: @@ -103,21 +184,14 @@ void xen_mc_flush(void) pr_err("%d of %d multicall(s) failed: cpu %d\n", ret, b->mcidx, smp_processor_id()); for (i = 0; i < b->mcidx; i++) { - if (b->entries[i].result < 0) { -#if MC_DEBUG - pr_err(" call %2d: op=%lu arg=[%lx] result=%ld\t%pS\n", - i + 1, - b->debug[i].op, - b->debug[i].args[0], - b->entries[i].result, - b->caller[i]); -#else + if (static_key_false(&mc_debug)) { + print_debug_data(b, mcdb, i); + } else if (b->entries[i].result < 0) { pr_err(" call %2d: op=%lu arg=[%lx] result=%ld\n", i + 1, b->entries[i].op, b->entries[i].args[0], b->entries[i].result); -#endif } } } @@ -155,9 +229,13 @@ struct multicall_space __xen_mc_entry(size_t args) } ret.mc = &b->entries[b->mcidx]; -#if MC_DEBUG - b->caller[b->mcidx] = __builtin_return_address(0); -#endif + if (static_key_false(&mc_debug)) { + struct mc_debug_data *mcdb = this_cpu_ptr(mc_debug_data); + + mcdb->caller[b->mcidx] = __builtin_return_address(0); + mcdb->argsz[b->mcidx] = args; + mcdb->args[b->mcidx] = (unsigned long *)(&b->args[argidx]); + } b->mcidx++; ret.args = &b->args[argidx]; b->argidx = argidx + args; diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h deleted file mode 100644 index c3867b585e0d..000000000000 --- a/arch/x86/xen/multicalls.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _XEN_MULTICALLS_H -#define _XEN_MULTICALLS_H - -#include - -#include "xen-ops.h" - -/* Multicalls */ -struct multicall_space -{ - struct multicall_entry *mc; - void *args; -}; - -/* Allocate room for a multicall and its args */ -struct multicall_space __xen_mc_entry(size_t args); - -DECLARE_PER_CPU(unsigned long, xen_mc_irq_flags); - -/* Call to start a batch of multiple __xen_mc_entry()s. Must be - paired with xen_mc_issue() */ -static inline void xen_mc_batch(void) -{ - unsigned long flags; - - /* need to disable interrupts until this entry is complete */ - local_irq_save(flags); - trace_xen_mc_batch(xen_get_lazy_mode()); - __this_cpu_write(xen_mc_irq_flags, flags); -} - -static inline struct multicall_space xen_mc_entry(size_t args) -{ - xen_mc_batch(); - return __xen_mc_entry(args); -} - -/* Flush all pending multicalls */ -void xen_mc_flush(void); - -/* Issue a multicall if we're not in a lazy mode */ -static inline void xen_mc_issue(unsigned mode) -{ - trace_xen_mc_issue(mode); - - if ((xen_get_lazy_mode() & mode) == 0) - xen_mc_flush(); - - /* restore flags saved in xen_mc_batch */ - local_irq_restore(this_cpu_read(xen_mc_irq_flags)); -} - -/* Set up a callback to be called when the current batch is flushed */ -void xen_mc_callback(void (*fn)(void *), void *data); - -/* - * Try to extend the arguments of the previous multicall command. The - * previous command's op must match. If it does, then it attempts to - * extend the argument space allocated to the multicall entry by - * arg_size bytes. - * - * The returned multicall_space will return with mc pointing to the - * command on success, or NULL on failure, and args pointing to the - * newly allocated space. - */ -struct multicall_space xen_mc_extend_args(unsigned long op, size_t arg_size); - -#endif /* _XEN_MULTICALLS_H */ diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 99918beccd80..7c735b730acd 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -81,7 +81,6 @@ #include #include -#include "multicalls.h" #include "xen-ops.h" #define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *)) @@ -730,7 +729,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, * immediate unmapping. */ map_ops[i].status = GNTST_general_error; - unmap[0].host_addr = map_ops[i].host_addr, + unmap[0].host_addr = map_ops[i].host_addr; unmap[0].handle = map_ops[i].handle; map_ops[i].handle = INVALID_GRANT_HANDLE; if (map_ops[i].flags & GNTMAP_device_map) @@ -740,7 +739,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, if (kmap_ops) { kmap_ops[i].status = GNTST_general_error; - unmap[1].host_addr = kmap_ops[i].host_addr, + unmap[1].host_addr = kmap_ops[i].host_addr; unmap[1].handle = kmap_ops[i].handle; kmap_ops[i].handle = INVALID_GRANT_HANDLE; if (kmap_ops[i].flags & GNTMAP_device_map) @@ -795,7 +794,6 @@ int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, #ifdef CONFIG_XEN_DEBUG_FS #include -#include "debugfs.h" static int p2m_dump_show(struct seq_file *m, void *v) { static const char * const type_name[] = { diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 246d67dab510..f06987b0efc3 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -10,7 +10,6 @@ #include #include "xen-ops.h" -#include "pmu.h" /* x86_pmu.handle_irq definition */ #include "../events/perf_event.h" diff --git a/arch/x86/xen/pmu.h b/arch/x86/xen/pmu.h deleted file mode 100644 index 65c58894fc79..000000000000 --- a/arch/x86/xen/pmu.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __XEN_PMU_H -#define __XEN_PMU_H - -#include - -extern bool is_xen_pmu; - -irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id); -#ifdef CONFIG_XEN_HAVE_VPMU -void xen_pmu_init(int cpu); -void xen_pmu_finish(int cpu); -#else -static inline void xen_pmu_init(int cpu) {} -static inline void xen_pmu_finish(int cpu) {} -#endif -bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err); -bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err); -int pmu_apic_update(uint32_t reg); -unsigned long long xen_read_pmc(int counter); - -#endif /* __XEN_PMU_H */ diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 380591028cb8..a0c3e77e3d5b 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -34,7 +34,6 @@ #include #include #include "xen-ops.h" -#include "mmu.h" #define GB(x) ((uint64_t)(x) * 1024 * 1024 * 1024) diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 935771726f9c..05f92c812ac8 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -9,7 +9,6 @@ #include #include "xen-ops.h" -#include "smp.h" static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 }; static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 }; diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h deleted file mode 100644 index b8efdbc693f7..000000000000 --- a/arch/x86/xen/smp.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _XEN_SMP_H - -#ifdef CONFIG_SMP - -void asm_cpu_bringup_and_idle(void); -asmlinkage void cpu_bringup_and_idle(void); - -extern void xen_send_IPI_mask(const struct cpumask *mask, - int vector); -extern void xen_send_IPI_mask_allbutself(const struct cpumask *mask, - int vector); -extern void xen_send_IPI_allbutself(int vector); -extern void xen_send_IPI_all(int vector); -extern void xen_send_IPI_self(int vector); - -extern int xen_smp_intr_init(unsigned int cpu); -extern void xen_smp_intr_free(unsigned int cpu); -int xen_smp_intr_init_pv(unsigned int cpu); -void xen_smp_intr_free_pv(unsigned int cpu); - -void xen_smp_count_cpus(void); -void xen_smp_cpus_done(unsigned int max_cpus); - -void xen_smp_send_reschedule(int cpu); -void xen_smp_send_call_function_ipi(const struct cpumask *mask); -void xen_smp_send_call_function_single_ipi(int cpu); - -void __noreturn xen_cpu_bringup_again(unsigned long stack); - -struct xen_common_irq { - int irq; - char *name; -}; -#else /* CONFIG_SMP */ - -static inline int xen_smp_intr_init(unsigned int cpu) -{ - return 0; -} -static inline void xen_smp_intr_free(unsigned int cpu) {} - -static inline int xen_smp_intr_init_pv(unsigned int cpu) -{ - return 0; -} -static inline void xen_smp_intr_free_pv(unsigned int cpu) {} -static inline void xen_smp_count_cpus(void) { } -#endif /* CONFIG_SMP */ - -#endif diff --git a/arch/x86/xen/smp_hvm.c b/arch/x86/xen/smp_hvm.c index ac95d1981cc0..485c1d8804f7 100644 --- a/arch/x86/xen/smp_hvm.c +++ b/arch/x86/xen/smp_hvm.c @@ -5,8 +5,6 @@ #include #include "xen-ops.h" -#include "smp.h" - static void __init xen_hvm_smp_prepare_boot_cpu(void) { diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index ac41d83b38d3..7ea57f728b89 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -46,9 +46,6 @@ #include #include "xen-ops.h" -#include "mmu.h" -#include "smp.h" -#include "pmu.h" cpumask_var_t xen_cpu_initialized_map; diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 5c6fc16e4b92..8e4efe0fb6f9 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -18,7 +18,6 @@ static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; static DEFINE_PER_CPU(char *, irq_name); static DEFINE_PER_CPU(atomic_t, xen_qlock_wait_nest); -static bool xen_pvspin = true; static void xen_qlock_kick(int cpu) { @@ -68,7 +67,7 @@ void xen_init_lock_cpu(int cpu) int irq; char *name; - if (!xen_pvspin) + if (nopvspin) return; WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n", @@ -95,7 +94,7 @@ void xen_uninit_lock_cpu(int cpu) { int irq; - if (!xen_pvspin) + if (nopvspin) return; kfree(per_cpu(irq_name, cpu)); @@ -125,10 +124,10 @@ PV_CALLEE_SAVE_REGS_THUNK(xen_vcpu_stolen); void __init xen_init_spinlocks(void) { /* Don't need to use pvqspinlock code if there is only 1 vCPU. */ - if (num_possible_cpus() == 1 || nopvspin) - xen_pvspin = false; + if (num_possible_cpus() == 1) + nopvspin = true; - if (!xen_pvspin) { + if (nopvspin) { printk(KERN_DEBUG "xen: PV spinlocks disabled\n"); static_branch_disable(&virt_spin_lock_key); return; @@ -143,12 +142,3 @@ void __init xen_init_spinlocks(void) pv_ops.lock.kick = xen_qlock_kick; pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); } - -static __init int xen_parse_nopvspin(char *arg) -{ - pr_notice("\"xen_nopvspin\" is deprecated, please use \"nopvspin\" instead\n"); - xen_pvspin = false; - return 0; -} -early_param("xen_nopvspin", xen_parse_nopvspin); - diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c index 1d83152c761b..77a6ea1c60e4 100644 --- a/arch/x86/xen/suspend.c +++ b/arch/x86/xen/suspend.c @@ -15,8 +15,6 @@ #include #include "xen-ops.h" -#include "mmu.h" -#include "pmu.h" static DEFINE_PER_CPU(u64, spec_ctrl); diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 52fa5609b7f6..96521b1874ac 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -30,7 +30,7 @@ #include "xen-ops.h" /* Minimum amount of time until next clock event fires */ -#define TIMER_SLOP 100000 +#define TIMER_SLOP 1 static u64 xen_sched_clock_offset __read_mostly; diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 79cf93f2c92f..e7775dff9452 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -5,8 +5,15 @@ #include #include #include +#include + +#include #include +#include + +#include + /* These are code, but not functions. Defined in entry.S */ extern const char xen_failsafe_callback[]; @@ -23,14 +30,11 @@ void xen_copy_trap_info(struct trap_info *traps); DECLARE_PER_CPU_ALIGNED(struct vcpu_info, xen_vcpu_info); DECLARE_PER_CPU(unsigned long, xen_cr3); -DECLARE_PER_CPU(unsigned long, xen_current_cr3); extern struct start_info *xen_start_info; extern struct shared_info xen_dummy_shared_info; extern struct shared_info *HYPERVISOR_shared_info; -extern bool xen_fifo_events; - void xen_setup_mfn_list_list(void); void xen_build_mfn_list_list(void); void xen_setup_machphys_mapping(void); @@ -177,4 +181,142 @@ static inline void xen_hvm_post_suspend(int suspend_cancelled) {} void xen_add_extra_mem(unsigned long start_pfn, unsigned long n_pfns); +struct dentry * __init xen_init_debugfs(void); + +enum pt_level { + PT_PGD, + PT_P4D, + PT_PUD, + PT_PMD, + PT_PTE +}; + +bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); +void set_pte_mfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags); +unsigned long xen_read_cr2_direct(void); +void xen_init_mmu_ops(void); +void xen_hvm_init_mmu_ops(void); + +/* Multicalls */ +struct multicall_space +{ + struct multicall_entry *mc; + void *args; +}; + +/* Allocate room for a multicall and its args */ +struct multicall_space __xen_mc_entry(size_t args); + +DECLARE_PER_CPU(unsigned long, xen_mc_irq_flags); + +/* Call to start a batch of multiple __xen_mc_entry()s. Must be + paired with xen_mc_issue() */ +static inline void xen_mc_batch(void) +{ + unsigned long flags; + + /* need to disable interrupts until this entry is complete */ + local_irq_save(flags); + trace_xen_mc_batch(xen_get_lazy_mode()); + __this_cpu_write(xen_mc_irq_flags, flags); +} + +static inline struct multicall_space xen_mc_entry(size_t args) +{ + xen_mc_batch(); + return __xen_mc_entry(args); +} + +/* Flush all pending multicalls */ +void xen_mc_flush(void); + +/* Issue a multicall if we're not in a lazy mode */ +static inline void xen_mc_issue(unsigned mode) +{ + trace_xen_mc_issue(mode); + + if ((xen_get_lazy_mode() & mode) == 0) + xen_mc_flush(); + + /* restore flags saved in xen_mc_batch */ + local_irq_restore(this_cpu_read(xen_mc_irq_flags)); +} + +/* Set up a callback to be called when the current batch is flushed */ +void xen_mc_callback(void (*fn)(void *), void *data); + +/* + * Try to extend the arguments of the previous multicall command. The + * previous command's op must match. If it does, then it attempts to + * extend the argument space allocated to the multicall entry by + * arg_size bytes. + * + * The returned multicall_space will return with mc pointing to the + * command on success, or NULL on failure, and args pointing to the + * newly allocated space. + */ +struct multicall_space xen_mc_extend_args(unsigned long op, size_t arg_size); + +extern bool is_xen_pmu; + +irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id); +#ifdef CONFIG_XEN_HAVE_VPMU +void xen_pmu_init(int cpu); +void xen_pmu_finish(int cpu); +#else +static inline void xen_pmu_init(int cpu) {} +static inline void xen_pmu_finish(int cpu) {} +#endif +bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err); +bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err); +int pmu_apic_update(uint32_t reg); +unsigned long long xen_read_pmc(int counter); + +#ifdef CONFIG_SMP + +void asm_cpu_bringup_and_idle(void); +asmlinkage void cpu_bringup_and_idle(void); + +extern void xen_send_IPI_mask(const struct cpumask *mask, + int vector); +extern void xen_send_IPI_mask_allbutself(const struct cpumask *mask, + int vector); +extern void xen_send_IPI_allbutself(int vector); +extern void xen_send_IPI_all(int vector); +extern void xen_send_IPI_self(int vector); + +extern int xen_smp_intr_init(unsigned int cpu); +extern void xen_smp_intr_free(unsigned int cpu); +int xen_smp_intr_init_pv(unsigned int cpu); +void xen_smp_intr_free_pv(unsigned int cpu); + +void xen_smp_count_cpus(void); +void xen_smp_cpus_done(unsigned int max_cpus); + +void xen_smp_send_reschedule(int cpu); +void xen_smp_send_call_function_ipi(const struct cpumask *mask); +void xen_smp_send_call_function_single_ipi(int cpu); + +void __noreturn xen_cpu_bringup_again(unsigned long stack); + +struct xen_common_irq { + int irq; + char *name; +}; +#else /* CONFIG_SMP */ + +static inline int xen_smp_intr_init(unsigned int cpu) +{ + return 0; +} +static inline void xen_smp_intr_free(unsigned int cpu) {} + +static inline int xen_smp_intr_init_pv(unsigned int cpu) +{ + return 0; +} +static inline void xen_smp_intr_free_pv(unsigned int cpu) {} +static inline void xen_smp_count_cpus(void) { } +#endif /* CONFIG_SMP */ + #endif /* XEN_OPS_H */ diff --git a/drivers/xen/evtchn.c b/drivers/xen/evtchn.c index f6a2216c2c87..9b7fcc7dbb38 100644 --- a/drivers/xen/evtchn.c +++ b/drivers/xen/evtchn.c @@ -729,4 +729,5 @@ static void __exit evtchn_cleanup(void) module_init(evtchn_init); module_exit(evtchn_cleanup); +MODULE_DESCRIPTION("Xen /dev/xen/evtchn device driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/xen/manage.c b/drivers/xen/manage.c index c16df629907e..b4b4ebed68da 100644 --- a/drivers/xen/manage.c +++ b/drivers/xen/manage.c @@ -208,7 +208,7 @@ static void do_reboot(void) orderly_reboot(); } -static struct shutdown_handler shutdown_handlers[] = { +static const struct shutdown_handler shutdown_handlers[] = { { "poweroff", true, do_poweroff }, { "halt", false, do_poweroff }, { "reboot", true, do_reboot }, diff --git a/drivers/xen/privcmd-buf.c b/drivers/xen/privcmd-buf.c index 2fa10ca5be14..0f0dad427d7e 100644 --- a/drivers/xen/privcmd-buf.c +++ b/drivers/xen/privcmd-buf.c @@ -19,6 +19,7 @@ #include "privcmd.h" +MODULE_DESCRIPTION("Xen Mmap of hypercall buffers"); MODULE_LICENSE("GPL"); struct privcmd_buf_private { diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 67dfa4778864..9563650dfbaf 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,7 @@ #include "privcmd.h" +MODULE_DESCRIPTION("Xen hypercall passthrough driver"); MODULE_LICENSE("GPL"); #define PRIV_VMA_LOCKED ((void *)1) @@ -845,7 +847,8 @@ out: #ifdef CONFIG_XEN_PRIVCMD_EVENTFD /* Irqfd support */ static struct workqueue_struct *irqfd_cleanup_wq; -static DEFINE_MUTEX(irqfds_lock); +static DEFINE_SPINLOCK(irqfds_lock); +DEFINE_STATIC_SRCU(irqfds_srcu); static LIST_HEAD(irqfds_list); struct privcmd_kernel_irqfd { @@ -873,6 +876,9 @@ static void irqfd_shutdown(struct work_struct *work) container_of(work, struct privcmd_kernel_irqfd, shutdown); u64 cnt; + /* Make sure irqfd has been initialized in assign path */ + synchronize_srcu(&irqfds_srcu); + eventfd_ctx_remove_wait_queue(kirqfd->eventfd, &kirqfd->wait, &cnt); eventfd_ctx_put(kirqfd->eventfd); kfree(kirqfd); @@ -909,9 +915,11 @@ irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key) irqfd_inject(kirqfd); if (flags & EPOLLHUP) { - mutex_lock(&irqfds_lock); + unsigned long flags; + + spin_lock_irqsave(&irqfds_lock, flags); irqfd_deactivate(kirqfd); - mutex_unlock(&irqfds_lock); + spin_unlock_irqrestore(&irqfds_lock, flags); } return 0; @@ -929,10 +937,11 @@ irqfd_poll_func(struct file *file, wait_queue_head_t *wqh, poll_table *pt) static int privcmd_irqfd_assign(struct privcmd_irqfd *irqfd) { struct privcmd_kernel_irqfd *kirqfd, *tmp; + unsigned long flags; __poll_t events; struct fd f; void *dm_op; - int ret; + int ret, idx; kirqfd = kzalloc(sizeof(*kirqfd) + irqfd->size, GFP_KERNEL); if (!kirqfd) @@ -968,18 +977,19 @@ static int privcmd_irqfd_assign(struct privcmd_irqfd *irqfd) init_waitqueue_func_entry(&kirqfd->wait, irqfd_wakeup); init_poll_funcptr(&kirqfd->pt, irqfd_poll_func); - mutex_lock(&irqfds_lock); + spin_lock_irqsave(&irqfds_lock, flags); list_for_each_entry(tmp, &irqfds_list, list) { if (kirqfd->eventfd == tmp->eventfd) { ret = -EBUSY; - mutex_unlock(&irqfds_lock); + spin_unlock_irqrestore(&irqfds_lock, flags); goto error_eventfd; } } + idx = srcu_read_lock(&irqfds_srcu); list_add_tail(&kirqfd->list, &irqfds_list); - mutex_unlock(&irqfds_lock); + spin_unlock_irqrestore(&irqfds_lock, flags); /* * Check if there was an event already pending on the eventfd before we @@ -989,6 +999,8 @@ static int privcmd_irqfd_assign(struct privcmd_irqfd *irqfd) if (events & EPOLLIN) irqfd_inject(kirqfd); + srcu_read_unlock(&irqfds_srcu, idx); + /* * Do not drop the file until the kirqfd is fully initialized, otherwise * we might race against the EPOLLHUP. @@ -1011,12 +1023,13 @@ static int privcmd_irqfd_deassign(struct privcmd_irqfd *irqfd) { struct privcmd_kernel_irqfd *kirqfd; struct eventfd_ctx *eventfd; + unsigned long flags; eventfd = eventfd_ctx_fdget(irqfd->fd); if (IS_ERR(eventfd)) return PTR_ERR(eventfd); - mutex_lock(&irqfds_lock); + spin_lock_irqsave(&irqfds_lock, flags); list_for_each_entry(kirqfd, &irqfds_list, list) { if (kirqfd->eventfd == eventfd) { @@ -1025,7 +1038,7 @@ static int privcmd_irqfd_deassign(struct privcmd_irqfd *irqfd) } } - mutex_unlock(&irqfds_lock); + spin_unlock_irqrestore(&irqfds_lock, flags); eventfd_ctx_put(eventfd); @@ -1073,13 +1086,14 @@ static int privcmd_irqfd_init(void) static void privcmd_irqfd_exit(void) { struct privcmd_kernel_irqfd *kirqfd, *tmp; + unsigned long flags; - mutex_lock(&irqfds_lock); + spin_lock_irqsave(&irqfds_lock, flags); list_for_each_entry_safe(kirqfd, tmp, &irqfds_list, list) irqfd_deactivate(kirqfd); - mutex_unlock(&irqfds_lock); + spin_unlock_irqrestore(&irqfds_lock, flags); destroy_workqueue(irqfd_cleanup_wq); } diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index e34b623e4b41..4faebbb84999 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -1708,5 +1708,6 @@ static void __exit xen_pcibk_cleanup(void) module_init(xen_pcibk_init); module_exit(xen_pcibk_cleanup); +MODULE_DESCRIPTION("Xen PCI-device stub driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_ALIAS("xen-backend:pci"); diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt index f5693164ca9a..bd2f530e5740 100644 --- a/fs/Kconfig.binfmt +++ b/fs/Kconfig.binfmt @@ -176,4 +176,12 @@ config COREDUMP certainly want to say Y here. Not necessary on systems that never need debugging or only ever run flawless code. +config EXEC_KUNIT_TEST + bool "Build execve tests" if !KUNIT_ALL_TESTS + depends on KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the exec KUnit tests, which tests boundary conditions + of various aspects of the exec internals. + endmenu diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index a43897b03ce9..40111451aa95 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1003,7 +1003,8 @@ out_free_interp: if (elf_read_implies_exec(*elf_ex, executable_stack)) current->personality |= READ_IMPLIES_EXEC; - if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) + const int snapshot_randomize_va_space = READ_ONCE(randomize_va_space); + if (!(current->personality & ADDR_NO_RANDOMIZE) && snapshot_randomize_va_space) current->flags |= PF_RANDOMIZE; setup_new_exec(bprm); @@ -1061,47 +1062,7 @@ out_free_interp: * Header for ET_DYN binaries to calculate the * randomization (load_bias) for all the LOAD * Program Headers. - * - * There are effectively two types of ET_DYN - * binaries: programs (i.e. PIE: ET_DYN with INTERP) - * and loaders (ET_DYN without INTERP, since they - * _are_ the ELF interpreter). The loaders must - * be loaded away from programs since the program - * may otherwise collide with the loader (especially - * for ET_EXEC which does not have a randomized - * position). For example to handle invocations of - * "./ld.so someprog" to test out a new version of - * the loader, the subsequent program that the - * loader loads must avoid the loader itself, so - * they cannot share the same load range. Sufficient - * room for the brk must be allocated with the - * loader as well, since brk must be available with - * the loader. - * - * Therefore, programs are loaded offset from - * ELF_ET_DYN_BASE and loaders are loaded into the - * independently randomized mmap region (0 load_bias - * without MAP_FIXED nor MAP_FIXED_NOREPLACE). */ - if (interpreter) { - load_bias = ELF_ET_DYN_BASE; - if (current->flags & PF_RANDOMIZE) - load_bias += arch_mmap_rnd(); - alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); - if (alignment) - load_bias &= ~(alignment - 1); - elf_flags |= MAP_FIXED_NOREPLACE; - } else - load_bias = 0; - - /* - * Since load_bias is used for all subsequent loading - * calculations, we must lower it by the first vaddr - * so that the remaining calculations based on the - * ELF vaddrs will be correctly offset. The result - * is then page aligned. - */ - load_bias = ELF_PAGESTART(load_bias - vaddr); /* * Calculate the entire size of the ELF mapping @@ -1127,6 +1088,80 @@ out_free_interp: retval = -EINVAL; goto out_free_dentry; } + + /* Calculate any requested alignment. */ + alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); + + /* + * There are effectively two types of ET_DYN + * binaries: programs (i.e. PIE: ET_DYN with PT_INTERP) + * and loaders (ET_DYN without PT_INTERP, since they + * _are_ the ELF interpreter). The loaders must + * be loaded away from programs since the program + * may otherwise collide with the loader (especially + * for ET_EXEC which does not have a randomized + * position). For example to handle invocations of + * "./ld.so someprog" to test out a new version of + * the loader, the subsequent program that the + * loader loads must avoid the loader itself, so + * they cannot share the same load range. Sufficient + * room for the brk must be allocated with the + * loader as well, since brk must be available with + * the loader. + * + * Therefore, programs are loaded offset from + * ELF_ET_DYN_BASE and loaders are loaded into the + * independently randomized mmap region (0 load_bias + * without MAP_FIXED nor MAP_FIXED_NOREPLACE). + */ + if (interpreter) { + /* On ET_DYN with PT_INTERP, we do the ASLR. */ + load_bias = ELF_ET_DYN_BASE; + if (current->flags & PF_RANDOMIZE) + load_bias += arch_mmap_rnd(); + /* Adjust alignment as requested. */ + if (alignment) + load_bias &= ~(alignment - 1); + elf_flags |= MAP_FIXED_NOREPLACE; + } else { + /* + * For ET_DYN without PT_INTERP, we rely on + * the architectures's (potentially ASLR) mmap + * base address (via a load_bias of 0). + * + * When a large alignment is requested, we + * must do the allocation at address "0" right + * now to discover where things will load so + * that we can adjust the resulting alignment. + * In this case (load_bias != 0), we can use + * MAP_FIXED_NOREPLACE to make sure the mapping + * doesn't collide with anything. + */ + if (alignment > ELF_MIN_ALIGN) { + load_bias = elf_load(bprm->file, 0, elf_ppnt, + elf_prot, elf_flags, total_size); + if (BAD_ADDR(load_bias)) { + retval = IS_ERR_VALUE(load_bias) ? + PTR_ERR((void*)load_bias) : -EINVAL; + goto out_free_dentry; + } + vm_munmap(load_bias, total_size); + /* Adjust alignment as requested. */ + if (alignment) + load_bias &= ~(alignment - 1); + elf_flags |= MAP_FIXED_NOREPLACE; + } else + load_bias = 0; + } + + /* + * Since load_bias is used for all subsequent loading + * calculations, we must lower it by the first vaddr + * so that the remaining calculations based on the + * ELF vaddrs will be correctly offset. The result + * is then page aligned. + */ + load_bias = ELF_PAGESTART(load_bias - vaddr); } error = elf_load(bprm->file, load_bias + vaddr, elf_ppnt, @@ -1251,7 +1286,7 @@ out_free_interp: mm->end_data = end_data; mm->start_stack = bprm->p; - if ((current->flags & PF_RANDOMIZE) && (randomize_va_space > 1)) { + if ((current->flags & PF_RANDOMIZE) && (snapshot_randomize_va_space > 1)) { /* * For architectures with ELF randomization, when executing * a loader directly (i.e. no interpreter listed in ELF diff --git a/fs/exec.c b/fs/exec.c index 40073142288f..5b580ff8d955 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -486,6 +486,35 @@ static int count_strings_kernel(const char *const *argv) return i; } +static inline int bprm_set_stack_limit(struct linux_binprm *bprm, + unsigned long limit) +{ +#ifdef CONFIG_MMU + /* Avoid a pathological bprm->p. */ + if (bprm->p < limit) + return -E2BIG; + bprm->argmin = bprm->p - limit; +#endif + return 0; +} +static inline bool bprm_hit_stack_limit(struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return bprm->p < bprm->argmin; +#else + return false; +#endif +} + +/* + * Calculate bprm->argmin from: + * - _STK_LIM + * - ARG_MAX + * - bprm->rlim_stack.rlim_cur + * - bprm->argc + * - bprm->envc + * - bprm->p + */ static int bprm_stack_limits(struct linux_binprm *bprm) { unsigned long limit, ptr_size; @@ -505,6 +534,9 @@ static int bprm_stack_limits(struct linux_binprm *bprm) * of argument strings even with small stacks */ limit = max_t(unsigned long, limit, ARG_MAX); + /* Reject totally pathological counts. */ + if (bprm->argc < 0 || bprm->envc < 0) + return -E2BIG; /* * We must account for the size of all the argv and envp pointers to * the argv and envp strings, since they will also take up space in @@ -518,13 +550,14 @@ static int bprm_stack_limits(struct linux_binprm *bprm) * argc can never be 0, to keep them from walking envp by accident. * See do_execveat_common(). */ - ptr_size = (max(bprm->argc, 1) + bprm->envc) * sizeof(void *); + if (check_add_overflow(max(bprm->argc, 1), bprm->envc, &ptr_size) || + check_mul_overflow(ptr_size, sizeof(void *), &ptr_size)) + return -E2BIG; if (limit <= ptr_size) return -E2BIG; limit -= ptr_size; - bprm->argmin = bprm->p - limit; - return 0; + return bprm_set_stack_limit(bprm, limit); } /* @@ -562,10 +595,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, pos = bprm->p; str += len; bprm->p -= len; -#ifdef CONFIG_MMU - if (bprm->p < bprm->argmin) + if (bprm_hit_stack_limit(bprm)) goto out; -#endif while (len > 0) { int offset, bytes_to_copy; @@ -640,7 +671,7 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) /* We're going to work our way backwards. */ arg += len; bprm->p -= len; - if (IS_ENABLED(CONFIG_MMU) && bprm->p < bprm->argmin) + if (bprm_hit_stack_limit(bprm)) return -E2BIG; while (len > 0) { @@ -2211,3 +2242,7 @@ static int __init init_fs_exec_sysctls(void) fs_initcall(init_fs_exec_sysctls); #endif /* CONFIG_SYSCTL */ + +#ifdef CONFIG_EXEC_KUNIT_TEST +#include "exec_test.c" +#endif diff --git a/fs/exec_test.c b/fs/exec_test.c new file mode 100644 index 000000000000..7c77d039680b --- /dev/null +++ b/fs/exec_test.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +struct bprm_stack_limits_result { + struct linux_binprm bprm; + int expected_rc; + unsigned long expected_argmin; +}; + +static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { + /* Negative argc/envc counts produce -E2BIG */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = INT_MIN, .envc = INT_MIN }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 5, .envc = -1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = -1, .envc = 10 }, .expected_rc = -E2BIG }, + /* The max value of argc or envc is MAX_ARG_STRINGS. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = INT_MAX, .envc = INT_MAX }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = MAX_ARG_STRINGS, .envc = MAX_ARG_STRINGS }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 0, .envc = MAX_ARG_STRINGS }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = MAX_ARG_STRINGS, .envc = 0 }, .expected_rc = -E2BIG }, + /* + * On 32-bit system these argc and envc counts, while likely impossible + * to represent within the associated TASK_SIZE, could overflow the + * limit calculation, and bypass the ptr_size <= limit check. + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 0x20000001, .envc = 0x20000001 }, .expected_rc = -E2BIG }, +#ifdef CONFIG_MMU + /* Make sure a pathological bprm->p doesn't cause an overflow. */ + { { .p = sizeof(void *), .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 10, .envc = 10 }, .expected_rc = -E2BIG }, +#endif + /* + * 0 rlim_stack will get raised to ARG_MAX. With 1 string pointer, + * we should see p - ARG_MAX + sizeof(void *). + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 1, .envc = 0 }, .expected_argmin = ULONG_MAX - ARG_MAX + sizeof(void *)}, + /* Validate that argc is always raised to a minimum of 1. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - ARG_MAX + sizeof(void *)}, + /* + * 0 rlim_stack will get raised to ARG_MAX. With pointers filling ARG_MAX, + * we should see -E2BIG. (Note argc is always raised to at least 1.) + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = ARG_MAX / sizeof(void *) + 1, .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = ARG_MAX / sizeof(void *) }, .expected_rc = -E2BIG }, + /* And with one less, we see space for exactly 1 pointer. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = (ARG_MAX / sizeof(void *)) - 1, .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = (ARG_MAX / sizeof(void *)) - 2, }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* If we raise rlim_stack / 4 to exactly ARG_MAX, nothing changes. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = ARG_MAX / sizeof(void *) + 1, .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = (ARG_MAX / sizeof(void *)) - 1, .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = (ARG_MAX / sizeof(void *)) - 2, }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* But raising it another pointer * 4 will provide space for 1 more pointer. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = (ARG_MAX + sizeof(void *)) * 4, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = (ARG_MAX + sizeof(void *)) * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + /* But raising it any further will see no increase. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3 + sizeof(void *)), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * + sizeof(void *)), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, +}; + +static void exec_test_bprm_stack_limits(struct kunit *test) +{ + /* Double-check the constants. */ + KUNIT_EXPECT_EQ(test, _STK_LIM, SZ_8M); + KUNIT_EXPECT_EQ(test, ARG_MAX, 32 * SZ_4K); + KUNIT_EXPECT_EQ(test, MAX_ARG_STRINGS, 0x7FFFFFFF); + + for (int i = 0; i < ARRAY_SIZE(bprm_stack_limits_results); i++) { + const struct bprm_stack_limits_result *result = &bprm_stack_limits_results[i]; + struct linux_binprm bprm = result->bprm; + int rc; + + rc = bprm_stack_limits(&bprm); + KUNIT_EXPECT_EQ_MSG(test, rc, result->expected_rc, "on loop %d", i); +#ifdef CONFIG_MMU + KUNIT_EXPECT_EQ_MSG(test, bprm.argmin, result->expected_argmin, "on loop %d", i); +#endif + } +} + +static struct kunit_case exec_test_cases[] = { + KUNIT_CASE(exec_test_bprm_stack_limits), + {}, +}; + +static struct kunit_suite exec_test_suite = { + .name = "exec", + .test_cases = exec_test_cases, +}; + +kunit_test_suite(exec_test_suite); diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 70f97f685bff..e6c00e860951 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -19,13 +19,13 @@ struct linux_binprm { #ifdef CONFIG_MMU struct vm_area_struct *vma; unsigned long vma_pages; + unsigned long argmin; /* rlimit marker for copy_strings() */ #else # define MAX_ARG_PAGES 32 struct page *page[MAX_ARG_PAGES]; #endif struct mm_struct *mm; unsigned long p; /* current top of mem */ - unsigned long argmin; /* rlimit marker for copy_strings() */ unsigned int /* Should an execfd be passed to userspace? */ have_execfd:1, diff --git a/include/xen/events.h b/include/xen/events.h index 3b07409f8032..de5da58a0205 100644 --- a/include/xen/events.h +++ b/include/xen/events.h @@ -144,4 +144,6 @@ static inline void xen_evtchn_close(evtchn_port_t port) BUG(); } +extern bool xen_fifo_events; + #endif /* _XEN_EVENTS_H */ diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index 1df5fef8a656..7d96bed718e4 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -583,7 +583,7 @@ EXPORT_SYMBOL(queued_spin_lock_slowpath); #include "qspinlock_paravirt.h" #include "qspinlock.c" -bool nopvspin __initdata; +bool nopvspin; static __init int parse_nopvspin(char *arg) { nopvspin = true; diff --git a/tools/testing/selftests/exec/Makefile b/tools/testing/selftests/exec/Makefile index fb4472ddffd8..ab67d58cfab7 100644 --- a/tools/testing/selftests/exec/Makefile +++ b/tools/testing/selftests/exec/Makefile @@ -3,8 +3,13 @@ CFLAGS = -Wall CFLAGS += -Wno-nonnull CFLAGS += -D_GNU_SOURCE +ALIGNS := 0x1000 0x200000 0x1000000 +ALIGN_PIES := $(patsubst %,load_address.%,$(ALIGNS)) +ALIGN_STATIC_PIES := $(patsubst %,load_address.static.%,$(ALIGNS)) +ALIGNMENT_TESTS := $(ALIGN_PIES) $(ALIGN_STATIC_PIES) + TEST_PROGS := binfmt_script.py -TEST_GEN_PROGS := execveat load_address_4096 load_address_2097152 load_address_16777216 non-regular +TEST_GEN_PROGS := execveat non-regular $(ALIGNMENT_TESTS) TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir # Makefile is a run-time dependency, since it's accessed by the execveat test TEST_FILES := Makefile @@ -28,9 +33,9 @@ $(OUTPUT)/execveat.symlink: $(OUTPUT)/execveat $(OUTPUT)/execveat.denatured: $(OUTPUT)/execveat cp $< $@ chmod -x $@ -$(OUTPUT)/load_address_4096: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000 -pie -static $< -o $@ -$(OUTPUT)/load_address_2097152: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x200000 -pie -static $< -o $@ -$(OUTPUT)/load_address_16777216: load_address.c - $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=0x1000000 -pie -static $< -o $@ +$(OUTPUT)/load_address.0x%: load_address.c + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ + -fPIE -pie $< -o $@ +$(OUTPUT)/load_address.static.0x%: load_address.c + $(CC) $(CFLAGS) $(LDFLAGS) -Wl,-z,max-page-size=$(lastword $(subst ., ,$@)) \ + -fPIE -static-pie $< -o $@ diff --git a/tools/testing/selftests/exec/load_address.c b/tools/testing/selftests/exec/load_address.c index 17e3207d34ae..8257fddba8c8 100644 --- a/tools/testing/selftests/exec/load_address.c +++ b/tools/testing/selftests/exec/load_address.c @@ -5,11 +5,13 @@ #include #include #include +#include #include "../kselftest.h" struct Statistics { unsigned long long load_address; unsigned long long alignment; + bool interp; }; int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) @@ -26,11 +28,20 @@ int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) stats->alignment = 0; for (i = 0; i < info->dlpi_phnum; i++) { + unsigned long long align; + + if (info->dlpi_phdr[i].p_type == PT_INTERP) { + stats->interp = true; + continue; + } + if (info->dlpi_phdr[i].p_type != PT_LOAD) continue; - if (info->dlpi_phdr[i].p_align > stats->alignment) - stats->alignment = info->dlpi_phdr[i].p_align; + align = info->dlpi_phdr[i].p_align; + + if (align > stats->alignment) + stats->alignment = align; } return 1; // Terminate dl_iterate_phdr. @@ -38,27 +49,57 @@ int ExtractStatistics(struct dl_phdr_info *info, size_t size, void *data) int main(int argc, char **argv) { - struct Statistics extracted; - unsigned long long misalign; + struct Statistics extracted = { }; + unsigned long long misalign, pow2; + bool interp_needed; + char buf[1024]; + FILE *maps; int ret; ksft_print_header(); - ksft_set_plan(1); + ksft_set_plan(4); + /* Dump maps file for debugging reference. */ + maps = fopen("/proc/self/maps", "r"); + if (!maps) + ksft_exit_fail_msg("FAILED: /proc/self/maps: %s\n", strerror(errno)); + while (fgets(buf, sizeof(buf), maps)) { + ksft_print_msg("%s", buf); + } + fclose(maps); + + /* Walk the program headers. */ ret = dl_iterate_phdr(ExtractStatistics, &extracted); if (ret != 1) ksft_exit_fail_msg("FAILED: dl_iterate_phdr\n"); - if (extracted.alignment == 0) - ksft_exit_fail_msg("FAILED: No alignment found\n"); - else if (extracted.alignment & (extracted.alignment - 1)) - ksft_exit_fail_msg("FAILED: Alignment is not a power of 2\n"); + /* Report our findings. */ + ksft_print_msg("load_address=%#llx alignment=%#llx\n", + extracted.load_address, extracted.alignment); + /* If we're named with ".static." we expect no INTERP. */ + interp_needed = strstr(argv[0], ".static.") == NULL; + + /* Were we built as expected? */ + ksft_test_result(interp_needed == extracted.interp, + "%s INTERP program header %s\n", + interp_needed ? "Wanted" : "Unwanted", + extracted.interp ? "seen" : "missing"); + + /* Did we find an alignment? */ + ksft_test_result(extracted.alignment != 0, + "Alignment%s found\n", extracted.alignment ? "" : " NOT"); + + /* Is the alignment sane? */ + pow2 = extracted.alignment & (extracted.alignment - 1); + ksft_test_result(pow2 == 0, + "Alignment is%s a power of 2: %#llx\n", + pow2 == 0 ? "" : " NOT", extracted.alignment); + + /* Is the load address aligned? */ misalign = extracted.load_address & (extracted.alignment - 1); - if (misalign) - ksft_exit_fail_msg("FAILED: alignment = %llu, load_address = %llu\n", - extracted.alignment, extracted.load_address); + ksft_test_result(misalign == 0, "Load Address is %saligned (%#llx)\n", + misalign ? "MIS" : "", misalign); - ksft_test_result_pass("Completed\n"); ksft_finished(); }