diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 77942eedf4f6..183a07c4f191 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -556,6 +556,7 @@ Description: Control Symmetric Multi Threading (SMT) ================ ========================================= "on" SMT is enabled "off" SMT is disabled + "" SMT is enabled with N threads per core. "forceoff" SMT is force disabled. Cannot be changed. "notsupported" SMT is not supported by the CPU "notimplemented" SMT runtime toggling is not diff --git a/Documentation/RCU/lockdep-splat.rst b/Documentation/RCU/lockdep-splat.rst index 2a5c79db57dc..bcbc4b3c88d7 100644 --- a/Documentation/RCU/lockdep-splat.rst +++ b/Documentation/RCU/lockdep-splat.rst @@ -10,7 +10,7 @@ misuses of the RCU API, most notably using one of the rcu_dereference() family to access an RCU-protected pointer without the proper protection. When such misuse is detected, an lockdep-RCU splat is emitted. -The usual cause of a lockdep-RCU slat is someone accessing an +The usual cause of a lockdep-RCU splat is someone accessing an RCU-protected data structure without either (1) being in the right kind of RCU read-side critical section or (2) holding the right update-side lock. This problem can therefore be serious: it might result in random memory diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 9a734bf54b76..21e40fcc08de 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -18,7 +18,16 @@ to solve following problem. Without 'nulls', a typical RCU linked list managing objects which are allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can use the following -algorithms: +algorithms. Following examples assume 'obj' is a pointer to such +objects, which is having below type. + +:: + + struct object { + struct hlist_node obj_node; + atomic_t refcnt; + unsigned int key; + }; 1) Lookup algorithm ------------------- @@ -26,11 +35,13 @@ algorithms: :: begin: - rcu_read_lock() + rcu_read_lock(); obj = lockless_lookup(key); if (obj) { - if (!try_get_ref(obj)) // might fail for free objects + if (!try_get_ref(obj)) { // might fail for free objects + rcu_read_unlock(); goto begin; + } /* * Because a writer could delete object, and a writer could * reuse these object before the RCU grace period, we @@ -54,7 +65,7 @@ but a version with an additional memory barrier (smp_rmb()) struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(next)) if (obj->key == key) return obj; @@ -66,10 +77,10 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); pos && ({ prefetch(pos->next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(pos->next)) - if (obj->key == key) - return obj; + if (obj->key == key) + return obj; return NULL; Quoting Corey Minyard:: @@ -86,7 +97,7 @@ Quoting Corey Minyard:: 2) Insertion algorithm ---------------------- -We need to make sure a reader cannot read the new 'obj->obj_next' value +We need to make sure a reader cannot read the new 'obj->obj_node.next' value and previous value of 'obj->key'. Otherwise, an item could be deleted from a chain, and inserted into another chain. If new chain was empty before the move, 'next' pointer is NULL, and lockless reader can not @@ -129,8 +140,7 @@ very very fast (before the end of RCU grace period) Avoiding extra smp_rmb() ======================== -With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup() -and extra _release() in insert function. +With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup(). For example, if we choose to store the slot number as the 'nulls' end-of-list marker for each slot of the hash table, we can detect @@ -142,6 +152,9 @@ the beginning. If the object was moved to the same chain, then the reader doesn't care: It might occasionally scan the list again without harm. +Note that using hlist_nulls means the type of 'obj_node' field of +'struct object' becomes 'struct hlist_nulls_node'. + 1) lookup algorithm ------------------- @@ -151,7 +164,7 @@ scan the list again without harm. head = &table[slot]; begin: rcu_read_lock(); - hlist_nulls_for_each_entry_rcu(obj, node, head, member) { + hlist_nulls_for_each_entry_rcu(obj, node, head, obj_node) { if (obj->key == key) { if (!try_get_ref(obj)) { // might fail for free objects rcu_read_unlock(); @@ -182,6 +195,9 @@ scan the list again without harm. 2) Insert algorithm ------------------- +Same to the above one, but uses hlist_nulls_add_head_rcu() instead of +hlist_add_head_rcu(). + :: /* diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 116221ac78a3..b38a5f79ffa9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2942,6 +2942,10 @@ locktorture.torture_type= [KNL] Specify the locking implementation to test. + locktorture.writer_fifo= [KNL] + Run the write-side locktorture kthreads at + sched_set_fifo() real-time priority. + locktorture.verbose= [KNL] Enable additional printk() statements. @@ -4953,6 +4957,15 @@ test until boot completes in order to avoid interference. + rcuscale.kfree_by_call_rcu= [KNL] + In kernels built with CONFIG_RCU_LAZY=y, test + call_rcu() instead of kfree_rcu(). + + rcuscale.kfree_mult= [KNL] + Instead of allocating an object of size kfree_obj, + allocate one of kfree_mult * sizeof(kfree_obj). + Defaults to 1. + rcuscale.kfree_rcu_test= [KNL] Set to measure performance of kfree_rcu() flooding. @@ -4978,6 +4991,12 @@ Number of loops doing rcuscale.kfree_alloc_num number of allocations and frees. + rcuscale.minruntime= [KNL] + Set the minimum test run time in seconds. This + does not affect the data-collection interval, + but instead allows better measurement of things + like CPU consumption. + rcuscale.nreaders= [KNL] Set number of RCU readers. The value -1 selects N, where N is the number of CPUs. A value @@ -4992,7 +5011,7 @@ the same as for rcuscale.nreaders. N, where N is the number of CPUs - rcuscale.perf_type= [KNL] + rcuscale.scale_type= [KNL] Specify the RCU implementation to test. rcuscale.shutdown= [KNL] @@ -5008,6 +5027,11 @@ in microseconds. The default of zero says no holdoff. + rcuscale.writer_holdoff_jiffies= [KNL] + Additional write-side holdoff between grace + periods, but in jiffies. The default of zero + says no holdoff. + rcutorture.fqs_duration= [KNL] Set duration of force_quiescent_state bursts in microseconds. @@ -5289,6 +5313,13 @@ number avoids disturbing real-time workloads, but lengthens grace periods. + rcupdate.rcu_task_lazy_lim= [KNL] + Number of callbacks on a given CPU that will + cancel laziness on that CPU. Use -1 to disable + cancellation of laziness, but be advised that + doing so increases the danger of OOM due to + callback flooding. + rcupdate.rcu_task_stall_info= [KNL] Set initial timeout in jiffies for RCU task stall informational messages, which give some indication @@ -5318,6 +5349,29 @@ A change in value does not take effect until the beginning of the next grace period. + rcupdate.rcu_tasks_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks asynchronous + callback batching for call_rcu_tasks(). + A negative value will take the default. A value + of zero will disable batching. Batching is + always disabled for synchronize_rcu_tasks(). + + rcupdate.rcu_tasks_rude_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Rude asynchronous callback batching for + call_rcu_tasks_rude(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_rude(). + + rcupdate.rcu_tasks_trace_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Trace asynchronous callback batching for + call_rcu_tasks_trace(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_trace(). + rcupdate.rcu_self_test= [KNL] Run the RCU early boot self tests diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst index e6f5bc39cf5c..b9ae591d0b18 100644 --- a/Documentation/core-api/cpu_hotplug.rst +++ b/Documentation/core-api/cpu_hotplug.rst @@ -395,8 +395,8 @@ multi-instance state the following function is available: * cpuhp_setup_state_multi(state, name, startup, teardown) The @state argument is either a statically allocated state or one of the -constants for dynamically allocated states - CPUHP_PREPARE_DYN, -CPUHP_ONLINE_DYN - depending on the state section (PREPARE, ONLINE) for +constants for dynamically allocated states - CPUHP_BP_PREPARE_DYN, +CPUHP_AP_ONLINE_DYN - depending on the state section (PREPARE, ONLINE) for which a dynamic state should be allocated. The @name argument is used for sysfs output and for instrumentation. The @@ -588,7 +588,7 @@ notifications on online and offline operations:: Setup and teardown a dynamically allocated state in the ONLINE section for notifications on offline operations:: - state = cpuhp_setup_state(CPUHP_ONLINE_DYN, "subsys:offline", NULL, subsys_cpu_offline); + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "subsys:offline", NULL, subsys_cpu_offline); if (state < 0) return state; .... @@ -597,7 +597,7 @@ for notifications on offline operations:: Setup and teardown a dynamically allocated state in the ONLINE section for notifications on online operations without invoking the callbacks:: - state = cpuhp_setup_state_nocalls(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, NULL); + state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "subsys:online", subsys_cpu_online, NULL); if (state < 0) return state; .... @@ -606,7 +606,7 @@ for notifications on online operations without invoking the callbacks:: Setup, use and teardown a dynamically allocated multi-instance state in the ONLINE section for notifications on online and offline operation:: - state = cpuhp_setup_state_multi(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, subsys_cpu_offline); + state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "subsys:online", subsys_cpu_online, subsys_cpu_offline); if (state < 0) return state; .... diff --git a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml index e84e4f33b358..3d06db98e978 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml @@ -35,6 +35,7 @@ properties: - amlogic,meson-sm1-gpio-intc - amlogic,meson-a1-gpio-intc - amlogic,meson-s4-gpio-intc + - amlogic,c3-gpio-intc - const: amlogic,meson-gpio-intc reg: diff --git a/arch/Kconfig b/arch/Kconfig index aff2746c8af2..63c5d6a2022b 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -34,6 +34,9 @@ config ARCH_HAS_SUBPAGE_FAULTS config HOTPLUG_SMT bool +config SMT_NUM_THREADS_DYNAMIC + bool + # Selected by HOTPLUG_CORE_SYNC_DEAD or HOTPLUG_CORE_SYNC_FULL config HOTPLUG_CORE_SYNC bool diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index d98701ee36c6..5db88b627439 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -97,7 +97,7 @@ struct osf_dirent { unsigned int d_ino; unsigned short d_reclen; unsigned short d_namlen; - char d_name[1]; + char d_name[]; }; struct osf_dirent_callback { diff --git a/arch/arm/include/asm/syscall.h b/arch/arm/include/asm/syscall.h index dfeed440254a..fe4326d938c1 100644 --- a/arch/arm/include/asm/syscall.h +++ b/arch/arm/include/asm/syscall.h @@ -25,6 +25,9 @@ static inline int syscall_get_nr(struct task_struct *task, if (IS_ENABLED(CONFIG_AEABI) && !IS_ENABLED(CONFIG_OABI_COMPAT)) return task_thread_info(task)->abi_syscall; + if (task_thread_info(task)->abi_syscall == -1) + return -1; + return task_thread_info(task)->abi_syscall & __NR_SYSCALL_MASK; } diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index bcc4c9ec3aa4..5c31e9de7a60 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -90,6 +90,7 @@ slow_work_pending: cmp r0, #0 beq no_work_pending movlt scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE) + str scno, [tsk, #TI_ABI_SYSCALL] @ make sure tracers see update ldmia sp, {r0 - r6} @ have to reload r0 - r6 b local_restart @ ... and off we go ENDPROC(ret_fast_syscall) diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 2d8e2516906b..fef32d73f912 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -783,8 +783,9 @@ long arch_ptrace(struct task_struct *child, long request, break; case PTRACE_SET_SYSCALL: - task_thread_info(child)->abi_syscall = data & - __NR_SYSCALL_MASK; + if (data != -1) + data &= __NR_SYSCALL_MASK; + task_thread_info(child)->abi_syscall = data; ret = 0; break; diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 9ddc025e4b86..2250253a6429 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -25,7 +25,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o -hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o +hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o hyp-obj-y += $(lib-objs) ## diff --git a/arch/arm64/kvm/hyp/nvhe/list_debug.c b/arch/arm64/kvm/hyp/nvhe/list_debug.c index d68abd7ea124..46a2d4f2b3c6 100644 --- a/arch/arm64/kvm/hyp/nvhe/list_debug.c +++ b/arch/arm64/kvm/hyp/nvhe/list_debug.c @@ -26,8 +26,9 @@ static inline __must_check bool nvhe_check_data_corruption(bool v) /* The predicates checked here are taken from lib/list_debug.c. */ -bool __list_add_valid(struct list_head *new, struct list_head *prev, - struct list_head *next) +__list_valid_slowpath +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) { if (NVHE_CHECK_DATA_CORRUPTION(next->prev != prev) || NVHE_CHECK_DATA_CORRUPTION(prev->next != next) || @@ -37,7 +38,8 @@ bool __list_add_valid(struct list_head *new, struct list_head *prev, return true; } -bool __list_del_entry_valid(struct list_head *entry) +__list_valid_slowpath +bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c index 5026e7b9adfe..ff4bda95b9c7 100644 --- a/arch/um/drivers/mconsole_kern.c +++ b/arch/um/drivers/mconsole_kern.c @@ -554,7 +554,7 @@ struct mconsole_output { static DEFINE_SPINLOCK(client_lock); static LIST_HEAD(clients); -static char console_buf[MCONSOLE_MAX_DATA]; +static char console_buf[MCONSOLE_MAX_DATA] __nonstring; static void console_write(struct console *console, const char *string, unsigned int len) @@ -567,7 +567,7 @@ static void console_write(struct console *console, const char *string, while (len > 0) { n = min((size_t) len, ARRAY_SIZE(console_buf)); - strncpy(console_buf, string, n); + memcpy(console_buf, string, n); string += n; len -= n; diff --git a/arch/um/drivers/vector_user.c b/arch/um/drivers/vector_user.c index c650e428432b..c719e1ec4645 100644 --- a/arch/um/drivers/vector_user.c +++ b/arch/um/drivers/vector_user.c @@ -141,7 +141,7 @@ static int create_tap_fd(char *iface) } memset(&ifr, 0, sizeof(ifr)); ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR; - strncpy((char *)&ifr.ifr_name, iface, sizeof(ifr.ifr_name) - 1); + strscpy(ifr.ifr_name, iface, sizeof(ifr.ifr_name)); err = ioctl(fd, TUNSETIFF, (void *) &ifr); if (err != 0) { @@ -171,7 +171,7 @@ static int create_raw_fd(char *iface, int flags, int proto) goto raw_fd_cleanup; } memset(&ifr, 0, sizeof(ifr)); - strncpy((char *)&ifr.ifr_name, iface, sizeof(ifr.ifr_name) - 1); + strscpy(ifr.ifr_name, iface, sizeof(ifr.ifr_name)); if (ioctl(fd, SIOCGIFINDEX, (void *) &ifr) < 0) { err = -errno; goto raw_fd_cleanup; diff --git a/arch/um/include/shared/user.h b/arch/um/include/shared/user.h index 0347a190429c..981e11d8e025 100644 --- a/arch/um/include/shared/user.h +++ b/arch/um/include/shared/user.h @@ -50,7 +50,6 @@ static inline int printk(const char *fmt, ...) #endif extern int in_aton(char *str); -extern size_t strlcpy(char *, const char *, size_t); extern size_t strlcat(char *, const char *, size_t); extern size_t strscpy(char *, const char *, size_t); diff --git a/arch/um/os-Linux/umid.c b/arch/um/os-Linux/umid.c index 7a1abb829930..288c422bfa96 100644 --- a/arch/um/os-Linux/umid.c +++ b/arch/um/os-Linux/umid.c @@ -40,7 +40,7 @@ static int __init make_uml_dir(void) __func__); goto err; } - strlcpy(dir, home, sizeof(dir)); + strscpy(dir, home, sizeof(dir)); uml_dir++; } strlcat(dir, uml_dir, sizeof(dir)); @@ -243,7 +243,7 @@ int __init set_umid(char *name) if (strlen(name) > UMID_LEN - 1) return -E2BIG; - strlcpy(umid, name, sizeof(umid)); + strscpy(umid, name, sizeof(umid)); return 0; } @@ -262,7 +262,7 @@ static int __init make_umid(void) make_uml_dir(); if (*umid == '\0') { - strlcpy(tmp, uml_dir, sizeof(tmp)); + strscpy(tmp, uml_dir, sizeof(tmp)); strlcat(tmp, "XXXXXX", sizeof(tmp)); fd = mkstemp(tmp); if (fd < 0) { diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index caf41c4869a0..3235ba1e5b06 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -136,10 +136,11 @@ static inline int topology_max_smt_threads(void) return __max_smt_threads; } +#include + int topology_update_package_map(unsigned int apicid, unsigned int cpu); int topology_update_die_map(unsigned int dieid, unsigned int cpu); int topology_phys_to_logical_pkg(unsigned int pkg); -bool topology_smt_supported(void); extern struct cpumask __cpu_primary_thread_mask; #define cpu_primary_thread_mask ((const struct cpumask *)&__cpu_primary_thread_mask) @@ -162,7 +163,6 @@ static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; } static inline int topology_max_die_per_package(void) { return 1; } static inline int topology_max_smt_threads(void) { return 1; } static inline bool topology_is_primary_thread(unsigned int cpu) { return true; } -static inline bool topology_smt_supported(void) { return false; } #endif /* !CONFIG_SMP */ static inline void arch_fix_phys_package_id(int num, u32 slot) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index e3a65e9fc750..281fc3f6ea6b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2343,7 +2343,7 @@ void __init arch_cpu_finalize_init(void) * identify_boot_cpu() initialized SMT support information, let the * core code know. */ - cpu_smt_check_topology(); + cpu_smt_set_num_threads(smp_num_siblings, smp_num_siblings); if (!IS_ENABLED(CONFIG_SMP)) { pr_info("CPU: "); diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index ac10b46c5832..23d4d7114473 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -79,6 +79,11 @@ void __init native_pv_lock_init(void) static_branch_disable(&virt_spin_lock_key); } +static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + tlb_remove_page(tlb, table); +} + unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, unsigned int len) { @@ -295,8 +300,7 @@ struct paravirt_patch_template pv_ops = { .mmu.flush_tlb_kernel = native_flush_tlb_global, .mmu.flush_tlb_one_user = native_flush_tlb_one_user, .mmu.flush_tlb_multi = native_flush_tlb_multi, - .mmu.tlb_remove_table = - (void (*)(struct mmu_gather *, void *))tlb_remove_page, + .mmu.tlb_remove_table = native_tlb_remove_table, .mmu.exit_mmap = paravirt_nop, .mmu.notify_page_enc_status_changed = paravirt_nop, diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index e1aa2cd7734b..d4e897b820c4 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -326,14 +326,6 @@ static void notrace start_secondary(void *unused) cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } -/** - * topology_smt_supported - Check whether SMT is supported by the CPUs - */ -bool topology_smt_supported(void) -{ - return smp_num_siblings > 1; -} - /** * topology_phys_to_logical_pkg - Map a physical package id to a logical * @phys_pkg: The physical package id to map diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 3425c6a943e4..15f97c0abc9d 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1258,7 +1258,7 @@ static void __init check_system_tsc_reliable(void) if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && boot_cpu_has(X86_FEATURE_NONSTOP_TSC) && boot_cpu_has(X86_FEATURE_TSC_ADJUST) && - nr_online_nodes <= 2) + nr_online_nodes <= 4) tsc_disable_clocksource_watchdog(); } diff --git a/drivers/eisa/eisa-bus.c b/drivers/eisa/eisa-bus.c index 713582cc27d1..33f0ba11c6ad 100644 --- a/drivers/eisa/eisa-bus.c +++ b/drivers/eisa/eisa-bus.c @@ -60,7 +60,7 @@ static void __init eisa_name_device(struct eisa_device *edev) int i; for (i = 0; i < EISA_INFOS; i++) { if (!strcmp(edev->id.sig, eisa_table[i].id.sig)) { - strlcpy(edev->pretty_name, + strscpy(edev->pretty_name, eisa_table[i].name, sizeof(edev->pretty_name)); return; diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c index 6341c0167c4a..9745a119d0e6 100644 --- a/drivers/irqchip/irq-bcm6345-l1.c +++ b/drivers/irqchip/irq-bcm6345-l1.c @@ -60,7 +60,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index a62b96237b82..24ca1d656adc 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 091b0fe7e324..5559c943f03f 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-gic-pm.c b/drivers/irqchip/irq-gic-pm.c index 3989d16f997b..a275a8071a25 100644 --- a/drivers/irqchip/irq-gic-pm.c +++ b/drivers/irqchip/irq-gic-pm.c @@ -4,7 +4,7 @@ */ #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c index 634263dfd7b5..8e87fc35f8aa 100644 --- a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c +++ b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c @@ -9,8 +9,6 @@ #include #include -#include -#include #include #include #include diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c index b70ce0d3c092..115bdcffab24 100644 --- a/drivers/irqchip/irq-i8259.c +++ b/drivers/irqchip/irq-i8259.c @@ -340,7 +340,7 @@ static void i8259_irq_dispatch(struct irq_desc *desc) generic_handle_domain_irq(domain, hwirq); } -int __init i8259_of_init(struct device_node *node, struct device_node *parent) +static int __init i8259_of_init(struct device_node *node, struct device_node *parent) { struct irq_domain *domain; unsigned int parent_irq; diff --git a/drivers/irqchip/irq-imx-intmux.c b/drivers/irqchip/irq-imx-intmux.c index 80aaea82468a..6d9a08238c9d 100644 --- a/drivers/irqchip/irq-imx-intmux.c +++ b/drivers/irqchip/irq-imx-intmux.c @@ -50,8 +50,9 @@ #include #include #include +#include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-imx-irqsteer.c b/drivers/irqchip/irq-imx-irqsteer.c index 96230a04ec23..bd9543314539 100644 --- a/drivers/irqchip/irq-imx-irqsteer.c +++ b/drivers/irqchip/irq-imx-irqsteer.c @@ -10,8 +10,9 @@ #include #include #include +#include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c index 229039eda1b1..90d41c1407ac 100644 --- a/drivers/irqchip/irq-imx-mu-msi.c +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -339,8 +339,8 @@ static int __init imx_mu_of_init(struct device_node *dn, msi_data->msiir_addr = res->start + msi_data->cfg->xTR; irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return -ENODEV; + if (irq < 0) + return irq; platform_set_drvdata(pdev, msi_data); diff --git a/drivers/irqchip/irq-keystone.c b/drivers/irqchip/irq-keystone.c index ba9792e60329..a36396db4b08 100644 --- a/drivers/irqchip/irq-keystone.c +++ b/drivers/irqchip/irq-keystone.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index 92d8aa28bdf5..1623cd779175 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -144,7 +144,7 @@ static int eiointc_router_init(unsigned int cpu) int i, bit; uint32_t data; uint32_t node = cpu_to_eio_node(cpu); - uint32_t index = eiointc_index(node); + int index = eiointc_index(node); if (index < 0) { pr_err("Error: invalid nodemap!\n"); diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index fc8bf1f5d41b..0bff728b25e3 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -15,7 +15,6 @@ #include #include #include -#include #include /* Registers */ diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index 93a71f66efeb..63db8e2172e0 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -12,9 +12,9 @@ #include #include #include +#include #include #include -#include #include /* Registers */ diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c index f5ba3f9f8415..f31a262fe438 100644 --- a/drivers/irqchip/irq-ls-scfg-msi.c +++ b/drivers/irqchip/irq-ls-scfg-msi.c @@ -349,8 +349,7 @@ static int ls_scfg_msi_probe(struct platform_device *pdev) msi_data->cfg = (struct ls_scfg_msi_cfg *) match->data; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - msi_data->regs = devm_ioremap_resource(&pdev->dev, res); + msi_data->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(msi_data->regs)) { dev_err(&pdev->dev, "failed to initialize 'regs'\n"); return PTR_ERR(msi_data->regs); diff --git a/drivers/irqchip/irq-madera.c b/drivers/irqchip/irq-madera.c index 8b81271c823c..3eb1f8cdf674 100644 --- a/drivers/irqchip/irq-madera.c +++ b/drivers/irqchip/irq-madera.c @@ -10,12 +10,10 @@ #include #include #include +#include #include #include #include -#include -#include -#include #include #include #include diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index 7da18ef95211..f88df39f4129 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -150,6 +150,10 @@ static const struct meson_gpio_irq_params s4_params = { INIT_MESON_S4_COMMON_DATA(82) }; +static const struct meson_gpio_irq_params c3_params = { + INIT_MESON_S4_COMMON_DATA(55) +}; + static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson8-gpio-intc", .data = &meson8_params }, { .compatible = "amlogic,meson8b-gpio-intc", .data = &meson8b_params }, @@ -160,6 +164,7 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson-sm1-gpio-intc", .data = &sm1_params }, { .compatible = "amlogic,meson-a1-gpio-intc", .data = &a1_params }, { .compatible = "amlogic,meson-s4-gpio-intc", .data = &s4_params }, + { .compatible = "amlogic,c3-gpio-intc", .data = &c3_params }, { } }; diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index 6d5ecc10a870..76253e864f23 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -557,7 +557,7 @@ static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq, return gic_irq_domain_map(d, virq, hwirq); } -void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, +static void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { } diff --git a/drivers/irqchip/irq-mvebu-sei.c b/drivers/irqchip/irq-mvebu-sei.c index 4ecef6d83777..a48dbe91b036 100644 --- a/drivers/irqchip/irq-mvebu-sei.c +++ b/drivers/irqchip/irq-mvebu-sei.c @@ -377,8 +377,7 @@ static int mvebu_sei_probe(struct platform_device *pdev) mutex_init(&sei->cp_msi_lock); raw_spin_lock_init(&sei->mask_lock); - sei->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - sei->base = devm_ioremap_resource(sei->dev, sei->res); + sei->base = devm_platform_get_and_ioremap_resource(pdev, 0, &sei->res); if (IS_ERR(sei->base)) return PTR_ERR(sei->base); diff --git a/drivers/irqchip/irq-orion.c b/drivers/irqchip/irq-orion.c index 17c2c7a07f10..4e4e874e09a8 100644 --- a/drivers/irqchip/irq-orion.c +++ b/drivers/irqchip/irq-orion.c @@ -57,8 +57,7 @@ static int __init orion_irq_init(struct device_node *np, struct resource r; /* count number of irq chips by valid reg addresses */ - while (of_address_to_resource(np, num_chips, &r) == 0) - num_chips++; + num_chips = of_address_count(np); orion_irq_domain = irq_domain_add_linear(np, num_chips * ORION_IRQS_PER_CHIP, diff --git a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c index fa8d89b02ec0..0f64ecb9b1f4 100644 --- a/drivers/irqchip/irq-pruss-intc.c +++ b/drivers/irqchip/irq-pruss-intc.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include /* @@ -565,8 +565,8 @@ static int pruss_intc_probe(struct platform_device *pdev) continue; irq = platform_get_irq_byname(pdev, irq_names[i]); - if (irq <= 0) { - ret = (irq == 0) ? -EINVAL : irq; + if (irq < 0) { + ret = irq; goto fail_irq; } diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index d30614661eea..7124565234a5 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c index 26e4c17a7bf2..fa19585f3dee 100644 --- a/drivers/irqchip/irq-renesas-intc-irqpin.c +++ b/drivers/irqchip/irq-renesas-intc-irqpin.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #define INTC_IRQPIN_MAX 8 /* maximum 8 interrupts per driver instance */ diff --git a/drivers/irqchip/irq-st.c b/drivers/irqchip/irq-st.c index 819a12297b58..de71bb350d57 100644 --- a/drivers/irqchip/irq-st.c +++ b/drivers/irqchip/irq-st.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c index b5fa76ce5046..d8ba5fba7450 100644 --- a/drivers/irqchip/irq-stm32-exti.c +++ b/drivers/irqchip/irq-stm32-exti.c @@ -14,10 +14,11 @@ #include #include #include +#include #include #include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c index 21d49791f855..e760b1278143 100644 --- a/drivers/irqchip/irq-sunxi-nmi.c +++ b/drivers/irqchip/irq-sunxi-nmi.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include diff --git a/drivers/irqchip/irq-tb10x.c b/drivers/irqchip/irq-tb10x.c index 8a0e69298e83..680586354d12 100644 --- a/drivers/irqchip/irq-tb10x.c +++ b/drivers/irqchip/irq-tb10x.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c index 7133f9fa6fd9..b83f5cbab123 100644 --- a/drivers/irqchip/irq-ti-sci-inta.c +++ b/drivers/irqchip/irq-ti-sci-inta.c @@ -15,9 +15,9 @@ #include #include #include -#include +#include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c index 1186f1e431a3..c027cd9e4a69 100644 --- a/drivers/irqchip/irq-ti-sci-intr.c +++ b/drivers/irqchip/irq-ti-sci-intr.c @@ -12,9 +12,9 @@ #include #include #include -#include -#include +#include #include +#include #include /** diff --git a/drivers/irqchip/irq-uniphier-aidet.c b/drivers/irqchip/irq-uniphier-aidet.c index 716b1bb88bf2..601f9343d5b3 100644 --- a/drivers/irqchip/irq-uniphier-aidet.c +++ b/drivers/irqchip/irq-uniphier-aidet.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-xtensa-pic.c b/drivers/irqchip/irq-xtensa-pic.c index ab12328be5ee..0c18d1f1e264 100644 --- a/drivers/irqchip/irq-xtensa-pic.c +++ b/drivers/irqchip/irq-xtensa-pic.c @@ -16,6 +16,7 @@ #include #include #include +#include #include unsigned int cached_irq_mask; diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 7899607fbee8..1eeb0d0156ce 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index d96916cf6a41..a32c0d28d038 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 3c95600ab2f7..c66cc05a68c4 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -273,8 +273,8 @@ static void lkdtm_HUNG_TASK(void) schedule(); } -volatile unsigned int huge = INT_MAX - 2; -volatile unsigned int ignored; +static volatile unsigned int huge = INT_MAX - 2; +static volatile unsigned int ignored; static void lkdtm_OVERFLOW_SIGNED(void) { @@ -305,7 +305,7 @@ static void lkdtm_OVERFLOW_UNSIGNED(void) ignored = value; } -/* Intentionally using old-style flex array definition of 1 byte. */ +/* Intentionally using unannotated flex array definition. */ struct array_bounds_flex_array { int one; int two; @@ -357,6 +357,46 @@ static void lkdtm_ARRAY_BOUNDS(void) pr_expected_config(CONFIG_UBSAN_BOUNDS); } +struct lkdtm_annotated { + unsigned long flags; + int count; + int array[] __counted_by(count); +}; + +static volatile int fam_count = 4; + +static void lkdtm_FAM_BOUNDS(void) +{ + struct lkdtm_annotated *inst; + + inst = kzalloc(struct_size(inst, array, fam_count + 1), GFP_KERNEL); + if (!inst) { + pr_err("FAIL: could not allocate test struct!\n"); + return; + } + + inst->count = fam_count; + pr_info("Array access within bounds ...\n"); + inst->array[1] = fam_count; + ignored = inst->array[1]; + + pr_info("Array access beyond bounds ...\n"); + inst->array[fam_count] = fam_count; + ignored = inst->array[fam_count]; + + kfree(inst); + + pr_err("FAIL: survived access of invalid flexible array member index!\n"); + + if (!__has_attribute(__counted_by__)) + pr_warn("This is expected since this %s was built a compiler supporting __counted_by\n", + lkdtm_kernel_info); + else if (IS_ENABLED(CONFIG_UBSAN_BOUNDS)) + pr_expected_config(CONFIG_UBSAN_TRAP); + else + pr_expected_config(CONFIG_UBSAN_BOUNDS); +} + static void lkdtm_CORRUPT_LIST_ADD(void) { /* @@ -393,7 +433,7 @@ static void lkdtm_CORRUPT_LIST_ADD(void) pr_err("Overwrite did not happen, but no BUG?!\n"); else { pr_err("list_add() corruption not detected!\n"); - pr_expected_config(CONFIG_DEBUG_LIST); + pr_expected_config(CONFIG_LIST_HARDENED); } } @@ -420,7 +460,7 @@ static void lkdtm_CORRUPT_LIST_DEL(void) pr_err("Overwrite did not happen, but no BUG?!\n"); else { pr_err("list_del() corruption not detected!\n"); - pr_expected_config(CONFIG_DEBUG_LIST); + pr_expected_config(CONFIG_LIST_HARDENED); } } @@ -616,6 +656,7 @@ static struct crashtype crashtypes[] = { CRASHTYPE(OVERFLOW_SIGNED), CRASHTYPE(OVERFLOW_UNSIGNED), CRASHTYPE(ARRAY_BOUNDS), + CRASHTYPE(FAM_BOUNDS), CRASHTYPE(CORRUPT_LIST_ADD), CRASHTYPE(CORRUPT_LIST_DEL), CRASHTYPE(STACK_GUARD_PAGE_LEADING), diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c index b3c226eb5292..58746e570d14 100644 --- a/drivers/soc/fsl/qe/qe.c +++ b/drivers/soc/fsl/qe/qe.c @@ -524,7 +524,7 @@ int qe_upload_firmware(const struct qe_firmware *firmware) * saved microcode information and put in the new. */ memset(&qe_firmware_info, 0, sizeof(qe_firmware_info)); - strlcpy(qe_firmware_info.id, firmware->id, sizeof(qe_firmware_info.id)); + strscpy(qe_firmware_info.id, firmware->id, sizeof(qe_firmware_info.id)); qe_firmware_info.extended_modes = be64_to_cpu(firmware->extended_modes); memcpy(qe_firmware_info.vtraps, firmware->vtraps, sizeof(firmware->vtraps)); @@ -599,7 +599,7 @@ struct qe_firmware_info *qe_get_firmware_info(void) /* Copy the data into qe_firmware_info*/ sprop = of_get_property(fw, "id", NULL); if (sprop) - strlcpy(qe_firmware_info.id, sprop, + strscpy(qe_firmware_info.id, sprop, sizeof(qe_firmware_info.id)); of_property_read_u64(fw, "extended-modes", diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 00efa35c350f..28566624f008 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -94,6 +94,19 @@ # define __copy(symbol) #endif +/* + * Optional: only supported since gcc >= 14 + * Optional: only supported since clang >= 18 + * + * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 + * clang: https://reviews.llvm.org/D148381 + */ +#if __has_attribute(__counted_by__) +# define __counted_by(member) __attribute__((__counted_by__(member))) +#else +# define __counted_by(member) +#endif + /* * Optional: not supported by gcc * Optional: only supported since clang >= 14.0 @@ -129,19 +142,6 @@ # define __designated_init #endif -/* - * Optional: only supported since gcc >= 14 - * Optional: only supported since clang >= 17 - * - * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 - * clang: https://reviews.llvm.org/D148381 - */ -#if __has_attribute(__element_count__) -# define __counted_by(member) __attribute__((__element_count__(#member))) -#else -# define __counted_by(member) -#endif - /* * Optional: only supported since clang >= 14.0 * diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 547ea1ff806e..c523c6683789 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -106,6 +106,34 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } #define __cold #endif +/* + * On x86-64 and arm64 targets, __preserve_most changes the calling convention + * of a function to make the code in the caller as unintrusive as possible. This + * convention behaves identically to the C calling convention on how arguments + * and return values are passed, but uses a different set of caller- and callee- + * saved registers. + * + * The purpose is to alleviates the burden of saving and recovering a large + * register set before and after the call in the caller. This is beneficial for + * rarely taken slow paths, such as error-reporting functions that may be called + * from hot paths. + * + * Note: This may conflict with instrumentation inserted on function entry which + * does not use __preserve_most or equivalent convention (if in assembly). Since + * function tracing assumes the normal C calling convention, where the attribute + * is supported, __preserve_most implies notrace. It is recommended to restrict + * use of the attribute to functions that should or already disable tracing. + * + * Optional: not supported by gcc. + * + * clang: https://clang.llvm.org/docs/AttributeReference.html#preserve-most + */ +#if __has_attribute(__preserve_most__) && (defined(CONFIG_X86_64) || defined(CONFIG_ARM64)) +# define __preserve_most notrace __attribute__((__preserve_most__)) +#else +# define __preserve_most +#endif + /* Builtins */ /* diff --git a/include/linux/completion.h b/include/linux/completion.h index 62b32b19e0a8..fb2915676574 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -116,6 +116,7 @@ extern bool try_wait_for_completion(struct completion *x); extern bool completion_done(struct completion *x); extern void complete(struct completion *); +extern void complete_on_current_cpu(struct completion *x); extern void complete_all(struct completion *); #endif diff --git a/include/linux/cpu.h b/include/linux/cpu.h index e006c719182b..0abd60a7987b 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -18,6 +18,7 @@ #include #include #include +#include struct device; struct device_node; @@ -194,7 +195,6 @@ void arch_cpu_finalize_init(void); static inline void arch_cpu_finalize_init(void) { } #endif -void cpu_set_state_online(int cpu); void play_idle_precise(u64 duration_ns, u64 latency_ns); static inline void play_idle(unsigned long duration_us) @@ -208,30 +208,6 @@ void cpuhp_report_idle_dead(void); static inline void cpuhp_report_idle_dead(void) { } #endif /* #ifdef CONFIG_HOTPLUG_CPU */ -enum cpuhp_smt_control { - CPU_SMT_ENABLED, - CPU_SMT_DISABLED, - CPU_SMT_FORCE_DISABLED, - CPU_SMT_NOT_SUPPORTED, - CPU_SMT_NOT_IMPLEMENTED, -}; - -#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) -extern enum cpuhp_smt_control cpu_smt_control; -extern void cpu_smt_disable(bool force); -extern void cpu_smt_check_topology(void); -extern bool cpu_smt_possible(void); -extern int cpuhp_smt_enable(void); -extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); -#else -# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) -static inline void cpu_smt_disable(bool force) { } -static inline void cpu_smt_check_topology(void) { } -static inline bool cpu_smt_possible(void) { return false; } -static inline int cpuhp_smt_enable(void) { return 0; } -static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } -#endif - extern bool cpu_mitigations_off(void); extern bool cpu_mitigations_auto_nosmt(void); diff --git a/include/linux/cpu_smt.h b/include/linux/cpu_smt.h new file mode 100644 index 000000000000..0c1664294b57 --- /dev/null +++ b/include/linux/cpu_smt.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CPU_SMT_H_ +#define _LINUX_CPU_SMT_H_ + +enum cpuhp_smt_control { + CPU_SMT_ENABLED, + CPU_SMT_DISABLED, + CPU_SMT_FORCE_DISABLED, + CPU_SMT_NOT_SUPPORTED, + CPU_SMT_NOT_IMPLEMENTED, +}; + +#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) +extern enum cpuhp_smt_control cpu_smt_control; +extern unsigned int cpu_smt_num_threads; +extern void cpu_smt_disable(bool force); +extern void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads); +extern bool cpu_smt_possible(void); +extern int cpuhp_smt_enable(void); +extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); +#else +# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) +# define cpu_smt_num_threads 1 +static inline void cpu_smt_disable(bool force) { } +static inline void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { } +static inline bool cpu_smt_possible(void) { return false; } +static inline int cpuhp_smt_enable(void) { return 0; } +static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } +#endif + +#endif /* _LINUX_CPU_SMT_H_ */ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 25b6e6e6ba6b..06dda85f0424 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -48,7 +48,7 @@ * same section. * * If neither #1 nor #2 apply, please use the dynamic state space when - * setting up a state by using CPUHP_PREPARE_DYN or CPUHP_PREPARE_ONLINE + * setting up a state by using CPUHP_BP_PREPARE_DYN or CPUHP_AP_ONLINE_DYN * for the @state argument of the setup function. * * See Documentation/core-api/cpu_hotplug.rst for further information and diff --git a/include/linux/dm-verity-loadpin.h b/include/linux/dm-verity-loadpin.h index 552b817ab102..3ac6dbaeaa37 100644 --- a/include/linux/dm-verity-loadpin.h +++ b/include/linux/dm-verity-loadpin.h @@ -12,7 +12,7 @@ extern struct list_head dm_verity_loadpin_trusted_root_digests; struct dm_verity_loadpin_trusted_root_digest { struct list_head node; unsigned int len; - u8 data[]; + u8 data[] __counted_by(len); }; #if IS_ENABLED(CONFIG_SECURITY_LOADPIN_VERITY) diff --git a/include/linux/list.h b/include/linux/list.h index f10344dbad4d..164b4d0e9d2a 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -38,11 +38,92 @@ static inline void INIT_LIST_HEAD(struct list_head *list) WRITE_ONCE(list->prev, list); } +#ifdef CONFIG_LIST_HARDENED + #ifdef CONFIG_DEBUG_LIST -extern bool __list_add_valid(struct list_head *new, - struct list_head *prev, - struct list_head *next); -extern bool __list_del_entry_valid(struct list_head *entry); +# define __list_valid_slowpath +#else +# define __list_valid_slowpath __cold __preserve_most +#endif + +/* + * Performs the full set of list corruption checks before __list_add(). + * On list corruption reports a warning, and returns false. + */ +extern bool __list_valid_slowpath __list_add_valid_or_report(struct list_head *new, + struct list_head *prev, + struct list_head *next); + +/* + * Performs list corruption checks before __list_add(). Returns false if a + * corruption is detected, true otherwise. + * + * With CONFIG_LIST_HARDENED only, performs minimal list integrity checking + * inline to catch non-faulting corruptions, and only if a corruption is + * detected calls the reporting function __list_add_valid_or_report(). + */ +static __always_inline bool __list_add_valid(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + bool ret = true; + + if (!IS_ENABLED(CONFIG_DEBUG_LIST)) { + /* + * With the hardening version, elide checking if next and prev + * are NULL, since the immediate dereference of them below would + * result in a fault if NULL. + * + * With the reduced set of checks, we can afford to inline the + * checks, which also gives the compiler a chance to elide some + * of them completely if they can be proven at compile-time. If + * one of the pre-conditions does not hold, the slow-path will + * show a report which pre-condition failed. + */ + if (likely(next->prev == prev && prev->next == next && new != prev && new != next)) + return true; + ret = false; + } + + ret &= __list_add_valid_or_report(new, prev, next); + return ret; +} + +/* + * Performs the full set of list corruption checks before __list_del_entry(). + * On list corruption reports a warning, and returns false. + */ +extern bool __list_valid_slowpath __list_del_entry_valid_or_report(struct list_head *entry); + +/* + * Performs list corruption checks before __list_del_entry(). Returns false if a + * corruption is detected, true otherwise. + * + * With CONFIG_LIST_HARDENED only, performs minimal list integrity checking + * inline to catch non-faulting corruptions, and only if a corruption is + * detected calls the reporting function __list_del_entry_valid_or_report(). + */ +static __always_inline bool __list_del_entry_valid(struct list_head *entry) +{ + bool ret = true; + + if (!IS_ENABLED(CONFIG_DEBUG_LIST)) { + struct list_head *prev = entry->prev; + struct list_head *next = entry->next; + + /* + * With the hardening version, elide checking if next and prev + * are NULL, LIST_POISON1 or LIST_POISON2, since the immediate + * dereference of them below would result in a fault. + */ + if (likely(prev->next == entry && next->prev == entry)) + return true; + ret = false; + } + + ret &= __list_del_entry_valid_or_report(entry); + return ret; +} #else static inline bool __list_add_valid(struct list_head *new, struct list_head *prev, diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 86544707236a..45702bdcbceb 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -73,9 +73,7 @@ struct raw_notifier_head { struct srcu_notifier_head { struct mutex mutex; -#ifdef CONFIG_TREE_SRCU struct srcu_usage srcuu; -#endif struct srcu_struct srcu; struct notifier_block __rcu *head; }; @@ -106,7 +104,6 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); #define RAW_NOTIFIER_INIT(name) { \ .head = NULL } -#ifdef CONFIG_TREE_SRCU #define SRCU_NOTIFIER_INIT(name, pcpu) \ { \ .mutex = __MUTEX_INITIALIZER(name.mutex), \ @@ -114,14 +111,6 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); .srcuu = __SRCU_USAGE_INIT(name.srcuu), \ .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ } -#else -#define SRCU_NOTIFIER_INIT(name, pcpu) \ - { \ - .mutex = __MUTEX_INITIALIZER(name.mutex), \ - .head = NULL, \ - .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ - } -#endif #define ATOMIC_NOTIFIER_HEAD(name) \ struct atomic_notifier_head name = \ diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h index fee881cded01..771cb0285872 100644 --- a/include/linux/nsproxy.h +++ b/include/linux/nsproxy.h @@ -29,7 +29,7 @@ struct fs_struct; * nsproxy is copied. */ struct nsproxy { - atomic_t count; + refcount_t count; struct uts_namespace *uts_ns; struct ipc_namespace *ipc_ns; struct mnt_namespace *mnt_ns; @@ -102,14 +102,13 @@ int __init nsproxy_cache_init(void); static inline void put_nsproxy(struct nsproxy *ns) { - if (atomic_dec_and_test(&ns->count)) { + if (refcount_dec_and_test(&ns->count)) free_nsproxy(ns); - } } static inline void get_nsproxy(struct nsproxy *ns) { - atomic_inc(&ns->count); + refcount_inc(&ns->count); } #endif diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index ba4c00dd8005..89186c499dd4 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h @@ -101,7 +101,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, { struct hlist_nulls_node *first = h->first; - n->next = first; + WRITE_ONCE(n->next, first); WRITE_ONCE(n->pprev, &h->first); rcu_assign_pointer(hlist_nulls_first_rcu(h), n); if (!is_a_nulls(first)) @@ -137,7 +137,7 @@ static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, last = i; if (last) { - n->next = last->next; + WRITE_ONCE(n->next, last->next); n->pprev = &last->next; rcu_assign_pointer(hlist_nulls_next_rcu(last), n); } else { diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 9bc8cbb33340..eda493200663 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -87,6 +87,7 @@ static inline void rcu_read_unlock_trace(void) void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); void synchronize_rcu_tasks_trace(void); void rcu_barrier_tasks_trace(void); +struct task_struct *get_rcu_tasks_trace_gp_kthread(void); #else /* * The BPF JIT forms these addresses even when it doesn't call these diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 699b938358bf..5e0f74f2f8ca 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -42,6 +42,11 @@ do { \ * call_srcu() function, with this wrapper supplying the pointer to the * corresponding srcu_struct. * + * Note that call_rcu_hurry() should be used instead of call_rcu() + * because in kernels built with CONFIG_RCU_LAZY=y the delay between the + * invocation of call_rcu() and that of the corresponding RCU callback + * can be multiple seconds. + * * The first argument tells Tiny RCU's _wait_rcu_gp() not to * bother waiting for RCU. The reason for this is because anywhere * synchronize_rcu_mult() can be called is automatically already a full diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index bd023dd38ae6..386ab580b839 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -249,18 +249,19 @@ static inline void seq_show_option(struct seq_file *m, const char *name, /** * seq_show_option_n - display mount options with appropriate escapes - * where @value must be a specific length. + * where @value must be a specific length (i.e. + * not NUL-terminated). * @m: the seq_file handle * @name: the mount option name * @value: the mount option name's value, cannot be NULL - * @length: the length of @value to display + * @length: the exact length of @value to display, must be constant expression * * This is a macro since this uses "length" to define the size of the * stack buffer. */ #define seq_show_option_n(m, name, value, length) { \ char val_buf[length + 1]; \ - strncpy(val_buf, value, length); \ + memcpy(val_buf, value, length); \ val_buf[length] = '\0'; \ seq_show_option(m, name, val_buf); \ } diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index ebd72491af99..447133171d95 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -48,6 +48,10 @@ void srcu_drive_gp(struct work_struct *wp); #define DEFINE_STATIC_SRCU(name) \ static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) +// Dummy structure for srcu_notifier_head. +struct srcu_usage { }; +#define __SRCU_USAGE_INIT(name) { } + void synchronize_srcu(struct srcu_struct *ssp); /* diff --git a/include/linux/swait.h b/include/linux/swait.h index 6a8c22b8c2a5..d324419482a0 100644 --- a/include/linux/swait.h +++ b/include/linux/swait.h @@ -146,7 +146,7 @@ static inline bool swq_has_sleeper(struct swait_queue_head *wq) extern void swake_up_one(struct swait_queue_head *q); extern void swake_up_all(struct swait_queue_head *q); -extern void swake_up_locked(struct swait_queue_head *q); +extern void swake_up_locked(struct swait_queue_head *q, int wake_flags); extern void prepare_to_swait_exclusive(struct swait_queue_head *q, struct swait_queue *wait, int state); extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state); diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 5690aef7b3a8..c0cb22cd607d 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -283,22 +283,6 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event) #define SYSCALL32_DEFINE6 SYSCALL_DEFINE6 #endif -/* - * Called before coming back to user-mode. Returning to user-mode with an - * address limit different than USER_DS can allow to overwrite kernel memory. - */ -static inline void addr_limit_user_check(void) -{ -#ifdef TIF_FSCHECK - if (!test_thread_flag(TIF_FSCHECK)) - return; -#endif - -#ifdef TIF_FSCHECK - clear_thread_flag(TIF_FSCHECK); -#endif -} - /* * These syscall function prototypes are kept in the same order as * include/uapi/asm-generic/unistd.h. Architecture specific entries go below, diff --git a/include/linux/torture.h b/include/linux/torture.h index 7038104463e4..bb466eec01e4 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -108,12 +108,15 @@ bool torture_must_stop(void); bool torture_must_stop_irq(void); void torture_kthread_stopping(char *title); int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, - char *f, struct task_struct **tp); + char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)); void _torture_stop_kthread(char *m, struct task_struct **tp); #define torture_create_kthread(n, arg, tp) \ _torture_create_kthread(n, (arg), #n, "Creating " #n " task", \ - "Failed to create " #n, &(tp)) + "Failed to create " #n, &(tp), NULL) +#define torture_create_kthread_cb(n, arg, tp, cbf) \ + _torture_create_kthread(n, (arg), #n, "Creating " #n " task", \ + "Failed to create " #n, &(tp), cbf) #define torture_stop_kthread(n, tp) \ _torture_stop_kthread("Stopping " #n " task", &(tp)) diff --git a/include/linux/wait.h b/include/linux/wait.h index 40dbc01cecb8..8a51ae0f02ec 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -210,6 +210,7 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq } int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key); +void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key); void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key); void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head, unsigned int mode, void *key, wait_queue_entry_t *bookmark); @@ -238,6 +239,8 @@ void __wake_up_pollfree(struct wait_queue_head *wq_head); #define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m)) #define wake_up_poll(x, m) \ __wake_up(x, TASK_NORMAL, 1, poll_to_key(m)) +#define wake_up_poll_on_current_cpu(x, m) \ + __wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m)) #define wake_up_locked_poll(x, m) \ __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m)) #define wake_up_interruptible_poll(x, m) \ diff --git a/include/uapi/linux/seccomp.h b/include/uapi/linux/seccomp.h index 0fdc6ef02b94..dbfc9b37fcae 100644 --- a/include/uapi/linux/seccomp.h +++ b/include/uapi/linux/seccomp.h @@ -115,6 +115,8 @@ struct seccomp_notif_resp { __u32 flags; }; +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) + /* valid flags for seccomp_notif_addfd */ #define SECCOMP_ADDFD_FLAG_SETFD (1UL << 0) /* Specify remote fd */ #define SECCOMP_ADDFD_FLAG_SEND (1UL << 1) /* Addfd and return it, atomically */ @@ -150,4 +152,6 @@ struct seccomp_notif_addfd { #define SECCOMP_IOCTL_NOTIF_ADDFD SECCOMP_IOW(3, \ struct seccomp_notif_addfd) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) + #endif /* _UAPI_LINUX_SECCOMP_H */ diff --git a/include/uapi/linux/stddef.h b/include/uapi/linux/stddef.h index 7837ba4fe728..7c3fc3980881 100644 --- a/include/uapi/linux/stddef.h +++ b/include/uapi/linux/stddef.h @@ -45,3 +45,7 @@ TYPE NAME[]; \ } #endif + +#ifndef __counted_by +#define __counted_by(m) +#endif diff --git a/kernel/cpu.c b/kernel/cpu.c index 88a7ede322bd..f6811c857102 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -592,7 +592,10 @@ static void lockdep_release_cpus_lock(void) void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMT + enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; +static unsigned int cpu_smt_max_threads __ro_after_init; +unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX; void __init cpu_smt_disable(bool force) { @@ -606,16 +609,33 @@ void __init cpu_smt_disable(bool force) pr_info("SMT: disabled\n"); cpu_smt_control = CPU_SMT_DISABLED; } + cpu_smt_num_threads = 1; } /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code. */ -void __init cpu_smt_check_topology(void) +void __init cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { - if (!topology_smt_supported()) + WARN_ON(!num_threads || (num_threads > max_threads)); + + if (max_threads == 1) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; + + cpu_smt_max_threads = max_threads; + + /* + * If SMT has been disabled via the kernel command line or SMT is + * not supported, set cpu_smt_num_threads to 1 for consistency. + * If enabled, take the architecture requested number of threads + * to bring up into account. + */ + if (cpu_smt_control != CPU_SMT_ENABLED) + cpu_smt_num_threads = 1; + else if (num_threads < cpu_smt_num_threads) + cpu_smt_num_threads = num_threads; } static int __init smt_cmdline_disable(char *str) @@ -625,9 +645,23 @@ static int __init smt_cmdline_disable(char *str) } early_param("nosmt", smt_cmdline_disable); +/* + * For Archicture supporting partial SMT states check if the thread is allowed. + * Otherwise this has already been checked through cpu_smt_max_threads when + * setting the SMT level. + */ +static inline bool cpu_smt_thread_allowed(unsigned int cpu) +{ +#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC + return topology_smt_thread_allowed(cpu); +#else + return true; +#endif +} + static inline bool cpu_smt_allowed(unsigned int cpu) { - if (cpu_smt_control == CPU_SMT_ENABLED) + if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu)) return true; if (topology_is_primary_thread(cpu)) @@ -642,7 +676,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu) return !cpumask_test_cpu(cpu, &cpus_booted_once_mask); } -/* Returns true if SMT is not supported of forcefully (irreversibly) disabled */ +/* Returns true if SMT is supported and not forcefully (irreversibly) disabled */ bool cpu_smt_possible(void) { return cpu_smt_control != CPU_SMT_FORCE_DISABLED && @@ -650,22 +684,8 @@ bool cpu_smt_possible(void) } EXPORT_SYMBOL_GPL(cpu_smt_possible); -static inline bool cpuhp_smt_aware(void) -{ - return topology_smt_supported(); -} - -static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) -{ - return cpu_primary_thread_mask; -} #else static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } -static inline bool cpuhp_smt_aware(void) { return false; } -static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) -{ - return cpu_present_mask; -} #endif static inline enum cpuhp_state @@ -1793,6 +1813,16 @@ static int __init parallel_bringup_parse_param(char *arg) } early_param("cpuhp.parallel", parallel_bringup_parse_param); +static inline bool cpuhp_smt_aware(void) +{ + return cpu_smt_max_threads > 1; +} + +static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) +{ + return cpu_primary_thread_mask; +} + /* * On architectures which have enabled parallel bringup this invokes all BP * prepare states for each of the to be onlined APs first. The last state @@ -2626,6 +2656,12 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) for_each_online_cpu(cpu) { if (topology_is_primary_thread(cpu)) continue; + /* + * Disable can be called with CPU_SMT_ENABLED when changing + * from a higher to lower number of SMT threads per core. + */ + if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu)) + continue; ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE); if (ret) break; @@ -2660,6 +2696,8 @@ int cpuhp_smt_enable(void) /* Skip online CPUs and CPUs on offline nodes */ if (cpu_online(cpu) || !node_online(cpu_to_node(cpu))) continue; + if (!cpu_smt_thread_allowed(cpu)) + continue; ret = _cpu_up(cpu, 0, CPUHP_ONLINE); if (ret) break; @@ -2838,20 +2876,19 @@ static const struct attribute_group cpuhp_cpu_root_attr_group = { #ifdef CONFIG_HOTPLUG_SMT +static bool cpu_smt_num_threads_valid(unsigned int threads) +{ + if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC)) + return threads >= 1 && threads <= cpu_smt_max_threads; + return threads == 1 || threads == cpu_smt_max_threads; +} + static ssize_t __store_smt_control(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - int ctrlval, ret; - - if (sysfs_streq(buf, "on")) - ctrlval = CPU_SMT_ENABLED; - else if (sysfs_streq(buf, "off")) - ctrlval = CPU_SMT_DISABLED; - else if (sysfs_streq(buf, "forceoff")) - ctrlval = CPU_SMT_FORCE_DISABLED; - else - return -EINVAL; + int ctrlval, ret, num_threads, orig_threads; + bool force_off; if (cpu_smt_control == CPU_SMT_FORCE_DISABLED) return -EPERM; @@ -2859,21 +2896,39 @@ __store_smt_control(struct device *dev, struct device_attribute *attr, if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED) return -ENODEV; + if (sysfs_streq(buf, "on")) { + ctrlval = CPU_SMT_ENABLED; + num_threads = cpu_smt_max_threads; + } else if (sysfs_streq(buf, "off")) { + ctrlval = CPU_SMT_DISABLED; + num_threads = 1; + } else if (sysfs_streq(buf, "forceoff")) { + ctrlval = CPU_SMT_FORCE_DISABLED; + num_threads = 1; + } else if (kstrtoint(buf, 10, &num_threads) == 0) { + if (num_threads == 1) + ctrlval = CPU_SMT_DISABLED; + else if (cpu_smt_num_threads_valid(num_threads)) + ctrlval = CPU_SMT_ENABLED; + else + return -EINVAL; + } else { + return -EINVAL; + } + ret = lock_device_hotplug_sysfs(); if (ret) return ret; - if (ctrlval != cpu_smt_control) { - switch (ctrlval) { - case CPU_SMT_ENABLED: - ret = cpuhp_smt_enable(); - break; - case CPU_SMT_DISABLED: - case CPU_SMT_FORCE_DISABLED: - ret = cpuhp_smt_disable(ctrlval); - break; - } - } + orig_threads = cpu_smt_num_threads; + cpu_smt_num_threads = num_threads; + + force_off = ctrlval != cpu_smt_control && ctrlval == CPU_SMT_FORCE_DISABLED; + + if (num_threads > orig_threads) + ret = cpuhp_smt_enable(); + else if (num_threads < orig_threads || force_off) + ret = cpuhp_smt_disable(ctrlval); unlock_device_hotplug(); return ret ? ret : count; @@ -2901,6 +2956,17 @@ static ssize_t control_show(struct device *dev, { const char *state = smt_states[cpu_smt_control]; +#ifdef CONFIG_HOTPLUG_SMT + /* + * If SMT is enabled but not all threads are enabled then show the + * number of threads. If all threads are enabled show "on". Otherwise + * show the state name. + */ + if (cpu_smt_control == CPU_SMT_ENABLED && + cpu_smt_num_threads != cpu_smt_max_threads) + return sysfs_emit(buf, "%d\n", cpu_smt_num_threads); +#endif + return snprintf(buf, PAGE_SIZE - 2, "%s\n", state); } diff --git a/kernel/entry/common.c b/kernel/entry/common.c index be61332c66b5..d7ee4bc3f2ba 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -205,8 +205,7 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) arch_exit_to_user_mode_prepare(regs, ti_work); - /* Ensure that the address limit is intact and no locks are held */ - addr_limit_user_check(); + /* Ensure that kernel state is sane for a return to userspace */ kmap_assert_nomap(); lockdep_assert_irqs_disabled(); lockdep_sys_exit(); diff --git a/kernel/events/core.c b/kernel/events/core.c index 60eedeaa16bd..23f67c2ba749 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8235,7 +8235,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) unsigned int size; memset(comm, 0, sizeof(comm)); - strlcpy(comm, comm_event->task->comm, sizeof(comm)); + strscpy(comm, comm_event->task->comm, sizeof(comm)); size = ALIGN(strlen(comm)+1, sizeof(u64)); comm_event->comm = comm; @@ -8690,7 +8690,7 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) } cpy_name: - strlcpy(tmp, name, sizeof(tmp)); + strscpy(tmp, name, sizeof(tmp)); name = tmp; got_name: /* @@ -9114,7 +9114,7 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN) goto err; - strlcpy(name, sym, KSYM_NAME_LEN); + strscpy(name, sym, KSYM_NAME_LEN); name_len = strlen(name) + 1; while (!IS_ALIGNED(name_len, sizeof(u64))) name[name_len++] = '\0'; diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index ddc7d4930bb0..6df0f0c83957 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -474,11 +474,12 @@ void handle_nested_irq(unsigned int irq) action = desc->action; if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) { desc->istate |= IRQS_PENDING; - goto out_unlock; + raw_spin_unlock_irq(&desc->lock); + return; } kstat_incr_irqs_this_cpu(desc); - irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS); + atomic_inc(&desc->threads_active); raw_spin_unlock_irq(&desc->lock); action_ret = IRQ_NONE; @@ -488,11 +489,7 @@ void handle_nested_irq(unsigned int irq) if (!irq_settings_no_debug(desc)) note_interrupt(desc, action_ret); - raw_spin_lock_irq(&desc->lock); - irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); - -out_unlock: - raw_spin_unlock_irq(&desc->lock); + wake_threads_waitq(desc); } EXPORT_SYMBOL_GPL(handle_nested_irq); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index bdd35bb9c735..bcc7f21db9ee 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -108,8 +108,6 @@ extern int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which, bool *state); -extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr); - irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc); irqreturn_t handle_irq_event_percpu(struct irq_desc *desc); irqreturn_t handle_irq_event(struct irq_desc *desc); @@ -121,6 +119,8 @@ void irq_resend_init(struct irq_desc *desc); bool irq_wait_for_poll(struct irq_desc *desc); void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action); +void wake_threads_waitq(struct irq_desc *desc); + #ifdef CONFIG_PROC_FS extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index d2742af0f0fd..d309ba84e08a 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -108,6 +108,16 @@ bool synchronize_hardirq(unsigned int irq) } EXPORT_SYMBOL(synchronize_hardirq); +static void __synchronize_irq(struct irq_desc *desc) +{ + __synchronize_hardirq(desc, true); + /* + * We made sure that no hardirq handler is running. Now verify that no + * threaded handlers are active. + */ + wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active)); +} + /** * synchronize_irq - wait for pending IRQ handlers (on other CPUs) * @irq: interrupt number to wait for @@ -127,16 +137,8 @@ void synchronize_irq(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); - if (desc) { - __synchronize_hardirq(desc, true); - /* - * We made sure that no hardirq handler is - * running. Now verify that no threaded handlers are - * active. - */ - wait_event(desc->wait_for_threads, - !atomic_read(&desc->threads_active)); - } + if (desc) + __synchronize_irq(desc); } EXPORT_SYMBOL(synchronize_irq); @@ -1216,7 +1218,7 @@ static irqreturn_t irq_thread_fn(struct irq_desc *desc, return ret; } -static void wake_threads_waitq(struct irq_desc *desc) +void wake_threads_waitq(struct irq_desc *desc) { if (atomic_dec_and_test(&desc->threads_active)) wake_up(&desc->wait_for_threads); @@ -1944,7 +1946,7 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) * supports it also make sure that there is no (not yet serviced) * interrupt in flight at the hardware level. */ - __synchronize_hardirq(desc, true); + __synchronize_irq(desc); #ifdef CONFIG_DEBUG_SHIRQ /* diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 016d997131d4..18edd57b5fe8 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -163,12 +163,12 @@ unsigned long kallsyms_sym_address(int idx) return kallsyms_relative_base - 1 - kallsyms_offsets[idx]; } -static bool cleanup_symbol_name(char *s) +static void cleanup_symbol_name(char *s) { char *res; if (!IS_ENABLED(CONFIG_LTO_CLANG)) - return false; + return; /* * LLVM appends various suffixes for local functions and variables that @@ -178,26 +178,21 @@ static bool cleanup_symbol_name(char *s) * - foo.llvm.[0-9a-f]+ */ res = strstr(s, ".llvm."); - if (res) { + if (res) *res = '\0'; - return true; - } - return false; + return; } static int compare_symbol_name(const char *name, char *namebuf) { - int ret; - - ret = strcmp(name, namebuf); - if (!ret) - return ret; - - if (cleanup_symbol_name(namebuf) && !strcmp(name, namebuf)) - return 0; - - return ret; + /* The kallsyms_seqs_of_names is sorted based on names after + * cleanup_symbol_name() (see scripts/kallsyms.c) if clang lto is enabled. + * To ensure correct bisection in kallsyms_lookup_names(), do + * cleanup_symbol_name(namebuf) before comparing name and namebuf. + */ + cleanup_symbol_name(namebuf); + return strcmp(name, namebuf); } static unsigned int get_symbol_seq(int index) diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c index a2e3745d15c4..e05ddc33a752 100644 --- a/kernel/kallsyms_selftest.c +++ b/kernel/kallsyms_selftest.c @@ -196,7 +196,7 @@ static bool match_cleanup_name(const char *s, const char *name) if (!IS_ENABLED(CONFIG_LTO_CLANG)) return false; - p = strchr(s, '.'); + p = strstr(s, ".llvm."); if (!p) return false; @@ -344,27 +344,6 @@ static int test_kallsyms_basic_function(void) goto failed; } - /* - * The first '.' may be the initial letter, in which case the - * entire symbol name will be truncated to an empty string in - * cleanup_symbol_name(). Do not test these symbols. - * - * For example: - * cat /proc/kallsyms | awk '{print $3}' | grep -E "^\." | head - * .E_read_words - * .E_leading_bytes - * .E_trailing_bytes - * .E_write_words - * .E_copy - * .str.292.llvm.12122243386960820698 - * .str.24.llvm.12122243386960820698 - * .str.29.llvm.12122243386960820698 - * .str.75.llvm.12122243386960820698 - * .str.99.llvm.12122243386960820698 - */ - if (IS_ENABLED(CONFIG_LTO_CLANG) && !namebuf[0]) - continue; - lookup_addr = kallsyms_lookup_name(namebuf); memset(stat, 0, sizeof(*stat)); diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 949d3deae506..270c7f80ce84 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -45,6 +45,7 @@ torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); torture_param(int, rt_boost, 2, "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); +torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority"); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ @@ -809,7 +810,8 @@ static int lock_torture_writer(void *arg) bool skip_main_lock; VERBOSE_TOROUT_STRING("lock_torture_writer task started"); - set_user_nice(current, MAX_NICE); + if (!rt_task(current)) + set_user_nice(current, MAX_NICE); do { if ((torture_random(&rand) & 0xfffff) == 0) @@ -1015,8 +1017,7 @@ static void lock_torture_cleanup(void) if (writer_tasks) { for (i = 0; i < cxt.nrealwriters_stress; i++) - torture_stop_kthread(lock_torture_writer, - writer_tasks[i]); + torture_stop_kthread(lock_torture_writer, writer_tasks[i]); kfree(writer_tasks); writer_tasks = NULL; } @@ -1244,8 +1245,9 @@ static int __init lock_torture_init(void) goto create_reader; /* Create writer. */ - firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], - writer_tasks[i]); + firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i], + writer_tasks[i], + writer_fifo ? sched_set_fifo : NULL); if (torture_init_error(firsterr)) goto unwind; diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index 80d9c6d77a45..15781acaac1c 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -30,7 +30,7 @@ static struct kmem_cache *nsproxy_cachep; struct nsproxy init_nsproxy = { - .count = ATOMIC_INIT(1), + .count = REFCOUNT_INIT(1), .uts_ns = &init_uts_ns, #if defined(CONFIG_POSIX_MQUEUE) || defined(CONFIG_SYSVIPC) .ipc_ns = &init_ipc_ns, @@ -55,7 +55,7 @@ static inline struct nsproxy *create_nsproxy(void) nsproxy = kmem_cache_alloc(nsproxy_cachep, GFP_KERNEL); if (nsproxy) - atomic_set(&nsproxy->count, 1); + refcount_set(&nsproxy->count, 1); return nsproxy; } diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 98c1544cf572..5befd8780dcd 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -511,6 +511,14 @@ static inline void show_rcu_tasks_gp_kthreads(void) {} void rcu_request_urgent_qs_task(struct task_struct *t); #endif /* #else #ifdef CONFIG_TINY_RCU */ +#ifdef CONFIG_TASKS_RCU +struct task_struct *get_rcu_tasks_gp_kthread(void); +#endif // # ifdef CONFIG_TASKS_RCU + +#ifdef CONFIG_TASKS_RUDE_RCU +struct task_struct *get_rcu_tasks_rude_gp_kthread(void); +#endif // # ifdef CONFIG_TASKS_RUDE_RCU + #define RCU_SCHEDULER_INACTIVE 0 #define RCU_SCHEDULER_INIT 1 #define RCU_SCHEDULER_RUNNING 2 diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index d1221731c7cf..ffdb30495e3c 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -84,15 +84,17 @@ MODULE_AUTHOR("Paul E. McKenney "); #endif torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); -torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader"); +torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); +torture_param(int, minruntime, 0, "Minimum run time (s)"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(int, nwriters, -1, "Number of RCU updater threads"); torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, "Shutdown at end of scalability tests."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); +torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); @@ -139,6 +141,7 @@ struct rcu_scale_ops { void (*gp_barrier)(void); void (*sync)(void); void (*exp_sync)(void); + struct task_struct *(*rso_gp_kthread)(void); const char *name; }; @@ -295,6 +298,7 @@ static struct rcu_scale_ops tasks_ops = { .gp_barrier = rcu_barrier_tasks, .sync = synchronize_rcu_tasks, .exp_sync = synchronize_rcu_tasks, + .rso_gp_kthread = get_rcu_tasks_gp_kthread, .name = "tasks" }; @@ -306,6 +310,44 @@ static struct rcu_scale_ops tasks_ops = { #endif // #else // #ifdef CONFIG_TASKS_RCU +#ifdef CONFIG_TASKS_RUDE_RCU + +/* + * Definitions for RCU-tasks-rude scalability testing. + */ + +static int tasks_rude_scale_read_lock(void) +{ + return 0; +} + +static void tasks_rude_scale_read_unlock(int idx) +{ +} + +static struct rcu_scale_ops tasks_rude_ops = { + .ptype = RCU_TASKS_RUDE_FLAVOR, + .init = rcu_sync_scale_init, + .readlock = tasks_rude_scale_read_lock, + .readunlock = tasks_rude_scale_read_unlock, + .get_gp_seq = rcu_no_completed, + .gp_diff = rcu_seq_diff, + .async = call_rcu_tasks_rude, + .gp_barrier = rcu_barrier_tasks_rude, + .sync = synchronize_rcu_tasks_rude, + .exp_sync = synchronize_rcu_tasks_rude, + .rso_gp_kthread = get_rcu_tasks_rude_gp_kthread, + .name = "tasks-rude" +}; + +#define TASKS_RUDE_OPS &tasks_rude_ops, + +#else // #ifdef CONFIG_TASKS_RUDE_RCU + +#define TASKS_RUDE_OPS + +#endif // #else // #ifdef CONFIG_TASKS_RUDE_RCU + #ifdef CONFIG_TASKS_TRACE_RCU /* @@ -334,6 +376,7 @@ static struct rcu_scale_ops tasks_tracing_ops = { .gp_barrier = rcu_barrier_tasks_trace, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, + .rso_gp_kthread = get_rcu_tasks_trace_gp_kthread, .name = "tasks-tracing" }; @@ -410,10 +453,12 @@ rcu_scale_writer(void *arg) { int i = 0; int i_max; + unsigned long jdone; long me = (long)arg; struct rcu_head *rhp = NULL; bool started = false, done = false, alldone = false; u64 t; + DEFINE_TORTURE_RANDOM(tr); u64 *wdp; u64 *wdpp = writer_durations[me]; @@ -424,7 +469,7 @@ rcu_scale_writer(void *arg) sched_set_fifo_low(current); if (holdoff) - schedule_timeout_uninterruptible(holdoff * HZ); + schedule_timeout_idle(holdoff * HZ); /* * Wait until rcu_end_inkernel_boot() is called for normal GP tests @@ -445,9 +490,12 @@ rcu_scale_writer(void *arg) } } + jdone = jiffies + minruntime * HZ; do { if (writer_holdoff) udelay(writer_holdoff); + if (writer_holdoff_jiffies) + schedule_timeout_idle(torture_random(&tr) % writer_holdoff_jiffies + 1); wdp = &wdpp[i]; *wdp = ktime_get_mono_fast_ns(); if (gp_async) { @@ -475,7 +523,7 @@ retry: if (!started && atomic_read(&n_rcu_scale_writer_started) >= nrealwriters) started = true; - if (!done && i >= MIN_MEAS) { + if (!done && i >= MIN_MEAS && time_after(jiffies, jdone)) { done = true; sched_set_normal(current, 0); pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n", @@ -518,8 +566,8 @@ static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { pr_alert("%s" SCALE_FLAG - "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n", - scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown); + "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown=%d\n", + scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown); } /* @@ -556,6 +604,8 @@ static struct task_struct **kfree_reader_tasks; static int kfree_nrealthreads; static atomic_t n_kfree_scale_thread_started; static atomic_t n_kfree_scale_thread_ended; +static struct task_struct *kthread_tp; +static u64 kthread_stime; struct kfree_obj { char kfree_obj[8]; @@ -701,6 +751,10 @@ kfree_scale_init(void) unsigned long jif_start; unsigned long orig_jif; + pr_alert("%s" SCALE_FLAG + "--- kfree_rcu_test: kfree_mult=%d kfree_by_call_rcu=%d kfree_nthreads=%d kfree_alloc_num=%d kfree_loops=%d kfree_rcu_test_double=%d kfree_rcu_test_single=%d\n", + scale_type, kfree_mult, kfree_by_call_rcu, kfree_nthreads, kfree_alloc_num, kfree_loops, kfree_rcu_test_double, kfree_rcu_test_single); + // Also, do a quick self-test to ensure laziness is as much as // expected. if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) { @@ -797,6 +851,18 @@ rcu_scale_cleanup(void) if (gp_exp && gp_async) SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); + // If built-in, just report all of the GP kthread's CPU time. + if (IS_BUILTIN(CONFIG_RCU_SCALE_TEST) && !kthread_tp && cur_ops->rso_gp_kthread) + kthread_tp = cur_ops->rso_gp_kthread(); + if (kthread_tp) { + u32 ns; + u64 us; + + kthread_stime = kthread_tp->stime - kthread_stime; + us = div_u64_rem(kthread_stime, 1000, &ns); + pr_info("rcu_scale: Grace-period kthread CPU time: %llu.%03u us\n", us, ns); + show_rcu_gp_kthreads(); + } if (kfree_rcu_test) { kfree_scale_cleanup(); return; @@ -885,7 +951,7 @@ rcu_scale_init(void) long i; int firsterr = 0; static struct rcu_scale_ops *scale_ops[] = { - &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_TRACING_OPS + &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS }; if (!torture_init_begin(scale_type, verbose)) @@ -910,6 +976,11 @@ rcu_scale_init(void) if (cur_ops->init) cur_ops->init(); + if (cur_ops->rso_gp_kthread) { + kthread_tp = cur_ops->rso_gp_kthread(); + if (kthread_tp) + kthread_stime = kthread_tp->stime; + } if (kfree_rcu_test) return kfree_scale_init(); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 147551c23baf..ade42d6a9d9b 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1581,6 +1581,7 @@ rcu_torture_writer(void *arg) rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { tracing_off(); + show_rcu_gp_kthreads(); WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); rcu_ftrace_dump(DUMP_ALL); } @@ -1876,7 +1877,7 @@ static int rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) { int mask = rcutorture_extend_mask_max(); - unsigned long randmask1 = torture_random(trsp) >> 8; + unsigned long randmask1 = torture_random(trsp); unsigned long randmask2 = randmask1 >> 3; unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; @@ -1935,7 +1936,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, if (!((mask - 1) & mask)) return rtrsp; /* Current RCU reader not extendable. */ /* Bias towards larger numbers of loops. */ - i = (torture_random(trsp) >> 3); + i = torture_random(trsp); i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; for (j = 0; j < i; j++) { mask = rcutorture_extend_mask(*readstate, trsp); @@ -2136,7 +2137,7 @@ static int rcu_nocb_toggle(void *arg) toggle_fuzz = NSEC_PER_USEC; do { r = torture_random(&rand); - cpu = (r >> 4) % (maxcpu + 1); + cpu = (r >> 1) % (maxcpu + 1); if (r & 0x1) { rcu_nocb_cpu_offload(cpu); atomic_long_inc(&n_nocb_offload); diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 1970ce5f22d4..91a0fd0d4d9a 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -528,6 +528,38 @@ static struct ref_scale_ops clock_ops = { .name = "clock" }; +static void ref_jiffies_section(const int nloops) +{ + u64 x = 0; + int i; + + preempt_disable(); + for (i = nloops; i >= 0; i--) + x += jiffies; + preempt_enable(); + stopopts = x; +} + +static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl) +{ + u64 x = 0; + int i; + + preempt_disable(); + for (i = nloops; i >= 0; i--) { + x += jiffies; + un_delay(udl, ndl); + } + preempt_enable(); + stopopts = x; +} + +static struct ref_scale_ops jiffies_ops = { + .readsection = ref_jiffies_section, + .delaysection = ref_jiffies_delay_section, + .name = "jiffies" +}; + //////////////////////////////////////////////////////////////////////// // // Methods leveraging SLAB_TYPESAFE_BY_RCU. @@ -1047,7 +1079,7 @@ ref_scale_init(void) int firsterr = 0; static struct ref_scale_ops *scale_ops[] = { &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, - &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, + &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; @@ -1107,12 +1139,11 @@ ref_scale_init(void) VERBOSE_SCALEOUT("Starting %d reader threads", nreaders); for (i = 0; i < nreaders; i++) { + init_waitqueue_head(&reader_tasks[i].wq); firsterr = torture_create_kthread(ref_scale_reader, (void *)i, reader_tasks[i].task); if (torture_init_error(firsterr)) goto unwind; - - init_waitqueue_head(&(reader_tasks[i].wq)); } // Main Task diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index b770add3f843..8d65f7d576a3 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -25,6 +25,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @cblist: Callback list. * @lock: Lock protecting per-CPU callback list. * @rtp_jiffies: Jiffies counter value for statistics. + * @lazy_timer: Timer to unlazify callbacks. + * @urgent_gp: Number of additional non-lazy grace periods. * @rtp_n_lock_retries: Rough lock-contention statistic. * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. @@ -38,6 +40,8 @@ struct rcu_tasks_percpu { raw_spinlock_t __private lock; unsigned long rtp_jiffies; unsigned long rtp_n_lock_retries; + struct timer_list lazy_timer; + unsigned int urgent_gp; struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; @@ -51,7 +55,6 @@ struct rcu_tasks_percpu { * @cbs_wait: RCU wait allowing a new callback to get kthread's attention. * @cbs_gbl_lock: Lock protecting callback list. * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone. - * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping. @@ -61,6 +64,8 @@ struct rcu_tasks_percpu { * @tasks_gp_seq: Number of grace periods completed since boot. * @n_ipis: Number of IPIs sent to encourage grace periods to end. * @n_ipis_fails: Number of IPI-send failures. + * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. + * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). * @pertask_func: This flavor's per-task scan function (optional). * @postscan_func: This flavor's post-task scan function (optional). @@ -92,6 +97,7 @@ struct rcu_tasks { unsigned long n_ipis; unsigned long n_ipis_fails; struct task_struct *kthread_ptr; + unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; pregp_func_t pregp_func; pertask_func_t pertask_func; @@ -127,6 +133,7 @@ static struct rcu_tasks rt_name = \ .gp_func = gp, \ .call_func = call, \ .rtpcpu = &rt_name ## __percpu, \ + .lazy_jiffies = DIV_ROUND_UP(HZ, 4), \ .name = n, \ .percpu_enqueue_shift = order_base_2(CONFIG_NR_CPUS), \ .percpu_enqueue_lim = 1, \ @@ -139,9 +146,7 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Track exiting tasks in order to allow them to be waited for. */ DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu); -#endif -#ifdef CONFIG_TASKS_RCU /* Report delay in synchronize_srcu() completion in rcu_tasks_postscan(). */ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); @@ -171,6 +176,8 @@ static int rcu_task_contend_lim __read_mostly = 100; module_param(rcu_task_contend_lim, int, 0444); static int rcu_task_collapse_lim __read_mostly = 10; module_param(rcu_task_collapse_lim, int, 0444); +static int rcu_task_lazy_lim __read_mostly = 32; +module_param(rcu_task_lazy_lim, int, 0444); /* RCU tasks grace-period state for debugging. */ #define RTGS_INIT 0 @@ -229,7 +236,7 @@ static const char *tasks_gp_state_getname(struct rcu_tasks *rtp) #endif /* #ifndef CONFIG_TINY_RCU */ // Initialize per-CPU callback lists for the specified flavor of -// Tasks RCU. +// Tasks RCU. Do not enqueue callbacks before this function is invoked. static void cblist_init_generic(struct rcu_tasks *rtp) { int cpu; @@ -237,7 +244,6 @@ static void cblist_init_generic(struct rcu_tasks *rtp) int lim; int shift; - raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rcu_task_enqueue_lim < 0) { rcu_task_enqueue_lim = 1; rcu_task_cb_adjust = true; @@ -260,22 +266,48 @@ static void cblist_init_generic(struct rcu_tasks *rtp) WARN_ON_ONCE(!rtpcp); if (cpu) raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock)); - raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. + local_irq_save(flags); // serialize initialization if (rcu_segcblist_empty(&rtpcp->cblist)) rcu_segcblist_init(&rtpcp->cblist); + local_irq_restore(flags); INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; if (!rtpcp->rtp_blkd_tasks.next) INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); - raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. } - raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d.\n", rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust); } +// Compute wakeup time for lazy callback timer. +static unsigned long rcu_tasks_lazy_time(struct rcu_tasks *rtp) +{ + return jiffies + rtp->lazy_jiffies; +} + +// Timer handler that unlazifies lazy callbacks. +static void call_rcu_tasks_generic_timer(struct timer_list *tlp) +{ + unsigned long flags; + bool needwake = false; + struct rcu_tasks *rtp; + struct rcu_tasks_percpu *rtpcp = from_timer(rtpcp, tlp, lazy_timer); + + rtp = rtpcp->rtpp; + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) { + if (!rtpcp->urgent_gp) + rtpcp->urgent_gp = 1; + needwake = true; + mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); + } + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); + if (needwake) + rcuwait_wake_up(&rtp->cbs_wait); +} + // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic(). static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp) { @@ -292,6 +324,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, { int chosen_cpu; unsigned long flags; + bool havekthread = smp_load_acquire(&rtp->kthread_ptr); int ideal_cpu; unsigned long j; bool needadjust = false; @@ -316,12 +349,19 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids) needadjust = true; // Defer adjustment to avoid deadlock. } - if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) { - raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. - cblist_init_generic(rtp); - raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. + // Queuing callbacks before initialization not yet supported. + if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) + rcu_segcblist_init(&rtpcp->cblist); + needwake = (func == wakeme_after_rcu) || + (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); + if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { + if (rtp->lazy_jiffies) + mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); + else + needwake = rcu_segcblist_empty(&rtpcp->cblist); } - needwake = rcu_segcblist_empty(&rtpcp->cblist); + if (needwake) + rtpcp->urgent_gp = 3; rcu_segcblist_enqueue(&rtpcp->cblist, rhp); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (unlikely(needadjust)) { @@ -415,9 +455,14 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) } rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); - if (rcu_segcblist_pend_cbs(&rtpcp->cblist)) + if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { + if (rtp->lazy_jiffies) + rtpcp->urgent_gp--; needgpcb |= 0x3; - if (!rcu_segcblist_empty(&rtpcp->cblist)) + } else if (rcu_segcblist_empty(&rtpcp->cblist)) { + rtpcp->urgent_gp = 0; + } + if (rcu_segcblist_ready_cbs(&rtpcp->cblist)) needgpcb |= 0x1; raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } @@ -525,10 +570,12 @@ static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot) if (unlikely(midboot)) { needgpcb = 0x2; } else { + mutex_unlock(&rtp->tasks_gp_mutex); set_tasks_gp_state(rtp, RTGS_WAIT_CBS); rcuwait_wait_event(&rtp->cbs_wait, (needgpcb = rcu_tasks_need_gpcb(rtp)), TASK_IDLE); + mutex_lock(&rtp->tasks_gp_mutex); } if (needgpcb & 0x2) { @@ -549,11 +596,19 @@ static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot) // RCU-tasks kthread that detects grace periods and invokes callbacks. static int __noreturn rcu_tasks_kthread(void *arg) { + int cpu; struct rcu_tasks *rtp = arg; + for_each_possible_cpu(cpu) { + struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); + + timer_setup(&rtpcp->lazy_timer, call_rcu_tasks_generic_timer, 0); + rtpcp->urgent_gp = 1; + } + /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ housekeeping_affine(current, HK_TYPE_RCU); - WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start! + smp_store_release(&rtp->kthread_ptr, current); // Let GPs start! /* * Each pass through the following loop makes one check for @@ -635,16 +690,22 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) { int cpu; bool havecbs = false; + bool haveurgent = false; + bool haveurgentcbs = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); - if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) { + if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) havecbs = true; + if (data_race(rtpcp->urgent_gp)) + haveurgent = true; + if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) + haveurgentcbs = true; + if (havecbs && haveurgent && haveurgentcbs) break; - } } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), @@ -652,6 +713,9 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], + ".u"[haveurgent], + ".U"[haveurgentcbs], + rtp->lazy_jiffies, s); } #endif // #ifndef CONFIG_TINY_RCU @@ -1020,11 +1084,16 @@ void rcu_barrier_tasks(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks); +int rcu_tasks_lazy_ms = -1; +module_param(rcu_tasks_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_kthread(void) { cblist_init_generic(&rcu_tasks); rcu_tasks.gp_sleep = HZ / 10; rcu_tasks.init_fract = HZ / 10; + if (rcu_tasks_lazy_ms >= 0) + rcu_tasks.lazy_jiffies = msecs_to_jiffies(rcu_tasks_lazy_ms); rcu_tasks.pregp_func = rcu_tasks_pregp_step; rcu_tasks.pertask_func = rcu_tasks_pertask; rcu_tasks.postscan_func = rcu_tasks_postscan; @@ -1042,6 +1111,12 @@ void show_rcu_tasks_classic_gp_kthread(void) EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) +struct task_struct *get_rcu_tasks_gp_kthread(void) +{ + return rcu_tasks.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_gp_kthread); + /* * Contribute to protect against tasklist scan blind spot while the * task is exiting and may be removed from the tasklist. See @@ -1173,10 +1248,15 @@ void rcu_barrier_tasks_rude(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude); +int rcu_tasks_rude_lazy_ms = -1; +module_param(rcu_tasks_rude_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_rude_kthread(void) { cblist_init_generic(&rcu_tasks_rude); rcu_tasks_rude.gp_sleep = HZ / 10; + if (rcu_tasks_rude_lazy_ms >= 0) + rcu_tasks_rude.lazy_jiffies = msecs_to_jiffies(rcu_tasks_rude_lazy_ms); rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude); return 0; } @@ -1188,6 +1268,13 @@ void show_rcu_tasks_rude_gp_kthread(void) } EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) + +struct task_struct *get_rcu_tasks_rude_gp_kthread(void) +{ + return rcu_tasks_rude.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_rude_gp_kthread); + #endif /* #ifdef CONFIG_TASKS_RUDE_RCU */ //////////////////////////////////////////////////////////////////////// @@ -1793,6 +1880,9 @@ void rcu_barrier_tasks_trace(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace); +int rcu_tasks_trace_lazy_ms = -1; +module_param(rcu_tasks_trace_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_trace_kthread(void) { cblist_init_generic(&rcu_tasks_trace); @@ -1807,6 +1897,8 @@ static int __init rcu_spawn_tasks_trace_kthread(void) if (rcu_tasks_trace.init_fract <= 0) rcu_tasks_trace.init_fract = 1; } + if (rcu_tasks_trace_lazy_ms >= 0) + rcu_tasks_trace.lazy_jiffies = msecs_to_jiffies(rcu_tasks_trace_lazy_ms); rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; rcu_tasks_trace.holdouts_func = check_all_holdout_tasks_trace; @@ -1830,6 +1922,12 @@ void show_rcu_tasks_trace_gp_kthread(void) EXPORT_SYMBOL_GPL(show_rcu_tasks_trace_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) +struct task_struct *get_rcu_tasks_trace_gp_kthread(void) +{ + return rcu_tasks_trace.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_trace_gp_kthread); + #else /* #ifdef CONFIG_TASKS_TRACE_RCU */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { } #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 1449cb69a0e0..cb1caefa8bd0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -632,7 +632,7 @@ void __rcu_irq_enter_check_tick(void) // prevents self-deadlock. So we can safely recheck under the lock. // Note that the nohz_full state currently cannot change. raw_spin_lock_rcu_node(rdp->mynode); - if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { + if (READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) { // A nohz_full CPU is in the kernel and RCU needs a // quiescent state. Turn on the tick! WRITE_ONCE(rdp->rcu_forced_tick, true); @@ -677,12 +677,16 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) } /** - * rcu_is_watching - see if RCU thinks that the current CPU is not idle + * rcu_is_watching - RCU read-side critical sections permitted on current CPU? * - * Return true if RCU is watching the running CPU, which means that this - * CPU can safely enter RCU read-side critical sections. In other words, - * if the current CPU is not in its idle loop or is in an interrupt or - * NMI handler, return true. + * Return @true if RCU is watching the running CPU and @false otherwise. + * An @true return means that this CPU can safely enter RCU read-side + * critical sections. + * + * Although calls to rcu_is_watching() from most parts of the kernel + * will return @true, there are important exceptions. For example, if the + * current CPU is deep within its idle loop, in kernel entry/exit code, + * or offline, rcu_is_watching() will return @false. * * Make notrace because it can be called by the internal functions of * ftrace, and making this notrace removes unnecessary recursion calls. diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 43229d2b0c44..5598212d1f27 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -77,9 +77,9 @@ __setup("rcu_nocbs", rcu_nocb_setup); static int __init parse_rcu_nocb_poll(char *arg) { rcu_nocb_poll = true; - return 0; + return 1; } -early_param("rcu_nocb_poll", parse_rcu_nocb_poll); +__setup("rcu_nocb_poll", parse_rcu_nocb_poll); /* * Don't bother bypassing ->cblist if the call_rcu() rate is low. diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 5d113aa59e77..59032aaccd18 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -171,7 +171,8 @@ static void scf_torture_stats_print(void) scfs.n_all_wait += scf_stats_p[i].n_all_wait; } if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || - atomic_read(&n_mb_out_errs) || atomic_read(&n_alloc_errs)) + atomic_read(&n_mb_out_errs) || + (!IS_ENABLED(CONFIG_KASAN) && atomic_read(&n_alloc_errs))) bangstr = "!!! "; pr_alert("%s %sscf_invoked_count %s: %lld resched: %lld single: %lld/%lld single_ofl: %lld/%lld single_rpc: %lld single_rpc_ofl: %lld many: %lld/%lld all: %lld/%lld ", SCFTORT_FLAG, bangstr, isdone ? "VER" : "ver", invoked_count, scfs.n_resched, @@ -312,6 +313,7 @@ static void scf_handler_1(void *scfc_in) // Randomly do an smp_call_function*() invocation. static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp) { + bool allocfail = false; uintptr_t cpu; int ret = 0; struct scf_check *scfcp = NULL; @@ -323,8 +325,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra preempt_disable(); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); - if (WARN_ON_ONCE(!scfcp)) { + if (!scfcp) { + WARN_ON_ONCE(!IS_ENABLED(CONFIG_KASAN)); atomic_inc(&n_alloc_errs); + allocfail = true; } else { scfcp->scfc_cpu = -1; scfcp->scfc_wait = scfsp->scfs_wait; @@ -431,7 +435,9 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra cpus_read_unlock(); else preempt_enable(); - if (!(torture_random(trsp) & 0xfff)) + if (allocfail) + schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete. + else if (!(torture_random(trsp) & 0xfff)) schedule_timeout_uninterruptible(1); } diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c index d57a5c1c1cd9..3561ab533dd4 100644 --- a/kernel/sched/completion.c +++ b/kernel/sched/completion.c @@ -13,6 +13,23 @@ * Waiting for completion is a typically sync point, but not an exclusion point. */ +static void complete_with_flags(struct completion *x, int wake_flags) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&x->wait.lock, flags); + + if (x->done != UINT_MAX) + x->done++; + swake_up_locked(&x->wait, wake_flags); + raw_spin_unlock_irqrestore(&x->wait.lock, flags); +} + +void complete_on_current_cpu(struct completion *x) +{ + return complete_with_flags(x, WF_CURRENT_CPU); +} + /** * complete: - signals a single thread waiting on this completion * @x: holds the state of this particular completion @@ -27,14 +44,7 @@ */ void complete(struct completion *x) { - unsigned long flags; - - raw_spin_lock_irqsave(&x->wait.lock, flags); - - if (x->done != UINT_MAX) - x->done++; - swake_up_locked(&x->wait); - raw_spin_unlock_irqrestore(&x->wait.lock, flags); + complete_with_flags(x, 0); } EXPORT_SYMBOL(complete); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2a562043f89c..93ab5151f770 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4264,8 +4264,7 @@ bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) * Return: %true if @p->state changes (an actual wakeup was done), * %false otherwise. */ -static int -try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) +int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; int cpu, success = 0; @@ -7120,7 +7119,7 @@ asmlinkage __visible void __sched preempt_schedule_irq(void) int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, void *key) { - WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); + WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~(WF_SYNC|WF_CURRENT_CPU)); return try_to_wake_up(curr->private, mode, wake_flags); } EXPORT_SYMBOL(default_wake_function); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 685c6b9ea1b2..f212c81fa455 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7788,6 +7788,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) if (wake_flags & WF_TTWU) { record_wakee(p); + if ((wake_flags & WF_CURRENT_CPU) && + cpumask_test_cpu(cpu, p->cpus_ptr)) + return cpu; + if (sched_energy_enabled()) { new_cpu = find_energy_efficient_cpu(p, prev_cpu, sync); if (new_cpu >= 0) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 59522c0304ee..909d4bbef203 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2146,12 +2146,13 @@ static inline int task_on_rq_migrating(struct task_struct *p) } /* Wake flags. The first three directly map to some SD flag value */ -#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ -#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ -#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ +#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ +#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ +#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ -#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ -#define WF_MIGRATED 0x20 /* Internal use, task got migrated */ +#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ +#define WF_MIGRATED 0x20 /* Internal use, task got migrated */ +#define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ #ifdef CONFIG_SMP static_assert(WF_EXEC == SD_BALANCE_EXEC); @@ -3245,6 +3246,8 @@ static inline bool is_per_cpu_kthread(struct task_struct *p) extern void swake_up_all_locked(struct swait_queue_head *q); extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); +extern int try_to_wake_up(struct task_struct *tsk, unsigned int state, int wake_flags); + #ifdef CONFIG_PREEMPT_DYNAMIC extern int preempt_dynamic_mode; extern int sched_dynamic_mode(const char *str); diff --git a/kernel/sched/swait.c b/kernel/sched/swait.c index 76b9b796e695..72505cd3b60a 100644 --- a/kernel/sched/swait.c +++ b/kernel/sched/swait.c @@ -18,7 +18,7 @@ EXPORT_SYMBOL(__init_swait_queue_head); * If for some reason it would return 0, that means the previously waiting * task is already running, so it will observe condition true (or has already). */ -void swake_up_locked(struct swait_queue_head *q) +void swake_up_locked(struct swait_queue_head *q, int wake_flags) { struct swait_queue *curr; @@ -26,7 +26,7 @@ void swake_up_locked(struct swait_queue_head *q) return; curr = list_first_entry(&q->task_list, typeof(*curr), task_list); - wake_up_process(curr->task); + try_to_wake_up(curr->task, TASK_NORMAL, wake_flags); list_del_init(&curr->task_list); } EXPORT_SYMBOL(swake_up_locked); @@ -41,7 +41,7 @@ EXPORT_SYMBOL(swake_up_locked); void swake_up_all_locked(struct swait_queue_head *q) { while (!list_empty(&q->task_list)) - swake_up_locked(q); + swake_up_locked(q, 0); } void swake_up_one(struct swait_queue_head *q) @@ -49,7 +49,7 @@ void swake_up_one(struct swait_queue_head *q) unsigned long flags; raw_spin_lock_irqsave(&q->lock, flags); - swake_up_locked(q); + swake_up_locked(q, 0); raw_spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(swake_up_one); diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 48c53e4739ea..802d98cf2de3 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -161,6 +161,11 @@ int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, } EXPORT_SYMBOL(__wake_up); +void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key) +{ + __wake_up_common_lock(wq_head, mode, 1, WF_CURRENT_CPU, key); +} + /* * Same as __wake_up but called with the spinlock in wait_queue_head_t held. */ diff --git a/kernel/seccomp.c b/kernel/seccomp.c index d3e584065c7f..255999ba9190 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -110,11 +110,13 @@ struct seccomp_knotif { * @flags: The flags for the new file descriptor. At the moment, only O_CLOEXEC * is allowed. * @ioctl_flags: The flags used for the seccomp_addfd ioctl. + * @setfd: whether or not SECCOMP_ADDFD_FLAG_SETFD was set during notify_addfd * @ret: The return value of the installing process. It is set to the fd num * upon success (>= 0). * @completion: Indicates that the installing process has completed fd * installation, or gone away (either due to successful * reply, or signal) + * @list: list_head for chaining seccomp_kaddfd together. * */ struct seccomp_kaddfd { @@ -138,14 +140,17 @@ struct seccomp_kaddfd { * structure is fairly large, we store the notification-specific stuff in a * separate structure. * - * @request: A semaphore that users of this notification can wait on for - * changes. Actual reads and writes are still controlled with - * filter->notify_lock. + * @requests: A semaphore that users of this notification can wait on for + * changes. Actual reads and writes are still controlled with + * filter->notify_lock. + * @flags: A set of SECCOMP_USER_NOTIF_FD_* flags. * @next_id: The id of the next request. * @notifications: A list of struct seccomp_knotif elements. */ + struct notification { - struct semaphore request; + atomic_t requests; + u32 flags; u64 next_id; struct list_head notifications; }; @@ -555,6 +560,8 @@ static void __seccomp_filter_release(struct seccomp_filter *orig) * drop its reference count, and notify * about unused filters * + * @tsk: task the filter should be released from. + * * This function should only be called when the task is exiting as * it detaches it from its filter tree. As such, READ_ONCE() and * barriers are not needed here, as would normally be needed. @@ -574,6 +581,8 @@ void seccomp_filter_release(struct task_struct *tsk) /** * seccomp_sync_threads: sets all threads to use current's filter * + * @flags: SECCOMP_FILTER_FLAG_* flags to set during sync. + * * Expects sighand and cred_guard_mutex locks to be held, and for * seccomp_can_sync_threads() to have returned success already * without dropping the locks. @@ -1116,8 +1125,11 @@ static int seccomp_do_user_notification(int this_syscall, list_add_tail(&n.list, &match->notif->notifications); INIT_LIST_HEAD(&n.addfd); - up(&match->notif->request); - wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM); + atomic_inc(&match->notif->requests); + if (match->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + wake_up_poll_on_current_cpu(&match->wqh, EPOLLIN | EPOLLRDNORM); + else + wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM); /* * This is where we wait for a reply from userspace. @@ -1450,6 +1462,37 @@ find_notification(struct seccomp_filter *filter, u64 id) return NULL; } +static int recv_wake_function(wait_queue_entry_t *wait, unsigned int mode, int sync, + void *key) +{ + /* Avoid a wakeup if event not interesting for us. */ + if (key && !(key_to_poll(key) & (EPOLLIN | EPOLLERR))) + return 0; + return autoremove_wake_function(wait, mode, sync, key); +} + +static int recv_wait_event(struct seccomp_filter *filter) +{ + DEFINE_WAIT_FUNC(wait, recv_wake_function); + int ret; + + if (atomic_dec_if_positive(&filter->notif->requests) >= 0) + return 0; + + for (;;) { + ret = prepare_to_wait_event(&filter->wqh, &wait, TASK_INTERRUPTIBLE); + + if (atomic_dec_if_positive(&filter->notif->requests) >= 0) + break; + + if (ret) + return ret; + + schedule(); + } + finish_wait(&filter->wqh, &wait); + return 0; +} static long seccomp_notify_recv(struct seccomp_filter *filter, void __user *buf) @@ -1467,7 +1510,7 @@ static long seccomp_notify_recv(struct seccomp_filter *filter, memset(&unotif, 0, sizeof(unotif)); - ret = down_interruptible(&filter->notif->request); + ret = recv_wait_event(filter); if (ret < 0) return ret; @@ -1515,7 +1558,8 @@ out: if (should_sleep_killable(filter, knotif)) complete(&knotif->ready); knotif->state = SECCOMP_NOTIFY_INIT; - up(&filter->notif->request); + atomic_inc(&filter->notif->requests); + wake_up_poll(&filter->wqh, EPOLLIN | EPOLLRDNORM); } mutex_unlock(&filter->notify_lock); } @@ -1561,7 +1605,10 @@ static long seccomp_notify_send(struct seccomp_filter *filter, knotif->error = resp.error; knotif->val = resp.val; knotif->flags = resp.flags; - complete(&knotif->ready); + if (filter->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + complete_on_current_cpu(&knotif->ready); + else + complete(&knotif->ready); out: mutex_unlock(&filter->notify_lock); return ret; @@ -1591,6 +1638,22 @@ static long seccomp_notify_id_valid(struct seccomp_filter *filter, return ret; } +static long seccomp_notify_set_flags(struct seccomp_filter *filter, + unsigned long flags) +{ + long ret; + + if (flags & ~SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + return -EINVAL; + + ret = mutex_lock_interruptible(&filter->notify_lock); + if (ret < 0) + return ret; + filter->notif->flags = flags; + mutex_unlock(&filter->notify_lock); + return 0; +} + static long seccomp_notify_addfd(struct seccomp_filter *filter, struct seccomp_notif_addfd __user *uaddfd, unsigned int size) @@ -1720,6 +1783,8 @@ static long seccomp_notify_ioctl(struct file *file, unsigned int cmd, case SECCOMP_IOCTL_NOTIF_ID_VALID_WRONG_DIR: case SECCOMP_IOCTL_NOTIF_ID_VALID: return seccomp_notify_id_valid(filter, buf); + case SECCOMP_IOCTL_NOTIF_SET_FLAGS: + return seccomp_notify_set_flags(filter, arg); } /* Extensible Argument ioctls */ @@ -1777,7 +1842,6 @@ static struct file *init_listener(struct seccomp_filter *filter) if (!filter->notif) goto out; - sema_init(&filter->notif->request, 0); filter->notif->next_id = get_random_u64(); INIT_LIST_HEAD(&filter->notif->notifications); diff --git a/kernel/smp.c b/kernel/smp.c index 385179dae360..8455a53465af 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -46,6 +46,8 @@ static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); +static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1); + static void __flush_smp_call_function_queue(bool warn_cpu_offline); int smpcfd_prepare_cpu(unsigned int cpu) @@ -253,13 +255,15 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 * *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); } if (cpu >= 0) { - dump_cpu_task(cpu); + if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0)) + dump_cpu_task(cpu); if (!cpu_cur_csd) { pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); } } - dump_stack(); + if (firsttime) + dump_stack(); *ts1 = ts2; return false; @@ -433,9 +437,14 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) struct llist_node *entry, *prev; struct llist_head *head; static bool warned; + atomic_t *tbt; lockdep_assert_irqs_disabled(); + /* Allow waiters to send backtrace NMI from here onwards */ + tbt = this_cpu_ptr(&trigger_backtrace); + atomic_set_release(tbt, 1); + head = this_cpu_ptr(&call_single_queue); entry = llist_del_all(head); entry = llist_reverse_order(entry); diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 88cbc1181b23..c108ed8a9804 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -473,8 +473,8 @@ static void clocksource_watchdog(struct timer_list *unused) /* Check the deviation from the watchdog clocksource. */ md = cs->uncertainty_margin + watchdog->uncertainty_margin; if (abs(cs_nsec - wd_nsec) > md) { - u64 cs_wd_msec; - u64 wd_msec; + s64 cs_wd_msec; + s64 wd_msec; u32 wd_rem; pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n", @@ -483,8 +483,8 @@ static void clocksource_watchdog(struct timer_list *unused) watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask); pr_warn(" '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx\n", cs->name, cs_nsec, csnow, cslast, cs->mask); - cs_wd_msec = div_u64_rem(cs_nsec - wd_nsec, 1000U * 1000U, &wd_rem); - wd_msec = div_u64_rem(wd_nsec, 1000U * 1000U, &wd_rem); + cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem); + wd_msec = div_s64_rem(wd_nsec, 1000 * 1000, &wd_rem); pr_warn(" Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms)\n", cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec); if (curr_clocksource == cs) diff --git a/kernel/torture.c b/kernel/torture.c index 1a0519b836ac..b28b05bbef02 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "rcu/rcu.h" MODULE_LICENSE("GPL"); @@ -54,6 +55,9 @@ module_param(verbose_sleep_frequency, int, 0444); static int verbose_sleep_duration = 1; module_param(verbose_sleep_duration, int, 0444); +static int random_shuffle; +module_param(random_shuffle, int, 0444); + static char *torture_type; static int verbose; @@ -88,8 +92,8 @@ int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_s ktime_t hto = baset_ns; if (trsp) - hto += (torture_random(trsp) >> 3) % fuzzt_ns; - set_current_state(TASK_UNINTERRUPTIBLE); + hto += torture_random(trsp) % fuzzt_ns; + set_current_state(TASK_IDLE); return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); } EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); @@ -350,22 +354,22 @@ torture_onoff(void *arg) if (onoff_holdoff > 0) { VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); - schedule_timeout_interruptible(onoff_holdoff); + torture_hrtimeout_jiffies(onoff_holdoff, &rand); VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); } while (!torture_must_stop()) { if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { - schedule_timeout_interruptible(HZ / 10); + torture_hrtimeout_jiffies(HZ / 10, &rand); continue; } - cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); + cpu = torture_random(&rand) % (maxcpu + 1); if (!torture_offline(cpu, &n_offline_attempts, &n_offline_successes, &sum_offline, &min_offline, &max_offline)) torture_online(cpu, &n_online_attempts, &n_online_successes, &sum_online, &min_online, &max_online); - schedule_timeout_interruptible(onoff_interval); + torture_hrtimeout_jiffies(onoff_interval, &rand); } stop: @@ -518,6 +522,7 @@ static void torture_shuffle_task_unregister_all(void) */ static void torture_shuffle_tasks(void) { + DEFINE_TORTURE_RANDOM(rand); struct shuffle_task *stp; cpumask_setall(shuffle_tmp_mask); @@ -537,8 +542,10 @@ static void torture_shuffle_tasks(void) cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); mutex_lock(&shuffle_task_mutex); - list_for_each_entry(stp, &shuffle_task_list, st_l) - set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); + list_for_each_entry(stp, &shuffle_task_list, st_l) { + if (!random_shuffle || torture_random(&rand) & 0x1) + set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); + } mutex_unlock(&shuffle_task_mutex); cpus_read_unlock(); @@ -550,9 +557,11 @@ static void torture_shuffle_tasks(void) */ static int torture_shuffle(void *arg) { + DEFINE_TORTURE_RANDOM(rand); + VERBOSE_TOROUT_STRING("torture_shuffle task started"); do { - schedule_timeout_interruptible(shuffle_interval); + torture_hrtimeout_jiffies(shuffle_interval, &rand); torture_shuffle_tasks(); torture_shutdown_absorb("torture_shuffle"); } while (!torture_must_stop()); @@ -728,12 +737,12 @@ bool stutter_wait(const char *title) cond_resched_tasks_rcu_qs(); spt = READ_ONCE(stutter_pause_test); for (; spt; spt = READ_ONCE(stutter_pause_test)) { - if (!ret) { + if (!ret && !rt_task(current)) { sched_set_normal(current, MAX_NICE); ret = true; } if (spt == 1) { - schedule_timeout_interruptible(1); + torture_hrtimeout_jiffies(1, NULL); } else if (spt == 2) { while (READ_ONCE(stutter_pause_test)) { if (!(i++ & 0xffff)) @@ -741,7 +750,7 @@ bool stutter_wait(const char *title) cond_resched(); } } else { - schedule_timeout_interruptible(round_jiffies_relative(HZ)); + torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL); } torture_shutdown_absorb(title); } @@ -926,7 +935,7 @@ EXPORT_SYMBOL_GPL(torture_kthread_stopping); * it starts, you will need to open-code your own. */ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, - char *f, struct task_struct **tp) + char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)) { int ret = 0; @@ -938,6 +947,10 @@ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, *tp = NULL; return ret; } + + if (cbf) + cbf(*tp); + wake_up_process(*tp); // Process is sleeping, so ordering provided. torture_shuffle_task_register(*tp); return ret; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index d6798513a8c2..73a0c8e41559 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1673,10 +1673,15 @@ menu "Debug kernel data structures" config DEBUG_LIST bool "Debug linked list manipulation" - depends on DEBUG_KERNEL || BUG_ON_DATA_CORRUPTION + depends on DEBUG_KERNEL + select LIST_HARDENED help - Enable this to turn on extended checks in the linked-list - walking routines. + Enable this to turn on extended checks in the linked-list walking + routines. + + This option trades better quality error reports for performance, and + is more suitable for kernel debugging. If you care about performance, + you should only enable CONFIG_LIST_HARDENED instead. If unsure, say N. @@ -1710,16 +1715,6 @@ config DEBUG_NOTIFIERS This is a relatively cheap check but if you care about maximum performance, say N. -config BUG_ON_DATA_CORRUPTION - bool "Trigger a BUG when data corruption is detected" - select DEBUG_LIST - help - Select this option if the kernel should BUG when it encounters - data corruption in kernel memory structures when they get checked - for validity. - - If unsure, say N. - config DEBUG_MAPLE_TREE bool "Debug maple trees" depends on DEBUG_KERNEL diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index efae7e011956..59e21bfec188 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -13,7 +13,7 @@ menuconfig UBSAN if UBSAN config UBSAN_TRAP - bool "On Sanitizer warnings, abort the running kernel code" + bool "Abort on Sanitizer warnings (smaller kernel but less verbose)" depends on !COMPILE_TEST help Building kernels with Sanitizer features enabled tends to grow @@ -26,6 +26,14 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. + Also note that selecting Y will cause your kernel to Oops + with an "illegal instruction" error with no further details + when a UBSAN violation occurs. (Except on arm64, which will + report which Sanitizer failed.) This may make it hard to + determine whether an Oops was caused by UBSAN or to figure + out the details of a UBSAN violation. It makes the kernel log + output less useful for bug reports. + config CC_HAS_UBSAN_BOUNDS_STRICT def_bool $(cc-option,-fsanitize=bounds-strict) help diff --git a/lib/Makefile b/lib/Makefile index 1ffae65bb7ee..d1397785ec16 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -167,7 +167,7 @@ obj-$(CONFIG_BTREE) += btree.o obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o -obj-$(CONFIG_DEBUG_LIST) += list_debug.o +obj-$(CONFIG_LIST_HARDENED) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o obj-$(CONFIG_BITREVERSE) += bitrev.o diff --git a/lib/list_debug.c b/lib/list_debug.c index d98d43f80958..db602417febf 100644 --- a/lib/list_debug.c +++ b/lib/list_debug.c @@ -2,7 +2,8 @@ * Copyright 2006, Red Hat, Inc., Dave Jones * Released under the General Public License (GPL). * - * This file contains the linked list validation for DEBUG_LIST. + * This file contains the linked list validation and error reporting for + * LIST_HARDENED and DEBUG_LIST. */ #include @@ -17,8 +18,9 @@ * attempt). */ -bool __list_add_valid(struct list_head *new, struct list_head *prev, - struct list_head *next) +__list_valid_slowpath +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) { if (CHECK_DATA_CORRUPTION(prev == NULL, "list_add corruption. prev is NULL.\n") || @@ -37,9 +39,10 @@ bool __list_add_valid(struct list_head *new, struct list_head *prev, return true; } -EXPORT_SYMBOL(__list_add_valid); +EXPORT_SYMBOL(__list_add_valid_or_report); -bool __list_del_entry_valid(struct list_head *entry) +__list_valid_slowpath +bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; @@ -65,6 +68,5 @@ bool __list_del_entry_valid(struct list_head *entry) return false; return true; - } -EXPORT_SYMBOL(__list_del_entry_valid); +EXPORT_SYMBOL(__list_del_entry_valid_or_report); diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 880fde13d9b8..a9841148cde2 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -7457,6 +7457,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 84c730da36dd..1ae39b9f4a95 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -440,4 +440,8 @@ static inline void debug_gimple_stmt(const_gimple s) #define SET_DECL_MODE(decl, mode) DECL_MODE(decl) = (mode) #endif +#if BUILDING_GCC_VERSION >= 14000 +#define last_stmt(x) last_nondebug_stmt(x) +#endif + #endif diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index 0f295961e773..2cff851ebfd7 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -279,6 +279,29 @@ config ZERO_CALL_USED_REGS endmenu +menu "Hardening of kernel data structures" + +config LIST_HARDENED + bool "Check integrity of linked list manipulation" + help + Minimal integrity checking in the linked-list manipulation routines + to catch memory corruptions that are not guaranteed to result in an + immediate access fault. + + If unsure, say N. + +config BUG_ON_DATA_CORRUPTION + bool "Trigger a BUG when data corruption is detected" + select LIST_HARDENED + help + Select this option if the kernel should BUG when it encounters + data corruption in kernel memory structures when they get checked + for validity. + + If unsure, say N. + +endmenu + config CC_HAS_RANDSTRUCT def_bool $(cc-option,-frandomize-layout-seed-file=/dev/null) # Randstruct was first added in Clang 15, but it isn't safe to use until diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index c9b3bd8f1bb9..7a0420cf1a6a 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -68,7 +68,7 @@ enum policy_rule_list { IMA_DEFAULT_POLICY = 1, IMA_CUSTOM_POLICY }; struct ima_rule_opt_list { size_t count; - char *items[]; + char *items[] __counted_by(count); }; /* @@ -342,6 +342,7 @@ static struct ima_rule_opt_list *ima_alloc_rule_opt_list(const substring_t *src) kfree(src_copy); return ERR_PTR(-ENOMEM); } + opt_list->count = count; /* * strsep() has already replaced all instances of '|' with '\0', @@ -357,7 +358,6 @@ static struct ima_rule_opt_list *ima_alloc_rule_opt_list(const substring_t *src) opt_list->items[i] = cur; cur = strchr(cur, '\0') + 1; } - opt_list->count = count; return opt_list; } diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index ebae964f7cc9..a9d40456a064 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -336,6 +336,7 @@ static int read_trusted_verity_root_digests(unsigned int fd) rc = -ENOMEM; goto err; } + trd->len = len; if (hex2bin(trd->data, d, len)) { kfree(trd); @@ -343,8 +344,6 @@ static int read_trusted_verity_root_digests(unsigned int fd) goto err; } - trd->len = len; - list_add_tail(&trd->node, &dm_verity_loadpin_trusted_root_digests); } diff --git a/tools/arch/x86/include/uapi/asm/unistd_32.h b/tools/arch/x86/include/uapi/asm/unistd_32.h index bc48a4dabe5d..4798f9d18fe8 100644 --- a/tools/arch/x86/include/uapi/asm/unistd_32.h +++ b/tools/arch/x86/include/uapi/asm/unistd_32.h @@ -26,3 +26,6 @@ #ifndef __NR_setns #define __NR_setns 346 #endif +#ifdef __NR_seccomp +#define __NR_seccomp 354 +#endif diff --git a/tools/arch/x86/include/uapi/asm/unistd_64.h b/tools/arch/x86/include/uapi/asm/unistd_64.h index f70d2cada256..d0f2043d7132 100644 --- a/tools/arch/x86/include/uapi/asm/unistd_64.h +++ b/tools/arch/x86/include/uapi/asm/unistd_64.h @@ -26,3 +26,6 @@ #ifndef __NR_getcpu #define __NR_getcpu 309 #endif +#ifndef __NR_seccomp +#define __NR_seccomp 317 +#endif diff --git a/tools/perf/bench/Build b/tools/perf/bench/Build index 0f158dc8139b..07bbc449329e 100644 --- a/tools/perf/bench/Build +++ b/tools/perf/bench/Build @@ -1,5 +1,6 @@ perf-y += sched-messaging.o perf-y += sched-pipe.o +perf-y += sched-seccomp-notify.o perf-y += syscall.o perf-y += mem-functions.o perf-y += futex-hash.o diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h index 0d2b65976212..a0625c77bea3 100644 --- a/tools/perf/bench/bench.h +++ b/tools/perf/bench/bench.h @@ -21,6 +21,7 @@ extern struct timeval bench__start, bench__end, bench__runtime; int bench_numa(int argc, const char **argv); int bench_sched_messaging(int argc, const char **argv); int bench_sched_pipe(int argc, const char **argv); +int bench_sched_seccomp_notify(int argc, const char **argv); int bench_syscall_basic(int argc, const char **argv); int bench_syscall_getpgid(int argc, const char **argv); int bench_syscall_fork(int argc, const char **argv); diff --git a/tools/perf/bench/sched-seccomp-notify.c b/tools/perf/bench/sched-seccomp-notify.c new file mode 100644 index 000000000000..b04ebcde4036 --- /dev/null +++ b/tools/perf/bench/sched-seccomp-notify.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "bench.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOOPS_DEFAULT 1000000UL +static uint64_t loops = LOOPS_DEFAULT; +static bool sync_mode; + +static const struct option options[] = { + OPT_U64('l', "loop", &loops, "Specify number of loops"), + OPT_BOOLEAN('s', "sync-mode", &sync_mode, + "Enable the synchronious mode for seccomp notifications"), + OPT_END() +}; + +static const char * const bench_seccomp_usage[] = { + "perf bench sched secccomp-notify ", + NULL +}; + +static int seccomp(unsigned int op, unsigned int flags, void *args) +{ + return syscall(__NR_seccomp, op, flags, args); +} + +static int user_notif_syscall(int nr, unsigned int flags) +{ + struct sock_filter filter[] = { + BPF_STMT(BPF_LD|BPF_W|BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP|BPF_JEQ|BPF_K, nr, 0, 1), + BPF_STMT(BPF_RET|BPF_K, SECCOMP_RET_USER_NOTIF), + BPF_STMT(BPF_RET|BPF_K, SECCOMP_RET_ALLOW), + }; + + struct sock_fprog prog = { + .len = (unsigned short)ARRAY_SIZE(filter), + .filter = filter, + }; + + return seccomp(SECCOMP_SET_MODE_FILTER, flags, &prog); +} + +#define USER_NOTIF_MAGIC INT_MAX +static void user_notification_sync_loop(int listener) +{ + struct seccomp_notif_resp resp; + struct seccomp_notif req; + uint64_t nr; + + for (nr = 0; nr < loops; nr++) { + memset(&req, 0, sizeof(req)); + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req)) + err(EXIT_FAILURE, "SECCOMP_IOCTL_NOTIF_RECV failed"); + + if (req.data.nr != __NR_gettid) + errx(EXIT_FAILURE, "unexpected syscall: %d", req.data.nr); + + resp.id = req.id; + resp.error = 0; + resp.val = USER_NOTIF_MAGIC; + resp.flags = 0; + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &resp)) + err(EXIT_FAILURE, "SECCOMP_IOCTL_NOTIF_SEND failed"); + } +} + +#ifndef SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) +#endif +int bench_sched_seccomp_notify(int argc, const char **argv) +{ + struct timeval start, stop, diff; + unsigned long long result_usec = 0; + int status, listener; + pid_t pid; + long ret; + + argc = parse_options(argc, argv, options, bench_seccomp_usage, 0); + + gettimeofday(&start, NULL); + + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + listener = user_notif_syscall(__NR_gettid, + SECCOMP_FILTER_FLAG_NEW_LISTENER); + if (listener < 0) + err(EXIT_FAILURE, "can't create a notification descriptor"); + + pid = fork(); + if (pid < 0) + err(EXIT_FAILURE, "fork"); + if (pid == 0) { + if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0)) + err(EXIT_FAILURE, "can't set the parent death signal"); + while (1) { + ret = syscall(__NR_gettid); + if (ret == USER_NOTIF_MAGIC) + continue; + break; + } + _exit(1); + } + + if (sync_mode) { + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP, 0)) + err(EXIT_FAILURE, + "can't set SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP"); + } + user_notification_sync_loop(listener); + + kill(pid, SIGKILL); + if (waitpid(pid, &status, 0) != pid) + err(EXIT_FAILURE, "waitpid(%d) failed", pid); + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) + errx(EXIT_FAILURE, "unexpected exit code: %d", status); + + gettimeofday(&stop, NULL); + timersub(&stop, &start, &diff); + + switch (bench_format) { + case BENCH_FORMAT_DEFAULT: + printf("# Executed %" PRIu64 " system calls\n\n", + loops); + + result_usec = diff.tv_sec * USEC_PER_SEC; + result_usec += diff.tv_usec; + + printf(" %14s: %lu.%03lu [sec]\n\n", "Total time", + (unsigned long) diff.tv_sec, + (unsigned long) (diff.tv_usec / USEC_PER_MSEC)); + + printf(" %14lf usecs/op\n", + (double)result_usec / (double)loops); + printf(" %14d ops/sec\n", + (int)((double)loops / + ((double)result_usec / (double)USEC_PER_SEC))); + break; + + case BENCH_FORMAT_SIMPLE: + printf("%lu.%03lu\n", + (unsigned long) diff.tv_sec, + (unsigned long) (diff.tv_usec / USEC_PER_MSEC)); + break; + + default: + /* reaching here is something disaster */ + fprintf(stderr, "Unknown format:%d\n", bench_format); + exit(1); + break; + } + + return 0; +} diff --git a/tools/perf/builtin-bench.c b/tools/perf/builtin-bench.c index db435b791a09..5033e8bab276 100644 --- a/tools/perf/builtin-bench.c +++ b/tools/perf/builtin-bench.c @@ -47,6 +47,7 @@ static struct bench numa_benchmarks[] = { static struct bench sched_benchmarks[] = { { "messaging", "Benchmark for scheduling and IPC", bench_sched_messaging }, { "pipe", "Benchmark for pipe() between two processes", bench_sched_pipe }, + { "seccomp-notify", "Benchmark for seccomp user notify", bench_sched_seccomp_notify}, { "all", "Run all scheduler benchmarks", NULL }, { NULL, NULL, NULL } }; diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 5fd49ad0c696..e05ac8261046 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -938,7 +938,11 @@ void __wait_for_test(struct __test_metadata *t) fprintf(TH_LOG_STREAM, "# %s: Test terminated by timeout\n", t->name); } else if (WIFEXITED(status)) { - if (t->termsig != -1) { + if (WEXITSTATUS(status) == 255) { + /* SKIP */ + t->passed = 1; + t->skip = 1; + } else if (t->termsig != -1) { t->passed = 0; fprintf(TH_LOG_STREAM, "# %s: Test exited normally instead of by signal (code: %d)\n", @@ -950,11 +954,6 @@ void __wait_for_test(struct __test_metadata *t) case 0: t->passed = 1; break; - /* SKIP */ - case 255: - t->passed = 1; - t->skip = 1; - break; /* Other failure, assume step report. */ default: t->passed = 0; diff --git a/tools/testing/selftests/rcutorture/bin/configcheck.sh b/tools/testing/selftests/rcutorture/bin/configcheck.sh index b92dfeb7fbbf..99162d18bad3 100755 --- a/tools/testing/selftests/rcutorture/bin/configcheck.sh +++ b/tools/testing/selftests/rcutorture/bin/configcheck.sh @@ -3,6 +3,8 @@ # # Usage: configcheck.sh .config .config-template # +# Non-empty output if errors detected. +# # Copyright (C) IBM Corporation, 2011 # # Authors: Paul E. McKenney @@ -10,32 +12,35 @@ T="`mktemp -d ${TMPDIR-/tmp}/configcheck.sh.XXXXXX`" trap 'rm -rf $T' 0 -sed -e 's/"//g' < $1 > $T/.config +# function test_kconfig_enabled ( Kconfig-var=val ) +function test_kconfig_enabled () { + if ! grep -q "^$1$" $T/.config + then + echo :$1: improperly set + return 1 + fi + return 0 +} -sed -e 's/"//g' -e 's/\(.*\)=n/# \1 is not set/' -e 's/^#CHECK#//' < $2 | -awk ' -{ - print "if grep -q \"" $0 "\" < '"$T/.config"'"; - print "then"; - print "\t:"; - print "else"; - if ($1 == "#") { - print "\tif grep -q \"" $2 "\" < '"$T/.config"'"; - print "\tthen"; - print "\t\tif test \"$firsttime\" = \"\"" - print "\t\tthen" - print "\t\t\tfirsttime=1" - print "\t\tfi" - print "\t\techo \":" $2 ": improperly set\""; - print "\telse"; - print "\t\t:"; - print "\tfi"; - } else { - print "\tif test \"$firsttime\" = \"\"" - print "\tthen" - print "\t\tfirsttime=1" - print "\tfi" - print "\techo \":" $0 ": improperly set\""; - } - print "fi"; - }' | sh +# function test_kconfig_disabled ( Kconfig-var ) +function test_kconfig_disabled () { + if grep -q "^$1=n$" $T/.config + then + return 0 + fi + if grep -q "^$1=" $T/.config + then + echo :$1=n: improperly set + return 1 + fi + return 0 +} + +sed -e 's/"//g' < $1 > $T/.config +sed -e 's/^#CHECK#//' < $2 > $T/ConfigFragment +grep '^CONFIG_.*=n$' $T/ConfigFragment | + sed -e 's/^/test_kconfig_disabled /' -e 's/=n$//' > $T/kconfig-n.sh +. $T/kconfig-n.sh +grep -v '^CONFIG_.*=n$' $T/ConfigFragment | grep '^CONFIG_' | + sed -e 's/^/test_kconfig_enabled /' > $T/kconfig-not-n.sh +. $T/kconfig-not-n.sh diff --git a/tools/testing/selftests/rcutorture/bin/functions.sh b/tools/testing/selftests/rcutorture/bin/functions.sh index 48b9147e8c91..b8e2ea23cb3f 100644 --- a/tools/testing/selftests/rcutorture/bin/functions.sh +++ b/tools/testing/selftests/rcutorture/bin/functions.sh @@ -45,7 +45,7 @@ checkarg () { configfrag_boot_params () { if test -r "$2.boot" then - echo $1 `grep -v '^#' "$2.boot" | tr '\012' ' '` + echo `grep -v '^#' "$2.boot" | tr '\012' ' '` $1 else echo $1 fi diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh index b582113178ac..f683e424ddd5 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh @@ -40,6 +40,10 @@ awk ' sum += $5 / 1000.; } +/rcu_scale: Grace-period kthread CPU time/ { + cputime = $6; +} + END { newNR = asort(gptimes); if (newNR <= 0) { @@ -78,6 +82,8 @@ END { print "90th percentile grace-period duration: " gptimes[pct90]; print "99th percentile grace-period duration: " gptimes[pct99]; print "Maximum grace-period duration: " gptimes[newNR]; - print "Grace periods: " ngps + 0 " Batches: " nbatches + 0 " Ratio: " ngps / nbatches; + if (cputime != "") + cpustr = " CPU: " cputime; + print "Grace periods: " ngps + 0 " Batches: " nbatches + 0 " Ratio: " ngps / nbatches cpustr; print "Computed from rcuscale printk output."; }' diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh index 1df7e695edf7..5be670dd4009 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh @@ -16,6 +16,8 @@ T=/tmp/kvm-recheck.sh.$$ trap 'rm -f $T' 0 2 +configerrors=0 + PATH=`pwd`/tools/testing/selftests/rcutorture/bin:$PATH; export PATH . functions.sh for rd in "$@" @@ -32,7 +34,7 @@ do fi TORTURE_SUITE="`cat $i/../torture_suite`" ; export TORTURE_SUITE configfile=`echo $i | sed -e 's,^.*/,,'` - rm -f $i/console.log.*.diags + rm -f $i/console.log.*.diags $i/ConfigFragment.diags case "${TORTURE_SUITE}" in X*) ;; @@ -49,8 +51,21 @@ do then echo QEMU killed fi - configcheck.sh $i/.config $i/ConfigFragment > $T 2>&1 - cat $T + configcheck.sh $i/.config $i/ConfigFragment > $i/ConfigFragment.diags 2>&1 + if grep -q '^CONFIG_KCSAN=y$' $i/ConfigFragment.input + then + # KCSAN forces a number of Kconfig options, so remove + # complaints about those Kconfig options in KCSAN runs. + mv $i/ConfigFragment.diags $i/ConfigFragment.diags.kcsan + grep -v -E 'CONFIG_PROVE_RCU|CONFIG_PREEMPT_COUNT' $i/ConfigFragment.diags.kcsan > $i/ConfigFragment.diags + fi + if test -s $i/ConfigFragment.diags + then + cat $i/ConfigFragment.diags + configerrors=$((configerrors+1)) + else + rm $i/ConfigFragment.diags + fi if test -r $i/Make.oldconfig.err then cat $i/Make.oldconfig.err @@ -65,7 +80,14 @@ do if test -f "$i/buildonly" then echo Build-only run, no boot/test - configcheck.sh $i/.config $i/ConfigFragment + configcheck.sh $i/.config $i/ConfigFragment > $i/ConfigFragment.diags 2>&1 + if test -s $i/ConfigFragment.diags + then + cat $i/ConfigFragment.diags + configerrors=$((configerrors+1)) + else + rm $i/ConfigFragment.diags + fi parse-build.sh $i/Make.out $configfile elif test -f "$i/qemu-cmd" then @@ -79,10 +101,10 @@ do done if test -f "$rd/kcsan.sum" then - if ! test -f $T + if ! test -f $i/ConfigFragment.diags then : - elif grep -q CONFIG_KCSAN=y $T + elif grep -q CONFIG_KCSAN=y $i/ConfigFragment.diags then echo "Compiler or architecture does not support KCSAN!" echo Did you forget to switch your compiler with '--kmake-arg CC='? @@ -94,17 +116,23 @@ do fi fi done + +if test "$configerrors" -gt 0 +then + echo $configerrors runs with .config errors. + ret=1 +fi EDITOR=echo kvm-find-errors.sh "${@: -1}" > $T 2>&1 builderrors="`tr ' ' '\012' < $T | grep -c '/Make.out.diags'`" if test "$builderrors" -gt 0 then echo $builderrors runs with build errors. - ret=1 + ret=2 fi runerrors="`tr ' ' '\012' < $T | grep -c '/console.log.diags'`" if test "$runerrors" -gt 0 then echo $runerrors runs with runtime errors. - ret=2 + ret=3 fi exit $ret diff --git a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh index a2328163eba1..134cdef5a6e0 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh @@ -137,14 +137,20 @@ chmod +x $T/bin/kvm-remote-*.sh # Check first to avoid the need for cleanup for system-name typos for i in $systems do - ncpus="`ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN 2> /dev/null`" + ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr ret=$? if test "$ret" -ne 0 then - echo System $i unreachable, giving up. | tee -a "$oldrun/remote-log" + echo "System $i unreachable ($ret), giving up." | tee -a "$oldrun/remote-log" + echo ' --- ssh stdout: vvv' | tee -a "$oldrun/remote-log" + cat $T/ssh.stdout | tee -a "$oldrun/remote-log" + echo ' --- ssh stdout: ^^^' | tee -a "$oldrun/remote-log" + echo ' --- ssh stderr: vvv' | tee -a "$oldrun/remote-log" + cat $T/ssh.stderr | tee -a "$oldrun/remote-log" + echo ' --- ssh stderr: ^^^' | tee -a "$oldrun/remote-log" exit 4 fi - echo $i: $ncpus CPUs " " `date` | tee -a "$oldrun/remote-log" + echo $i: `cat $T/ssh.stdout` CPUs " " `date` | tee -a "$oldrun/remote-log" done # Download and expand the tarball on all systems. diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index d2a3710a5f2a..b33cd8753689 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh @@ -9,9 +9,10 @@ # # Usage: kvm-test-1-run.sh config resdir seconds qemu-args boot_args_in # -# qemu-args defaults to "-enable-kvm -nographic", along with arguments -# specifying the number of CPUs and other options -# generated from the underlying CPU architecture. +# qemu-args defaults to "-enable-kvm -display none -no-reboot", along +# with arguments specifying the number of CPUs +# and other options generated from the underlying +# CPU architecture. # boot_args_in defaults to value returned by the per_version_boot_params # shell function. # @@ -57,7 +58,6 @@ config_override_param () { cat $T/Kconfig_args >> $resdir/ConfigFragment.input config_override.sh $T/$2 $T/Kconfig_args > $T/$2.tmp mv $T/$2.tmp $T/$2 - # Note that "#CHECK#" is not permitted on commandline. fi } @@ -140,7 +140,7 @@ then fi # Generate -smp qemu argument. -qemu_args="-enable-kvm -nographic $qemu_args" +qemu_args="-enable-kvm -display none -no-reboot $qemu_args" cpu_count=`configNR_CPUS.sh $resdir/ConfigFragment` cpu_count=`configfrag_boot_cpus "$boot_args_in" "$config_template" "$cpu_count"` if test "$cpu_count" -gt "$TORTURE_ALLOTED_CPUS" @@ -163,7 +163,7 @@ boot_args="`configfrag_boot_params "$boot_args_in" "$config_template"`" boot_args="`per_version_boot_params "$boot_args" $resdir/.config $seconds`" if test -n "$TORTURE_BOOT_GDB_ARG" then - boot_args="$boot_args $TORTURE_BOOT_GDB_ARG" + boot_args="$TORTURE_BOOT_GDB_ARG $boot_args" fi # Give bare-metal advice diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index d3cdc2d33d4b..b0f36a638a69 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -186,7 +186,7 @@ do fi ;; --kconfig|--kconfigs) - checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\)*$' '^error$' + checkarg --kconfig "(Kconfig options)" $# "$2" '^\(#CHECK#\)\?CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\( \(#CHECK#\)\?CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\)*$' '^error$' TORTURE_KCONFIG_ARG="`echo "$TORTURE_KCONFIG_ARG $2" | sed -e 's/^ *//' -e 's/ *$//'`" shift ;; diff --git a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh index 71f0dfbb2a6d..212c52ca90b5 100755 --- a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh +++ b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh @@ -10,7 +10,6 @@ D=tools/testing/selftests/rcutorture # Prerequisite checks -[ -z "$D" ] && echo >&2 "No argument supplied" && exit 1 if [ ! -d "$D" ]; then echo >&2 "$D does not exist: Malformed kernel source tree?" exit 1 @@ -34,12 +33,16 @@ cat > init.c << '___EOF___' volatile unsigned long delaycount; -int main(int argc, int argv[]) +int main(int argc, char *argv[]) { int i; struct timeval tv; struct timeval tvb; + printf("Torture-test rudimentary init program started, command line:\n"); + for (i = 0; i < argc; i++) + printf(" %s", argv[i]); + printf("\n"); for (;;) { sleep(1); /* Need some userspace time. */ @@ -64,15 +67,23 @@ ___EOF___ # build using nolibc on supported archs (smaller executable) and fall # back to regular glibc on other ones. if echo -e "#if __x86_64__||__i386__||__i486__||__i586__||__i686__" \ - "||__ARM_EABI__||__aarch64__||__s390x__\nyes\n#endif" \ + "||__ARM_EABI__||__aarch64__||__s390x__||__loongarch__\nyes\n#endif" \ | ${CROSS_COMPILE}gcc -E -nostdlib -xc - \ | grep -q '^yes'; then # architecture supported by nolibc ${CROSS_COMPILE}gcc -fno-asynchronous-unwind-tables -fno-ident \ -nostdlib -include ../../../../include/nolibc/nolibc.h \ -s -static -Os -o init init.c -lgcc + ret=$? else ${CROSS_COMPILE}gcc -s -static -Os -o init init.c + ret=$? +fi + +if [ "$ret" -ne 0 ] +then + echo "Failed to create a statically linked C-language initrd" + exit "$ret" fi rm init.c diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index 5a2ae2264403..12b50a4a881a 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -55,6 +55,8 @@ do_kasan=yes do_kcsan=no do_clocksourcewd=yes do_rt=yes +do_rcutasksflavors=yes +do_srcu_lockdep=yes # doyesno - Helper function for yes/no arguments function doyesno () { @@ -73,18 +75,20 @@ usage () { echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\"" echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\"" echo " --do-all" - echo " --do-allmodconfig / --do-no-allmodconfig" - echo " --do-clocksourcewd / --do-no-clocksourcewd" - echo " --do-kasan / --do-no-kasan" - echo " --do-kcsan / --do-no-kcsan" - echo " --do-kvfree / --do-no-kvfree" - echo " --do-locktorture / --do-no-locktorture" + echo " --do-allmodconfig / --do-no-allmodconfig / --no-allmodconfig" + echo " --do-clocksourcewd / --do-no-clocksourcewd / --no-clocksourcewd" + echo " --do-kasan / --do-no-kasan / --no-kasan" + echo " --do-kcsan / --do-no-kcsan / --no-kcsan" + echo " --do-kvfree / --do-no-kvfree / --no-kvfree" + echo " --do-locktorture / --do-no-locktorture / --no-locktorture" echo " --do-none" - echo " --do-rcuscale / --do-no-rcuscale" - echo " --do-rcutorture / --do-no-rcutorture" - echo " --do-refscale / --do-no-refscale" - echo " --do-rt / --do-no-rt" - echo " --do-scftorture / --do-no-scftorture" + echo " --do-rcuscale / --do-no-rcuscale / --no-rcuscale" + echo " --do-rcutasksflavors / --do-no-rcutasksflavors / --no-rcutasksflavors" + echo " --do-rcutorture / --do-no-rcutorture / --no-rcutorture" + echo " --do-refscale / --do-no-refscale / --no-refscale" + echo " --do-rt / --do-no-rt / --no-rt" + echo " --do-scftorture / --do-no-scftorture / --no-scftorture" + echo " --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep" echo " --duration [ | h | d ]" echo " --kcsan-kmake-arg kernel-make-arguments" exit 1 @@ -115,6 +119,7 @@ do ;; --do-all|--doall) do_allmodconfig=yes + do_rcutasksflavor=yes do_rcutorture=yes do_locktorture=yes do_scftorture=yes @@ -125,27 +130,29 @@ do do_kasan=yes do_kcsan=yes do_clocksourcewd=yes + do_srcu_lockdep=yes ;; - --do-allmodconfig|--do-no-allmodconfig) + --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig) do_allmodconfig=`doyesno "$1" --do-allmodconfig` ;; - --do-clocksourcewd|--do-no-clocksourcewd) + --do-clocksourcewd|--do-no-clocksourcewd|--no-clocksourcewd) do_clocksourcewd=`doyesno "$1" --do-clocksourcewd` ;; - --do-kasan|--do-no-kasan) + --do-kasan|--do-no-kasan|--no-kasan) do_kasan=`doyesno "$1" --do-kasan` ;; - --do-kcsan|--do-no-kcsan) + --do-kcsan|--do-no-kcsan|--no-kcsan) do_kcsan=`doyesno "$1" --do-kcsan` ;; - --do-kvfree|--do-no-kvfree) + --do-kvfree|--do-no-kvfree|--no-kvfree) do_kvfree=`doyesno "$1" --do-kvfree` ;; - --do-locktorture|--do-no-locktorture) + --do-locktorture|--do-no-locktorture|--no-locktorture) do_locktorture=`doyesno "$1" --do-locktorture` ;; --do-none|--donone) do_allmodconfig=no + do_rcutasksflavors=no do_rcutorture=no do_locktorture=no do_scftorture=no @@ -156,22 +163,29 @@ do do_kasan=no do_kcsan=no do_clocksourcewd=no + do_srcu_lockdep=no ;; - --do-rcuscale|--do-no-rcuscale) + --do-rcuscale|--do-no-rcuscale|--no-rcuscale) do_rcuscale=`doyesno "$1" --do-rcuscale` ;; - --do-rcutorture|--do-no-rcutorture) + --do-rcutasksflavors|--do-no-rcutasksflavors|--no-rcutasksflavors) + do_rcutasksflavors=`doyesno "$1" --do-rcutasksflavors` + ;; + --do-rcutorture|--do-no-rcutorture|--no-rcutorture) do_rcutorture=`doyesno "$1" --do-rcutorture` ;; - --do-refscale|--do-no-refscale) + --do-refscale|--do-no-refscale|--no-refscale) do_refscale=`doyesno "$1" --do-refscale` ;; - --do-rt|--do-no-rt) + --do-rt|--do-no-rt|--no-rt) do_rt=`doyesno "$1" --do-rt` ;; - --do-scftorture|--do-no-scftorture) + --do-scftorture|--do-no-scftorture|--no-scftorture) do_scftorture=`doyesno "$1" --do-scftorture` ;; + --do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep) + do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep` + ;; --duration) checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' mult=1 @@ -361,6 +375,40 @@ then fi fi +# Test building RCU Tasks flavors in isolation, both SMP and !SMP +if test "$do_rcutasksflavors" = "yes" +then + echo " --- rcutasksflavors:" Start `date` | tee -a $T/log + rtfdir="tools/testing/selftests/rcutorture/res/$ds/results-rcutasksflavors" + mkdir -p "$rtfdir" + cat > $T/rcutasksflavors << __EOF__ +#CHECK#CONFIG_TASKS_RCU=n +#CHECK#CONFIG_TASKS_RUDE_RCU=n +#CHECK#CONFIG_TASKS_TRACE_RCU=n +__EOF__ + for flavor in CONFIG_TASKS_RCU CONFIG_TASKS_RUDE_RCU CONFIG_TASKS_TRACE_RCU + do + forceflavor="`echo $flavor | sed -e 's/^CONFIG/CONFIG_FORCE/'`" + deselectedflavors="`grep -v $flavor $T/rcutasksflavors | tr '\012' ' ' | tr -s ' ' | sed -e 's/ *$//'`" + echo " --- Running RCU Tasks Trace flavor $flavor `date`" >> $rtfdir/log + tools/testing/selftests/rcutorture/bin/kvm.sh --datestamp "$ds/results-rcutasksflavors/$flavor" --buildonly --configs "TINY01 TREE04" --kconfig "CONFIG_RCU_EXPERT=y CONFIG_RCU_SCALE_TEST=y $forceflavor=y $deselectedflavors" --trust-make > $T/$flavor.out 2>&1 + retcode=$? + if test "$retcode" -ne 0 + then + break + fi + done + if test "$retcode" -eq 0 + then + echo "rcutasksflavors($retcode)" $rtfdir >> $T/successes + echo Success >> $rtfdir/log + else + echo "rcutasksflavors($retcode)" $rtfdir >> $T/failures + echo " --- rcutasksflavors Test summary:" >> $rtfdir/log + echo " --- Summary: Exit code $retcode from $flavor, see Make.out" >> $rtfdir/log + fi +fi + # --torture rcu if test "$do_rcutorture" = "yes" then @@ -376,8 +424,10 @@ fi if test "$do_scftorture" = "yes" then + # Scale memory based on the number of CPUs. + scfmem=$((2+HALF_ALLOTED_CPUS/16)) torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1" - torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make + torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory ${scfmem}G --trust-make fi if test "$do_rt" = "yes" @@ -391,6 +441,23 @@ then torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make fi +if test "$do_srcu_lockdep" = "yes" +then + echo " --- do-srcu-lockdep:" Start `date` | tee -a $T/log + tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh --datestamp "$ds/results-srcu-lockdep" > $T/srcu_lockdep.sh.out 2>&1 + retcode=$? + cp $T/srcu_lockdep.sh.out "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + if test "$retcode" -eq 0 + then + echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/successes + echo Success >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + else + echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/failures + echo " --- srcu_lockdep Test Summary:" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + echo " --- Summary: Exit code $retcode from srcu_lockdep.sh, see ds/results-srcu-lockdep" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + fi +fi + if test "$do_refscale" = yes then primlist="`grep '\.name[ ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" @@ -541,11 +608,23 @@ then fi echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log +tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" +find "$tdir" -name 'ConfigFragment.diags' -print > $T/configerrors +find "$tdir" -name 'Make.out.diags' -print > $T/builderrors +if test -s "$T/configerrors" +then + echo " Scenarios with .config errors: `wc -l "$T/configerrors" | awk '{ print $1 }'`" + nonkcsanbug="yes" +fi +if test -s "$T/builderrors" +then + echo " Scenarios with build errors: `wc -l "$T/builderrors" | awk '{ print $1 }'`" + nonkcsanbug="yes" +fi if test -z "$nonkcsanbug" && test -s "$T/failuresum" then echo " All bugs were KCSAN failures." fi -tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" if test -n "$tdir" && test $compress_concurrency -gt 0 then # KASAN vmlinux files can approach 1GB in size, so compress them. diff --git a/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh index d3e4b2971f92..e7bb32709d78 100644 --- a/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh @@ -22,8 +22,9 @@ locktorture_param_onoff () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `locktorture_param_onoff "$1" "$2"` \ + echo `locktorture_param_onoff "$1" "$2"` \ locktorture.stat_interval=15 \ locktorture.shutdown_secs=$3 \ - locktorture.verbose=1 + locktorture.verbose=1 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 b/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 index dea26c568678..2ef2fb69c360 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 @@ -6,6 +6,5 @@ CONFIG_PREEMPT=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=n CONFIG_NO_HZ_FULL=y -#CHECK#CONFIG_RCU_EXPERT=n CONFIG_TASKS_RCU=y CONFIG_RCU_EXPERT=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 index 04831ef1f9b5..8ae41d5f81a3 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 @@ -15,4 +15,3 @@ CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n CONFIG_RCU_EXPERT=y -CONFIG_BOOTPARAM_HOTPLUG_CPU0=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh index e2bc99c785e7..c044df386876 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh @@ -46,10 +46,11 @@ rcutorture_param_stat_interval () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `rcutorture_param_onoff "$1" "$2"` \ + echo `rcutorture_param_onoff "$1" "$2"` \ `rcutorture_param_n_barrier_cbs "$1"` \ `rcutorture_param_stat_interval "$1"` \ rcutorture.shutdown_secs=$3 \ rcutorture.test_no_idle_hz=1 \ - rcutorture.verbose=1 + rcutorture.verbose=1 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon b/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon index 6a00157bee5b..b1ffd7c67604 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon @@ -2,5 +2,7 @@ CONFIG_RCU_SCALE_TEST=y CONFIG_PRINTK_TIME=y CONFIG_FORCE_TASKS_RCU=y #CHECK#CONFIG_TASKS_RCU=y +CONFIG_FORCE_TASKS_RUDE_RCU=y +#CHECK#CONFIG_TASKS_RUDE_RCU=y CONFIG_FORCE_TASKS_TRACE_RCU=y #CHECK#CONFIG_TASKS_TRACE_RCU=y diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 b/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 index 227aba7783af..0059592c7408 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 @@ -2,6 +2,8 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +#CHECK#CONFIG_TREE_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh index ffbe15109f0d..28070b43f017 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh @@ -11,6 +11,7 @@ # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 rcuscale.shutdown=1 \ - rcuscale.verbose=0 + echo rcuscale.shutdown=1 \ + rcuscale.verbose=0 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT index ef2b501a6971..67f9d2998afd 100644 --- a/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT +++ b/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT @@ -2,6 +2,7 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n #CHECK#CONFIG_PREEMPT_RCU=n CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y diff --git a/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh index f81fa2c541a6..748465627601 100644 --- a/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh @@ -11,6 +11,7 @@ # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 refscale.shutdown=1 \ - refscale.verbose=0 + echo refscale.shutdown=1 \ + refscale.verbose=0 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT index 3a59346b3de7..6133f54ce2a7 100644 --- a/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT +++ b/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT @@ -2,6 +2,8 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +#CHECK#CONFIG_PREEMPT_RCU=n CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=n CONFIG_NO_HZ_FULL=y diff --git a/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh index 2d949e58f5a5..7637f68ef0ce 100644 --- a/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh @@ -22,8 +22,9 @@ scftorture_param_onoff () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `scftorture_param_onoff "$1" "$2"` \ + echo `scftorture_param_onoff "$1" "$2"` \ scftorture.stat_interval=15 \ scftorture.shutdown_secs=$3 \ - scftorture.verbose=1 + scftorture.verbose=1 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore deleted file mode 100644 index 24e27957efcc..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -srcu.c diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile deleted file mode 100644 index 4bed0b678f8b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -all: srcu.c store_buffering - -LINUX_SOURCE = ../../../../../.. - -modified_srcu_input = $(LINUX_SOURCE)/include/linux/srcu.h \ - $(LINUX_SOURCE)/kernel/rcu/srcu.c - -modified_srcu_output = include/linux/srcu.h srcu.c - -include/linux/srcu.h: srcu.c - -srcu.c: modify_srcu.awk Makefile $(modified_srcu_input) - awk -f modify_srcu.awk $(modified_srcu_input) $(modified_srcu_output) - -store_buffering: - @cd tests/store_buffering; make diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/delay.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/delay.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/export.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/export.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/mutex.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/mutex.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/percpu.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/percpu.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/preempt.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/preempt.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/rcupdate.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/rcupdate.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/sched.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/sched.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/smp.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/smp.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/workqueue.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/workqueue.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/uapi/linux/types.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/uapi/linux/types.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore deleted file mode 100644 index 57d296341304..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -srcu.h diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h deleted file mode 100644 index f2860dd1b407..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h deleted file mode 100644 index 8bc960e5e713..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h +++ /dev/null @@ -1,152 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This header has been modifies to remove definitions of types that - * are defined in standard userspace headers or are problematic for some - * other reason. - */ - -#ifndef _LINUX_TYPES_H -#define _LINUX_TYPES_H - -#define __EXPORTED_HEADERS__ -#include - -#ifndef __ASSEMBLY__ - -#define DECLARE_BITMAP(name, bits) \ - unsigned long name[BITS_TO_LONGS(bits)] - -typedef __u32 __kernel_dev_t; - -/* bsd */ -typedef unsigned char u_char; -typedef unsigned short u_short; -typedef unsigned int u_int; -typedef unsigned long u_long; - -/* sysv */ -typedef unsigned char unchar; -typedef unsigned short ushort; -typedef unsigned int uint; -typedef unsigned long ulong; - -#ifndef __BIT_TYPES_DEFINED__ -#define __BIT_TYPES_DEFINED__ - -typedef __u8 u_int8_t; -typedef __s8 int8_t; -typedef __u16 u_int16_t; -typedef __s16 int16_t; -typedef __u32 u_int32_t; -typedef __s32 int32_t; - -#endif /* !(__BIT_TYPES_DEFINED__) */ - -typedef __u8 uint8_t; -typedef __u16 uint16_t; -typedef __u32 uint32_t; - -/* this is a special 64bit data type that is 8-byte aligned */ -#define aligned_u64 __u64 __attribute__((aligned(8))) -#define aligned_be64 __be64 __attribute__((aligned(8))) -#define aligned_le64 __le64 __attribute__((aligned(8))) - -/** - * The type used for indexing onto a disc or disc partition. - * - * Linux always considers sectors to be 512 bytes long independently - * of the devices real block size. - * - * blkcnt_t is the type of the inode's block count. - */ -typedef u64 sector_t; - -/* - * The type of an index into the pagecache. - */ -#define pgoff_t unsigned long - -/* - * A dma_addr_t can hold any valid DMA address, i.e., any address returned - * by the DMA API. - * - * If the DMA API only uses 32-bit addresses, dma_addr_t need only be 32 - * bits wide. Bus addresses, e.g., PCI BARs, may be wider than 32 bits, - * but drivers do memory-mapped I/O to ioremapped kernel virtual addresses, - * so they don't care about the size of the actual bus addresses. - */ -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT -typedef u64 dma_addr_t; -#else -typedef u32 dma_addr_t; -#endif - -#ifdef CONFIG_PHYS_ADDR_T_64BIT -typedef u64 phys_addr_t; -#else -typedef u32 phys_addr_t; -#endif - -typedef phys_addr_t resource_size_t; - -/* - * This type is the placeholder for a hardware interrupt number. It has to be - * big enough to enclose whatever representation is used by a given platform. - */ -typedef unsigned long irq_hw_number_t; - -typedef struct { - int counter; -} atomic_t; - -#ifdef CONFIG_64BIT -typedef struct { - long counter; -} atomic64_t; -#endif - -struct list_head { - struct list_head *next, *prev; -}; - -struct hlist_head { - struct hlist_node *first; -}; - -struct hlist_node { - struct hlist_node *next, **pprev; -}; - -/** - * struct callback_head - callback structure for use with RCU and task_work - * @next: next update requests in a list - * @func: actual update function to call after the grace period. - * - * The struct is aligned to size of pointer. On most architectures it happens - * naturally due ABI requirements, but some architectures (like CRIS) have - * weird ABI and we need to ask it explicitly. - * - * The alignment is required to guarantee that bits 0 and 1 of @next will be - * clear under normal conditions -- as long as we use call_rcu() or - * call_srcu() to queue callback. - * - * This guarantee is important for few reasons: - * - future call_rcu_lazy() will make use of lower bits in the pointer; - * - the structure shares storage spacer in struct page with @compound_head, - * which encode PageTail() in bit 0. The guarantee is needed to avoid - * false-positive PageTail(). - */ -struct callback_head { - struct callback_head *next; - void (*func)(struct callback_head *head); -} __attribute__((aligned(sizeof(void *)))); -#define rcu_head callback_head - -typedef void (*rcu_callback_t)(struct rcu_head *head); -typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); - -/* clocksource cycle base type */ -typedef u64 cycle_t; - -#endif /* __ASSEMBLY__ */ -#endif /* _LINUX_TYPES_H */ diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk deleted file mode 100755 index e05182d3e47d..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/awk -f -# SPDX-License-Identifier: GPL-2.0 - -# Modify SRCU for formal verification. The first argument should be srcu.h and -# the second should be srcu.c. Outputs modified srcu.h and srcu.c into the -# current directory. - -BEGIN { - if (ARGC != 5) { - print "Usange: input.h input.c output.h output.c" > "/dev/stderr"; - exit 1; - } - h_output = ARGV[3]; - c_output = ARGV[4]; - ARGC = 3; - - # Tokenize using FS and not RS as FS supports regular expressions. Each - # record is one line of source, except that backslashed lines are - # combined. Comments are treated as field separators, as are quotes. - quote_regexp="\"([^\\\\\"]|\\\\.)*\""; - comment_regexp="\\/\\*([^*]|\\*+[^*/])*\\*\\/|\\/\\/.*(\n|$)"; - FS="([ \\\\\t\n\v\f;,.=(){}+*/<>&|^-]|\\[|\\]|" comment_regexp "|" quote_regexp ")+"; - - inside_srcu_struct = 0; - inside_srcu_init_def = 0; - srcu_init_param_name = ""; - in_macro = 0; - brace_nesting = 0; - paren_nesting = 0; - - # Allow the manipulation of the last field separator after has been - # seen. - last_fs = ""; - # Whether the last field separator was intended to be output. - last_fs_print = 0; - - # rcu_batches stores the initialization for each instance of struct - # rcu_batch - - in_comment = 0; - - outputfile = ""; -} - -{ - prev_outputfile = outputfile; - if (FILENAME ~ /\.h$/) { - outputfile = h_output; - if (FNR != NR) { - print "Incorrect file order" > "/dev/stderr"; - exit 1; - } - } - else - outputfile = c_output; - - if (prev_outputfile && outputfile != prev_outputfile) { - new_outputfile = outputfile; - outputfile = prev_outputfile; - update_fieldsep("", 0); - outputfile = new_outputfile; - } -} - -# Combine the next line into $0. -function combine_line() { - ret = getline next_line; - if (ret == 0) { - # Don't allow two consecutive getlines at the end of the file - if (eof_found) { - print "Error: expected more input." > "/dev/stderr"; - exit 1; - } else { - eof_found = 1; - } - } else if (ret == -1) { - print "Error reading next line of file" FILENAME > "/dev/stderr"; - exit 1; - } - $0 = $0 "\n" next_line; -} - -# Combine backslashed lines and multiline comments. -function combine_backslashes() { - while (/\\$|\/\*([^*]|\*+[^*\/])*\**$/) { - combine_line(); - } -} - -function read_line() { - combine_line(); - combine_backslashes(); -} - -# Print out field separators and update variables that depend on them. Only -# print if p is true. Call with sep="" and p=0 to print out the last field -# separator. -function update_fieldsep(sep, p) { - # Count braces - sep_tmp = sep; - gsub(quote_regexp "|" comment_regexp, "", sep_tmp); - while (1) - { - if (sub("[^{}()]*\\{", "", sep_tmp)) { - brace_nesting++; - continue; - } - if (sub("[^{}()]*\\}", "", sep_tmp)) { - brace_nesting--; - if (brace_nesting < 0) { - print "Unbalanced braces!" > "/dev/stderr"; - exit 1; - } - continue; - } - if (sub("[^{}()]*\\(", "", sep_tmp)) { - paren_nesting++; - continue; - } - if (sub("[^{}()]*\\)", "", sep_tmp)) { - paren_nesting--; - if (paren_nesting < 0) { - print "Unbalanced parenthesis!" > "/dev/stderr"; - exit 1; - } - continue; - } - - break; - } - - if (last_fs_print) - printf("%s", last_fs) > outputfile; - last_fs = sep; - last_fs_print = p; -} - -# Shifts the fields down by n positions. Calls next if there are no more. If p -# is true then print out field separators. -function shift_fields(n, p) { - do { - if (match($0, FS) > 0) { - update_fieldsep(substr($0, RSTART, RLENGTH), p); - if (RSTART + RLENGTH <= length()) - $0 = substr($0, RSTART + RLENGTH); - else - $0 = ""; - } else { - update_fieldsep("", 0); - print "" > outputfile; - next; - } - } while (--n > 0); -} - -# Shifts and prints the first n fields. -function print_fields(n) { - do { - update_fieldsep("", 0); - printf("%s", $1) > outputfile; - shift_fields(1, 1); - } while (--n > 0); -} - -{ - combine_backslashes(); -} - -# Print leading FS -{ - if (match($0, "^(" FS ")+") > 0) { - update_fieldsep(substr($0, RSTART, RLENGTH), 1); - if (RSTART + RLENGTH <= length()) - $0 = substr($0, RSTART + RLENGTH); - else - $0 = ""; - } -} - -# Parse the line. -{ - while (NF > 0) { - if ($1 == "struct" && NF < 3) { - read_line(); - continue; - } - - if (FILENAME ~ /\.h$/ && !inside_srcu_struct && - brace_nesting == 0 && paren_nesting == 0 && - $1 == "struct" && $2 == "srcu_struct" && - $0 ~ "^struct(" FS ")+srcu_struct(" FS ")+\\{") { - inside_srcu_struct = 1; - print_fields(2); - continue; - } - if (inside_srcu_struct && brace_nesting == 0 && - paren_nesting == 0) { - inside_srcu_struct = 0; - update_fieldsep("", 0); - for (name in rcu_batches) - print "extern struct rcu_batch " name ";" > outputfile; - } - - if (inside_srcu_struct && $1 == "struct" && $2 == "rcu_batch") { - # Move rcu_batches outside of the struct. - rcu_batches[$3] = ""; - shift_fields(3, 1); - sub(/;[[:space:]]*$/, "", last_fs); - continue; - } - - if (FILENAME ~ /\.h$/ && !inside_srcu_init_def && - $1 == "#define" && $2 == "__SRCU_STRUCT_INIT") { - inside_srcu_init_def = 1; - srcu_init_param_name = $3; - in_macro = 1; - print_fields(3); - continue; - } - if (inside_srcu_init_def && brace_nesting == 0 && - paren_nesting == 0) { - inside_srcu_init_def = 0; - in_macro = 0; - continue; - } - - if (inside_srcu_init_def && brace_nesting == 1 && - paren_nesting == 0 && last_fs ~ /\.[[:space:]]*$/ && - $1 ~ /^[[:alnum:]_]+$/) { - name = $1; - if (name in rcu_batches) { - # Remove the dot. - sub(/\.[[:space:]]*$/, "", last_fs); - - old_record = $0; - do - shift_fields(1, 0); - while (last_fs !~ /,/ || paren_nesting > 0); - end_loc = length(old_record) - length($0); - end_loc += index(last_fs, ",") - length(last_fs); - - last_fs = substr(last_fs, index(last_fs, ",") + 1); - last_fs_print = 1; - - match(old_record, "^"name"("FS")+="); - start_loc = RSTART + RLENGTH; - - len = end_loc - start_loc; - initializer = substr(old_record, start_loc, len); - gsub(srcu_init_param_name "\\.", "", initializer); - rcu_batches[name] = initializer; - continue; - } - } - - # Don't include a nonexistent file - if (!in_macro && $1 == "#include" && /^#include[[:space:]]+"rcu\.h"/) { - update_fieldsep("", 0); - next; - } - - # Ignore most preprocessor stuff. - if (!in_macro && $1 ~ /#/) { - break; - } - - if (brace_nesting > 0 && $1 ~ "^[[:alnum:]_]+$" && NF < 2) { - read_line(); - continue; - } - if (brace_nesting > 0 && - $0 ~ "^[[:alnum:]_]+[[:space:]]*(\\.|->)[[:space:]]*[[:alnum:]_]+" && - $2 in rcu_batches) { - # Make uses of rcu_batches global. Somewhat unreliable. - shift_fields(1, 0); - print_fields(1); - continue; - } - - if ($1 == "static" && NF < 3) { - read_line(); - continue; - } - if ($1 == "static" && ($2 == "bool" && $3 == "try_check_zero" || - $2 == "void" && $3 == "srcu_flip")) { - shift_fields(1, 1); - print_fields(2); - continue; - } - - # Distinguish between read-side and write-side memory barriers. - if ($1 == "smp_mb" && NF < 2) { - read_line(); - continue; - } - if (match($0, /^smp_mb[[:space:]();\/*]*[[:alnum:]]/)) { - barrier_letter = substr($0, RLENGTH, 1); - if (barrier_letter ~ /A|D/) - new_barrier_name = "sync_smp_mb"; - else if (barrier_letter ~ /B|C/) - new_barrier_name = "rs_smp_mb"; - else { - print "Unrecognized memory barrier." > "/dev/null"; - exit 1; - } - - shift_fields(1, 1); - printf("%s", new_barrier_name) > outputfile; - continue; - } - - # Skip definition of rcu_synchronize, since it is already - # defined in misc.h. Only present in old versions of srcu. - if (brace_nesting == 0 && paren_nesting == 0 && - $1 == "struct" && $2 == "rcu_synchronize" && - $0 ~ "^struct(" FS ")+rcu_synchronize(" FS ")+\\{") { - shift_fields(2, 0); - while (brace_nesting) { - if (NF < 2) - read_line(); - shift_fields(1, 0); - } - } - - # Skip definition of wakeme_after_rcu for the same reason - if (brace_nesting == 0 && $1 == "static" && $2 == "void" && - $3 == "wakeme_after_rcu") { - while (NF < 5) - read_line(); - shift_fields(3, 0); - do { - while (NF < 3) - read_line(); - shift_fields(1, 0); - } while (paren_nesting || brace_nesting); - } - - if ($1 ~ /^(unsigned|long)$/ && NF < 3) { - read_line(); - continue; - } - - # Give srcu_batches_completed the correct type for old SRCU. - if (brace_nesting == 0 && $1 == "long" && - $2 == "srcu_batches_completed") { - update_fieldsep("", 0); - printf("unsigned ") > outputfile; - print_fields(2); - continue; - } - if (brace_nesting == 0 && $1 == "unsigned" && $2 == "long" && - $3 == "srcu_batches_completed") { - print_fields(3); - continue; - } - - # Just print out the input code by default. - print_fields(1); - } - update_fieldsep("", 0); - print > outputfile; - next; -} - -END { - update_fieldsep("", 0); - - if (brace_nesting != 0) { - print "Unbalanced braces!" > "/dev/stderr"; - exit 1; - } - - # Define the rcu_batches - for (name in rcu_batches) - print "struct rcu_batch " name " = " rcu_batches[name] ";" > c_output; -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h deleted file mode 100644 index 570a49d9da7e..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASSUME_H -#define ASSUME_H - -/* Provide an assumption macro that can be disabled for gcc. */ -#ifdef RUN -#define assume(x) \ - do { \ - /* Evaluate x to suppress warnings. */ \ - (void) (x); \ - } while (0) - -#else -#define assume(x) __CPROVER_assume(x) -#endif - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h deleted file mode 100644 index 3f95a768a03b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef BARRIERS_H -#define BARRIERS_H - -#define barrier() __asm__ __volatile__("" : : : "memory") - -#ifdef RUN -#define smp_mb() __sync_synchronize() -#define smp_mb__after_unlock_lock() __sync_synchronize() -#else -/* - * Copied from CBMC's implementation of __sync_synchronize(), which - * seems to be disabled by default. - */ -#define smp_mb() __CPROVER_fence("WWfence", "RRfence", "RWfence", "WRfence", \ - "WWcumul", "RRcumul", "RWcumul", "WRcumul") -#define smp_mb__after_unlock_lock() __CPROVER_fence("WWfence", "RRfence", "RWfence", "WRfence", \ - "WWcumul", "RRcumul", "RWcumul", "WRcumul") -#endif - -/* - * Allow memory barriers to be disabled in either the read or write side - * of SRCU individually. - */ - -#ifndef NO_SYNC_SMP_MB -#define sync_smp_mb() smp_mb() -#else -#define sync_smp_mb() do {} while (0) -#endif - -#ifndef NO_READ_SIDE_SMP_MB -#define rs_smp_mb() smp_mb() -#else -#define rs_smp_mb() do {} while (0) -#endif - -#define READ_ONCE(x) (*(volatile typeof(x) *) &(x)) -#define WRITE_ONCE(x) ((*(volatile typeof(x) *) &(x)) = (val)) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h deleted file mode 100644 index 5e7912c6a521..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef BUG_ON_H -#define BUG_ON_H - -#include - -#define BUG() assert(0) -#define BUG_ON(x) assert(!(x)) - -/* Does it make sense to treat warnings as errors? */ -#define WARN() BUG() -#define WARN_ON(x) (BUG_ON(x), false) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c deleted file mode 100644 index e67ee5b3dd7c..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -/* Include all source files. */ - -#include "include_srcu.c" - -#include "preempt.c" -#include "misc.c" - -/* Used by test.c files */ -#include -#include -#include diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h deleted file mode 100644 index 283d7103334f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* "Cheater" definitions based on restricted Kconfig choices. */ - -#undef CONFIG_TINY_RCU -#undef __CHECKER__ -#undef CONFIG_DEBUG_LOCK_ALLOC -#undef CONFIG_DEBUG_OBJECTS_RCU_HEAD -#undef CONFIG_HOTPLUG_CPU -#undef CONFIG_MODULES -#undef CONFIG_NO_HZ_FULL_SYSIDLE -#undef CONFIG_PREEMPT_COUNT -#undef CONFIG_PREEMPT_RCU -#undef CONFIG_PROVE_RCU -#undef CONFIG_RCU_NOCB_CPU -#undef CONFIG_RCU_NOCB_CPU_ALL -#undef CONFIG_RCU_STALL_COMMON -#undef CONFIG_RCU_TRACE -#undef CONFIG_RCU_USER_QS -#undef CONFIG_TASKS_RCU -#define CONFIG_TREE_RCU - -#define CONFIG_GENERIC_ATOMIC64 - -#if NR_CPUS > 1 -#define CONFIG_SMP -#else -#undef CONFIG_SMP -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c deleted file mode 100644 index e5202d4cff30..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "int_typedefs.h" - -#include "barriers.h" -#include "bug_on.h" -#include "locks.h" -#include "misc.h" -#include "preempt.h" -#include "percpu.h" -#include "workqueues.h" - -#ifdef USE_SIMPLE_SYNC_SRCU -#define synchronize_srcu(sp) synchronize_srcu_original(sp) -#endif - -#include - -#ifdef USE_SIMPLE_SYNC_SRCU -#undef synchronize_srcu - -#include "simple_sync_srcu.c" -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h deleted file mode 100644 index 0dd27aa517a7..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef INT_TYPEDEFS_H -#define INT_TYPEDEFS_H - -#include - -typedef int8_t s8; -typedef uint8_t u8; -typedef int16_t s16; -typedef uint16_t u16; -typedef int32_t s32; -typedef uint32_t u32; -typedef int64_t s64; -typedef uint64_t u64; - -typedef int8_t __s8; -typedef uint8_t __u8; -typedef int16_t __s16; -typedef uint16_t __u16; -typedef int32_t __s32; -typedef uint32_t __u32; -typedef int64_t __s64; -typedef uint64_t __u64; - -#define S8_C(x) INT8_C(x) -#define U8_C(x) UINT8_C(x) -#define S16_C(x) INT16_C(x) -#define U16_C(x) UINT16_C(x) -#define S32_C(x) INT32_C(x) -#define U32_C(x) UINT32_C(x) -#define S64_C(x) INT64_C(x) -#define U64_C(x) UINT64_C(x) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h deleted file mode 100644 index 1e24827f96f1..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h +++ /dev/null @@ -1,221 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef LOCKS_H -#define LOCKS_H - -#include -#include -#include - -#include "assume.h" -#include "bug_on.h" -#include "preempt.h" - -int nondet_int(void); - -#define __acquire(x) -#define __acquires(x) -#define __release(x) -#define __releases(x) - -/* Only use one lock mechanism. Select which one. */ -#ifdef PTHREAD_LOCK -struct lock_impl { - pthread_mutex_t mutex; -}; - -static inline void lock_impl_lock(struct lock_impl *lock) -{ - BUG_ON(pthread_mutex_lock(&lock->mutex)); -} - -static inline void lock_impl_unlock(struct lock_impl *lock) -{ - BUG_ON(pthread_mutex_unlock(&lock->mutex)); -} - -static inline bool lock_impl_trylock(struct lock_impl *lock) -{ - int err = pthread_mutex_trylock(&lock->mutex); - - if (!err) - return true; - else if (err == EBUSY) - return false; - BUG(); -} - -static inline void lock_impl_init(struct lock_impl *lock) -{ - pthread_mutex_init(&lock->mutex, NULL); -} - -#define LOCK_IMPL_INITIALIZER {.mutex = PTHREAD_MUTEX_INITIALIZER} - -#else /* !defined(PTHREAD_LOCK) */ -/* Spinlock that assumes that it always gets the lock immediately. */ - -struct lock_impl { - bool locked; -}; - -static inline bool lock_impl_trylock(struct lock_impl *lock) -{ -#ifdef RUN - /* TODO: Should this be a test and set? */ - return __sync_bool_compare_and_swap(&lock->locked, false, true); -#else - __CPROVER_atomic_begin(); - bool old_locked = lock->locked; - lock->locked = true; - __CPROVER_atomic_end(); - - /* Minimal barrier to prevent accesses leaking out of lock. */ - __CPROVER_fence("RRfence", "RWfence"); - - return !old_locked; -#endif -} - -static inline void lock_impl_lock(struct lock_impl *lock) -{ - /* - * CBMC doesn't support busy waiting, so just assume that the - * lock is available. - */ - assume(lock_impl_trylock(lock)); - - /* - * If the lock was already held by this thread then the assumption - * is unsatisfiable (deadlock). - */ -} - -static inline void lock_impl_unlock(struct lock_impl *lock) -{ -#ifdef RUN - BUG_ON(!__sync_bool_compare_and_swap(&lock->locked, true, false)); -#else - /* Minimal barrier to prevent accesses leaking out of lock. */ - __CPROVER_fence("RWfence", "WWfence"); - - __CPROVER_atomic_begin(); - bool old_locked = lock->locked; - lock->locked = false; - __CPROVER_atomic_end(); - - BUG_ON(!old_locked); -#endif -} - -static inline void lock_impl_init(struct lock_impl *lock) -{ - lock->locked = false; -} - -#define LOCK_IMPL_INITIALIZER {.locked = false} - -#endif /* !defined(PTHREAD_LOCK) */ - -/* - * Implement spinlocks using the lock mechanism. Wrap the lock to prevent mixing - * locks of different types. - */ -typedef struct { - struct lock_impl internal_lock; -} spinlock_t; - -#define SPIN_LOCK_UNLOCKED {.internal_lock = LOCK_IMPL_INITIALIZER} -#define __SPIN_LOCK_UNLOCKED(x) SPIN_LOCK_UNLOCKED -#define DEFINE_SPINLOCK(x) spinlock_t x = SPIN_LOCK_UNLOCKED - -static inline void spin_lock_init(spinlock_t *lock) -{ - lock_impl_init(&lock->internal_lock); -} - -static inline void spin_lock(spinlock_t *lock) -{ - /* - * Spin locks also need to be removed in order to eliminate all - * memory barriers. They are only used by the write side anyway. - */ -#ifndef NO_SYNC_SMP_MB - preempt_disable(); - lock_impl_lock(&lock->internal_lock); -#endif -} - -static inline void spin_unlock(spinlock_t *lock) -{ -#ifndef NO_SYNC_SMP_MB - lock_impl_unlock(&lock->internal_lock); - preempt_enable(); -#endif -} - -/* Don't bother with interrupts */ -#define spin_lock_irq(lock) spin_lock(lock) -#define spin_unlock_irq(lock) spin_unlock(lock) -#define spin_lock_irqsave(lock, flags) spin_lock(lock) -#define spin_unlock_irqrestore(lock, flags) spin_unlock(lock) - -/* - * This is supposed to return an int, but I think that a bool should work as - * well. - */ -static inline bool spin_trylock(spinlock_t *lock) -{ -#ifndef NO_SYNC_SMP_MB - preempt_disable(); - return lock_impl_trylock(&lock->internal_lock); -#else - return true; -#endif -} - -struct completion { - /* Hopefully this won't overflow. */ - unsigned int count; -}; - -#define COMPLETION_INITIALIZER(x) {.count = 0} -#define DECLARE_COMPLETION(x) struct completion x = COMPLETION_INITIALIZER(x) -#define DECLARE_COMPLETION_ONSTACK(x) DECLARE_COMPLETION(x) - -static inline void init_completion(struct completion *c) -{ - c->count = 0; -} - -static inline void wait_for_completion(struct completion *c) -{ - unsigned int prev_count = __sync_fetch_and_sub(&c->count, 1); - - assume(prev_count); -} - -static inline void complete(struct completion *c) -{ - unsigned int prev_count = __sync_fetch_and_add(&c->count, 1); - - BUG_ON(prev_count == UINT_MAX); -} - -/* This function probably isn't very useful for CBMC. */ -static inline bool try_wait_for_completion(struct completion *c) -{ - BUG(); -} - -static inline bool completion_done(struct completion *c) -{ - return c->count; -} - -/* TODO: Implement complete_all */ -static inline void complete_all(struct completion *c) -{ - BUG(); -} - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c deleted file mode 100644 index 9440cc39e3c6..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include "misc.h" -#include "bug_on.h" - -struct rcu_head; - -void wakeme_after_rcu(struct rcu_head *head) -{ - BUG(); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h deleted file mode 100644 index aca50030f954..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef MISC_H -#define MISC_H - -#include "assume.h" -#include "int_typedefs.h" -#include "locks.h" - -#include - -/* Probably won't need to deal with bottom halves. */ -static inline void local_bh_disable(void) {} -static inline void local_bh_enable(void) {} - -#define MODULE_ALIAS(X) -#define module_param(...) -#define EXPORT_SYMBOL_GPL(x) - -#define container_of(ptr, type, member) ({ \ - const typeof(((type *)0)->member) *__mptr = (ptr); \ - (type *)((char *)__mptr - offsetof(type, member)); \ -}) - -#ifndef USE_SIMPLE_SYNC_SRCU -/* Abuse udelay to make sure that busy loops terminate. */ -#define udelay(x) assume(0) - -#else - -/* The simple custom synchronize_srcu is ok with try_check_zero failing. */ -#define udelay(x) do { } while (0) -#endif - -#define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ - do { } while (0) - -#define notrace - -/* Avoid including rcupdate.h */ -struct rcu_synchronize { - struct rcu_head head; - struct completion completion; -}; - -void wakeme_after_rcu(struct rcu_head *head); - -#define rcu_lock_acquire(a) do { } while (0) -#define rcu_lock_release(a) do { } while (0) -#define rcu_lockdep_assert(c, s) do { } while (0) -#define RCU_LOCKDEP_WARN(c, s) do { } while (0) - -/* Let CBMC non-deterministically choose switch between normal and expedited. */ -bool rcu_gp_is_normal(void); -bool rcu_gp_is_expedited(void); - -/* Do the same for old versions of rcu. */ -#define rcu_expedited (rcu_gp_is_expedited()) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h deleted file mode 100644 index 27e67a3f291f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef PERCPU_H -#define PERCPU_H - -#include -#include "bug_on.h" -#include "preempt.h" - -#define __percpu - -/* Maximum size of any percpu data. */ -#define PERCPU_OFFSET (4 * sizeof(long)) - -/* Ignore alignment, as CBMC doesn't care about false sharing. */ -#define alloc_percpu(type) __alloc_percpu(sizeof(type), 1) - -static inline void *__alloc_percpu(size_t size, size_t align) -{ - BUG(); - return NULL; -} - -static inline void free_percpu(void *ptr) -{ - BUG(); -} - -#define per_cpu_ptr(ptr, cpu) \ - ((typeof(ptr)) ((char *) (ptr) + PERCPU_OFFSET * cpu)) - -#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1) -#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1) -#define __this_cpu_sub(pcp, n) __this_cpu_add(pcp, -(typeof(pcp)) (n)) - -#define this_cpu_inc(pcp) this_cpu_add(pcp, 1) -#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1) -#define this_cpu_sub(pcp, n) this_cpu_add(pcp, -(typeof(pcp)) (n)) - -/* Make CBMC use atomics to work around bug. */ -#ifdef RUN -#define THIS_CPU_ADD_HELPER(ptr, x) (*(ptr) += (x)) -#else -/* - * Split the atomic into a read and a write so that it has the least - * possible ordering. - */ -#define THIS_CPU_ADD_HELPER(ptr, x) \ - do { \ - typeof(ptr) this_cpu_add_helper_ptr = (ptr); \ - typeof(ptr) this_cpu_add_helper_x = (x); \ - typeof(*ptr) this_cpu_add_helper_temp; \ - __CPROVER_atomic_begin(); \ - this_cpu_add_helper_temp = *(this_cpu_add_helper_ptr); \ - __CPROVER_atomic_end(); \ - this_cpu_add_helper_temp += this_cpu_add_helper_x; \ - __CPROVER_atomic_begin(); \ - *(this_cpu_add_helper_ptr) = this_cpu_add_helper_temp; \ - __CPROVER_atomic_end(); \ - } while (0) -#endif - -/* - * For some reason CBMC needs an atomic operation even though this is percpu - * data. - */ -#define __this_cpu_add(pcp, n) \ - do { \ - BUG_ON(preemptible()); \ - THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), thread_cpu_id), \ - (typeof(pcp)) (n)); \ - } while (0) - -#define this_cpu_add(pcp, n) \ - do { \ - int this_cpu_add_impl_cpu = get_cpu(); \ - THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), this_cpu_add_impl_cpu), \ - (typeof(pcp)) (n)); \ - put_cpu(); \ - } while (0) - -/* - * This will cause a compiler warning because of the cast from char[][] to - * type*. This will cause a compile time error if type is too big. - */ -#define DEFINE_PER_CPU(type, name) \ - char name[NR_CPUS][PERCPU_OFFSET]; \ - typedef char percpu_too_big_##name \ - [sizeof(type) > PERCPU_OFFSET ? -1 : 1] - -#define for_each_possible_cpu(cpu) \ - for ((cpu) = 0; (cpu) < NR_CPUS; ++(cpu)) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c deleted file mode 100644 index b4083ae348fb..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include "preempt.h" - -#include "assume.h" -#include "locks.h" - -/* Support NR_CPUS of at most 64 */ -#define CPU_PREEMPTION_LOCKS_INIT0 LOCK_IMPL_INITIALIZER -#define CPU_PREEMPTION_LOCKS_INIT1 \ - CPU_PREEMPTION_LOCKS_INIT0, CPU_PREEMPTION_LOCKS_INIT0 -#define CPU_PREEMPTION_LOCKS_INIT2 \ - CPU_PREEMPTION_LOCKS_INIT1, CPU_PREEMPTION_LOCKS_INIT1 -#define CPU_PREEMPTION_LOCKS_INIT3 \ - CPU_PREEMPTION_LOCKS_INIT2, CPU_PREEMPTION_LOCKS_INIT2 -#define CPU_PREEMPTION_LOCKS_INIT4 \ - CPU_PREEMPTION_LOCKS_INIT3, CPU_PREEMPTION_LOCKS_INIT3 -#define CPU_PREEMPTION_LOCKS_INIT5 \ - CPU_PREEMPTION_LOCKS_INIT4, CPU_PREEMPTION_LOCKS_INIT4 - -/* - * Simulate disabling preemption by locking a particular cpu. NR_CPUS - * should be the actual number of cpus, not just the maximum. - */ -struct lock_impl cpu_preemption_locks[NR_CPUS] = { - CPU_PREEMPTION_LOCKS_INIT0 -#if (NR_CPUS - 1) & 1 - , CPU_PREEMPTION_LOCKS_INIT0 -#endif -#if (NR_CPUS - 1) & 2 - , CPU_PREEMPTION_LOCKS_INIT1 -#endif -#if (NR_CPUS - 1) & 4 - , CPU_PREEMPTION_LOCKS_INIT2 -#endif -#if (NR_CPUS - 1) & 8 - , CPU_PREEMPTION_LOCKS_INIT3 -#endif -#if (NR_CPUS - 1) & 16 - , CPU_PREEMPTION_LOCKS_INIT4 -#endif -#if (NR_CPUS - 1) & 32 - , CPU_PREEMPTION_LOCKS_INIT5 -#endif -}; - -#undef CPU_PREEMPTION_LOCKS_INIT0 -#undef CPU_PREEMPTION_LOCKS_INIT1 -#undef CPU_PREEMPTION_LOCKS_INIT2 -#undef CPU_PREEMPTION_LOCKS_INIT3 -#undef CPU_PREEMPTION_LOCKS_INIT4 -#undef CPU_PREEMPTION_LOCKS_INIT5 - -__thread int thread_cpu_id; -__thread int preempt_disable_count; - -void preempt_disable(void) -{ - BUG_ON(preempt_disable_count < 0 || preempt_disable_count == INT_MAX); - - if (preempt_disable_count++) - return; - - thread_cpu_id = nondet_int(); - assume(thread_cpu_id >= 0); - assume(thread_cpu_id < NR_CPUS); - lock_impl_lock(&cpu_preemption_locks[thread_cpu_id]); -} - -void preempt_enable(void) -{ - BUG_ON(preempt_disable_count < 1); - - if (--preempt_disable_count) - return; - - lock_impl_unlock(&cpu_preemption_locks[thread_cpu_id]); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h deleted file mode 100644 index f8b762cd214c..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef PREEMPT_H -#define PREEMPT_H - -#include - -#include "bug_on.h" - -/* This flag contains garbage if preempt_disable_count is 0. */ -extern __thread int thread_cpu_id; - -/* Support recursive preemption disabling. */ -extern __thread int preempt_disable_count; - -void preempt_disable(void); -void preempt_enable(void); - -static inline void preempt_disable_notrace(void) -{ - preempt_disable(); -} - -static inline void preempt_enable_no_resched(void) -{ - preempt_enable(); -} - -static inline void preempt_enable_notrace(void) -{ - preempt_enable(); -} - -static inline int preempt_count(void) -{ - return preempt_disable_count; -} - -static inline bool preemptible(void) -{ - return !preempt_count(); -} - -static inline int get_cpu(void) -{ - preempt_disable(); - return thread_cpu_id; -} - -static inline void put_cpu(void) -{ - preempt_enable(); -} - -static inline void might_sleep(void) -{ - BUG_ON(preempt_disable_count); -} - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c deleted file mode 100644 index 97f592048e0b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "int_typedefs.h" - -#include "barriers.h" -#include "bug_on.h" -#include "locks.h" -#include "misc.h" -#include "preempt.h" -#include "percpu.h" -#include "workqueues.h" - -#include - -/* Functions needed from modify_srcu.c */ -bool try_check_zero(struct srcu_struct *sp, int idx, int trycount); -void srcu_flip(struct srcu_struct *sp); - -/* Simpler implementation of synchronize_srcu that ignores batching. */ -void synchronize_srcu(struct srcu_struct *sp) -{ - int idx; - /* - * This code assumes that try_check_zero will succeed anyway, - * so there is no point in multiple tries. - */ - const int trycount = 1; - - might_sleep(); - - /* Ignore the lock, as multiple writers aren't working yet anyway. */ - - idx = 1 ^ (sp->completed & 1); - - /* For comments see srcu_advance_batches. */ - - assume(try_check_zero(sp, idx, trycount)); - - srcu_flip(sp); - - assume(try_check_zero(sp, idx^1, trycount)); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h deleted file mode 100644 index 28b960300971..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h +++ /dev/null @@ -1,103 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef WORKQUEUES_H -#define WORKQUEUES_H - -#include - -#include "barriers.h" -#include "bug_on.h" -#include "int_typedefs.h" - -#include - -/* Stub workqueue implementation. */ - -struct work_struct; -typedef void (*work_func_t)(struct work_struct *work); -void delayed_work_timer_fn(unsigned long __data); - -struct work_struct { -/* atomic_long_t data; */ - unsigned long data; - - struct list_head entry; - work_func_t func; -#ifdef CONFIG_LOCKDEP - struct lockdep_map lockdep_map; -#endif -}; - -struct timer_list { - struct hlist_node entry; - unsigned long expires; - void (*function)(unsigned long); - unsigned long data; - u32 flags; - int slack; -}; - -struct delayed_work { - struct work_struct work; - struct timer_list timer; - - /* target workqueue and CPU ->timer uses to queue ->work */ - struct workqueue_struct *wq; - int cpu; -}; - - -static inline bool schedule_work(struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool schedule_work_on(int cpu, struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool queue_work(struct workqueue_struct *wq, - struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool queue_delayed_work(struct workqueue_struct *wq, - struct delayed_work *dwork, - unsigned long delay) -{ - BUG(); - return true; -} - -#define INIT_WORK(w, f) \ - do { \ - (w)->data = 0; \ - (w)->func = (f); \ - } while (0) - -#define INIT_DELAYED_WORK(w, f) INIT_WORK(&(w)->work, (f)) - -#define __WORK_INITIALIZER(n, f) { \ - .data = 0, \ - .entry = { &(n).entry, &(n).entry }, \ - .func = f \ - } - -/* Don't bother initializing timer. */ -#define __DELAYED_WORK_INITIALIZER(n, f, tflags) { \ - .work = __WORK_INITIALIZER((n).work, (f)), \ - } - -#define DECLARE_WORK(n, f) \ - struct workqueue_struct n = __WORK_INITIALIZER - -#define DECLARE_DELAYED_WORK(n, f) \ - struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f, 0) - -#define system_power_efficient_wq ((struct workqueue_struct *) NULL) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore deleted file mode 100644 index d65462d64816..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -*.out diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile deleted file mode 100644 index ad21b925fbb4..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -CBMC_FLAGS = -I../.. -I../../src -I../../include -I../../empty_includes -32 -pointer-check -mm pso - -all: - for i in ./*.pass; do \ - echo $$i ; \ - CBMC_FLAGS="$(CBMC_FLAGS)" sh ../test_script.sh --should-pass $$i > $$i.out 2>&1 ; \ - done - for i in ./*.fail; do \ - echo $$i ; \ - CBMC_FLAGS="$(CBMC_FLAGS)" sh ../test_script.sh --should-fail $$i > $$i.out 2>&1 ; \ - done diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail deleted file mode 100644 index 40c8075919d1..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DASSERT_END" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail deleted file mode 100644 index ada5baf0b60d..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail deleted file mode 100644 index 8fe00c8db466..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE_2" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail deleted file mode 100644 index 612ed6772844..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE_3" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/main.pass b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/main.pass deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c deleted file mode 100644 index 2ce2016f7871..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -int x; -int y; - -int __unbuffered_tpr_x; -int __unbuffered_tpr_y; - -DEFINE_SRCU(ss); - -void rcu_reader(void) -{ - int idx; - -#ifndef FORCE_FAILURE_3 - idx = srcu_read_lock(&ss); -#endif - might_sleep(); - - __unbuffered_tpr_y = READ_ONCE(y); -#ifdef FORCE_FAILURE - srcu_read_unlock(&ss, idx); - idx = srcu_read_lock(&ss); -#endif - WRITE_ONCE(x, 1); - -#ifndef FORCE_FAILURE_3 - srcu_read_unlock(&ss, idx); -#endif - might_sleep(); -} - -void *thread_update(void *arg) -{ - WRITE_ONCE(y, 1); -#ifndef FORCE_FAILURE_2 - synchronize_srcu(&ss); -#endif - might_sleep(); - __unbuffered_tpr_x = READ_ONCE(x); - - return NULL; -} - -void *thread_process_reader(void *arg) -{ - rcu_reader(); - - return NULL; -} - -int main(int argc, char *argv[]) -{ - pthread_t tu; - pthread_t tpr; - - if (pthread_create(&tu, NULL, thread_update, NULL)) - abort(); - if (pthread_create(&tpr, NULL, thread_process_reader, NULL)) - abort(); - if (pthread_join(tu, NULL)) - abort(); - if (pthread_join(tpr, NULL)) - abort(); - assert(__unbuffered_tpr_y != 0 || __unbuffered_tpr_x != 0); - -#ifdef ASSERT_END - assert(0); -#endif - - return 0; -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh deleted file mode 100755 index 2fe1f0339b4f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# This script expects a mode (either --should-pass or --should-fail) followed by -# an input file. The script uses the following environment variables. The test C -# source file is expected to be named test.c in the directory containing the -# input file. -# -# CBMC: The command to run CBMC. Default: cbmc -# CBMC_FLAGS: Additional flags to pass to CBMC -# NR_CPUS: Number of cpus to run tests with. Default specified by the test -# SYNC_SRCU_MODE: Choose implementation of synchronize_srcu. Defaults to simple. -# kernel: Version included in the linux kernel source. -# simple: Use try_check_zero directly. -# -# The input file is a script that is sourced by this file. It can define any of -# the following variables to configure the test. -# -# test_cbmc_options: Extra options to pass to CBMC. -# min_cpus_fail: Minimum number of CPUs (NR_CPUS) for verification to fail. -# The test is expected to pass if it is run with fewer. (Only -# useful for .fail files) -# default_cpus: Quantity of CPUs to use for the test, if not specified on the -# command line. Default: Larger of 2 and MIN_CPUS_FAIL. - -set -e - -if test "$#" -ne 2; then - echo "Expected one option followed by an input file" 1>&2 - exit 99 -fi - -if test "x$1" = "x--should-pass"; then - should_pass="yes" -elif test "x$1" = "x--should-fail"; then - should_pass="no" -else - echo "Unrecognized argument '$1'" 1>&2 - - # Exit code 99 indicates a hard error. - exit 99 -fi - -CBMC=${CBMC:-cbmc} - -SYNC_SRCU_MODE=${SYNC_SRCU_MODE:-simple} - -case ${SYNC_SRCU_MODE} in -kernel) sync_srcu_mode_flags="" ;; -simple) sync_srcu_mode_flags="-DUSE_SIMPLE_SYNC_SRCU" ;; - -*) - echo "Unrecognized argument '${SYNC_SRCU_MODE}'" 1>&2 - exit 99 - ;; -esac - -min_cpus_fail=1 - -c_file=`dirname "$2"`/test.c - -# Source the input file. -. $2 - -if test ${min_cpus_fail} -gt 2; then - default_default_cpus=${min_cpus_fail} -else - default_default_cpus=2 -fi -default_cpus=${default_cpus:-${default_default_cpus}} -cpus=${NR_CPUS:-${default_cpus}} - -# Check if there are two few cpus to make the test fail. -if test $cpus -lt ${min_cpus_fail:-0}; then - should_pass="yes" -fi - -cbmc_opts="-DNR_CPUS=${cpus} ${sync_srcu_mode_flags} ${test_cbmc_options} ${CBMC_FLAGS}" - -echo "Running CBMC: ${CBMC} ${cbmc_opts} ${c_file}" -if ${CBMC} ${cbmc_opts} "${c_file}"; then - # Verification successful. Make sure that it was supposed to verify. - test "x${should_pass}" = xyes -else - cbmc_exit_status=$? - - # An exit status of 10 indicates a failed verification. - # (see cbmc_parse_optionst::do_bmc in the CBMC source code) - if test ${cbmc_exit_status} -eq 10 && test "x${should_pass}" = xno; then - : - else - echo "CBMC returned ${cbmc_exit_status} exit status" 1>&2 - - # Parse errors have exit status 6. Any other type of error - # should be considered a hard error. - if test ${cbmc_exit_status} -ne 6 && \ - test ${cbmc_exit_status} -ne 10; then - exit 99 - else - exit 1 - fi - fi -fi diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 43ec36b179dc..38f651469968 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -2184,6 +2184,9 @@ FIXTURE_TEARDOWN(TRACE_syscall) TEST(negative_ENOSYS) { +#if defined(__arm__) + SKIP(return, "arm32 does not support calling syscall -1"); +#endif /* * There should be no difference between an "internal" skip * and userspace asking for syscall "-1". @@ -3072,7 +3075,8 @@ TEST(syscall_restart) timeout.tv_sec = 1; errno = 0; EXPECT_EQ(0, nanosleep(&timeout, NULL)) { - TH_LOG("Call to nanosleep() failed (errno %d)", errno); + TH_LOG("Call to nanosleep() failed (errno %d: %s)", + errno, strerror(errno)); } /* Read final sync from parent. */ @@ -3908,6 +3912,9 @@ TEST(user_notification_filter_empty) TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); } + if (__NR_clone3 < 0) + SKIP(return, "Test not built with clone3 support"); + pid = sys_clone3(&args, sizeof(args)); ASSERT_GE(pid, 0); @@ -3962,6 +3969,9 @@ TEST(user_notification_filter_empty_threaded) TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); } + if (__NR_clone3 < 0) + SKIP(return, "Test not built with clone3 support"); + pid = sys_clone3(&args, sizeof(args)); ASSERT_GE(pid, 0); @@ -4255,6 +4265,61 @@ TEST(user_notification_addfd_rlimit) close(memfd); } +#ifndef SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) +#endif + +TEST(user_notification_sync) +{ + struct seccomp_notif req = {}; + struct seccomp_notif_resp resp = {}; + int status, listener; + pid_t pid; + long ret; + + ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + ASSERT_EQ(0, ret) { + TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); + } + + listener = user_notif_syscall(__NR_getppid, + SECCOMP_FILTER_FLAG_NEW_LISTENER); + ASSERT_GE(listener, 0); + + /* Try to set invalid flags. */ + EXPECT_SYSCALL_RETURN(-EINVAL, + ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, 0xffffffff, 0)); + + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP, 0), 0); + + pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + ret = syscall(__NR_getppid); + ASSERT_EQ(ret, USER_NOTIF_MAGIC) { + _exit(1); + } + _exit(0); + } + + req.pid = 0; + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req), 0); + + ASSERT_EQ(req.data.nr, __NR_getppid); + + resp.id = req.id; + resp.error = 0; + resp.val = USER_NOTIF_MAGIC; + resp.flags = 0; + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &resp), 0); + + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_EQ(status, 0); +} + + /* Make sure PTRACE_O_SUSPEND_SECCOMP requires CAP_SYS_ADMIN. */ FIXTURE(O_SUSPEND_SECCOMP) { pid_t pid;