From 5bd00f6db012f75b42434d39b7fec98b95c1afcc Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Tue, 9 May 2023 08:31:23 +1000 Subject: [PATCH 001/130] smp: Reduce logging due to dump_stack of CSD waiters If a waiter is waiting for CSD lock, its call stack will not change between first and subsequent hang detection for the same CSD lock. Therefore, do dump_stack only for first-time detection for a given waiter. This avoids excessive logging on systems with hundreds of CPUs where repetitive dump_stack from hundreds of CPUs would otherwise flood the console. Signed-off-by: Imran Khan Cc: Peter Zijlstra Cc: Juergen Gross Cc: Valentin Schneider Cc: Yury Norov Signed-off-by: Paul E. McKenney --- kernel/smp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/smp.c b/kernel/smp.c index 385179dae360..1d41a0cb54f1 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -259,7 +259,8 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 * arch_send_call_function_single_ipi(cpu); } } - dump_stack(); + if (firsttime) + dump_stack(); *ts1 = ts2; return false; From 0d3a00b370424f5f1b9fd037bc8a4a3e7cbf0939 Mon Sep 17 00:00:00 2001 From: Imran Khan Date: Tue, 9 May 2023 08:31:24 +1000 Subject: [PATCH 002/130] smp: Reduce NMI traffic from CSD waiters to CSD destination On systems with hundreds of CPUs, if most of the CPUs detect a CSD hang, then all of these waiting CPUs send an NMI to the destination CPU in order to dump its backtrace. Given enough NMIs, the destination CPU will spent much of its time producing backtraces, thus further delaying that CPU's response to the original CSD IPI. In the worst case, by the time destination CPU is done producing all of these backtrace NMIs, the CSD wait timeout will have elapsed so that the waiters resend their backtrace NMIs again, further delaying forward progress. Therefore, to avoid these delays, issue the backtrace NMI only from the first waiter. The destination CPU's other waiters can make use of backtrace obtained from the first waiter's NMI. Signed-off-by: Imran Khan Cc: Peter Zijlstra Cc: Juergen Gross Cc: Valentin Schneider Cc: Yury Norov Signed-off-by: Paul E. McKenney --- kernel/smp.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/smp.c b/kernel/smp.c index 1d41a0cb54f1..8455a53465af 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -46,6 +46,8 @@ static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); +static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1); + static void __flush_smp_call_function_queue(bool warn_cpu_offline); int smpcfd_prepare_cpu(unsigned int cpu) @@ -253,7 +255,8 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 * *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); } if (cpu >= 0) { - dump_cpu_task(cpu); + if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0)) + dump_cpu_task(cpu); if (!cpu_cur_csd) { pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); @@ -434,9 +437,14 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) struct llist_node *entry, *prev; struct llist_head *head; static bool warned; + atomic_t *tbt; lockdep_assert_irqs_disabled(); + /* Allow waiters to send backtrace NMI from here onwards */ + tbt = this_cpu_ptr(&trigger_backtrace); + atomic_set_release(tbt, 1); + head = this_cpu_ptr(&call_single_queue); entry = llist_del_all(head); entry = llist_reverse_order(entry); From 401013e2ad9e1ba329a91a7b66779d8cb780bcd0 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Wed, 7 Jun 2023 08:15:21 -0600 Subject: [PATCH 003/130] documentation/rcu: Fix typo s/slat/splat/ Signed-off-by: Tycho Andersen Signed-off-by: Paul E. McKenney --- Documentation/RCU/lockdep-splat.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/RCU/lockdep-splat.rst b/Documentation/RCU/lockdep-splat.rst index 2a5c79db57dc..bcbc4b3c88d7 100644 --- a/Documentation/RCU/lockdep-splat.rst +++ b/Documentation/RCU/lockdep-splat.rst @@ -10,7 +10,7 @@ misuses of the RCU API, most notably using one of the rcu_dereference() family to access an RCU-protected pointer without the proper protection. When such misuse is detected, an lockdep-RCU splat is emitted. -The usual cause of a lockdep-RCU slat is someone accessing an +The usual cause of a lockdep-RCU splat is someone accessing an RCU-protected data structure without either (1) being in the right kind of RCU read-side critical section or (2) holding the right update-side lock. This problem can therefore be serious: it might result in random memory From bc25e7c3598e56992120505ea09f6cf0651da63b Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Tue, 13 Jun 2023 00:57:01 +0000 Subject: [PATCH 004/130] docs/RCU: Add the missing rcu_read_unlock() We should exit the RCU read-side critical section before re-entering. Signed-off-by: Alan Huang Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 9a734bf54b76..0612a6387d8e 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -29,8 +29,10 @@ algorithms: rcu_read_lock() obj = lockless_lookup(key); if (obj) { - if (!try_get_ref(obj)) // might fail for free objects + if (!try_get_ref(obj)) { // might fail for free objects + rcu_read_unlock(); goto begin; + } /* * Because a writer could delete object, and a writer could * reuse these object before the RCU grace period, we From 3f831e38cecdd72a139a130da11f95be38481034 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 13 Jun 2023 18:24:31 +0000 Subject: [PATCH 005/130] Docs/RCU/rculist_nulls: Fix trivial coding style Lookup example of non-hlist_nulls management is missing a semicolon, and having inconsistent indentation (one line is using single space indentation while others are using two spaces indentation). Fix the trivial issues. Signed-off-by: SeongJae Park Reviewed-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 0612a6387d8e..25b739885cfa 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -26,7 +26,7 @@ algorithms: :: begin: - rcu_read_lock() + rcu_read_lock(); obj = lockless_lookup(key); if (obj) { if (!try_get_ref(obj)) { // might fail for free objects @@ -70,8 +70,8 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: pos && ({ prefetch(pos->next); 1; }) && ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); pos = rcu_dereference(pos->next)) - if (obj->key == key) - return obj; + if (obj->key == key) + return obj; return NULL; Quoting Corey Minyard:: From 674dd365736125b56297aaea8845a90e5ea990b9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 13 Jun 2023 18:24:32 +0000 Subject: [PATCH 006/130] Docs/RCU/rculist_nulls: Assign 'obj' before use from the examples Lookup example code snippets in rculist_nulls.rst are using 'obj' without assignment. Fix the code to assign it properly. Signed-off-by: SeongJae Park Reviewed-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 25b739885cfa..4d6f077552ed 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -56,7 +56,7 @@ but a version with an additional memory barrier (smp_rmb()) struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); pos = rcu_dereference(next)) if (obj->key == key) return obj; @@ -68,7 +68,7 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); pos && ({ prefetch(pos->next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); pos = rcu_dereference(pos->next)) if (obj->key == key) return obj; From 5326caa7a147743340f68b5b7ed4c2a91b4abb56 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:24 +0000 Subject: [PATCH 007/130] Docs/RCU/rculist_nulls: Specify type of the object in examples The type of 'obj' in example code of rculist_nulls.rst is implicit. Provide the specific type of it before the example code. Suggested-by: Paul E. McKenney Link: https://lore.kernel.org/rcu/43943609-f80c-4b6a-9844-994eef800757@paulmck-laptop/ Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 4d6f077552ed..479cedfec446 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -18,7 +18,16 @@ to solve following problem. Without 'nulls', a typical RCU linked list managing objects which are allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can use the following -algorithms: +algorithms. Following examples assume 'obj' is a pointer to such +objects, which is having below type. + +:: + + struct object { + struct hlist_node obj_node; + atomic_t refcnt; + unsigned int key; + }; 1) Lookup algorithm ------------------- @@ -144,6 +153,9 @@ the beginning. If the object was moved to the same chain, then the reader doesn't care: It might occasionally scan the list again without harm. +Note that using hlist_nulls means the type of 'obj_node' field of +'struct object' becomes 'struct hlist_nulls_node'. + 1) lookup algorithm ------------------- From d186204a98541c7d693290bfc91853bbfdfcf0cb Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:25 +0000 Subject: [PATCH 008/130] Docs/RCU/rculist_nulls: Fix hlist_[nulls]_head field names of 'obj' The example code snippets on rculist_nulls.rst are assuming 'obj' to have the 'hlist_head' or 'hlist_nulls_head' field named 'obj_node', but a sentence and some code snippets are wrongly calling 'obj->obj_node.next' as 'obj->obj_next', or 'obj->obj_node' as 'member'. Fix it. Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 479cedfec446..1fb086066377 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -65,7 +65,7 @@ but a version with an additional memory barrier (smp_rmb()) struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(next)) if (obj->key == key) return obj; @@ -77,7 +77,7 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); pos && ({ prefetch(pos->next); 1; }) && - ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(pos->next)) if (obj->key == key) return obj; @@ -97,7 +97,7 @@ Quoting Corey Minyard:: 2) Insertion algorithm ---------------------- -We need to make sure a reader cannot read the new 'obj->obj_next' value +We need to make sure a reader cannot read the new 'obj->obj_node.next' value and previous value of 'obj->key'. Otherwise, an item could be deleted from a chain, and inserted into another chain. If new chain was empty before the move, 'next' pointer is NULL, and lockless reader can not @@ -165,7 +165,7 @@ Note that using hlist_nulls means the type of 'obj_node' field of head = &table[slot]; begin: rcu_read_lock(); - hlist_nulls_for_each_entry_rcu(obj, node, head, member) { + hlist_nulls_for_each_entry_rcu(obj, node, head, obj_node) { if (obj->key == key) { if (!try_get_ref(obj)) { // might fail for free objects rcu_read_unlock(); From 47d63d7a3918d8bc1ccd0b0527da8dbb66429100 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:26 +0000 Subject: [PATCH 009/130] Docs/RCU/rculist_nulls: Fix text about atomic_set_release() The document says we can avoid extra _release() in insert function when hlist_nulls is used, but that's not true[1]. Drop it. [1] https://lore.kernel.org/rcu/46440869-644a-4982-b790-b71b43976c66@paulmck-laptop/ Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 1fb086066377..21e40fcc08de 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -140,8 +140,7 @@ very very fast (before the end of RCU grace period) Avoiding extra smp_rmb() ======================== -With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup() -and extra _release() in insert function. +With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup(). For example, if we choose to store the slot number as the 'nulls' end-of-list marker for each slot of the hash table, we can detect @@ -196,6 +195,9 @@ Note that using hlist_nulls means the type of 'obj_node' field of 2) Insert algorithm ------------------- +Same to the above one, but uses hlist_nulls_add_head_rcu() instead of +hlist_add_head_rcu(). + :: /* From d119357d0743548f90243315de905cf6a79e5221 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 14 May 2023 18:06:12 -0700 Subject: [PATCH 010/130] rcu-tasks: Treat only synchronous grace periods urgently The performance requirements on RCU Tasks, and in particular on RCU Tasks Trace, have evolved over time as the workloads have evolved. The current implementation is designed to provide low grace-period latencies, and also to accommodate short-duration floods of callbacks. However, current workloads can also provide a constant background callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations per second. This results in continuous back-to-back RCU Tasks Trace grace periods, which in turn can consume the better part of 10% of a CPU. One could take the attitude that there are several tens of other CPUs on the systems running such workloads, but energy efficiency is a thing. On these systems, although asynchronous grace-period requests happen every few milliseconds, synchronous grace-period requests are quite rare. This commit therefore arrranges for grace periods to be initiated immediately in response to calls to synchronize_rcu_tasks*() and also to calls to synchronize_rcu_mult() that are passed one of the call_rcu_tasks*() functions. These are recognized by the tell-tale wakeme_after_rcu callback function. In other cases, callbacks are gathered up for up to about 250 milliseconds before a grace period is initiated. This results in more than an order of magnitude reduction in RCU Tasks Trace grace periods, with corresponding reduction in consumption of CPU time. Reported-by: Alexei Starovoitov Reported-by: Martin KaFai Lau Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 81 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index b770add3f843..c88d7fe29d92 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -25,6 +25,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); * @cblist: Callback list. * @lock: Lock protecting per-CPU callback list. * @rtp_jiffies: Jiffies counter value for statistics. + * @lazy_timer: Timer to unlazify callbacks. + * @urgent_gp: Number of additional non-lazy grace periods. * @rtp_n_lock_retries: Rough lock-contention statistic. * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. @@ -38,6 +40,8 @@ struct rcu_tasks_percpu { raw_spinlock_t __private lock; unsigned long rtp_jiffies; unsigned long rtp_n_lock_retries; + struct timer_list lazy_timer; + unsigned int urgent_gp; struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; @@ -51,7 +55,6 @@ struct rcu_tasks_percpu { * @cbs_wait: RCU wait allowing a new callback to get kthread's attention. * @cbs_gbl_lock: Lock protecting callback list. * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone. - * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping. @@ -61,6 +64,8 @@ struct rcu_tasks_percpu { * @tasks_gp_seq: Number of grace periods completed since boot. * @n_ipis: Number of IPIs sent to encourage grace periods to end. * @n_ipis_fails: Number of IPI-send failures. + * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. + * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). * @pertask_func: This flavor's per-task scan function (optional). * @postscan_func: This flavor's post-task scan function (optional). @@ -92,6 +97,7 @@ struct rcu_tasks { unsigned long n_ipis; unsigned long n_ipis_fails; struct task_struct *kthread_ptr; + unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; pregp_func_t pregp_func; pertask_func_t pertask_func; @@ -127,6 +133,7 @@ static struct rcu_tasks rt_name = \ .gp_func = gp, \ .call_func = call, \ .rtpcpu = &rt_name ## __percpu, \ + .lazy_jiffies = DIV_ROUND_UP(HZ, 4), \ .name = n, \ .percpu_enqueue_shift = order_base_2(CONFIG_NR_CPUS), \ .percpu_enqueue_lim = 1, \ @@ -276,6 +283,33 @@ static void cblist_init_generic(struct rcu_tasks *rtp) data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust); } +// Compute wakeup time for lazy callback timer. +static unsigned long rcu_tasks_lazy_time(struct rcu_tasks *rtp) +{ + return jiffies + rtp->lazy_jiffies; +} + +// Timer handler that unlazifies lazy callbacks. +static void call_rcu_tasks_generic_timer(struct timer_list *tlp) +{ + unsigned long flags; + bool needwake = false; + struct rcu_tasks *rtp; + struct rcu_tasks_percpu *rtpcp = from_timer(rtpcp, tlp, lazy_timer); + + rtp = rtpcp->rtpp; + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); + if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) { + if (!rtpcp->urgent_gp) + rtpcp->urgent_gp = 1; + needwake = true; + mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); + } + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); + if (needwake) + rcuwait_wake_up(&rtp->cbs_wait); +} + // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic(). static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp) { @@ -292,6 +326,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, { int chosen_cpu; unsigned long flags; + bool havekthread = smp_load_acquire(&rtp->kthread_ptr); int ideal_cpu; unsigned long j; bool needadjust = false; @@ -321,7 +356,15 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, cblist_init_generic(rtp); raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. } - needwake = rcu_segcblist_empty(&rtpcp->cblist); + needwake = func == wakeme_after_rcu; + if (havekthread && !timer_pending(&rtpcp->lazy_timer)) { + if (rtp->lazy_jiffies) + mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); + else + needwake = rcu_segcblist_empty(&rtpcp->cblist); + } + if (needwake) + rtpcp->urgent_gp = 3; rcu_segcblist_enqueue(&rtpcp->cblist, rhp); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (unlikely(needadjust)) { @@ -415,9 +458,14 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) } rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); - if (rcu_segcblist_pend_cbs(&rtpcp->cblist)) + if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { + if (rtp->lazy_jiffies) + rtpcp->urgent_gp--; needgpcb |= 0x3; - if (!rcu_segcblist_empty(&rtpcp->cblist)) + } else if (rcu_segcblist_empty(&rtpcp->cblist)) { + rtpcp->urgent_gp = 0; + } + if (rcu_segcblist_ready_cbs(&rtpcp->cblist)) needgpcb |= 0x1; raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } @@ -549,11 +597,19 @@ static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot) // RCU-tasks kthread that detects grace periods and invokes callbacks. static int __noreturn rcu_tasks_kthread(void *arg) { + int cpu; struct rcu_tasks *rtp = arg; + for_each_possible_cpu(cpu) { + struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); + + timer_setup(&rtpcp->lazy_timer, call_rcu_tasks_generic_timer, 0); + rtpcp->urgent_gp = 1; + } + /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ housekeeping_affine(current, HK_TYPE_RCU); - WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start! + smp_store_release(&rtp->kthread_ptr, current); // Let GPs start! /* * Each pass through the following loop makes one check for @@ -635,16 +691,22 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) { int cpu; bool havecbs = false; + bool haveurgent = false; + bool haveurgentcbs = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); - if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) { + if (!data_race(rcu_segcblist_empty(&rtpcp->cblist))) havecbs = true; + if (data_race(rtpcp->urgent_gp)) + haveurgent = true; + if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) + haveurgentcbs = true; + if (havecbs && haveurgent && haveurgentcbs) break; - } } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), @@ -652,6 +714,9 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], + ".u"[haveurgent], + ".U"[haveurgentcbs], + rtp->lazy_jiffies, s); } #endif // #ifndef CONFIG_TINY_RCU From 5ae769c611e7d77c1d945cb132140deb93a32d49 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 17 May 2023 05:20:03 -0700 Subject: [PATCH 011/130] rcu-tasks: Remove redundant #ifdef CONFIG_TASKS_RCU The kernel/rcu/tasks.h file has a #endif immediately followed by an Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index c88d7fe29d92..7eae67fbe47c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -146,9 +146,7 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Track exiting tasks in order to allow them to be waited for. */ DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu); -#endif -#ifdef CONFIG_TASKS_RCU /* Report delay in synchronize_srcu() completion in rcu_tasks_postscan(). */ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); From 450d461aa629024c0f01022122838b5243fe0f2a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 17 May 2023 06:40:23 -0700 Subject: [PATCH 012/130] rcu-tasks: Add kernel boot parameters for callback laziness This commit adds kernel boot parameters for callback laziness, allowing the RCU Tasks flavors to be individually adjusted. Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 23 +++++++++++++++++++ kernel/rcu/tasks.h | 15 ++++++++++++ 2 files changed, 38 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..a0f427e1a562 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5293,6 +5293,29 @@ A change in value does not take effect until the beginning of the next grace period. + rcupdate.rcu_tasks_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks asynchronous + callback batching for call_rcu_tasks(). + A negative value will take the default. A value + of zero will disable batching. Batching is + always disabled for synchronize_rcu_tasks(). + + rcupdate.rcu_tasks_rude_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Rude asynchronous callback batching for + call_rcu_tasks_rude(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_rude(). + + rcupdate.rcu_tasks_trace_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Trace asynchronous callback batching for + call_rcu_tasks_trace(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_trace(). + rcupdate.rcu_self_test= [KNL] Run the RCU early boot self tests diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 7eae67fbe47c..28e986627e3f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1083,11 +1083,16 @@ void rcu_barrier_tasks(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks); +int rcu_tasks_lazy_ms = -1; +module_param(rcu_tasks_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_kthread(void) { cblist_init_generic(&rcu_tasks); rcu_tasks.gp_sleep = HZ / 10; rcu_tasks.init_fract = HZ / 10; + if (rcu_tasks_lazy_ms >= 0) + rcu_tasks.lazy_jiffies = msecs_to_jiffies(rcu_tasks_lazy_ms); rcu_tasks.pregp_func = rcu_tasks_pregp_step; rcu_tasks.pertask_func = rcu_tasks_pertask; rcu_tasks.postscan_func = rcu_tasks_postscan; @@ -1236,10 +1241,15 @@ void rcu_barrier_tasks_rude(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks_rude); +int rcu_tasks_rude_lazy_ms = -1; +module_param(rcu_tasks_rude_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_rude_kthread(void) { cblist_init_generic(&rcu_tasks_rude); rcu_tasks_rude.gp_sleep = HZ / 10; + if (rcu_tasks_rude_lazy_ms >= 0) + rcu_tasks_rude.lazy_jiffies = msecs_to_jiffies(rcu_tasks_rude_lazy_ms); rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude); return 0; } @@ -1856,6 +1866,9 @@ void rcu_barrier_tasks_trace(void) } EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace); +int rcu_tasks_trace_lazy_ms = -1; +module_param(rcu_tasks_trace_lazy_ms, int, 0444); + static int __init rcu_spawn_tasks_trace_kthread(void) { cblist_init_generic(&rcu_tasks_trace); @@ -1870,6 +1883,8 @@ static int __init rcu_spawn_tasks_trace_kthread(void) if (rcu_tasks_trace.init_fract <= 0) rcu_tasks_trace.init_fract = 1; } + if (rcu_tasks_trace_lazy_ms >= 0) + rcu_tasks_trace.lazy_jiffies = msecs_to_jiffies(rcu_tasks_trace_lazy_ms); rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; rcu_tasks_trace.holdouts_func = check_all_holdout_tasks_trace; From db13710a0365ba66f8f232eb3bdfaa1a10d820f7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 13:45:13 -0700 Subject: [PATCH 013/130] rcu-tasks: Cancel callback laziness if too many callbacks The various RCU Tasks flavors now do lazy grace periods when there are only asynchronous grace period requests. By default, the system will let 250 milliseconds elapse after the first call_rcu_tasks*() callbacki is queued before starting a grace period. In contrast, synchronous grace period requests such as synchronize_rcu_tasks*() will start a grace period immediately. However, invoking one of the call_rcu_tasks*() functions in a too-tight loop can result in a callback flood, which in turn can exhaust memory if grace periods are delayed for too long. This commit therefore sets a limit so that the grace-period kthread will be awakened when any CPU's callback list expands to contain rcupdate.rcu_task_lazy_lim callbacks elements (defaulting to 32, set to -1 to disable), the grace-period kthread will be awakened, thus cancelling any ongoing laziness and getting out in front of the potential callback flood. Cc: Alexei Starovoitov Cc: Martin KaFai Lau Cc: Andrii Nakryiko Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++++ kernel/rcu/tasks.h | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a0f427e1a562..44705a3618b9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5264,6 +5264,13 @@ number avoids disturbing real-time workloads, but lengthens grace periods. + rcupdate.rcu_task_lazy_lim= [KNL] + Number of callbacks on a given CPU that will + cancel laziness on that CPU. Use -1 to disable + cancellation of laziness, but be advised that + doing so increases the danger of OOM due to + callback flooding. + rcupdate.rcu_task_stall_info= [KNL] Set initial timeout in jiffies for RCU task stall informational messages, which give some indication diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 28e986627e3f..291d536b1789 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -176,6 +176,8 @@ static int rcu_task_contend_lim __read_mostly = 100; module_param(rcu_task_contend_lim, int, 0444); static int rcu_task_collapse_lim __read_mostly = 10; module_param(rcu_task_collapse_lim, int, 0444); +static int rcu_task_lazy_lim __read_mostly = 32; +module_param(rcu_task_lazy_lim, int, 0444); /* RCU tasks grace-period state for debugging. */ #define RTGS_INIT 0 @@ -354,8 +356,9 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, cblist_init_generic(rtp); raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. } - needwake = func == wakeme_after_rcu; - if (havekthread && !timer_pending(&rtpcp->lazy_timer)) { + needwake = (func == wakeme_after_rcu) || + (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); + if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); else From 8f78f046e27b72897f15ea62c8b7882a7d7b3bc6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 15:22:37 -0700 Subject: [PATCH 014/130] refscale: Add CONFIG_PREEMPT_DYNAMIC=n to NOPREEMPT scenario It is no longer possible to build a kernel with a preemption-disabled RCU without use of CONFIG_PREEMPT_DYNAMIC=n. This commit therefore adds CONFIG_PREEMPT_DYNAMIC=n to the refscale torture type's NOPREEMPT scenario file. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT index ef2b501a6971..67f9d2998afd 100644 --- a/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT +++ b/tools/testing/selftests/rcutorture/configs/refscale/NOPREEMPT @@ -2,6 +2,7 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n #CHECK#CONFIG_PREEMPT_RCU=n CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y From f5063e8948dad7f31adb007284a5d5038ae31bb8 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Fri, 7 Jul 2023 13:53:55 -0400 Subject: [PATCH 015/130] refscale: Fix uninitalized use of wait_queue_head_t Running the refscale test occasionally crashes the kernel with the following error: [ 8569.952896] BUG: unable to handle page fault for address: ffffffffffffffe8 [ 8569.952900] #PF: supervisor read access in kernel mode [ 8569.952902] #PF: error_code(0x0000) - not-present page [ 8569.952904] PGD c4b048067 P4D c4b049067 PUD c4b04b067 PMD 0 [ 8569.952910] Oops: 0000 [#1] PREEMPT_RT SMP NOPTI [ 8569.952916] Hardware name: Dell Inc. PowerEdge R750/0WMWCR, BIOS 1.2.4 05/28/2021 [ 8569.952917] RIP: 0010:prepare_to_wait_event+0x101/0x190 : [ 8569.952940] Call Trace: [ 8569.952941] [ 8569.952944] ref_scale_reader+0x380/0x4a0 [refscale] [ 8569.952959] kthread+0x10e/0x130 [ 8569.952966] ret_from_fork+0x1f/0x30 [ 8569.952973] The likely cause is that init_waitqueue_head() is called after the call to the torture_create_kthread() function that creates the ref_scale_reader kthread. Although this init_waitqueue_head() call will very likely complete before this kthread is created and starts running, it is possible that the calling kthread will be delayed between the calls to torture_create_kthread() and init_waitqueue_head(). In this case, the new kthread will use the waitqueue head before it is properly initialized, which is not good for the kernel's health and well-being. The above crash happened here: static inline void __add_wait_queue(...) { : if (!(wq->flags & WQ_FLAG_PRIORITY)) <=== Crash here The offset of flags from list_head entry in wait_queue_entry is -0x18. If reader_tasks[i].wq.head.next is NULL as allocated reader_task structure is zero initialized, the instruction will try to access address 0xffffffffffffffe8, which is exactly the fault address listed above. This commit therefore invokes init_waitqueue_head() before creating the kthread. Fixes: 653ed64b01dc ("refperf: Add a test to measure performance of read-side synchronization") Signed-off-by: Waiman Long Reviewed-by: Qiuxu Zhuo Reviewed-by: Davidlohr Bueso Acked-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 1970ce5f22d4..71d138573856 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -1107,12 +1107,11 @@ ref_scale_init(void) VERBOSE_SCALEOUT("Starting %d reader threads", nreaders); for (i = 0; i < nreaders; i++) { + init_waitqueue_head(&reader_tasks[i].wq); firsterr = torture_create_kthread(ref_scale_reader, (void *)i, reader_tasks[i].task); if (torture_init_error(firsterr)) goto unwind; - - init_waitqueue_head(&(reader_tasks[i].wq)); } // Main Task From b5a2801fc05350b3b81e6fdcefddd6cfdbf8b238 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 11 Jul 2023 18:01:38 -0700 Subject: [PATCH 016/130] refscale: Add a "jiffies" test This commit adds a "jiffies" test to refscale, allowing use of jiffies to be compared to ktime_get_real_fast_ns(). On my x86 laptop, jiffies is more than 20x faster. (Though for many uses, the tens-of-nanoseconds overhead of ktime_get_real_fast_ns() will be just fine.) Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 71d138573856..91a0fd0d4d9a 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -528,6 +528,38 @@ static struct ref_scale_ops clock_ops = { .name = "clock" }; +static void ref_jiffies_section(const int nloops) +{ + u64 x = 0; + int i; + + preempt_disable(); + for (i = nloops; i >= 0; i--) + x += jiffies; + preempt_enable(); + stopopts = x; +} + +static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl) +{ + u64 x = 0; + int i; + + preempt_disable(); + for (i = nloops; i >= 0; i--) { + x += jiffies; + un_delay(udl, ndl); + } + preempt_enable(); + stopopts = x; +} + +static struct ref_scale_ops jiffies_ops = { + .readsection = ref_jiffies_section, + .delaysection = ref_jiffies_delay_section, + .name = "jiffies" +}; + //////////////////////////////////////////////////////////////////////// // // Methods leveraging SLAB_TYPESAFE_BY_RCU. @@ -1047,7 +1079,7 @@ ref_scale_init(void) int firsterr = 0; static struct ref_scale_ops *scale_ops[] = { &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, - &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, + &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, }; From 2226f3dc05a988ce81308bfc3d125f5d0b0d592c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 07:02:01 -0700 Subject: [PATCH 017/130] rcuscale: Permit blocking delays between writers Some workloads do isolated RCU work, and this can affect operation latencies. This commit therefore adds a writer_holdoff_jiffies module parameter that causes writers to block for the specified number of jiffies between each pair of consecutive write-side operations. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ kernel/rcu/rcuscale.c | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..5ba231b786f8 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4983,6 +4983,11 @@ in microseconds. The default of zero says no holdoff. + rcuscale.writer_holdoff_jiffies= [KNL] + Additional write-side holdoff between grace + periods, but in jiffies. The default of zero + says no holdoff. + rcutorture.fqs_duration= [KNL] Set duration of force_quiescent_state bursts in microseconds. diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index d1221731c7cf..4277037e0996 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -93,6 +93,7 @@ torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, "Shutdown at end of scalability tests."); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); +torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); @@ -414,6 +415,7 @@ rcu_scale_writer(void *arg) struct rcu_head *rhp = NULL; bool started = false, done = false, alldone = false; u64 t; + DEFINE_TORTURE_RANDOM(tr); u64 *wdp; u64 *wdpp = writer_durations[me]; @@ -448,6 +450,8 @@ rcu_scale_writer(void *arg) do { if (writer_holdoff) udelay(writer_holdoff); + if (writer_holdoff_jiffies) + schedule_timeout_idle(torture_random(&tr) % writer_holdoff_jiffies + 1); wdp = &wdpp[i]; *wdp = ktime_get_mono_fast_ns(); if (gp_async) { From ee7516a16350ce2c1d45314a8c79bc93750fca8f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 08:21:17 -0700 Subject: [PATCH 018/130] rcuscale: Fix gp_async_max typo: s/reader/writer/ Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 4277037e0996..15edd8c82933 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -84,7 +84,7 @@ MODULE_AUTHOR("Paul E. McKenney "); #endif torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); -torture_param(int, gp_async_max, 1000, "Max # outstanding waits per reader"); +torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); From 7221f493c5ff8bee5c39de1e43cb8a8997c2cda5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 08:22:31 -0700 Subject: [PATCH 019/130] rcuscale: Add minruntime module parameter By default, rcuscale collects only 100 points of data per writer, but arranging for all kthreads to be actively collecting (if not recording) data during the time that any kthread might be recording. This works well, but does not allow much time to bring external performance tools to bear. This commit therefore adds a minruntime module parameter that specifies a minimum data-collection interval in seconds. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 6 ++++++ kernel/rcu/rcuscale.c | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 5ba231b786f8..a6888e3dfc20 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4953,6 +4953,12 @@ Number of loops doing rcuscale.kfree_alloc_num number of allocations and frees. + rcuscale.minruntime= [KNL] + Set the minimum test run time in seconds. This + does not affect the data-collection interval, + but instead allows better measurement of things + like CPU consumption. + rcuscale.nreaders= [KNL] Set number of RCU readers. The value -1 selects N, where N is the number of CPUs. A value diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 15edd8c82933..7c5bab5a4f19 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -87,6 +87,7 @@ torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives"); torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(int, holdoff, 10, "Holdoff time before test start (s)"); +torture_param(int, minruntime, 0, "Minimum run time (s)"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(int, nwriters, -1, "Number of RCU updater threads"); torture_param(bool, shutdown, RCUSCALE_SHUTDOWN, @@ -411,6 +412,7 @@ rcu_scale_writer(void *arg) { int i = 0; int i_max; + unsigned long jdone; long me = (long)arg; struct rcu_head *rhp = NULL; bool started = false, done = false, alldone = false; @@ -447,6 +449,7 @@ rcu_scale_writer(void *arg) } } + jdone = jiffies + minruntime * HZ; do { if (writer_holdoff) udelay(writer_holdoff); @@ -479,7 +482,7 @@ retry: if (!started && atomic_read(&n_rcu_scale_writer_started) >= nrealwriters) started = true; - if (!done && i >= MIN_MEAS) { + if (!done && i >= MIN_MEAS && time_after(jiffies, jdone)) { done = true; sched_set_normal(current, 0); pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n", From c68465dfaac3a36e2a72030f834811e651da8097 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 13:45:16 -0700 Subject: [PATCH 020/130] rcuscale: Print out full set of module parameters Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 7c5bab5a4f19..18f1b0e13604 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -525,8 +525,8 @@ static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { pr_alert("%s" SCALE_FLAG - "--- %s: nreaders=%d nwriters=%d verbose=%d shutdown=%d\n", - scale_type, tag, nrealreaders, nrealwriters, verbose, shutdown); + "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown=%d\n", + scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown); } /* From bb7bad3dae42b669cfe471716f0073ac611e7f75 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 16:04:52 -0700 Subject: [PATCH 021/130] rcuscale: Print out full set of kfree_rcu parameters Signed-off-by: Paul E. McKenney Cc: Uladzislau Rezki (Sony) --- kernel/rcu/rcuscale.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 18f1b0e13604..821a3e65c54a 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -708,6 +708,10 @@ kfree_scale_init(void) unsigned long jif_start; unsigned long orig_jif; + pr_alert("%s" SCALE_FLAG + "--- kfree_rcu_test: kfree_mult=%d kfree_by_call_rcu=%d kfree_nthreads=%d kfree_alloc_num=%d kfree_loops=%d kfree_rcu_test_double=%d kfree_rcu_test_single=%d\n", + scale_type, kfree_mult, kfree_by_call_rcu, kfree_nthreads, kfree_alloc_num, kfree_loops, kfree_rcu_test_double, kfree_rcu_test_single); + // Also, do a quick self-test to ensure laziness is as much as // expected. if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) { From 5f8e3202696f5e94d3fc54cfed75d270ae843ee9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 17 May 2023 05:01:28 -0700 Subject: [PATCH 022/130] rcuscale: Measure grace-period kthread CPU time This commit adds the ability to output the CPU time consumed by the grace-period kthread for the RCU variant under test. The CPU time is whatever is in the designated task's current->stime field, and thus is controlled by whatever CPU-time accounting scheme is in effect. This output appears in microseconds as follows on the console: rcu_scale: Grace-period kthread CPU time: 42367.037 [ paulmck: Apply feedback from Stephen Rothwell and kernel test robot. ] Signed-off-by: Paul E. McKenney Tested-by: Yujie Liu --- include/linux/rcupdate_trace.h | 1 + kernel/rcu/rcuscale.c | 21 +++++++++++++++++++++ kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 28 insertions(+) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 9bc8cbb33340..eda493200663 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -87,6 +87,7 @@ static inline void rcu_read_unlock_trace(void) void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); void synchronize_rcu_tasks_trace(void); void rcu_barrier_tasks_trace(void); +struct task_struct *get_rcu_tasks_trace_gp_kthread(void); #else /* * The BPF JIT forms these addresses even when it doesn't call these diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 821a3e65c54a..7fba3ab66e35 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -141,6 +141,7 @@ struct rcu_scale_ops { void (*gp_barrier)(void); void (*sync)(void); void (*exp_sync)(void); + struct task_struct *(*rso_gp_kthread)(void); const char *name; }; @@ -336,6 +337,7 @@ static struct rcu_scale_ops tasks_tracing_ops = { .gp_barrier = rcu_barrier_tasks_trace, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, + .rso_gp_kthread = get_rcu_tasks_trace_gp_kthread, .name = "tasks-tracing" }; @@ -563,6 +565,8 @@ static struct task_struct **kfree_reader_tasks; static int kfree_nrealthreads; static atomic_t n_kfree_scale_thread_started; static atomic_t n_kfree_scale_thread_ended; +static struct task_struct *kthread_tp; +static u64 kthread_stime; struct kfree_obj { char kfree_obj[8]; @@ -808,6 +812,18 @@ rcu_scale_cleanup(void) if (gp_exp && gp_async) SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!"); + // If built-in, just report all of the GP kthread's CPU time. + if (IS_BUILTIN(CONFIG_RCU_SCALE_TEST) && !kthread_tp && cur_ops->rso_gp_kthread) + kthread_tp = cur_ops->rso_gp_kthread(); + if (kthread_tp) { + u32 ns; + u64 us; + + kthread_stime = kthread_tp->stime - kthread_stime; + us = div_u64_rem(kthread_stime, 1000, &ns); + pr_info("rcu_scale: Grace-period kthread CPU time: %llu.%03u us\n", us, ns); + show_rcu_gp_kthreads(); + } if (kfree_rcu_test) { kfree_scale_cleanup(); return; @@ -921,6 +937,11 @@ rcu_scale_init(void) if (cur_ops->init) cur_ops->init(); + if (cur_ops->rso_gp_kthread) { + kthread_tp = cur_ops->rso_gp_kthread(); + if (kthread_tp) + kthread_stime = kthread_tp->stime; + } if (kfree_rcu_test) return kfree_scale_init(); diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index b770add3f843..990a6cf5fa35 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1830,6 +1830,12 @@ void show_rcu_tasks_trace_gp_kthread(void) EXPORT_SYMBOL_GPL(show_rcu_tasks_trace_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) +struct task_struct *get_rcu_tasks_trace_gp_kthread(void) +{ + return rcu_tasks_trace.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_trace_gp_kthread); + #else /* #ifdef CONFIG_TASKS_TRACE_RCU */ static void exit_tasks_rcu_finish_trace(struct task_struct *t) { } #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ From 2d7b2b344c0c665d47a3a9b540e03b834852b679 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 1 Jun 2023 09:38:50 -0700 Subject: [PATCH 023/130] rcuscale: Add kfree_by_call_rcu and kfree_mult to documentation This commit adds the kfree_by_call_rcu and kfree_mult rcuscale module parameters to kernel-parameters.txt. While in the area, it updates to rcuscale.scale_type. Signed-off-by: Paul E. McKenney Cc: Uladzislau Rezki (Sony) --- Documentation/admin-guide/kernel-parameters.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a6888e3dfc20..3329dc22f53b 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4928,6 +4928,15 @@ test until boot completes in order to avoid interference. + rcuscale.kfree_by_call_rcu= [KNL] + In kernels built with CONFIG_RCU_LAZY=y, test + call_rcu() instead of kfree_rcu(). + + rcuscale.kfree_mult= [KNL] + Instead of allocating an object of size kfree_obj, + allocate one of kfree_mult * sizeof(kfree_obj). + Defaults to 1. + rcuscale.kfree_rcu_test= [KNL] Set to measure performance of kfree_rcu() flooding. @@ -4973,7 +4982,7 @@ the same as for rcuscale.nreaders. N, where N is the number of CPUs - rcuscale.perf_type= [KNL] + rcuscale.scale_type= [KNL] Specify the RCU implementation to test. rcuscale.shutdown= [KNL] From 1d702e2eff8be1f39a1774f8a0f66f077b663b27 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 2 Jun 2023 14:01:44 -0700 Subject: [PATCH 024/130] rcuscale: Print grace-period kthread CPU time, if recorded This commit prints out the CPU time consumed by the grace-period kthread, if the specified RCU flavor supports this notion. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/kvm-recheck-rcuscale.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh index b582113178ac..f683e424ddd5 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcuscale.sh @@ -40,6 +40,10 @@ awk ' sum += $5 / 1000.; } +/rcu_scale: Grace-period kthread CPU time/ { + cputime = $6; +} + END { newNR = asort(gptimes); if (newNR <= 0) { @@ -78,6 +82,8 @@ END { print "90th percentile grace-period duration: " gptimes[pct90]; print "99th percentile grace-period duration: " gptimes[pct99]; print "Maximum grace-period duration: " gptimes[newNR]; - print "Grace periods: " ngps + 0 " Batches: " nbatches + 0 " Ratio: " ngps / nbatches; + if (cputime != "") + cpustr = " CPU: " cputime; + print "Grace periods: " ngps + 0 " Batches: " nbatches + 0 " Ratio: " ngps / nbatches cpustr; print "Computed from rcuscale printk output."; }' From 271a8467a5f7ab7654f06541d2483b5340bb7192 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 2 Jun 2023 14:38:44 -0700 Subject: [PATCH 025/130] rcuscale: Measure RCU Tasks Trace grace-period kthread CPU time This commit causes RCU Tasks Trace to output the CPU time consumed by its grace-period kthread. The CPU time is whatever is in the designated task's current->stime field, and thus is controlled by whatever CPU-time accounting scheme is in effect. This output appears in microseconds as follows on the console: rcu_scale: Grace-period kthread CPU time: 42367.037 [ paulmck: Apply Willy Tarreau feedback. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 3 +++ kernel/rcu/rcuscale.c | 1 + kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 98c1544cf572..d58bc0e86769 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -505,6 +505,9 @@ void rcu_async_relax(void); void rcupdate_announce_bootup_oddness(void); #ifdef CONFIG_TASKS_RCU_GENERIC void show_rcu_tasks_gp_kthreads(void); +# ifdef CONFIG_TASKS_RCU +struct task_struct *get_rcu_tasks_gp_kthread(void); +# endif // # ifdef CONFIG_TASKS_RCU #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ static inline void show_rcu_tasks_gp_kthreads(void) {} #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 7fba3ab66e35..35e06c86acc9 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -298,6 +298,7 @@ static struct rcu_scale_ops tasks_ops = { .gp_barrier = rcu_barrier_tasks, .sync = synchronize_rcu_tasks, .exp_sync = synchronize_rcu_tasks, + .rso_gp_kthread = get_rcu_tasks_gp_kthread, .name = "tasks" }; diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 990a6cf5fa35..0f03de023097 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1042,6 +1042,12 @@ void show_rcu_tasks_classic_gp_kthread(void) EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) +struct task_struct *get_rcu_tasks_gp_kthread(void) +{ + return rcu_tasks.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_gp_kthread); + /* * Contribute to protect against tasklist scan blind spot while the * task is exiting and may be removed from the tasklist. See From a15ec57cfcf83858aced058e00a7feb824322fa4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 3 Jun 2023 09:07:44 -0700 Subject: [PATCH 026/130] rcuscale: Add RCU Tasks Rude testing Add a "tasks-rude" option to the rcuscale.scale_type module parameter. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 3 ++ kernel/rcu/rcuscale.c | 31 ++++++++++++++++++- kernel/rcu/tasks.h | 7 +++++ .../rcutorture/configs/rcuscale/CFcommon | 2 ++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index d58bc0e86769..9829d8161b21 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -508,6 +508,9 @@ void show_rcu_tasks_gp_kthreads(void); # ifdef CONFIG_TASKS_RCU struct task_struct *get_rcu_tasks_gp_kthread(void); # endif // # ifdef CONFIG_TASKS_RCU +# ifdef CONFIG_TASKS_RUDE_RCU +struct task_struct *get_rcu_tasks_rude_gp_kthread(void); +# endif // # ifdef CONFIG_TASKS_RUDE_RCU #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ static inline void show_rcu_tasks_gp_kthreads(void) {} #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 35e06c86acc9..5ce3b4e7ce71 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -310,6 +310,35 @@ static struct rcu_scale_ops tasks_ops = { #endif // #else // #ifdef CONFIG_TASKS_RCU +#ifdef CONFIG_TASKS_RUDE_RCU + +/* + * Definitions for RCU-tasks-rude scalability testing. + */ + +static struct rcu_scale_ops tasks_rude_ops = { + .ptype = RCU_TASKS_RUDE_FLAVOR, + .init = rcu_sync_scale_init, + .readlock = tasks_scale_read_lock, + .readunlock = tasks_scale_read_unlock, + .get_gp_seq = rcu_no_completed, + .gp_diff = rcu_seq_diff, + .async = call_rcu_tasks_rude, + .gp_barrier = rcu_barrier_tasks_rude, + .sync = synchronize_rcu_tasks_rude, + .exp_sync = synchronize_rcu_tasks_rude, + .rso_gp_kthread = get_rcu_tasks_rude_gp_kthread, + .name = "tasks-rude" +}; + +#define TASKS_RUDE_OPS &tasks_rude_ops, + +#else // #ifdef CONFIG_TASKS_RUDE_RCU + +#define TASKS_RUDE_OPS + +#endif // #else // #ifdef CONFIG_TASKS_RUDE_RCU + #ifdef CONFIG_TASKS_TRACE_RCU /* @@ -913,7 +942,7 @@ rcu_scale_init(void) long i; int firsterr = 0; static struct rcu_scale_ops *scale_ops[] = { - &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_TRACING_OPS + &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS }; if (!torture_init_begin(scale_type, verbose)) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 0f03de023097..0372c5cbc83b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1194,6 +1194,13 @@ void show_rcu_tasks_rude_gp_kthread(void) } EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread); #endif // !defined(CONFIG_TINY_RCU) + +struct task_struct *get_rcu_tasks_rude_gp_kthread(void) +{ + return rcu_tasks_rude.kthread_ptr; +} +EXPORT_SYMBOL_GPL(get_rcu_tasks_rude_gp_kthread); + #endif /* #ifdef CONFIG_TASKS_RUDE_RCU */ //////////////////////////////////////////////////////////////////////// diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon b/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon index 6a00157bee5b..b1ffd7c67604 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/CFcommon @@ -2,5 +2,7 @@ CONFIG_RCU_SCALE_TEST=y CONFIG_PRINTK_TIME=y CONFIG_FORCE_TASKS_RCU=y #CHECK#CONFIG_TASKS_RCU=y +CONFIG_FORCE_TASKS_RUDE_RCU=y +#CHECK#CONFIG_TASKS_RUDE_RCU=y CONFIG_FORCE_TASKS_TRACE_RCU=y #CHECK#CONFIG_TASKS_TRACE_RCU=y From e0a34641eb551e3c51e1588ef7874f2de2844a06 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 9 Jun 2023 14:05:14 +0200 Subject: [PATCH 027/130] rcuscale: fix building with RCU_TINY Both the CONFIG_TASKS_RCU and CONFIG_TASKS_RUDE_RCU options are broken when RCU_TINY is enabled as well, as some functions are missing a declaration. In file included from kernel/rcu/update.c:649: kernel/rcu/tasks.h:1271:21: error: no previous prototype for 'get_rcu_tasks_rude_gp_kthread' [-Werror=missing-prototypes] 1271 | struct task_struct *get_rcu_tasks_rude_gp_kthread(void) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ kernel/rcu/rcuscale.c:330:27: error: 'get_rcu_tasks_rude_gp_kthread' undeclared here (not in a function); did you mean 'get_rcu_tasks_trace_gp_kthread'? 330 | .rso_gp_kthread = get_rcu_tasks_rude_gp_kthread, | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | get_rcu_tasks_trace_gp_kthread In file included from /home/arnd/arm-soc/kernel/rcu/update.c:649: kernel/rcu/tasks.h:1113:21: error: no previous prototype for 'get_rcu_tasks_gp_kthread' [-Werror=missing-prototypes] 1113 | struct task_struct *get_rcu_tasks_gp_kthread(void) | ^~~~~~~~~~~~~~~~~~~~~~~~ Also, building with CONFIG_TASKS_RUDE_RCU but not CONFIG_TASKS_RCU is broken because of some missing stub functions: kernel/rcu/rcuscale.c:322:27: error: 'tasks_scale_read_lock' undeclared here (not in a function); did you mean 'srcu_scale_read_lock'? 322 | .readlock = tasks_scale_read_lock, | ^~~~~~~~~~~~~~~~~~~~~ | srcu_scale_read_lock kernel/rcu/rcuscale.c:323:27: error: 'tasks_scale_read_unlock' undeclared here (not in a function); did you mean 'srcu_scale_read_unlock'? 323 | .readunlock = tasks_scale_read_unlock, | ^~~~~~~~~~~~~~~~~~~~~~~ | srcu_scale_read_unlock Move the declarations outside of the RCU_TINY #ifdef and duplicate the shared stub functions to address all of the above. Fixes: 88d7ff818f0ce ("rcuscale: Add RCU Tasks Rude testing") Fixes: 755f1c5eb416b ("rcuscale: Measure RCU Tasks Trace grace-period kthread CPU time") Signed-off-by: Arnd Bergmann Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 14 ++++++++------ kernel/rcu/rcuscale.c | 13 +++++++++++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 9829d8161b21..5befd8780dcd 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -505,18 +505,20 @@ void rcu_async_relax(void); void rcupdate_announce_bootup_oddness(void); #ifdef CONFIG_TASKS_RCU_GENERIC void show_rcu_tasks_gp_kthreads(void); -# ifdef CONFIG_TASKS_RCU -struct task_struct *get_rcu_tasks_gp_kthread(void); -# endif // # ifdef CONFIG_TASKS_RCU -# ifdef CONFIG_TASKS_RUDE_RCU -struct task_struct *get_rcu_tasks_rude_gp_kthread(void); -# endif // # ifdef CONFIG_TASKS_RUDE_RCU #else /* #ifdef CONFIG_TASKS_RCU_GENERIC */ static inline void show_rcu_tasks_gp_kthreads(void) {} #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ void rcu_request_urgent_qs_task(struct task_struct *t); #endif /* #else #ifdef CONFIG_TINY_RCU */ +#ifdef CONFIG_TASKS_RCU +struct task_struct *get_rcu_tasks_gp_kthread(void); +#endif // # ifdef CONFIG_TASKS_RCU + +#ifdef CONFIG_TASKS_RUDE_RCU +struct task_struct *get_rcu_tasks_rude_gp_kthread(void); +#endif // # ifdef CONFIG_TASKS_RUDE_RCU + #define RCU_SCHEDULER_INACTIVE 0 #define RCU_SCHEDULER_INIT 1 #define RCU_SCHEDULER_RUNNING 2 diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 5ce3b4e7ce71..a0eae1900708 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -316,11 +316,20 @@ static struct rcu_scale_ops tasks_ops = { * Definitions for RCU-tasks-rude scalability testing. */ +static int tasks_rude_scale_read_lock(void) +{ + return 0; +} + +static void tasks_rude_scale_read_unlock(int idx) +{ +} + static struct rcu_scale_ops tasks_rude_ops = { .ptype = RCU_TASKS_RUDE_FLAVOR, .init = rcu_sync_scale_init, - .readlock = tasks_scale_read_lock, - .readunlock = tasks_scale_read_unlock, + .readlock = tasks_rude_scale_read_lock, + .readunlock = tasks_rude_scale_read_unlock, .get_gp_seq = rcu_no_completed, .gp_diff = rcu_seq_diff, .async = call_rcu_tasks_rude, From e60c122a1614b4f65b29a7bef9d83b9fd30e937a Mon Sep 17 00:00:00 2001 From: Zqiang Date: Fri, 16 Jun 2023 15:39:26 +0800 Subject: [PATCH 028/130] rcuscale: Move rcu_scale_writer() schedule_timeout_uninterruptible() to _idle() The rcuscale.holdoff module parameter can be used to delay the start of rcu_scale_writer() kthread. However, the hung-task timeout will trigger when the timeout specified by rcuscale.holdoff is greater than hung_task_timeout_secs: runqemu kvm nographic slirp qemuparams="-smp 4 -m 2048M" bootparams="rcuscale.shutdown=0 rcuscale.holdoff=300" [ 247.071753] INFO: task rcu_scale_write:59 blocked for more than 122 seconds. [ 247.072529] Not tainted 6.4.0-rc1-00134-gb9ed6de8d4ff #7 [ 247.073400] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 247.074331] task:rcu_scale_write state:D stack:30144 pid:59 ppid:2 flags:0x00004000 [ 247.075346] Call Trace: [ 247.075660] [ 247.075965] __schedule+0x635/0x1280 [ 247.076448] ? __pfx___schedule+0x10/0x10 [ 247.076967] ? schedule_timeout+0x2dc/0x4d0 [ 247.077471] ? __pfx_lock_release+0x10/0x10 [ 247.078018] ? enqueue_timer+0xe2/0x220 [ 247.078522] schedule+0x84/0x120 [ 247.078957] schedule_timeout+0x2e1/0x4d0 [ 247.079447] ? __pfx_schedule_timeout+0x10/0x10 [ 247.080032] ? __pfx_rcu_scale_writer+0x10/0x10 [ 247.080591] ? __pfx_process_timeout+0x10/0x10 [ 247.081163] ? __pfx_sched_set_fifo_low+0x10/0x10 [ 247.081760] ? __pfx_rcu_scale_writer+0x10/0x10 [ 247.082287] rcu_scale_writer+0x6b1/0x7f0 [ 247.082773] ? mark_held_locks+0x29/0xa0 [ 247.083252] ? __pfx_rcu_scale_writer+0x10/0x10 [ 247.083865] ? __pfx_rcu_scale_writer+0x10/0x10 [ 247.084412] kthread+0x179/0x1c0 [ 247.084759] ? __pfx_kthread+0x10/0x10 [ 247.085098] ret_from_fork+0x2c/0x50 [ 247.085433] This commit therefore replaces schedule_timeout_uninterruptible() with schedule_timeout_idle(). Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index a0eae1900708..ffdb30495e3c 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -469,7 +469,7 @@ rcu_scale_writer(void *arg) sched_set_fifo_low(current); if (holdoff) - schedule_timeout_uninterruptible(holdoff * HZ); + schedule_timeout_idle(holdoff * HZ); /* * Wait until rcu_end_inkernel_boot() is called for normal GP tests From 8afeb54190e80cb4d1c9583b06dc16d15198dce0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 16:17:27 -0700 Subject: [PATCH 029/130] rcuscale: Add CONFIG_PREEMPT_DYNAMIC=n to TRACE01 scenario It is no longer possible to build a kernel with a preemption-disabled RCU without use of CONFIG_PREEMPT_DYNAMIC=n. This commit therefore adds CONFIG_PREEMPT_DYNAMIC=n to the rcuscale torture type's TRACE01 scenario file. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 b/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 index 227aba7783af..0059592c7408 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/TRACE01 @@ -2,6 +2,8 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +#CHECK#CONFIG_TREE_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n From 822e42509997cfc8582ab1108eef1a280524b26a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 06:18:32 -0700 Subject: [PATCH 030/130] torture: Scale scftorture memory based on number of CPUs As the number of CPUs increases, the number of outstanding no-wait smp_call_function() handlers also increases, so that the default of 2G of memory is not always sufficient on 80-CPU systems. This commit therefore scales the amount of memory specified to qemu based on the number of CPUs specified to the scftorture test instance. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/torture.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index 5a2ae2264403..9fa6526067f0 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -376,8 +376,10 @@ fi if test "$do_scftorture" = "yes" then + # Scale memory based on the number of CPUs. + scfmem=$((2+HALF_ALLOTED_CPUS/16)) torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1" - torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 2G --trust-make + torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory ${scfmem}G --trust-make fi if test "$do_rt" = "yes" From 013608cd0812bdb21fc26d39ed8fdd2fc76e8b9b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 15 May 2023 19:00:10 -0700 Subject: [PATCH 031/130] scftorture: Forgive memory-allocation failure if KASAN Kernels built with CONFIG_KASAN=y quarantine newly freed memory in order to better detect use-after-free errors. However, this can exhaust memory more quickly in allocator-heavy tests, which can result in spurious scftorture failure. This commit therefore forgives memory-allocation failure in kernels built with CONFIG_KASAN=y, but continues counting the errors for use in detailed test-result analyses. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 5d113aa59e77..83c33ba0ca7e 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -171,7 +171,8 @@ static void scf_torture_stats_print(void) scfs.n_all_wait += scf_stats_p[i].n_all_wait; } if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || - atomic_read(&n_mb_out_errs) || atomic_read(&n_alloc_errs)) + atomic_read(&n_mb_out_errs) || + (!IS_ENABLED(CONFIG_KASAN) && atomic_read(&n_alloc_errs))) bangstr = "!!! "; pr_alert("%s %sscf_invoked_count %s: %lld resched: %lld single: %lld/%lld single_ofl: %lld/%lld single_rpc: %lld single_rpc_ofl: %lld many: %lld/%lld all: %lld/%lld ", SCFTORT_FLAG, bangstr, isdone ? "VER" : "ver", invoked_count, scfs.n_resched, @@ -323,7 +324,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra preempt_disable(); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); - if (WARN_ON_ONCE(!scfcp)) { + if (!scfcp) { + WARN_ON_ONCE(!IS_ENABLED(CONFIG_KASAN)); atomic_inc(&n_alloc_errs); } else { scfcp->scfc_cpu = -1; From 4a71be93876171e2d30e413abd33126d8c526791 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 06:21:02 -0700 Subject: [PATCH 032/130] scftorture: Pause testing after memory-allocation failure The scftorture test can quickly execute a large number of calls to no-wait smp_call_function(), each of which holds a block of memory until the corresponding handler is invoked. Especially when the longwait module parameter is specified, this can chew up an arbitrarily large amount of memory. This commit therefore blocks after each memory-allocation failure, with the duration a function of longwait. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 83c33ba0ca7e..59032aaccd18 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -313,6 +313,7 @@ static void scf_handler_1(void *scfc_in) // Randomly do an smp_call_function*() invocation. static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp) { + bool allocfail = false; uintptr_t cpu; int ret = 0; struct scf_check *scfcp = NULL; @@ -327,6 +328,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra if (!scfcp) { WARN_ON_ONCE(!IS_ENABLED(CONFIG_KASAN)); atomic_inc(&n_alloc_errs); + allocfail = true; } else { scfcp->scfc_cpu = -1; scfcp->scfc_wait = scfsp->scfs_wait; @@ -433,7 +435,9 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra cpus_read_unlock(); else preempt_enable(); - if (!(torture_random(trsp) & 0xfff)) + if (allocfail) + schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete. + else if (!(torture_random(trsp) & 0xfff)) schedule_timeout_uninterruptible(1); } From 3f68f9c822ebe208f961ab0402e49a10465278ca Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 17:44:15 -0700 Subject: [PATCH 033/130] scftorture: Add CONFIG_PREEMPT_DYNAMIC=n to NOPREEMPT scenario It is no longer possible to build a kernel with a preemption-disabled RCU without use of CONFIG_PREEMPT_DYNAMIC=n. This commit therefore adds CONFIG_PREEMPT_DYNAMIC=n to the scf torture type's NOPREEMPT scenario file. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT index 3a59346b3de7..6133f54ce2a7 100644 --- a/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT +++ b/tools/testing/selftests/rcutorture/configs/scf/NOPREEMPT @@ -2,6 +2,8 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=y CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +#CHECK#CONFIG_PREEMPT_RCU=n CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=n CONFIG_NO_HZ_FULL=y From 9cafe974cf46425c18cf78971400ae12e416e166 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 15 May 2023 09:18:20 -0700 Subject: [PATCH 034/130] rcutorture: Dump grace-period state upon rtort_pipe_count incidents The rtort_pipe_count WARN() indicates that grace periods were unable to invoke all callbacks during a stutter_wait() interval. But it is sometimes helpful to have a bit more information as to why. This commit therefore invokes show_rcu_gp_kthreads() immediately before that WARN() in order to dump out some relevant information. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 147551c23baf..d291a1438c30 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1581,6 +1581,7 @@ rcu_torture_writer(void *arg) rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { tracing_off(); + show_rcu_gp_kthreads(); WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); rcu_ftrace_dump(DUMP_ALL); } From e2a0b786c50cf4f0c3abfcd05888ae583c98cf25 Mon Sep 17 00:00:00 2001 From: Connor O'Brien Date: Fri, 2 Jun 2023 22:02:09 +0000 Subject: [PATCH 035/130] torture: Support randomized shuffling for proxy exec testing Currently shuffling sets the same cpu affinities for all tasks, which makes us less likely to hit paths involving migrating blocked tasks onto a cpu where they can't run. This patch adds an element of randomness to allow affinities of different writer tasks to diverge. This has helped uncover issues in testing with Proxy Execution Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Signed-off-by: Connor O'Brien Signed-off-by: John Stultz Acked-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- kernel/torture.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/torture.c b/kernel/torture.c index 1a0519b836ac..8be83fdc6be1 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -54,6 +54,9 @@ module_param(verbose_sleep_frequency, int, 0444); static int verbose_sleep_duration = 1; module_param(verbose_sleep_duration, int, 0444); +static int random_shuffle; +module_param(random_shuffle, int, 0444); + static char *torture_type; static int verbose; @@ -518,6 +521,7 @@ static void torture_shuffle_task_unregister_all(void) */ static void torture_shuffle_tasks(void) { + DEFINE_TORTURE_RANDOM(rand); struct shuffle_task *stp; cpumask_setall(shuffle_tmp_mask); @@ -537,8 +541,10 @@ static void torture_shuffle_tasks(void) cpumask_clear_cpu(shuffle_idle_cpu, shuffle_tmp_mask); mutex_lock(&shuffle_task_mutex); - list_for_each_entry(stp, &shuffle_task_list, st_l) - set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); + list_for_each_entry(stp, &shuffle_task_list, st_l) { + if (!random_shuffle || torture_random(&rand) & 0x1) + set_cpus_allowed_ptr(stp->st_t, shuffle_tmp_mask); + } mutex_unlock(&shuffle_task_mutex); cpus_read_unlock(); From 965167e8e6c9877390bb9574f88da21dc42f03eb Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 21 Jun 2023 09:44:52 -0700 Subject: [PATCH 036/130] rcutorture: Remove obsolete BOOTPARAM_HOTPLUG_CPU0 Kconfig option Now that the BOOTPARAM_HOTPLUG_CPU0 Kconfig option is in the process of being removed, it is time to remove rcutorture's use of it. Link: https://lore.kernel.org/lkml/20230414232309.510911744@linutronix.de/ Signed-off-by: Paul E. McKenney Cc: Thomas Gleixner Cc: --- tools/testing/selftests/rcutorture/configs/rcu/TREE01 | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 index 04831ef1f9b5..8ae41d5f81a3 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 @@ -15,4 +15,3 @@ CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n CONFIG_RCU_EXPERT=y -CONFIG_BOOTPARAM_HOTPLUG_CPU0=y From 1304affd35739d83b4089a297d39ab08f1624158 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 11 May 2023 19:12:16 -0700 Subject: [PATCH 037/130] rcu: Remove formal-verification tests The CBMC-based formal-verification testing for SRCU was quite the thing back in 2016, but the problem is that SRCU changes too quickly for the scripting to keep up. In addition, more recently, SRCU's grace-period ordering has been formally modeled by a group of Linux-kernel memory-model litmus tests. This commit therefore removes the pioneering formal-verification tests. Signed-off-by: Paul E. McKenney --- .../rcutorture/formal/srcu-cbmc/.gitignore | 2 - .../rcutorture/formal/srcu-cbmc/Makefile | 17 - .../srcu-cbmc/empty_includes/linux/delay.h | 0 .../srcu-cbmc/empty_includes/linux/export.h | 0 .../srcu-cbmc/empty_includes/linux/mutex.h | 0 .../srcu-cbmc/empty_includes/linux/percpu.h | 0 .../srcu-cbmc/empty_includes/linux/preempt.h | 0 .../srcu-cbmc/empty_includes/linux/rcupdate.h | 0 .../srcu-cbmc/empty_includes/linux/sched.h | 0 .../srcu-cbmc/empty_includes/linux/smp.h | 0 .../empty_includes/linux/workqueue.h | 0 .../empty_includes/uapi/linux/types.h | 0 .../formal/srcu-cbmc/include/linux/.gitignore | 2 - .../formal/srcu-cbmc/include/linux/kconfig.h | 1 - .../formal/srcu-cbmc/include/linux/types.h | 152 ------- .../formal/srcu-cbmc/modify_srcu.awk | 376 ------------------ .../rcutorture/formal/srcu-cbmc/src/assume.h | 17 - .../formal/srcu-cbmc/src/barriers.h | 41 -- .../rcutorture/formal/srcu-cbmc/src/bug_on.h | 14 - .../formal/srcu-cbmc/src/combined_source.c | 14 - .../rcutorture/formal/srcu-cbmc/src/config.h | 28 -- .../formal/srcu-cbmc/src/include_srcu.c | 32 -- .../formal/srcu-cbmc/src/int_typedefs.h | 34 -- .../rcutorture/formal/srcu-cbmc/src/locks.h | 221 ---------- .../rcutorture/formal/srcu-cbmc/src/misc.c | 12 - .../rcutorture/formal/srcu-cbmc/src/misc.h | 58 --- .../rcutorture/formal/srcu-cbmc/src/percpu.h | 93 ----- .../rcutorture/formal/srcu-cbmc/src/preempt.c | 79 ---- .../rcutorture/formal/srcu-cbmc/src/preempt.h | 59 --- .../formal/srcu-cbmc/src/simple_sync_srcu.c | 51 --- .../formal/srcu-cbmc/src/workqueues.h | 103 ----- .../tests/store_buffering/.gitignore | 2 - .../srcu-cbmc/tests/store_buffering/Makefile | 12 - .../tests/store_buffering/assert_end.fail | 1 - .../tests/store_buffering/force.fail | 1 - .../tests/store_buffering/force2.fail | 1 - .../tests/store_buffering/force3.fail | 1 - .../srcu-cbmc/tests/store_buffering/main.pass | 0 .../srcu-cbmc/tests/store_buffering/test.c | 73 ---- .../formal/srcu-cbmc/tests/test_script.sh | 103 ----- 40 files changed, 1600 deletions(-) delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/delay.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/export.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/mutex.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/percpu.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/preempt.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/rcupdate.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/sched.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/smp.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/workqueue.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/uapi/linux/types.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h delete mode 100755 tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/main.pass delete mode 100644 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c delete mode 100755 tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore deleted file mode 100644 index 24e27957efcc..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -srcu.c diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile deleted file mode 100644 index 4bed0b678f8b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -all: srcu.c store_buffering - -LINUX_SOURCE = ../../../../../.. - -modified_srcu_input = $(LINUX_SOURCE)/include/linux/srcu.h \ - $(LINUX_SOURCE)/kernel/rcu/srcu.c - -modified_srcu_output = include/linux/srcu.h srcu.c - -include/linux/srcu.h: srcu.c - -srcu.c: modify_srcu.awk Makefile $(modified_srcu_input) - awk -f modify_srcu.awk $(modified_srcu_input) $(modified_srcu_output) - -store_buffering: - @cd tests/store_buffering; make diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/delay.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/delay.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/export.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/export.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/mutex.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/mutex.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/percpu.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/percpu.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/preempt.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/preempt.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/rcupdate.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/rcupdate.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/sched.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/sched.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/smp.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/smp.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/workqueue.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/linux/workqueue.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/uapi/linux/types.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/empty_includes/uapi/linux/types.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore deleted file mode 100644 index 57d296341304..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -srcu.h diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h deleted file mode 100644 index f2860dd1b407..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/kconfig.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h deleted file mode 100644 index 8bc960e5e713..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/include/linux/types.h +++ /dev/null @@ -1,152 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This header has been modifies to remove definitions of types that - * are defined in standard userspace headers or are problematic for some - * other reason. - */ - -#ifndef _LINUX_TYPES_H -#define _LINUX_TYPES_H - -#define __EXPORTED_HEADERS__ -#include - -#ifndef __ASSEMBLY__ - -#define DECLARE_BITMAP(name, bits) \ - unsigned long name[BITS_TO_LONGS(bits)] - -typedef __u32 __kernel_dev_t; - -/* bsd */ -typedef unsigned char u_char; -typedef unsigned short u_short; -typedef unsigned int u_int; -typedef unsigned long u_long; - -/* sysv */ -typedef unsigned char unchar; -typedef unsigned short ushort; -typedef unsigned int uint; -typedef unsigned long ulong; - -#ifndef __BIT_TYPES_DEFINED__ -#define __BIT_TYPES_DEFINED__ - -typedef __u8 u_int8_t; -typedef __s8 int8_t; -typedef __u16 u_int16_t; -typedef __s16 int16_t; -typedef __u32 u_int32_t; -typedef __s32 int32_t; - -#endif /* !(__BIT_TYPES_DEFINED__) */ - -typedef __u8 uint8_t; -typedef __u16 uint16_t; -typedef __u32 uint32_t; - -/* this is a special 64bit data type that is 8-byte aligned */ -#define aligned_u64 __u64 __attribute__((aligned(8))) -#define aligned_be64 __be64 __attribute__((aligned(8))) -#define aligned_le64 __le64 __attribute__((aligned(8))) - -/** - * The type used for indexing onto a disc or disc partition. - * - * Linux always considers sectors to be 512 bytes long independently - * of the devices real block size. - * - * blkcnt_t is the type of the inode's block count. - */ -typedef u64 sector_t; - -/* - * The type of an index into the pagecache. - */ -#define pgoff_t unsigned long - -/* - * A dma_addr_t can hold any valid DMA address, i.e., any address returned - * by the DMA API. - * - * If the DMA API only uses 32-bit addresses, dma_addr_t need only be 32 - * bits wide. Bus addresses, e.g., PCI BARs, may be wider than 32 bits, - * but drivers do memory-mapped I/O to ioremapped kernel virtual addresses, - * so they don't care about the size of the actual bus addresses. - */ -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT -typedef u64 dma_addr_t; -#else -typedef u32 dma_addr_t; -#endif - -#ifdef CONFIG_PHYS_ADDR_T_64BIT -typedef u64 phys_addr_t; -#else -typedef u32 phys_addr_t; -#endif - -typedef phys_addr_t resource_size_t; - -/* - * This type is the placeholder for a hardware interrupt number. It has to be - * big enough to enclose whatever representation is used by a given platform. - */ -typedef unsigned long irq_hw_number_t; - -typedef struct { - int counter; -} atomic_t; - -#ifdef CONFIG_64BIT -typedef struct { - long counter; -} atomic64_t; -#endif - -struct list_head { - struct list_head *next, *prev; -}; - -struct hlist_head { - struct hlist_node *first; -}; - -struct hlist_node { - struct hlist_node *next, **pprev; -}; - -/** - * struct callback_head - callback structure for use with RCU and task_work - * @next: next update requests in a list - * @func: actual update function to call after the grace period. - * - * The struct is aligned to size of pointer. On most architectures it happens - * naturally due ABI requirements, but some architectures (like CRIS) have - * weird ABI and we need to ask it explicitly. - * - * The alignment is required to guarantee that bits 0 and 1 of @next will be - * clear under normal conditions -- as long as we use call_rcu() or - * call_srcu() to queue callback. - * - * This guarantee is important for few reasons: - * - future call_rcu_lazy() will make use of lower bits in the pointer; - * - the structure shares storage spacer in struct page with @compound_head, - * which encode PageTail() in bit 0. The guarantee is needed to avoid - * false-positive PageTail(). - */ -struct callback_head { - struct callback_head *next; - void (*func)(struct callback_head *head); -} __attribute__((aligned(sizeof(void *)))); -#define rcu_head callback_head - -typedef void (*rcu_callback_t)(struct rcu_head *head); -typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); - -/* clocksource cycle base type */ -typedef u64 cycle_t; - -#endif /* __ASSEMBLY__ */ -#endif /* _LINUX_TYPES_H */ diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk deleted file mode 100755 index e05182d3e47d..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/modify_srcu.awk +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/awk -f -# SPDX-License-Identifier: GPL-2.0 - -# Modify SRCU for formal verification. The first argument should be srcu.h and -# the second should be srcu.c. Outputs modified srcu.h and srcu.c into the -# current directory. - -BEGIN { - if (ARGC != 5) { - print "Usange: input.h input.c output.h output.c" > "/dev/stderr"; - exit 1; - } - h_output = ARGV[3]; - c_output = ARGV[4]; - ARGC = 3; - - # Tokenize using FS and not RS as FS supports regular expressions. Each - # record is one line of source, except that backslashed lines are - # combined. Comments are treated as field separators, as are quotes. - quote_regexp="\"([^\\\\\"]|\\\\.)*\""; - comment_regexp="\\/\\*([^*]|\\*+[^*/])*\\*\\/|\\/\\/.*(\n|$)"; - FS="([ \\\\\t\n\v\f;,.=(){}+*/<>&|^-]|\\[|\\]|" comment_regexp "|" quote_regexp ")+"; - - inside_srcu_struct = 0; - inside_srcu_init_def = 0; - srcu_init_param_name = ""; - in_macro = 0; - brace_nesting = 0; - paren_nesting = 0; - - # Allow the manipulation of the last field separator after has been - # seen. - last_fs = ""; - # Whether the last field separator was intended to be output. - last_fs_print = 0; - - # rcu_batches stores the initialization for each instance of struct - # rcu_batch - - in_comment = 0; - - outputfile = ""; -} - -{ - prev_outputfile = outputfile; - if (FILENAME ~ /\.h$/) { - outputfile = h_output; - if (FNR != NR) { - print "Incorrect file order" > "/dev/stderr"; - exit 1; - } - } - else - outputfile = c_output; - - if (prev_outputfile && outputfile != prev_outputfile) { - new_outputfile = outputfile; - outputfile = prev_outputfile; - update_fieldsep("", 0); - outputfile = new_outputfile; - } -} - -# Combine the next line into $0. -function combine_line() { - ret = getline next_line; - if (ret == 0) { - # Don't allow two consecutive getlines at the end of the file - if (eof_found) { - print "Error: expected more input." > "/dev/stderr"; - exit 1; - } else { - eof_found = 1; - } - } else if (ret == -1) { - print "Error reading next line of file" FILENAME > "/dev/stderr"; - exit 1; - } - $0 = $0 "\n" next_line; -} - -# Combine backslashed lines and multiline comments. -function combine_backslashes() { - while (/\\$|\/\*([^*]|\*+[^*\/])*\**$/) { - combine_line(); - } -} - -function read_line() { - combine_line(); - combine_backslashes(); -} - -# Print out field separators and update variables that depend on them. Only -# print if p is true. Call with sep="" and p=0 to print out the last field -# separator. -function update_fieldsep(sep, p) { - # Count braces - sep_tmp = sep; - gsub(quote_regexp "|" comment_regexp, "", sep_tmp); - while (1) - { - if (sub("[^{}()]*\\{", "", sep_tmp)) { - brace_nesting++; - continue; - } - if (sub("[^{}()]*\\}", "", sep_tmp)) { - brace_nesting--; - if (brace_nesting < 0) { - print "Unbalanced braces!" > "/dev/stderr"; - exit 1; - } - continue; - } - if (sub("[^{}()]*\\(", "", sep_tmp)) { - paren_nesting++; - continue; - } - if (sub("[^{}()]*\\)", "", sep_tmp)) { - paren_nesting--; - if (paren_nesting < 0) { - print "Unbalanced parenthesis!" > "/dev/stderr"; - exit 1; - } - continue; - } - - break; - } - - if (last_fs_print) - printf("%s", last_fs) > outputfile; - last_fs = sep; - last_fs_print = p; -} - -# Shifts the fields down by n positions. Calls next if there are no more. If p -# is true then print out field separators. -function shift_fields(n, p) { - do { - if (match($0, FS) > 0) { - update_fieldsep(substr($0, RSTART, RLENGTH), p); - if (RSTART + RLENGTH <= length()) - $0 = substr($0, RSTART + RLENGTH); - else - $0 = ""; - } else { - update_fieldsep("", 0); - print "" > outputfile; - next; - } - } while (--n > 0); -} - -# Shifts and prints the first n fields. -function print_fields(n) { - do { - update_fieldsep("", 0); - printf("%s", $1) > outputfile; - shift_fields(1, 1); - } while (--n > 0); -} - -{ - combine_backslashes(); -} - -# Print leading FS -{ - if (match($0, "^(" FS ")+") > 0) { - update_fieldsep(substr($0, RSTART, RLENGTH), 1); - if (RSTART + RLENGTH <= length()) - $0 = substr($0, RSTART + RLENGTH); - else - $0 = ""; - } -} - -# Parse the line. -{ - while (NF > 0) { - if ($1 == "struct" && NF < 3) { - read_line(); - continue; - } - - if (FILENAME ~ /\.h$/ && !inside_srcu_struct && - brace_nesting == 0 && paren_nesting == 0 && - $1 == "struct" && $2 == "srcu_struct" && - $0 ~ "^struct(" FS ")+srcu_struct(" FS ")+\\{") { - inside_srcu_struct = 1; - print_fields(2); - continue; - } - if (inside_srcu_struct && brace_nesting == 0 && - paren_nesting == 0) { - inside_srcu_struct = 0; - update_fieldsep("", 0); - for (name in rcu_batches) - print "extern struct rcu_batch " name ";" > outputfile; - } - - if (inside_srcu_struct && $1 == "struct" && $2 == "rcu_batch") { - # Move rcu_batches outside of the struct. - rcu_batches[$3] = ""; - shift_fields(3, 1); - sub(/;[[:space:]]*$/, "", last_fs); - continue; - } - - if (FILENAME ~ /\.h$/ && !inside_srcu_init_def && - $1 == "#define" && $2 == "__SRCU_STRUCT_INIT") { - inside_srcu_init_def = 1; - srcu_init_param_name = $3; - in_macro = 1; - print_fields(3); - continue; - } - if (inside_srcu_init_def && brace_nesting == 0 && - paren_nesting == 0) { - inside_srcu_init_def = 0; - in_macro = 0; - continue; - } - - if (inside_srcu_init_def && brace_nesting == 1 && - paren_nesting == 0 && last_fs ~ /\.[[:space:]]*$/ && - $1 ~ /^[[:alnum:]_]+$/) { - name = $1; - if (name in rcu_batches) { - # Remove the dot. - sub(/\.[[:space:]]*$/, "", last_fs); - - old_record = $0; - do - shift_fields(1, 0); - while (last_fs !~ /,/ || paren_nesting > 0); - end_loc = length(old_record) - length($0); - end_loc += index(last_fs, ",") - length(last_fs); - - last_fs = substr(last_fs, index(last_fs, ",") + 1); - last_fs_print = 1; - - match(old_record, "^"name"("FS")+="); - start_loc = RSTART + RLENGTH; - - len = end_loc - start_loc; - initializer = substr(old_record, start_loc, len); - gsub(srcu_init_param_name "\\.", "", initializer); - rcu_batches[name] = initializer; - continue; - } - } - - # Don't include a nonexistent file - if (!in_macro && $1 == "#include" && /^#include[[:space:]]+"rcu\.h"/) { - update_fieldsep("", 0); - next; - } - - # Ignore most preprocessor stuff. - if (!in_macro && $1 ~ /#/) { - break; - } - - if (brace_nesting > 0 && $1 ~ "^[[:alnum:]_]+$" && NF < 2) { - read_line(); - continue; - } - if (brace_nesting > 0 && - $0 ~ "^[[:alnum:]_]+[[:space:]]*(\\.|->)[[:space:]]*[[:alnum:]_]+" && - $2 in rcu_batches) { - # Make uses of rcu_batches global. Somewhat unreliable. - shift_fields(1, 0); - print_fields(1); - continue; - } - - if ($1 == "static" && NF < 3) { - read_line(); - continue; - } - if ($1 == "static" && ($2 == "bool" && $3 == "try_check_zero" || - $2 == "void" && $3 == "srcu_flip")) { - shift_fields(1, 1); - print_fields(2); - continue; - } - - # Distinguish between read-side and write-side memory barriers. - if ($1 == "smp_mb" && NF < 2) { - read_line(); - continue; - } - if (match($0, /^smp_mb[[:space:]();\/*]*[[:alnum:]]/)) { - barrier_letter = substr($0, RLENGTH, 1); - if (barrier_letter ~ /A|D/) - new_barrier_name = "sync_smp_mb"; - else if (barrier_letter ~ /B|C/) - new_barrier_name = "rs_smp_mb"; - else { - print "Unrecognized memory barrier." > "/dev/null"; - exit 1; - } - - shift_fields(1, 1); - printf("%s", new_barrier_name) > outputfile; - continue; - } - - # Skip definition of rcu_synchronize, since it is already - # defined in misc.h. Only present in old versions of srcu. - if (brace_nesting == 0 && paren_nesting == 0 && - $1 == "struct" && $2 == "rcu_synchronize" && - $0 ~ "^struct(" FS ")+rcu_synchronize(" FS ")+\\{") { - shift_fields(2, 0); - while (brace_nesting) { - if (NF < 2) - read_line(); - shift_fields(1, 0); - } - } - - # Skip definition of wakeme_after_rcu for the same reason - if (brace_nesting == 0 && $1 == "static" && $2 == "void" && - $3 == "wakeme_after_rcu") { - while (NF < 5) - read_line(); - shift_fields(3, 0); - do { - while (NF < 3) - read_line(); - shift_fields(1, 0); - } while (paren_nesting || brace_nesting); - } - - if ($1 ~ /^(unsigned|long)$/ && NF < 3) { - read_line(); - continue; - } - - # Give srcu_batches_completed the correct type for old SRCU. - if (brace_nesting == 0 && $1 == "long" && - $2 == "srcu_batches_completed") { - update_fieldsep("", 0); - printf("unsigned ") > outputfile; - print_fields(2); - continue; - } - if (brace_nesting == 0 && $1 == "unsigned" && $2 == "long" && - $3 == "srcu_batches_completed") { - print_fields(3); - continue; - } - - # Just print out the input code by default. - print_fields(1); - } - update_fieldsep("", 0); - print > outputfile; - next; -} - -END { - update_fieldsep("", 0); - - if (brace_nesting != 0) { - print "Unbalanced braces!" > "/dev/stderr"; - exit 1; - } - - # Define the rcu_batches - for (name in rcu_batches) - print "struct rcu_batch " name " = " rcu_batches[name] ";" > c_output; -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h deleted file mode 100644 index 570a49d9da7e..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/assume.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASSUME_H -#define ASSUME_H - -/* Provide an assumption macro that can be disabled for gcc. */ -#ifdef RUN -#define assume(x) \ - do { \ - /* Evaluate x to suppress warnings. */ \ - (void) (x); \ - } while (0) - -#else -#define assume(x) __CPROVER_assume(x) -#endif - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h deleted file mode 100644 index 3f95a768a03b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/barriers.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef BARRIERS_H -#define BARRIERS_H - -#define barrier() __asm__ __volatile__("" : : : "memory") - -#ifdef RUN -#define smp_mb() __sync_synchronize() -#define smp_mb__after_unlock_lock() __sync_synchronize() -#else -/* - * Copied from CBMC's implementation of __sync_synchronize(), which - * seems to be disabled by default. - */ -#define smp_mb() __CPROVER_fence("WWfence", "RRfence", "RWfence", "WRfence", \ - "WWcumul", "RRcumul", "RWcumul", "WRcumul") -#define smp_mb__after_unlock_lock() __CPROVER_fence("WWfence", "RRfence", "RWfence", "WRfence", \ - "WWcumul", "RRcumul", "RWcumul", "WRcumul") -#endif - -/* - * Allow memory barriers to be disabled in either the read or write side - * of SRCU individually. - */ - -#ifndef NO_SYNC_SMP_MB -#define sync_smp_mb() smp_mb() -#else -#define sync_smp_mb() do {} while (0) -#endif - -#ifndef NO_READ_SIDE_SMP_MB -#define rs_smp_mb() smp_mb() -#else -#define rs_smp_mb() do {} while (0) -#endif - -#define READ_ONCE(x) (*(volatile typeof(x) *) &(x)) -#define WRITE_ONCE(x) ((*(volatile typeof(x) *) &(x)) = (val)) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h deleted file mode 100644 index 5e7912c6a521..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/bug_on.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef BUG_ON_H -#define BUG_ON_H - -#include - -#define BUG() assert(0) -#define BUG_ON(x) assert(!(x)) - -/* Does it make sense to treat warnings as errors? */ -#define WARN() BUG() -#define WARN_ON(x) (BUG_ON(x), false) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c deleted file mode 100644 index e67ee5b3dd7c..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/combined_source.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -/* Include all source files. */ - -#include "include_srcu.c" - -#include "preempt.c" -#include "misc.c" - -/* Used by test.c files */ -#include -#include -#include diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h deleted file mode 100644 index 283d7103334f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* "Cheater" definitions based on restricted Kconfig choices. */ - -#undef CONFIG_TINY_RCU -#undef __CHECKER__ -#undef CONFIG_DEBUG_LOCK_ALLOC -#undef CONFIG_DEBUG_OBJECTS_RCU_HEAD -#undef CONFIG_HOTPLUG_CPU -#undef CONFIG_MODULES -#undef CONFIG_NO_HZ_FULL_SYSIDLE -#undef CONFIG_PREEMPT_COUNT -#undef CONFIG_PREEMPT_RCU -#undef CONFIG_PROVE_RCU -#undef CONFIG_RCU_NOCB_CPU -#undef CONFIG_RCU_NOCB_CPU_ALL -#undef CONFIG_RCU_STALL_COMMON -#undef CONFIG_RCU_TRACE -#undef CONFIG_RCU_USER_QS -#undef CONFIG_TASKS_RCU -#define CONFIG_TREE_RCU - -#define CONFIG_GENERIC_ATOMIC64 - -#if NR_CPUS > 1 -#define CONFIG_SMP -#else -#undef CONFIG_SMP -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c deleted file mode 100644 index e5202d4cff30..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/include_srcu.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "int_typedefs.h" - -#include "barriers.h" -#include "bug_on.h" -#include "locks.h" -#include "misc.h" -#include "preempt.h" -#include "percpu.h" -#include "workqueues.h" - -#ifdef USE_SIMPLE_SYNC_SRCU -#define synchronize_srcu(sp) synchronize_srcu_original(sp) -#endif - -#include - -#ifdef USE_SIMPLE_SYNC_SRCU -#undef synchronize_srcu - -#include "simple_sync_srcu.c" -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h deleted file mode 100644 index 0dd27aa517a7..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/int_typedefs.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef INT_TYPEDEFS_H -#define INT_TYPEDEFS_H - -#include - -typedef int8_t s8; -typedef uint8_t u8; -typedef int16_t s16; -typedef uint16_t u16; -typedef int32_t s32; -typedef uint32_t u32; -typedef int64_t s64; -typedef uint64_t u64; - -typedef int8_t __s8; -typedef uint8_t __u8; -typedef int16_t __s16; -typedef uint16_t __u16; -typedef int32_t __s32; -typedef uint32_t __u32; -typedef int64_t __s64; -typedef uint64_t __u64; - -#define S8_C(x) INT8_C(x) -#define U8_C(x) UINT8_C(x) -#define S16_C(x) INT16_C(x) -#define U16_C(x) UINT16_C(x) -#define S32_C(x) INT32_C(x) -#define U32_C(x) UINT32_C(x) -#define S64_C(x) INT64_C(x) -#define U64_C(x) UINT64_C(x) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h deleted file mode 100644 index 1e24827f96f1..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/locks.h +++ /dev/null @@ -1,221 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef LOCKS_H -#define LOCKS_H - -#include -#include -#include - -#include "assume.h" -#include "bug_on.h" -#include "preempt.h" - -int nondet_int(void); - -#define __acquire(x) -#define __acquires(x) -#define __release(x) -#define __releases(x) - -/* Only use one lock mechanism. Select which one. */ -#ifdef PTHREAD_LOCK -struct lock_impl { - pthread_mutex_t mutex; -}; - -static inline void lock_impl_lock(struct lock_impl *lock) -{ - BUG_ON(pthread_mutex_lock(&lock->mutex)); -} - -static inline void lock_impl_unlock(struct lock_impl *lock) -{ - BUG_ON(pthread_mutex_unlock(&lock->mutex)); -} - -static inline bool lock_impl_trylock(struct lock_impl *lock) -{ - int err = pthread_mutex_trylock(&lock->mutex); - - if (!err) - return true; - else if (err == EBUSY) - return false; - BUG(); -} - -static inline void lock_impl_init(struct lock_impl *lock) -{ - pthread_mutex_init(&lock->mutex, NULL); -} - -#define LOCK_IMPL_INITIALIZER {.mutex = PTHREAD_MUTEX_INITIALIZER} - -#else /* !defined(PTHREAD_LOCK) */ -/* Spinlock that assumes that it always gets the lock immediately. */ - -struct lock_impl { - bool locked; -}; - -static inline bool lock_impl_trylock(struct lock_impl *lock) -{ -#ifdef RUN - /* TODO: Should this be a test and set? */ - return __sync_bool_compare_and_swap(&lock->locked, false, true); -#else - __CPROVER_atomic_begin(); - bool old_locked = lock->locked; - lock->locked = true; - __CPROVER_atomic_end(); - - /* Minimal barrier to prevent accesses leaking out of lock. */ - __CPROVER_fence("RRfence", "RWfence"); - - return !old_locked; -#endif -} - -static inline void lock_impl_lock(struct lock_impl *lock) -{ - /* - * CBMC doesn't support busy waiting, so just assume that the - * lock is available. - */ - assume(lock_impl_trylock(lock)); - - /* - * If the lock was already held by this thread then the assumption - * is unsatisfiable (deadlock). - */ -} - -static inline void lock_impl_unlock(struct lock_impl *lock) -{ -#ifdef RUN - BUG_ON(!__sync_bool_compare_and_swap(&lock->locked, true, false)); -#else - /* Minimal barrier to prevent accesses leaking out of lock. */ - __CPROVER_fence("RWfence", "WWfence"); - - __CPROVER_atomic_begin(); - bool old_locked = lock->locked; - lock->locked = false; - __CPROVER_atomic_end(); - - BUG_ON(!old_locked); -#endif -} - -static inline void lock_impl_init(struct lock_impl *lock) -{ - lock->locked = false; -} - -#define LOCK_IMPL_INITIALIZER {.locked = false} - -#endif /* !defined(PTHREAD_LOCK) */ - -/* - * Implement spinlocks using the lock mechanism. Wrap the lock to prevent mixing - * locks of different types. - */ -typedef struct { - struct lock_impl internal_lock; -} spinlock_t; - -#define SPIN_LOCK_UNLOCKED {.internal_lock = LOCK_IMPL_INITIALIZER} -#define __SPIN_LOCK_UNLOCKED(x) SPIN_LOCK_UNLOCKED -#define DEFINE_SPINLOCK(x) spinlock_t x = SPIN_LOCK_UNLOCKED - -static inline void spin_lock_init(spinlock_t *lock) -{ - lock_impl_init(&lock->internal_lock); -} - -static inline void spin_lock(spinlock_t *lock) -{ - /* - * Spin locks also need to be removed in order to eliminate all - * memory barriers. They are only used by the write side anyway. - */ -#ifndef NO_SYNC_SMP_MB - preempt_disable(); - lock_impl_lock(&lock->internal_lock); -#endif -} - -static inline void spin_unlock(spinlock_t *lock) -{ -#ifndef NO_SYNC_SMP_MB - lock_impl_unlock(&lock->internal_lock); - preempt_enable(); -#endif -} - -/* Don't bother with interrupts */ -#define spin_lock_irq(lock) spin_lock(lock) -#define spin_unlock_irq(lock) spin_unlock(lock) -#define spin_lock_irqsave(lock, flags) spin_lock(lock) -#define spin_unlock_irqrestore(lock, flags) spin_unlock(lock) - -/* - * This is supposed to return an int, but I think that a bool should work as - * well. - */ -static inline bool spin_trylock(spinlock_t *lock) -{ -#ifndef NO_SYNC_SMP_MB - preempt_disable(); - return lock_impl_trylock(&lock->internal_lock); -#else - return true; -#endif -} - -struct completion { - /* Hopefully this won't overflow. */ - unsigned int count; -}; - -#define COMPLETION_INITIALIZER(x) {.count = 0} -#define DECLARE_COMPLETION(x) struct completion x = COMPLETION_INITIALIZER(x) -#define DECLARE_COMPLETION_ONSTACK(x) DECLARE_COMPLETION(x) - -static inline void init_completion(struct completion *c) -{ - c->count = 0; -} - -static inline void wait_for_completion(struct completion *c) -{ - unsigned int prev_count = __sync_fetch_and_sub(&c->count, 1); - - assume(prev_count); -} - -static inline void complete(struct completion *c) -{ - unsigned int prev_count = __sync_fetch_and_add(&c->count, 1); - - BUG_ON(prev_count == UINT_MAX); -} - -/* This function probably isn't very useful for CBMC. */ -static inline bool try_wait_for_completion(struct completion *c) -{ - BUG(); -} - -static inline bool completion_done(struct completion *c) -{ - return c->count; -} - -/* TODO: Implement complete_all */ -static inline void complete_all(struct completion *c) -{ - BUG(); -} - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c deleted file mode 100644 index 9440cc39e3c6..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.c +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include "misc.h" -#include "bug_on.h" - -struct rcu_head; - -void wakeme_after_rcu(struct rcu_head *head) -{ - BUG(); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h deleted file mode 100644 index aca50030f954..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/misc.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef MISC_H -#define MISC_H - -#include "assume.h" -#include "int_typedefs.h" -#include "locks.h" - -#include - -/* Probably won't need to deal with bottom halves. */ -static inline void local_bh_disable(void) {} -static inline void local_bh_enable(void) {} - -#define MODULE_ALIAS(X) -#define module_param(...) -#define EXPORT_SYMBOL_GPL(x) - -#define container_of(ptr, type, member) ({ \ - const typeof(((type *)0)->member) *__mptr = (ptr); \ - (type *)((char *)__mptr - offsetof(type, member)); \ -}) - -#ifndef USE_SIMPLE_SYNC_SRCU -/* Abuse udelay to make sure that busy loops terminate. */ -#define udelay(x) assume(0) - -#else - -/* The simple custom synchronize_srcu is ok with try_check_zero failing. */ -#define udelay(x) do { } while (0) -#endif - -#define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ - do { } while (0) - -#define notrace - -/* Avoid including rcupdate.h */ -struct rcu_synchronize { - struct rcu_head head; - struct completion completion; -}; - -void wakeme_after_rcu(struct rcu_head *head); - -#define rcu_lock_acquire(a) do { } while (0) -#define rcu_lock_release(a) do { } while (0) -#define rcu_lockdep_assert(c, s) do { } while (0) -#define RCU_LOCKDEP_WARN(c, s) do { } while (0) - -/* Let CBMC non-deterministically choose switch between normal and expedited. */ -bool rcu_gp_is_normal(void); -bool rcu_gp_is_expedited(void); - -/* Do the same for old versions of rcu. */ -#define rcu_expedited (rcu_gp_is_expedited()) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h deleted file mode 100644 index 27e67a3f291f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/percpu.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef PERCPU_H -#define PERCPU_H - -#include -#include "bug_on.h" -#include "preempt.h" - -#define __percpu - -/* Maximum size of any percpu data. */ -#define PERCPU_OFFSET (4 * sizeof(long)) - -/* Ignore alignment, as CBMC doesn't care about false sharing. */ -#define alloc_percpu(type) __alloc_percpu(sizeof(type), 1) - -static inline void *__alloc_percpu(size_t size, size_t align) -{ - BUG(); - return NULL; -} - -static inline void free_percpu(void *ptr) -{ - BUG(); -} - -#define per_cpu_ptr(ptr, cpu) \ - ((typeof(ptr)) ((char *) (ptr) + PERCPU_OFFSET * cpu)) - -#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1) -#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1) -#define __this_cpu_sub(pcp, n) __this_cpu_add(pcp, -(typeof(pcp)) (n)) - -#define this_cpu_inc(pcp) this_cpu_add(pcp, 1) -#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1) -#define this_cpu_sub(pcp, n) this_cpu_add(pcp, -(typeof(pcp)) (n)) - -/* Make CBMC use atomics to work around bug. */ -#ifdef RUN -#define THIS_CPU_ADD_HELPER(ptr, x) (*(ptr) += (x)) -#else -/* - * Split the atomic into a read and a write so that it has the least - * possible ordering. - */ -#define THIS_CPU_ADD_HELPER(ptr, x) \ - do { \ - typeof(ptr) this_cpu_add_helper_ptr = (ptr); \ - typeof(ptr) this_cpu_add_helper_x = (x); \ - typeof(*ptr) this_cpu_add_helper_temp; \ - __CPROVER_atomic_begin(); \ - this_cpu_add_helper_temp = *(this_cpu_add_helper_ptr); \ - __CPROVER_atomic_end(); \ - this_cpu_add_helper_temp += this_cpu_add_helper_x; \ - __CPROVER_atomic_begin(); \ - *(this_cpu_add_helper_ptr) = this_cpu_add_helper_temp; \ - __CPROVER_atomic_end(); \ - } while (0) -#endif - -/* - * For some reason CBMC needs an atomic operation even though this is percpu - * data. - */ -#define __this_cpu_add(pcp, n) \ - do { \ - BUG_ON(preemptible()); \ - THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), thread_cpu_id), \ - (typeof(pcp)) (n)); \ - } while (0) - -#define this_cpu_add(pcp, n) \ - do { \ - int this_cpu_add_impl_cpu = get_cpu(); \ - THIS_CPU_ADD_HELPER(per_cpu_ptr(&(pcp), this_cpu_add_impl_cpu), \ - (typeof(pcp)) (n)); \ - put_cpu(); \ - } while (0) - -/* - * This will cause a compiler warning because of the cast from char[][] to - * type*. This will cause a compile time error if type is too big. - */ -#define DEFINE_PER_CPU(type, name) \ - char name[NR_CPUS][PERCPU_OFFSET]; \ - typedef char percpu_too_big_##name \ - [sizeof(type) > PERCPU_OFFSET ? -1 : 1] - -#define for_each_possible_cpu(cpu) \ - for ((cpu) = 0; (cpu) < NR_CPUS; ++(cpu)) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c deleted file mode 100644 index b4083ae348fb..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.c +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include "preempt.h" - -#include "assume.h" -#include "locks.h" - -/* Support NR_CPUS of at most 64 */ -#define CPU_PREEMPTION_LOCKS_INIT0 LOCK_IMPL_INITIALIZER -#define CPU_PREEMPTION_LOCKS_INIT1 \ - CPU_PREEMPTION_LOCKS_INIT0, CPU_PREEMPTION_LOCKS_INIT0 -#define CPU_PREEMPTION_LOCKS_INIT2 \ - CPU_PREEMPTION_LOCKS_INIT1, CPU_PREEMPTION_LOCKS_INIT1 -#define CPU_PREEMPTION_LOCKS_INIT3 \ - CPU_PREEMPTION_LOCKS_INIT2, CPU_PREEMPTION_LOCKS_INIT2 -#define CPU_PREEMPTION_LOCKS_INIT4 \ - CPU_PREEMPTION_LOCKS_INIT3, CPU_PREEMPTION_LOCKS_INIT3 -#define CPU_PREEMPTION_LOCKS_INIT5 \ - CPU_PREEMPTION_LOCKS_INIT4, CPU_PREEMPTION_LOCKS_INIT4 - -/* - * Simulate disabling preemption by locking a particular cpu. NR_CPUS - * should be the actual number of cpus, not just the maximum. - */ -struct lock_impl cpu_preemption_locks[NR_CPUS] = { - CPU_PREEMPTION_LOCKS_INIT0 -#if (NR_CPUS - 1) & 1 - , CPU_PREEMPTION_LOCKS_INIT0 -#endif -#if (NR_CPUS - 1) & 2 - , CPU_PREEMPTION_LOCKS_INIT1 -#endif -#if (NR_CPUS - 1) & 4 - , CPU_PREEMPTION_LOCKS_INIT2 -#endif -#if (NR_CPUS - 1) & 8 - , CPU_PREEMPTION_LOCKS_INIT3 -#endif -#if (NR_CPUS - 1) & 16 - , CPU_PREEMPTION_LOCKS_INIT4 -#endif -#if (NR_CPUS - 1) & 32 - , CPU_PREEMPTION_LOCKS_INIT5 -#endif -}; - -#undef CPU_PREEMPTION_LOCKS_INIT0 -#undef CPU_PREEMPTION_LOCKS_INIT1 -#undef CPU_PREEMPTION_LOCKS_INIT2 -#undef CPU_PREEMPTION_LOCKS_INIT3 -#undef CPU_PREEMPTION_LOCKS_INIT4 -#undef CPU_PREEMPTION_LOCKS_INIT5 - -__thread int thread_cpu_id; -__thread int preempt_disable_count; - -void preempt_disable(void) -{ - BUG_ON(preempt_disable_count < 0 || preempt_disable_count == INT_MAX); - - if (preempt_disable_count++) - return; - - thread_cpu_id = nondet_int(); - assume(thread_cpu_id >= 0); - assume(thread_cpu_id < NR_CPUS); - lock_impl_lock(&cpu_preemption_locks[thread_cpu_id]); -} - -void preempt_enable(void) -{ - BUG_ON(preempt_disable_count < 1); - - if (--preempt_disable_count) - return; - - lock_impl_unlock(&cpu_preemption_locks[thread_cpu_id]); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h deleted file mode 100644 index f8b762cd214c..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/preempt.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef PREEMPT_H -#define PREEMPT_H - -#include - -#include "bug_on.h" - -/* This flag contains garbage if preempt_disable_count is 0. */ -extern __thread int thread_cpu_id; - -/* Support recursive preemption disabling. */ -extern __thread int preempt_disable_count; - -void preempt_disable(void); -void preempt_enable(void); - -static inline void preempt_disable_notrace(void) -{ - preempt_disable(); -} - -static inline void preempt_enable_no_resched(void) -{ - preempt_enable(); -} - -static inline void preempt_enable_notrace(void) -{ - preempt_enable(); -} - -static inline int preempt_count(void) -{ - return preempt_disable_count; -} - -static inline bool preemptible(void) -{ - return !preempt_count(); -} - -static inline int get_cpu(void) -{ - preempt_disable(); - return thread_cpu_id; -} - -static inline void put_cpu(void) -{ - preempt_enable(); -} - -static inline void might_sleep(void) -{ - BUG_ON(preempt_disable_count); -} - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c deleted file mode 100644 index 97f592048e0b..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/simple_sync_srcu.c +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "int_typedefs.h" - -#include "barriers.h" -#include "bug_on.h" -#include "locks.h" -#include "misc.h" -#include "preempt.h" -#include "percpu.h" -#include "workqueues.h" - -#include - -/* Functions needed from modify_srcu.c */ -bool try_check_zero(struct srcu_struct *sp, int idx, int trycount); -void srcu_flip(struct srcu_struct *sp); - -/* Simpler implementation of synchronize_srcu that ignores batching. */ -void synchronize_srcu(struct srcu_struct *sp) -{ - int idx; - /* - * This code assumes that try_check_zero will succeed anyway, - * so there is no point in multiple tries. - */ - const int trycount = 1; - - might_sleep(); - - /* Ignore the lock, as multiple writers aren't working yet anyway. */ - - idx = 1 ^ (sp->completed & 1); - - /* For comments see srcu_advance_batches. */ - - assume(try_check_zero(sp, idx, trycount)); - - srcu_flip(sp); - - assume(try_check_zero(sp, idx^1, trycount)); -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h deleted file mode 100644 index 28b960300971..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/workqueues.h +++ /dev/null @@ -1,103 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef WORKQUEUES_H -#define WORKQUEUES_H - -#include - -#include "barriers.h" -#include "bug_on.h" -#include "int_typedefs.h" - -#include - -/* Stub workqueue implementation. */ - -struct work_struct; -typedef void (*work_func_t)(struct work_struct *work); -void delayed_work_timer_fn(unsigned long __data); - -struct work_struct { -/* atomic_long_t data; */ - unsigned long data; - - struct list_head entry; - work_func_t func; -#ifdef CONFIG_LOCKDEP - struct lockdep_map lockdep_map; -#endif -}; - -struct timer_list { - struct hlist_node entry; - unsigned long expires; - void (*function)(unsigned long); - unsigned long data; - u32 flags; - int slack; -}; - -struct delayed_work { - struct work_struct work; - struct timer_list timer; - - /* target workqueue and CPU ->timer uses to queue ->work */ - struct workqueue_struct *wq; - int cpu; -}; - - -static inline bool schedule_work(struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool schedule_work_on(int cpu, struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool queue_work(struct workqueue_struct *wq, - struct work_struct *work) -{ - BUG(); - return true; -} - -static inline bool queue_delayed_work(struct workqueue_struct *wq, - struct delayed_work *dwork, - unsigned long delay) -{ - BUG(); - return true; -} - -#define INIT_WORK(w, f) \ - do { \ - (w)->data = 0; \ - (w)->func = (f); \ - } while (0) - -#define INIT_DELAYED_WORK(w, f) INIT_WORK(&(w)->work, (f)) - -#define __WORK_INITIALIZER(n, f) { \ - .data = 0, \ - .entry = { &(n).entry, &(n).entry }, \ - .func = f \ - } - -/* Don't bother initializing timer. */ -#define __DELAYED_WORK_INITIALIZER(n, f, tflags) { \ - .work = __WORK_INITIALIZER((n).work, (f)), \ - } - -#define DECLARE_WORK(n, f) \ - struct workqueue_struct n = __WORK_INITIALIZER - -#define DECLARE_DELAYED_WORK(n, f) \ - struct delayed_work n = __DELAYED_WORK_INITIALIZER(n, f, 0) - -#define system_power_efficient_wq ((struct workqueue_struct *) NULL) - -#endif diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore deleted file mode 100644 index d65462d64816..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -*.out diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile deleted file mode 100644 index ad21b925fbb4..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -CBMC_FLAGS = -I../.. -I../../src -I../../include -I../../empty_includes -32 -pointer-check -mm pso - -all: - for i in ./*.pass; do \ - echo $$i ; \ - CBMC_FLAGS="$(CBMC_FLAGS)" sh ../test_script.sh --should-pass $$i > $$i.out 2>&1 ; \ - done - for i in ./*.fail; do \ - echo $$i ; \ - CBMC_FLAGS="$(CBMC_FLAGS)" sh ../test_script.sh --should-fail $$i > $$i.out 2>&1 ; \ - done diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail deleted file mode 100644 index 40c8075919d1..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/assert_end.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DASSERT_END" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail deleted file mode 100644 index ada5baf0b60d..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail deleted file mode 100644 index 8fe00c8db466..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force2.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE_2" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail deleted file mode 100644 index 612ed6772844..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/force3.fail +++ /dev/null @@ -1 +0,0 @@ -test_cbmc_options="-DFORCE_FAILURE_3" diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/main.pass b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/main.pass deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c deleted file mode 100644 index 2ce2016f7871..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/store_buffering/test.c +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include - -int x; -int y; - -int __unbuffered_tpr_x; -int __unbuffered_tpr_y; - -DEFINE_SRCU(ss); - -void rcu_reader(void) -{ - int idx; - -#ifndef FORCE_FAILURE_3 - idx = srcu_read_lock(&ss); -#endif - might_sleep(); - - __unbuffered_tpr_y = READ_ONCE(y); -#ifdef FORCE_FAILURE - srcu_read_unlock(&ss, idx); - idx = srcu_read_lock(&ss); -#endif - WRITE_ONCE(x, 1); - -#ifndef FORCE_FAILURE_3 - srcu_read_unlock(&ss, idx); -#endif - might_sleep(); -} - -void *thread_update(void *arg) -{ - WRITE_ONCE(y, 1); -#ifndef FORCE_FAILURE_2 - synchronize_srcu(&ss); -#endif - might_sleep(); - __unbuffered_tpr_x = READ_ONCE(x); - - return NULL; -} - -void *thread_process_reader(void *arg) -{ - rcu_reader(); - - return NULL; -} - -int main(int argc, char *argv[]) -{ - pthread_t tu; - pthread_t tpr; - - if (pthread_create(&tu, NULL, thread_update, NULL)) - abort(); - if (pthread_create(&tpr, NULL, thread_process_reader, NULL)) - abort(); - if (pthread_join(tu, NULL)) - abort(); - if (pthread_join(tpr, NULL)) - abort(); - assert(__unbuffered_tpr_y != 0 || __unbuffered_tpr_x != 0); - -#ifdef ASSERT_END - assert(0); -#endif - - return 0; -} diff --git a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh b/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh deleted file mode 100755 index 2fe1f0339b4f..000000000000 --- a/tools/testing/selftests/rcutorture/formal/srcu-cbmc/tests/test_script.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 - -# This script expects a mode (either --should-pass or --should-fail) followed by -# an input file. The script uses the following environment variables. The test C -# source file is expected to be named test.c in the directory containing the -# input file. -# -# CBMC: The command to run CBMC. Default: cbmc -# CBMC_FLAGS: Additional flags to pass to CBMC -# NR_CPUS: Number of cpus to run tests with. Default specified by the test -# SYNC_SRCU_MODE: Choose implementation of synchronize_srcu. Defaults to simple. -# kernel: Version included in the linux kernel source. -# simple: Use try_check_zero directly. -# -# The input file is a script that is sourced by this file. It can define any of -# the following variables to configure the test. -# -# test_cbmc_options: Extra options to pass to CBMC. -# min_cpus_fail: Minimum number of CPUs (NR_CPUS) for verification to fail. -# The test is expected to pass if it is run with fewer. (Only -# useful for .fail files) -# default_cpus: Quantity of CPUs to use for the test, if not specified on the -# command line. Default: Larger of 2 and MIN_CPUS_FAIL. - -set -e - -if test "$#" -ne 2; then - echo "Expected one option followed by an input file" 1>&2 - exit 99 -fi - -if test "x$1" = "x--should-pass"; then - should_pass="yes" -elif test "x$1" = "x--should-fail"; then - should_pass="no" -else - echo "Unrecognized argument '$1'" 1>&2 - - # Exit code 99 indicates a hard error. - exit 99 -fi - -CBMC=${CBMC:-cbmc} - -SYNC_SRCU_MODE=${SYNC_SRCU_MODE:-simple} - -case ${SYNC_SRCU_MODE} in -kernel) sync_srcu_mode_flags="" ;; -simple) sync_srcu_mode_flags="-DUSE_SIMPLE_SYNC_SRCU" ;; - -*) - echo "Unrecognized argument '${SYNC_SRCU_MODE}'" 1>&2 - exit 99 - ;; -esac - -min_cpus_fail=1 - -c_file=`dirname "$2"`/test.c - -# Source the input file. -. $2 - -if test ${min_cpus_fail} -gt 2; then - default_default_cpus=${min_cpus_fail} -else - default_default_cpus=2 -fi -default_cpus=${default_cpus:-${default_default_cpus}} -cpus=${NR_CPUS:-${default_cpus}} - -# Check if there are two few cpus to make the test fail. -if test $cpus -lt ${min_cpus_fail:-0}; then - should_pass="yes" -fi - -cbmc_opts="-DNR_CPUS=${cpus} ${sync_srcu_mode_flags} ${test_cbmc_options} ${CBMC_FLAGS}" - -echo "Running CBMC: ${CBMC} ${cbmc_opts} ${c_file}" -if ${CBMC} ${cbmc_opts} "${c_file}"; then - # Verification successful. Make sure that it was supposed to verify. - test "x${should_pass}" = xyes -else - cbmc_exit_status=$? - - # An exit status of 10 indicates a failed verification. - # (see cbmc_parse_optionst::do_bmc in the CBMC source code) - if test ${cbmc_exit_status} -eq 10 && test "x${should_pass}" = xno; then - : - else - echo "CBMC returned ${cbmc_exit_status} exit status" 1>&2 - - # Parse errors have exit status 6. Any other type of error - # should be considered a hard error. - if test ${cbmc_exit_status} -ne 6 && \ - test ${cbmc_exit_status} -ne 10; then - exit 99 - else - exit 1 - fi - fi -fi From 0feda4da6da5ce5d2094df406a8c5090e8403bf3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 7 Jun 2023 14:10:40 -0700 Subject: [PATCH 038/130] torture: Add "--no-" as synonym for "--do-no-" in torture.sh In order to (for example) omit the real-time testing that torture.sh would otherwise carry out, you put "--do-no-rt" on the torture.sh command line. This works, but it is all too easy to instead type "--no-rt". This is unambiguous and easier to type, so this commit therefore allows all "--no-" arguments as synonyms for their "--do-no-" counterparts. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/torture.sh | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index 5a2ae2264403..c3190ca9c96f 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -73,18 +73,18 @@ usage () { echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\"" echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\"" echo " --do-all" - echo " --do-allmodconfig / --do-no-allmodconfig" - echo " --do-clocksourcewd / --do-no-clocksourcewd" - echo " --do-kasan / --do-no-kasan" - echo " --do-kcsan / --do-no-kcsan" - echo " --do-kvfree / --do-no-kvfree" - echo " --do-locktorture / --do-no-locktorture" + echo " --do-allmodconfig / --do-no-allmodconfig / --no-allmodconfig" + echo " --do-clocksourcewd / --do-no-clocksourcewd / --no-clocksourcewd" + echo " --do-kasan / --do-no-kasan / --no-kasan" + echo " --do-kcsan / --do-no-kcsan / --no-kcsan" + echo " --do-kvfree / --do-no-kvfree / --no-kvfree" + echo " --do-locktorture / --do-no-locktorture / --no-locktorture" echo " --do-none" - echo " --do-rcuscale / --do-no-rcuscale" - echo " --do-rcutorture / --do-no-rcutorture" - echo " --do-refscale / --do-no-refscale" - echo " --do-rt / --do-no-rt" - echo " --do-scftorture / --do-no-scftorture" + echo " --do-rcuscale / --do-no-rcuscale / --no-rcuscale" + echo " --do-rcutorture / --do-no-rcutorture / --no-rcutorture" + echo " --do-refscale / --do-no-refscale / --no-refscale" + echo " --do-rt / --do-no-rt / --no-rt" + echo " --do-scftorture / --do-no-scftorture / --no-scftorture" echo " --duration [ | h | d ]" echo " --kcsan-kmake-arg kernel-make-arguments" exit 1 @@ -126,22 +126,22 @@ do do_kcsan=yes do_clocksourcewd=yes ;; - --do-allmodconfig|--do-no-allmodconfig) + --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig) do_allmodconfig=`doyesno "$1" --do-allmodconfig` ;; - --do-clocksourcewd|--do-no-clocksourcewd) + --do-clocksourcewd|--do-no-clocksourcewd|--no-clocksourcewd) do_clocksourcewd=`doyesno "$1" --do-clocksourcewd` ;; - --do-kasan|--do-no-kasan) + --do-kasan|--do-no-kasan|--no-kasan) do_kasan=`doyesno "$1" --do-kasan` ;; - --do-kcsan|--do-no-kcsan) + --do-kcsan|--do-no-kcsan|--no-kcsan) do_kcsan=`doyesno "$1" --do-kcsan` ;; - --do-kvfree|--do-no-kvfree) + --do-kvfree|--do-no-kvfree|--no-kvfree) do_kvfree=`doyesno "$1" --do-kvfree` ;; - --do-locktorture|--do-no-locktorture) + --do-locktorture|--do-no-locktorture|--no-locktorture) do_locktorture=`doyesno "$1" --do-locktorture` ;; --do-none|--donone) @@ -157,19 +157,19 @@ do do_kcsan=no do_clocksourcewd=no ;; - --do-rcuscale|--do-no-rcuscale) + --do-rcuscale|--do-no-rcuscale|--no-rcuscale) do_rcuscale=`doyesno "$1" --do-rcuscale` ;; - --do-rcutorture|--do-no-rcutorture) + --do-rcutorture|--do-no-rcutorture|--no-rcutorture) do_rcutorture=`doyesno "$1" --do-rcutorture` ;; - --do-refscale|--do-no-refscale) + --do-refscale|--do-no-refscale|--no-refscale) do_refscale=`doyesno "$1" --do-refscale` ;; - --do-rt|--do-no-rt) + --do-rt|--do-no-rt|--no-rt) do_rt=`doyesno "$1" --do-rt` ;; - --do-scftorture|--do-no-scftorture) + --do-scftorture|--do-no-scftorture|--no-scftorture) do_scftorture=`doyesno "$1" --do-scftorture` ;; --duration) From 4f64435417b450e8c0e76fba11a5cd23d81610c6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 15 Jun 2023 06:49:17 -0700 Subject: [PATCH 039/130] rcutorture: Remove contradictory Kconfig constraint The TASKS03 scenario specifies both CONFIG_RCU_EXPERT=y and removes #CHECK#CONFIG_RCU_EXPERT=n in order to be consistent with CONFIG_RCU_EXPERT=y prerequisites. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/configs/rcu/TASKS03 | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 b/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 index dea26c568678..2ef2fb69c360 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TASKS03 @@ -6,6 +6,5 @@ CONFIG_PREEMPT=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=n CONFIG_NO_HZ_FULL=y -#CHECK#CONFIG_RCU_EXPERT=n CONFIG_TASKS_RCU=y CONFIG_RCU_EXPERT=y From 5cec64e49519320ef3dab451f52ca0855de5da37 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jun 2023 10:34:37 -0700 Subject: [PATCH 040/130] torture: Allow #CHECK# in --kconfig argument to kvm.sh Testing building of a given RCU Tasks flavor with the other two flavors disabled requires checking that the other two flavors are in fact disabled. This commit therefore modifies the scripting to permit things like "#CHECK#CONFIG_TASKS_TRACE_RCU=n" to be passed into the kvm.sh script's --kconfig parameter. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/configcheck.sh | 61 ++++++++++--------- .../rcutorture/bin/kvm-test-1-run.sh | 1 - tools/testing/selftests/rcutorture/bin/kvm.sh | 2 +- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/configcheck.sh b/tools/testing/selftests/rcutorture/bin/configcheck.sh index b92dfeb7fbbf..99162d18bad3 100755 --- a/tools/testing/selftests/rcutorture/bin/configcheck.sh +++ b/tools/testing/selftests/rcutorture/bin/configcheck.sh @@ -3,6 +3,8 @@ # # Usage: configcheck.sh .config .config-template # +# Non-empty output if errors detected. +# # Copyright (C) IBM Corporation, 2011 # # Authors: Paul E. McKenney @@ -10,32 +12,35 @@ T="`mktemp -d ${TMPDIR-/tmp}/configcheck.sh.XXXXXX`" trap 'rm -rf $T' 0 -sed -e 's/"//g' < $1 > $T/.config +# function test_kconfig_enabled ( Kconfig-var=val ) +function test_kconfig_enabled () { + if ! grep -q "^$1$" $T/.config + then + echo :$1: improperly set + return 1 + fi + return 0 +} -sed -e 's/"//g' -e 's/\(.*\)=n/# \1 is not set/' -e 's/^#CHECK#//' < $2 | -awk ' -{ - print "if grep -q \"" $0 "\" < '"$T/.config"'"; - print "then"; - print "\t:"; - print "else"; - if ($1 == "#") { - print "\tif grep -q \"" $2 "\" < '"$T/.config"'"; - print "\tthen"; - print "\t\tif test \"$firsttime\" = \"\"" - print "\t\tthen" - print "\t\t\tfirsttime=1" - print "\t\tfi" - print "\t\techo \":" $2 ": improperly set\""; - print "\telse"; - print "\t\t:"; - print "\tfi"; - } else { - print "\tif test \"$firsttime\" = \"\"" - print "\tthen" - print "\t\tfirsttime=1" - print "\tfi" - print "\techo \":" $0 ": improperly set\""; - } - print "fi"; - }' | sh +# function test_kconfig_disabled ( Kconfig-var ) +function test_kconfig_disabled () { + if grep -q "^$1=n$" $T/.config + then + return 0 + fi + if grep -q "^$1=" $T/.config + then + echo :$1=n: improperly set + return 1 + fi + return 0 +} + +sed -e 's/"//g' < $1 > $T/.config +sed -e 's/^#CHECK#//' < $2 > $T/ConfigFragment +grep '^CONFIG_.*=n$' $T/ConfigFragment | + sed -e 's/^/test_kconfig_disabled /' -e 's/=n$//' > $T/kconfig-n.sh +. $T/kconfig-n.sh +grep -v '^CONFIG_.*=n$' $T/ConfigFragment | grep '^CONFIG_' | + sed -e 's/^/test_kconfig_enabled /' > $T/kconfig-not-n.sh +. $T/kconfig-not-n.sh diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index d2a3710a5f2a..b9912de9d71e 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh @@ -57,7 +57,6 @@ config_override_param () { cat $T/Kconfig_args >> $resdir/ConfigFragment.input config_override.sh $T/$2 $T/Kconfig_args > $T/$2.tmp mv $T/$2.tmp $T/$2 - # Note that "#CHECK#" is not permitted on commandline. fi } diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index d3cdc2d33d4b..b0f36a638a69 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -186,7 +186,7 @@ do fi ;; --kconfig|--kconfigs) - checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\)*$' '^error$' + checkarg --kconfig "(Kconfig options)" $# "$2" '^\(#CHECK#\)\?CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\( \(#CHECK#\)\?CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\|"[^"]*"\)\)*$' '^error$' TORTURE_KCONFIG_ARG="`echo "$TORTURE_KCONFIG_ARG $2" | sed -e 's/^ *//' -e 's/ *$//'`" shift ;; From cd1955d050f81450a48aa07b60933bec990c6166 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jun 2023 11:27:04 -0700 Subject: [PATCH 041/130] torture: Make kvm-recheck.sh report .config errors Currently, kvm-recheck.sh will print out any .config errors with messages of the form: :CONFIG_TASKS_TRACE_RCU=y: improperly set However, if these are the only errors, the resulting exit code will declare the run successful. This commit therefore causes kvm-recheck.sh to record .config errors in the results directory in a file named ConfigFragment.diags and also returns a non-zero error code in that case. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/kvm-recheck.sh | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh index 1df7e695edf7..97bdacc3fc55 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh @@ -16,6 +16,8 @@ T=/tmp/kvm-recheck.sh.$$ trap 'rm -f $T' 0 2 +configerrors=0 + PATH=`pwd`/tools/testing/selftests/rcutorture/bin:$PATH; export PATH . functions.sh for rd in "$@" @@ -32,7 +34,7 @@ do fi TORTURE_SUITE="`cat $i/../torture_suite`" ; export TORTURE_SUITE configfile=`echo $i | sed -e 's,^.*/,,'` - rm -f $i/console.log.*.diags + rm -f $i/console.log.*.diags $i/ConfigFragment.diags case "${TORTURE_SUITE}" in X*) ;; @@ -49,8 +51,14 @@ do then echo QEMU killed fi - configcheck.sh $i/.config $i/ConfigFragment > $T 2>&1 - cat $T + configcheck.sh $i/.config $i/ConfigFragment > $i/ConfigFragment.diags 2>&1 + if test -s $i/ConfigFragment.diags + then + cat $i/ConfigFragment.diags + configerrors=$((configerrors+1)) + else + rm $i/ConfigFragment.diags + fi if test -r $i/Make.oldconfig.err then cat $i/Make.oldconfig.err @@ -65,7 +73,14 @@ do if test -f "$i/buildonly" then echo Build-only run, no boot/test - configcheck.sh $i/.config $i/ConfigFragment + configcheck.sh $i/.config $i/ConfigFragment > $i/ConfigFragment.diags 2>&1 + if test -s $i/ConfigFragment.diags + then + cat $i/ConfigFragment.diags + configerrors=$((configerrors+1)) + else + rm $i/ConfigFragment.diags + fi parse-build.sh $i/Make.out $configfile elif test -f "$i/qemu-cmd" then @@ -79,10 +94,10 @@ do done if test -f "$rd/kcsan.sum" then - if ! test -f $T + if ! test -f $i/ConfigFragment.diags then : - elif grep -q CONFIG_KCSAN=y $T + elif grep -q CONFIG_KCSAN=y $i/ConfigFragment.diags then echo "Compiler or architecture does not support KCSAN!" echo Did you forget to switch your compiler with '--kmake-arg CC='? @@ -94,17 +109,23 @@ do fi fi done + +if test "$configerrors" -gt 0 +then + echo $configerrors runs with .config errors. + ret=1 +fi EDITOR=echo kvm-find-errors.sh "${@: -1}" > $T 2>&1 builderrors="`tr ' ' '\012' < $T | grep -c '/Make.out.diags'`" if test "$builderrors" -gt 0 then echo $builderrors runs with build errors. - ret=1 + ret=2 fi runerrors="`tr ' ' '\012' < $T | grep -c '/console.log.diags'`" if test "$runerrors" -gt 0 then echo $runerrors runs with runtime errors. - ret=2 + ret=3 fi exit $ret From dd3ffd125f89cdb38093df56eecdfa5539d983b4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jun 2023 18:03:48 -0700 Subject: [PATCH 042/130] torture: Add RCU Tasks individual-flavor build tests This commit adds build tests of the individual RCU Tasks flavors in order to detect inadvertent dependencies among the flavors. Reported-by: Arnd Bergmann Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/torture.sh | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index c3190ca9c96f..dd3bfadb0639 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -55,6 +55,7 @@ do_kasan=yes do_kcsan=no do_clocksourcewd=yes do_rt=yes +do_rcutasksflavors=yes # doyesno - Helper function for yes/no arguments function doyesno () { @@ -81,6 +82,7 @@ usage () { echo " --do-locktorture / --do-no-locktorture / --no-locktorture" echo " --do-none" echo " --do-rcuscale / --do-no-rcuscale / --no-rcuscale" + echo " --do-rcutasksflavors / --do-no-rcutasksflavors / --no-rcutasksflavors" echo " --do-rcutorture / --do-no-rcutorture / --no-rcutorture" echo " --do-refscale / --do-no-refscale / --no-refscale" echo " --do-rt / --do-no-rt / --no-rt" @@ -115,6 +117,7 @@ do ;; --do-all|--doall) do_allmodconfig=yes + do_rcutasksflavor=yes do_rcutorture=yes do_locktorture=yes do_scftorture=yes @@ -146,6 +149,7 @@ do ;; --do-none|--donone) do_allmodconfig=no + do_rcutasksflavors=no do_rcutorture=no do_locktorture=no do_scftorture=no @@ -160,6 +164,9 @@ do --do-rcuscale|--do-no-rcuscale|--no-rcuscale) do_rcuscale=`doyesno "$1" --do-rcuscale` ;; + --do-rcutasksflavors|--do-no-rcutasksflavors|--no-rcutasksflavors) + do_rcutasksflavors=`doyesno "$1" --do-rcutasksflavors` + ;; --do-rcutorture|--do-no-rcutorture|--no-rcutorture) do_rcutorture=`doyesno "$1" --do-rcutorture` ;; @@ -361,6 +368,40 @@ then fi fi +# Test building RCU Tasks flavors in isolation, both SMP and !SMP +if test "$do_rcutasksflavors" = "yes" +then + echo " --- rcutasksflavors:" Start `date` | tee -a $T/log + rtfdir="tools/testing/selftests/rcutorture/res/$ds/results-rcutasksflavors" + mkdir -p "$rtfdir" + cat > $T/rcutasksflavors << __EOF__ +#CHECK#CONFIG_TASKS_RCU=n +#CHECK#CONFIG_TASKS_RUDE_RCU=n +#CHECK#CONFIG_TASKS_TRACE_RCU=n +__EOF__ + for flavor in CONFIG_TASKS_RCU CONFIG_TASKS_RUDE_RCU CONFIG_TASKS_TRACE_RCU + do + forceflavor="`echo $flavor | sed -e 's/^CONFIG/CONFIG_FORCE/'`" + deselectedflavors="`grep -v $flavor $T/rcutasksflavors | tr '\012' ' ' | tr -s ' ' | sed -e 's/ *$//'`" + echo " --- Running RCU Tasks Trace flavor $flavor `date`" >> $rtfdir/log + tools/testing/selftests/rcutorture/bin/kvm.sh --datestamp "$ds/results-rcutasksflavors/$flavor" --buildonly --configs "TINY01 TREE04" --kconfig "CONFIG_RCU_EXPERT=y CONFIG_RCU_SCALE_TEST=y $forceflavor=y $deselectedflavors" --trust-make > $T/$flavor.out 2>&1 + retcode=$? + if test "$retcode" -ne 0 + then + break + fi + done + if test "$retcode" -eq 0 + then + echo "rcutasksflavors($retcode)" $rtfdir >> $T/successes + echo Success >> $rtfdir/log + else + echo "rcutasksflavors($retcode)" $rtfdir >> $T/failures + echo " --- rcutasksflavors Test summary:" >> $rtfdir/log + echo " --- Summary: Exit code $retcode from $flavor, see Make.out" >> $rtfdir/log + fi +fi + # --torture rcu if test "$do_rcutorture" = "yes" then From 80021ffb6820baf77d17b7a3b7ec9e89faa69578 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 15 Jun 2023 20:11:21 -0700 Subject: [PATCH 043/130] torture: Make kvm-remote print diagnostics on initial ssh failure Currently, if the initial ssh fails, kvm-remote.sh gives up, printing a message saying so. But it would be nice to get a better idea as to why ssh failed. This commit therefore dumps out ssh's exit code, stdout, and stderr upon ssh failure for diagnostic purposes. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-remote.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh index a2328163eba1..134cdef5a6e0 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh @@ -137,14 +137,20 @@ chmod +x $T/bin/kvm-remote-*.sh # Check first to avoid the need for cleanup for system-name typos for i in $systems do - ncpus="`ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN 2> /dev/null`" + ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr ret=$? if test "$ret" -ne 0 then - echo System $i unreachable, giving up. | tee -a "$oldrun/remote-log" + echo "System $i unreachable ($ret), giving up." | tee -a "$oldrun/remote-log" + echo ' --- ssh stdout: vvv' | tee -a "$oldrun/remote-log" + cat $T/ssh.stdout | tee -a "$oldrun/remote-log" + echo ' --- ssh stdout: ^^^' | tee -a "$oldrun/remote-log" + echo ' --- ssh stderr: vvv' | tee -a "$oldrun/remote-log" + cat $T/ssh.stderr | tee -a "$oldrun/remote-log" + echo ' --- ssh stderr: ^^^' | tee -a "$oldrun/remote-log" exit 4 fi - echo $i: $ncpus CPUs " " `date` | tee -a "$oldrun/remote-log" + echo $i: `cat $T/ssh.stdout` CPUs " " `date` | tee -a "$oldrun/remote-log" done # Download and expand the tarball on all systems. From 2f25542c6f626011ecf52cf8eb4a9b666cb8da86 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 5 Jul 2023 15:49:29 -0700 Subject: [PATCH 044/130] rcutorture: Remove obsolete parameter check from mkinitrd.sh The mkinitrd.sh script no longer takes an argument, so this commit therefore removes the code that checks for the parameter being present. Signed-off-by: Paul E. McKenney Reviewed-by: Frederic Weisbecker --- tools/testing/selftests/rcutorture/bin/mkinitrd.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh index 71f0dfbb2a6d..3af3a86a0091 100755 --- a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh +++ b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh @@ -10,7 +10,6 @@ D=tools/testing/selftests/rcutorture # Prerequisite checks -[ -z "$D" ] && echo >&2 "No argument supplied" && exit 1 if [ ! -d "$D" ]; then echo >&2 "$D does not exist: Malformed kernel source tree?" exit 1 From 171cfa011ea4a3fd0fe5a1818fd6871daebabe3f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 14:19:26 -0700 Subject: [PATCH 045/130] torture: Place --bootargs parameters at end of -append list Currently, the kernel boot parameters specified by the kvm.sh --bootargs parameter are placed near the beginning of the -append list that is passed to qemu. This means that in the not-uncommon case of a kernel boot parameter where the last argument wins, the --bootargs list overrides neither the list in the .boot file nor the additional parameters supplied by the rcutorture scripting. This commit therefore places the kernel boot parameters specified by the kvm.sh --bootargs parameter at the end of qemu's -append list. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/functions.sh | 2 +- tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh | 2 +- .../selftests/rcutorture/configs/lock/ver_functions.sh | 5 +++-- .../selftests/rcutorture/configs/rcu/ver_functions.sh | 5 +++-- .../selftests/rcutorture/configs/rcuscale/ver_functions.sh | 5 +++-- .../selftests/rcutorture/configs/refscale/ver_functions.sh | 5 +++-- .../selftests/rcutorture/configs/scf/ver_functions.sh | 5 +++-- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/functions.sh b/tools/testing/selftests/rcutorture/bin/functions.sh index 48b9147e8c91..b8e2ea23cb3f 100644 --- a/tools/testing/selftests/rcutorture/bin/functions.sh +++ b/tools/testing/selftests/rcutorture/bin/functions.sh @@ -45,7 +45,7 @@ checkarg () { configfrag_boot_params () { if test -r "$2.boot" then - echo $1 `grep -v '^#' "$2.boot" | tr '\012' ' '` + echo `grep -v '^#' "$2.boot" | tr '\012' ' '` $1 else echo $1 fi diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index b9912de9d71e..1e7566997e51 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh @@ -162,7 +162,7 @@ boot_args="`configfrag_boot_params "$boot_args_in" "$config_template"`" boot_args="`per_version_boot_params "$boot_args" $resdir/.config $seconds`" if test -n "$TORTURE_BOOT_GDB_ARG" then - boot_args="$boot_args $TORTURE_BOOT_GDB_ARG" + boot_args="$TORTURE_BOOT_GDB_ARG $boot_args" fi # Give bare-metal advice diff --git a/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh index d3e4b2971f92..e7bb32709d78 100644 --- a/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/lock/ver_functions.sh @@ -22,8 +22,9 @@ locktorture_param_onoff () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `locktorture_param_onoff "$1" "$2"` \ + echo `locktorture_param_onoff "$1" "$2"` \ locktorture.stat_interval=15 \ locktorture.shutdown_secs=$3 \ - locktorture.verbose=1 + locktorture.verbose=1 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh index e2bc99c785e7..c044df386876 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/rcu/ver_functions.sh @@ -46,10 +46,11 @@ rcutorture_param_stat_interval () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `rcutorture_param_onoff "$1" "$2"` \ + echo `rcutorture_param_onoff "$1" "$2"` \ `rcutorture_param_n_barrier_cbs "$1"` \ `rcutorture_param_stat_interval "$1"` \ rcutorture.shutdown_secs=$3 \ rcutorture.test_no_idle_hz=1 \ - rcutorture.verbose=1 + rcutorture.verbose=1 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh index ffbe15109f0d..28070b43f017 100644 --- a/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/rcuscale/ver_functions.sh @@ -11,6 +11,7 @@ # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 rcuscale.shutdown=1 \ - rcuscale.verbose=0 + echo rcuscale.shutdown=1 \ + rcuscale.verbose=0 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh index f81fa2c541a6..748465627601 100644 --- a/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/refscale/ver_functions.sh @@ -11,6 +11,7 @@ # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 refscale.shutdown=1 \ - refscale.verbose=0 + echo refscale.shutdown=1 \ + refscale.verbose=0 \ + $1 } diff --git a/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh index 2d949e58f5a5..7637f68ef0ce 100644 --- a/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh +++ b/tools/testing/selftests/rcutorture/configs/scf/ver_functions.sh @@ -22,8 +22,9 @@ scftorture_param_onoff () { # # Adds per-version torture-module parameters to kernels supporting them. per_version_boot_params () { - echo $1 `scftorture_param_onoff "$1" "$2"` \ + echo `scftorture_param_onoff "$1" "$2"` \ scftorture.stat_interval=15 \ scftorture.shutdown_secs=$3 \ - scftorture.verbose=1 + scftorture.verbose=1 \ + $1 } From eb3156f78b490f7d9482c9f44004d8e7638e3e64 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 17:21:31 -0700 Subject: [PATCH 046/130] torture: Make torture.sh summarize config and build errors If some of the torture.sh runs had config and/or build errors, but all runs for which kernels were built ran successfully to completion, then torture.sh will incorrectly claim that all errors were KCSAN errors. This commit therefore makes torture.sh print the number of runs with config and build errors, and to refrain from claiming that all bugs were KCSAN bugs in that case. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/torture.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index dd3bfadb0639..d36b839bf586 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -582,11 +582,23 @@ then fi echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log +tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" +find "$tdir" -name 'ConfigFragment.diags' -print > $T/configerrors +find "$tdir" -name 'Make.out.diags' -print > $T/builderrors +if test -s "$T/configerrors" +then + echo " Scenarios with .config errors: `wc -l "$T/configerrors" | awk '{ print $1 }'`" + nonkcsanbug="yes" +fi +if test -s "$T/builderrors" +then + echo " Scenarios with build errors: `wc -l "$T/builderrors" | awk '{ print $1 }'`" + nonkcsanbug="yes" +fi if test -z "$nonkcsanbug" && test -s "$T/failuresum" then echo " All bugs were KCSAN failures." fi -tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" if test -n "$tdir" && test $compress_concurrency -gt 0 then # KASAN vmlinux files can approach 1GB in size, so compress them. From adec488031f60476fc2e2aa95ab569f6f0b07c72 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 10 Jul 2023 11:45:34 -0700 Subject: [PATCH 047/130] torture: Loosen .config checks for KCSAN kernels KCSAN enables some Kconfig options unilaterally and unconditionally, including CONFIG_PROVE_LOCKING. This in turn enables CONFIG_PROVE_RCU and CONFIG_PREEMPT_COUNT, which conflicts with constraints in SRCU-T, TRACE01, and TREE10, which in turn causes rcutorture to emit spurious configuration complaints. This commit therefore forgives configuration complaints involving CONFIG_PROVE_RCU and CONFIG_PREEMPT_COUNT. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-recheck.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh index 97bdacc3fc55..5be670dd4009 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh @@ -52,6 +52,13 @@ do echo QEMU killed fi configcheck.sh $i/.config $i/ConfigFragment > $i/ConfigFragment.diags 2>&1 + if grep -q '^CONFIG_KCSAN=y$' $i/ConfigFragment.input + then + # KCSAN forces a number of Kconfig options, so remove + # complaints about those Kconfig options in KCSAN runs. + mv $i/ConfigFragment.diags $i/ConfigFragment.diags.kcsan + grep -v -E 'CONFIG_PROVE_RCU|CONFIG_PREEMPT_COUNT' $i/ConfigFragment.diags.kcsan > $i/ConfigFragment.diags + fi if test -s $i/ConfigFragment.diags then cat $i/ConfigFragment.diags From e40806e9bcf8aaa86dbf0d484e7cf3cfa09cb86c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 7 Jun 2023 11:59:49 -0700 Subject: [PATCH 048/130] clocksource: Handle negative skews in "skew is too large" messages The nanosecond-to-millisecond skew computation uses unsigned arithmetic, which produces user-unfriendly large positive numbers for negative skews. Therefore, use signed arithmetic for this computation in order to preserve the negativity. Reported-by: Chris Bainbridge Reported-by: Feng Tang Fixes: dd029269947a ("clocksource: Improve "skew is too large" messages") Reviewed-by: Feng Tang Tested-by: Chris Bainbridge Signed-off-by: Paul E. McKenney --- kernel/time/clocksource.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 88cbc1181b23..c108ed8a9804 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -473,8 +473,8 @@ static void clocksource_watchdog(struct timer_list *unused) /* Check the deviation from the watchdog clocksource. */ md = cs->uncertainty_margin + watchdog->uncertainty_margin; if (abs(cs_nsec - wd_nsec) > md) { - u64 cs_wd_msec; - u64 wd_msec; + s64 cs_wd_msec; + s64 wd_msec; u32 wd_rem; pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n", @@ -483,8 +483,8 @@ static void clocksource_watchdog(struct timer_list *unused) watchdog->name, wd_nsec, wdnow, wdlast, watchdog->mask); pr_warn(" '%s' cs_nsec: %lld cs_now: %llx cs_last: %llx mask: %llx\n", cs->name, cs_nsec, csnow, cslast, cs->mask); - cs_wd_msec = div_u64_rem(cs_nsec - wd_nsec, 1000U * 1000U, &wd_rem); - wd_msec = div_u64_rem(wd_nsec, 1000U * 1000U, &wd_rem); + cs_wd_msec = div_s64_rem(cs_nsec - wd_nsec, 1000 * 1000, &wd_rem); + wd_msec = div_s64_rem(wd_nsec, 1000 * 1000, &wd_rem); pr_warn(" Clocksource '%s' skewed %lld ns (%lld ms) over watchdog '%s' interval of %lld ns (%lld ms)\n", cs->name, cs_nsec - wd_nsec, cs_wd_msec, watchdog->name, wd_nsec, wd_msec); if (curr_clocksource == cs) From 233756a640be811efae33763db718fe29753b1e9 Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Wed, 7 Jun 2023 15:54:33 +0800 Subject: [PATCH 049/130] x86/tsc: Extend watchdog check exemption to 4-Sockets platform There were reports again that the tsc clocksource on 4 sockets x86 servers was wrongly judged as 'unstable' by 'jiffies' and other watchdogs, and disabled [1][2]. Commit b50db7095fe0 ("x86/tsc: Disable clocksource watchdog for TSC on qualified platorms") was introduce to deal with these false alarms of tsc unstable issues, covering qualified platforms for 2 sockets or smaller ones. And from history of chasing TSC issues, Thomas and Peter only saw real TSC synchronization issue on 8 socket machines. So extend the exemption to 4 sockets to fix the issue. Rui also proposed another way to disable 'jiffies' as clocksource watchdog [3], which can also solve problem in [1]. in an architecture independent way, but can't cure the problem in [2]. whose watchdog is HPET or PMTIMER, while 'jiffies' is mostly used as watchdog in boot phase. 'nr_online_nodes' has known inaccurate problem for cases like platform with cpu-less memory nodes, sub numa cluster enabled, fakenuma, kernel cmdline parameter 'maxcpus=', etc. The harmful case is the 'maxcpus' one which could possibly under estimates the package number, and disable the watchdog, but bright side is it is mostly for debug usage. All these will be addressed in other patches, as discussed in thread [4]. [1]. https://lore.kernel.org/all/9d3bf570-3108-0336-9c52-9bee15767d29@huawei.com/ [2]. https://lore.kernel.org/lkml/06df410c-2177-4671-832f-339cff05b1d9@paulmck-laptop/ [3]. https://lore.kernel.org/all/bd5b97f89ab2887543fc262348d1c7cafcaae536.camel@intel.com/ [4]. https://lore.kernel.org/all/20221021062131.1826810-1-feng.tang@intel.com/ Reported-by: Yu Liao Reported-by: Paul E. McKenney Signed-off-by: Feng Tang Signed-off-by: Paul E. McKenney --- arch/x86/kernel/tsc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 3425c6a943e4..15f97c0abc9d 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -1258,7 +1258,7 @@ static void __init check_system_tsc_reliable(void) if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && boot_cpu_has(X86_FEATURE_NONSTOP_TSC) && boot_cpu_has(X86_FEATURE_TSC_ADJUST) && - nr_online_nodes <= 2) + nr_online_nodes <= 4) tsc_disable_clocksource_watchdog(); } From ce6616724fb425d6043d0dad6af996cd7c79bcc4 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Wed, 5 Jul 2023 23:51:27 +0200 Subject: [PATCH 050/130] ubsan: Clarify Kconfig text for CONFIG_UBSAN_TRAP Make it clearer in the one-line description and the verbose description text that CONFIG_UBSAN_TRAP as currently implemented involves a tradeoff of much less helpful oops messages in exchange for a smaller kernel image. (With the additional effect of turning UBSAN warnings into crashes, which may or may not be desired.) Signed-off-by: Jann Horn Link: https://lore.kernel.org/r/20230705215128.486054-1-jannh@google.com Signed-off-by: Kees Cook --- lib/Kconfig.ubsan | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index efae7e011956..59e21bfec188 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -13,7 +13,7 @@ menuconfig UBSAN if UBSAN config UBSAN_TRAP - bool "On Sanitizer warnings, abort the running kernel code" + bool "Abort on Sanitizer warnings (smaller kernel but less verbose)" depends on !COMPILE_TEST help Building kernels with Sanitizer features enabled tends to grow @@ -26,6 +26,14 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. + Also note that selecting Y will cause your kernel to Oops + with an "illegal instruction" error with no further details + when a UBSAN violation occurs. (Except on arm64, which will + report which Sanitizer failed.) This may make it hard to + determine whether an Oops was caused by UBSAN or to figure + out the details of a UBSAN violation. It makes the kernel log + output less useful for bug reports. + config CC_HAS_UBSAN_BOUNDS_STRICT def_bool $(cc-option,-fsanitize=bounds-strict) help From 8453e7924a1a9130f2b4d2c507de2cdc3892a5b5 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Tue, 23 May 2023 02:14:25 +0000 Subject: [PATCH 051/130] soc: fsl: qe: Replace all non-returning strlcpy with strscpy strlcpy() reads the entire source buffer first. This read may exceed the destination size limit. This is both inefficient and can lead to linear read overflows if a source string is not NUL-terminated [1]. In an effort to remove strlcpy() completely [2], replace strlcpy() here with strscpy(). No return values were used, so direct replacement is safe. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#strlcpy [2] https://github.com/KSPP/linux/issues/89 Signed-off-by: Azeem Shaikh Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230523021425.2406309-1-azeemshaikh38@gmail.com Signed-off-by: Kees Cook --- drivers/soc/fsl/qe/qe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c index b3c226eb5292..58746e570d14 100644 --- a/drivers/soc/fsl/qe/qe.c +++ b/drivers/soc/fsl/qe/qe.c @@ -524,7 +524,7 @@ int qe_upload_firmware(const struct qe_firmware *firmware) * saved microcode information and put in the new. */ memset(&qe_firmware_info, 0, sizeof(qe_firmware_info)); - strlcpy(qe_firmware_info.id, firmware->id, sizeof(qe_firmware_info.id)); + strscpy(qe_firmware_info.id, firmware->id, sizeof(qe_firmware_info.id)); qe_firmware_info.extended_modes = be64_to_cpu(firmware->extended_modes); memcpy(qe_firmware_info.vtraps, firmware->vtraps, sizeof(firmware->vtraps)); @@ -599,7 +599,7 @@ struct qe_firmware_info *qe_get_firmware_info(void) /* Copy the data into qe_firmware_info*/ sprop = of_get_property(fw, "id", NULL); if (sprop) - strlcpy(qe_firmware_info.id, sprop, + strscpy(qe_firmware_info.id, sprop, sizeof(qe_firmware_info.id)); of_property_read_u64(fw, "extended-modes", From 4943b66df18a0e8aedd006792ed73257cd2da8f8 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 7 Mar 2023 23:31:56 -0800 Subject: [PATCH 052/130] seccomp: don't use semaphore and wait_queue together The main reason is to use new wake_up helpers that will be added in the following patches. But here are a few other reasons: * if we use two different ways, we always need to call them both. This patch fixes seccomp_notify_recv where we forgot to call wake_up_poll in the error path. * If we use one primitive, we can control how many waiters are woken up for each request. Our goal is to wake up just one that will handle a request. Right now, wake_up_poll can wake up one waiter and up(&match->notif->request) can wake up one more. Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-2-avagin@google.com Signed-off-by: Kees Cook --- kernel/seccomp.c | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/kernel/seccomp.c b/kernel/seccomp.c index d3e584065c7f..1386dcedda1a 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -145,7 +145,7 @@ struct seccomp_kaddfd { * @notifications: A list of struct seccomp_knotif elements. */ struct notification { - struct semaphore request; + atomic_t requests; u64 next_id; struct list_head notifications; }; @@ -1116,7 +1116,7 @@ static int seccomp_do_user_notification(int this_syscall, list_add_tail(&n.list, &match->notif->notifications); INIT_LIST_HEAD(&n.addfd); - up(&match->notif->request); + atomic_inc(&match->notif->requests); wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM); /* @@ -1450,6 +1450,37 @@ find_notification(struct seccomp_filter *filter, u64 id) return NULL; } +static int recv_wake_function(wait_queue_entry_t *wait, unsigned int mode, int sync, + void *key) +{ + /* Avoid a wakeup if event not interesting for us. */ + if (key && !(key_to_poll(key) & (EPOLLIN | EPOLLERR))) + return 0; + return autoremove_wake_function(wait, mode, sync, key); +} + +static int recv_wait_event(struct seccomp_filter *filter) +{ + DEFINE_WAIT_FUNC(wait, recv_wake_function); + int ret; + + if (atomic_dec_if_positive(&filter->notif->requests) >= 0) + return 0; + + for (;;) { + ret = prepare_to_wait_event(&filter->wqh, &wait, TASK_INTERRUPTIBLE); + + if (atomic_dec_if_positive(&filter->notif->requests) >= 0) + break; + + if (ret) + return ret; + + schedule(); + } + finish_wait(&filter->wqh, &wait); + return 0; +} static long seccomp_notify_recv(struct seccomp_filter *filter, void __user *buf) @@ -1467,7 +1498,7 @@ static long seccomp_notify_recv(struct seccomp_filter *filter, memset(&unotif, 0, sizeof(unotif)); - ret = down_interruptible(&filter->notif->request); + ret = recv_wait_event(filter); if (ret < 0) return ret; @@ -1515,7 +1546,8 @@ out: if (should_sleep_killable(filter, knotif)) complete(&knotif->ready); knotif->state = SECCOMP_NOTIFY_INIT; - up(&filter->notif->request); + atomic_inc(&filter->notif->requests); + wake_up_poll(&filter->wqh, EPOLLIN | EPOLLRDNORM); } mutex_unlock(&filter->notify_lock); } @@ -1777,7 +1809,6 @@ static struct file *init_listener(struct seccomp_filter *filter) if (!filter->notif) goto out; - sema_init(&filter->notif->request, 0); filter->notif->next_id = get_random_u64(); INIT_LIST_HEAD(&filter->notif->notifications); From ab83f455f04df5b2f7c6d4de03b6d2eaeaa27b8a Mon Sep 17 00:00:00 2001 From: Peter Oskolkov Date: Tue, 7 Mar 2023 23:31:57 -0800 Subject: [PATCH 053/130] sched: add WF_CURRENT_CPU and externise ttwu Add WF_CURRENT_CPU wake flag that advices the scheduler to move the wakee to the current CPU. This is useful for fast on-CPU context switching use cases. In addition, make ttwu external rather than static so that the flag could be passed to it from outside of sched/core.c. Signed-off-by: Peter Oskolkov Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-3-avagin@google.com Signed-off-by: Kees Cook --- kernel/sched/core.c | 3 +-- kernel/sched/fair.c | 4 ++++ kernel/sched/sched.h | 13 ++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c52c2eba7c73..1574989627b0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4193,8 +4193,7 @@ bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) * Return: %true if @p->state changes (an actual wakeup was done), * %false otherwise. */ -static int -try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) +int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; int cpu, success = 0; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index b3e25be58e2b..ceb5d4c4738e 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7741,6 +7741,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) if (wake_flags & WF_TTWU) { record_wakee(p); + if ((wake_flags & WF_CURRENT_CPU) && + cpumask_test_cpu(cpu, p->cpus_ptr)) + return cpu; + if (sched_energy_enabled()) { new_cpu = find_energy_efficient_cpu(p, prev_cpu); if (new_cpu >= 0) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e93e006a942b..48d0be005f08 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2131,12 +2131,13 @@ static inline int task_on_rq_migrating(struct task_struct *p) } /* Wake flags. The first three directly map to some SD flag value */ -#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ -#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ -#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ +#define WF_EXEC 0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */ +#define WF_FORK 0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */ +#define WF_TTWU 0x08 /* Wakeup; maps to SD_BALANCE_WAKE */ -#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ -#define WF_MIGRATED 0x20 /* Internal use, task got migrated */ +#define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ +#define WF_MIGRATED 0x20 /* Internal use, task got migrated */ +#define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */ #ifdef CONFIG_SMP static_assert(WF_EXEC == SD_BALANCE_EXEC); @@ -3229,6 +3230,8 @@ static inline bool is_per_cpu_kthread(struct task_struct *p) extern void swake_up_all_locked(struct swait_queue_head *q); extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait); +extern int try_to_wake_up(struct task_struct *tsk, unsigned int state, int wake_flags); + #ifdef CONFIG_PREEMPT_DYNAMIC extern int preempt_dynamic_mode; extern int sched_dynamic_mode(const char *str); From 6f63904c8f3edb65bd85c1be01d69214ff8ca4c5 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 7 Mar 2023 23:31:58 -0800 Subject: [PATCH 054/130] sched: add a few helpers to wake up tasks on the current cpu Add complete_on_current_cpu, wake_up_poll_on_current_cpu helpers to wake up tasks on the current CPU. These two helpers are useful when the task needs to make a synchronous context switch to another task. In this context, synchronous means it wakes up the target task and falls asleep right after that. One example of such workloads is seccomp user notifies. This mechanism allows the supervisor process handles system calls on behalf of a target process. While the supervisor is handling an intercepted system call, the target process will be blocked in the kernel, waiting for a response to come back. On-CPU context switches are much faster than regular ones. Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-4-avagin@google.com Signed-off-by: Kees Cook --- include/linux/completion.h | 1 + include/linux/swait.h | 2 +- include/linux/wait.h | 3 +++ kernel/sched/completion.c | 26 ++++++++++++++++++-------- kernel/sched/core.c | 2 +- kernel/sched/swait.c | 8 ++++---- kernel/sched/wait.c | 5 +++++ 7 files changed, 33 insertions(+), 14 deletions(-) diff --git a/include/linux/completion.h b/include/linux/completion.h index 62b32b19e0a8..fb2915676574 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -116,6 +116,7 @@ extern bool try_wait_for_completion(struct completion *x); extern bool completion_done(struct completion *x); extern void complete(struct completion *); +extern void complete_on_current_cpu(struct completion *x); extern void complete_all(struct completion *); #endif diff --git a/include/linux/swait.h b/include/linux/swait.h index 6a8c22b8c2a5..d324419482a0 100644 --- a/include/linux/swait.h +++ b/include/linux/swait.h @@ -146,7 +146,7 @@ static inline bool swq_has_sleeper(struct swait_queue_head *wq) extern void swake_up_one(struct swait_queue_head *q); extern void swake_up_all(struct swait_queue_head *q); -extern void swake_up_locked(struct swait_queue_head *q); +extern void swake_up_locked(struct swait_queue_head *q, int wake_flags); extern void prepare_to_swait_exclusive(struct swait_queue_head *q, struct swait_queue *wait, int state); extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state); diff --git a/include/linux/wait.h b/include/linux/wait.h index a0307b516b09..5ec7739400f4 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -210,6 +210,7 @@ __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq } int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key); +void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key); void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key); void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head, unsigned int mode, void *key, wait_queue_entry_t *bookmark); @@ -237,6 +238,8 @@ void __wake_up_pollfree(struct wait_queue_head *wq_head); #define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m)) #define wake_up_poll(x, m) \ __wake_up(x, TASK_NORMAL, 1, poll_to_key(m)) +#define wake_up_poll_on_current_cpu(x, m) \ + __wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m)) #define wake_up_locked_poll(x, m) \ __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m)) #define wake_up_interruptible_poll(x, m) \ diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c index d57a5c1c1cd9..3561ab533dd4 100644 --- a/kernel/sched/completion.c +++ b/kernel/sched/completion.c @@ -13,6 +13,23 @@ * Waiting for completion is a typically sync point, but not an exclusion point. */ +static void complete_with_flags(struct completion *x, int wake_flags) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&x->wait.lock, flags); + + if (x->done != UINT_MAX) + x->done++; + swake_up_locked(&x->wait, wake_flags); + raw_spin_unlock_irqrestore(&x->wait.lock, flags); +} + +void complete_on_current_cpu(struct completion *x) +{ + return complete_with_flags(x, WF_CURRENT_CPU); +} + /** * complete: - signals a single thread waiting on this completion * @x: holds the state of this particular completion @@ -27,14 +44,7 @@ */ void complete(struct completion *x) { - unsigned long flags; - - raw_spin_lock_irqsave(&x->wait.lock, flags); - - if (x->done != UINT_MAX) - x->done++; - swake_up_locked(&x->wait); - raw_spin_unlock_irqrestore(&x->wait.lock, flags); + complete_with_flags(x, 0); } EXPORT_SYMBOL(complete); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1574989627b0..4d63e063608a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7029,7 +7029,7 @@ asmlinkage __visible void __sched preempt_schedule_irq(void) int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, void *key) { - WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC); + WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~(WF_SYNC|WF_CURRENT_CPU)); return try_to_wake_up(curr->private, mode, wake_flags); } EXPORT_SYMBOL(default_wake_function); diff --git a/kernel/sched/swait.c b/kernel/sched/swait.c index 76b9b796e695..72505cd3b60a 100644 --- a/kernel/sched/swait.c +++ b/kernel/sched/swait.c @@ -18,7 +18,7 @@ EXPORT_SYMBOL(__init_swait_queue_head); * If for some reason it would return 0, that means the previously waiting * task is already running, so it will observe condition true (or has already). */ -void swake_up_locked(struct swait_queue_head *q) +void swake_up_locked(struct swait_queue_head *q, int wake_flags) { struct swait_queue *curr; @@ -26,7 +26,7 @@ void swake_up_locked(struct swait_queue_head *q) return; curr = list_first_entry(&q->task_list, typeof(*curr), task_list); - wake_up_process(curr->task); + try_to_wake_up(curr->task, TASK_NORMAL, wake_flags); list_del_init(&curr->task_list); } EXPORT_SYMBOL(swake_up_locked); @@ -41,7 +41,7 @@ EXPORT_SYMBOL(swake_up_locked); void swake_up_all_locked(struct swait_queue_head *q) { while (!list_empty(&q->task_list)) - swake_up_locked(q); + swake_up_locked(q, 0); } void swake_up_one(struct swait_queue_head *q) @@ -49,7 +49,7 @@ void swake_up_one(struct swait_queue_head *q) unsigned long flags; raw_spin_lock_irqsave(&q->lock, flags); - swake_up_locked(q); + swake_up_locked(q, 0); raw_spin_unlock_irqrestore(&q->lock, flags); } EXPORT_SYMBOL(swake_up_one); diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 48c53e4739ea..802d98cf2de3 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -161,6 +161,11 @@ int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, } EXPORT_SYMBOL(__wake_up); +void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key) +{ + __wake_up_common_lock(wq_head, mode, 1, WF_CURRENT_CPU, key); +} + /* * Same as __wake_up but called with the spinlock in wait_queue_head_t held. */ From 48a1084a8b7423642b5f17ca6202f6f277c5392b Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 7 Mar 2023 23:31:59 -0800 Subject: [PATCH 055/130] seccomp: add the synchronous mode for seccomp_unotify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seccomp_unotify allows more privileged processes do actions on behalf of less privileged processes. In many cases, the workflow is fully synchronous. It means a target process triggers a system call and passes controls to a supervisor process that handles the system call and returns controls to the target process. In this context, "synchronous" means that only one process is running and another one is waiting. There is the WF_CURRENT_CPU flag that is used to advise the scheduler to move the wakee to the current CPU. For such synchronous workflows, it makes context switches a few times faster. Right now, each interaction takes 12µs. With this patch, it takes about 3µs. This change introduce the SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP flag that it used to enable the sync mode. Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-5-avagin@google.com Signed-off-by: Kees Cook --- include/uapi/linux/seccomp.h | 4 ++++ kernel/seccomp.c | 31 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/seccomp.h b/include/uapi/linux/seccomp.h index 0fdc6ef02b94..dbfc9b37fcae 100644 --- a/include/uapi/linux/seccomp.h +++ b/include/uapi/linux/seccomp.h @@ -115,6 +115,8 @@ struct seccomp_notif_resp { __u32 flags; }; +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) + /* valid flags for seccomp_notif_addfd */ #define SECCOMP_ADDFD_FLAG_SETFD (1UL << 0) /* Specify remote fd */ #define SECCOMP_ADDFD_FLAG_SEND (1UL << 1) /* Addfd and return it, atomically */ @@ -150,4 +152,6 @@ struct seccomp_notif_addfd { #define SECCOMP_IOCTL_NOTIF_ADDFD SECCOMP_IOW(3, \ struct seccomp_notif_addfd) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) + #endif /* _UAPI_LINUX_SECCOMP_H */ diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 1386dcedda1a..d3fdc0086168 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -143,9 +143,12 @@ struct seccomp_kaddfd { * filter->notify_lock. * @next_id: The id of the next request. * @notifications: A list of struct seccomp_knotif elements. + * @flags: A set of SECCOMP_USER_NOTIF_FD_* flags. */ + struct notification { atomic_t requests; + u32 flags; u64 next_id; struct list_head notifications; }; @@ -1117,7 +1120,10 @@ static int seccomp_do_user_notification(int this_syscall, INIT_LIST_HEAD(&n.addfd); atomic_inc(&match->notif->requests); - wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM); + if (match->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + wake_up_poll_on_current_cpu(&match->wqh, EPOLLIN | EPOLLRDNORM); + else + wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM); /* * This is where we wait for a reply from userspace. @@ -1593,7 +1599,10 @@ static long seccomp_notify_send(struct seccomp_filter *filter, knotif->error = resp.error; knotif->val = resp.val; knotif->flags = resp.flags; - complete(&knotif->ready); + if (filter->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + complete_on_current_cpu(&knotif->ready); + else + complete(&knotif->ready); out: mutex_unlock(&filter->notify_lock); return ret; @@ -1623,6 +1632,22 @@ static long seccomp_notify_id_valid(struct seccomp_filter *filter, return ret; } +static long seccomp_notify_set_flags(struct seccomp_filter *filter, + unsigned long flags) +{ + long ret; + + if (flags & ~SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP) + return -EINVAL; + + ret = mutex_lock_interruptible(&filter->notify_lock); + if (ret < 0) + return ret; + filter->notif->flags = flags; + mutex_unlock(&filter->notify_lock); + return 0; +} + static long seccomp_notify_addfd(struct seccomp_filter *filter, struct seccomp_notif_addfd __user *uaddfd, unsigned int size) @@ -1752,6 +1777,8 @@ static long seccomp_notify_ioctl(struct file *file, unsigned int cmd, case SECCOMP_IOCTL_NOTIF_ID_VALID_WRONG_DIR: case SECCOMP_IOCTL_NOTIF_ID_VALID: return seccomp_notify_id_valid(filter, buf); + case SECCOMP_IOCTL_NOTIF_SET_FLAGS: + return seccomp_notify_set_flags(filter, arg); } /* Extensible Argument ioctls */ From 8feae5adec17ec190118ad84da84a6ad9fa3e30b Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 7 Mar 2023 23:32:00 -0800 Subject: [PATCH 056/130] selftest/seccomp: add a new test for the sync mode of seccomp_user_notify Test output: # RUN global.user_notification_sync ... # OK global.user_notification_sync ok 51 global.user_notification_sync Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-6-avagin@google.com Signed-off-by: Kees Cook --- tools/testing/selftests/seccomp/seccomp_bpf.c | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 43ec36b179dc..f6a04d88e02f 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -4255,6 +4255,61 @@ TEST(user_notification_addfd_rlimit) close(memfd); } +#ifndef SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) +#endif + +TEST(user_notification_sync) +{ + struct seccomp_notif req = {}; + struct seccomp_notif_resp resp = {}; + int status, listener; + pid_t pid; + long ret; + + ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + ASSERT_EQ(0, ret) { + TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); + } + + listener = user_notif_syscall(__NR_getppid, + SECCOMP_FILTER_FLAG_NEW_LISTENER); + ASSERT_GE(listener, 0); + + /* Try to set invalid flags. */ + EXPECT_SYSCALL_RETURN(-EINVAL, + ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, 0xffffffff, 0)); + + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP, 0), 0); + + pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + ret = syscall(__NR_getppid); + ASSERT_EQ(ret, USER_NOTIF_MAGIC) { + _exit(1); + } + _exit(0); + } + + req.pid = 0; + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req), 0); + + ASSERT_EQ(req.data.nr, __NR_getppid); + + resp.id = req.id; + resp.error = 0; + resp.val = USER_NOTIF_MAGIC; + resp.flags = 0; + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &resp), 0); + + ASSERT_EQ(waitpid(pid, &status, 0), pid); + ASSERT_EQ(status, 0); +} + + /* Make sure PTRACE_O_SUSPEND_SECCOMP requires CAP_SYS_ADMIN. */ FIXTURE(O_SUSPEND_SECCOMP) { pid_t pid; From 7d5cb68af638fc370e89df4c2d8d8c4708600e67 Mon Sep 17 00:00:00 2001 From: Andrei Vagin Date: Tue, 7 Mar 2023 23:32:01 -0800 Subject: [PATCH 057/130] perf/benchmark: add a new benchmark for seccom_unotify The benchmark is similar to the pipe benchmark. It creates two processes, one is calling syscalls, and another process is handling them via seccomp user notifications. It measures the time required to run a specified number of interations. $ ./perf bench sched seccomp-notify --sync-mode --loop 1000000 # Running 'sched/seccomp-notify' benchmark: # Executed 1000000 system calls Total time: 2.769 [sec] 2.769629 usecs/op 361059 ops/sec $ ./perf bench sched seccomp-notify # Running 'sched/seccomp-notify' benchmark: # Executed 1000000 system calls Total time: 8.571 [sec] 8.571119 usecs/op 116670 ops/sec Signed-off-by: Andrei Vagin Acked-by: "Peter Zijlstra (Intel)" Link: https://lore.kernel.org/r/20230308073201.3102738-7-avagin@google.com Link: https://lore.kernel.org/r/20230630051953.454638-1-avagin@gmail.com [kees: Added PRIu64 format string] Signed-off-by: Kees Cook --- tools/arch/x86/include/uapi/asm/unistd_32.h | 3 + tools/arch/x86/include/uapi/asm/unistd_64.h | 3 + tools/perf/bench/Build | 1 + tools/perf/bench/bench.h | 1 + tools/perf/bench/sched-seccomp-notify.c | 178 ++++++++++++++++++++ tools/perf/builtin-bench.c | 1 + 6 files changed, 187 insertions(+) create mode 100644 tools/perf/bench/sched-seccomp-notify.c diff --git a/tools/arch/x86/include/uapi/asm/unistd_32.h b/tools/arch/x86/include/uapi/asm/unistd_32.h index bc48a4dabe5d..4798f9d18fe8 100644 --- a/tools/arch/x86/include/uapi/asm/unistd_32.h +++ b/tools/arch/x86/include/uapi/asm/unistd_32.h @@ -26,3 +26,6 @@ #ifndef __NR_setns #define __NR_setns 346 #endif +#ifdef __NR_seccomp +#define __NR_seccomp 354 +#endif diff --git a/tools/arch/x86/include/uapi/asm/unistd_64.h b/tools/arch/x86/include/uapi/asm/unistd_64.h index f70d2cada256..d0f2043d7132 100644 --- a/tools/arch/x86/include/uapi/asm/unistd_64.h +++ b/tools/arch/x86/include/uapi/asm/unistd_64.h @@ -26,3 +26,6 @@ #ifndef __NR_getcpu #define __NR_getcpu 309 #endif +#ifndef __NR_seccomp +#define __NR_seccomp 317 +#endif diff --git a/tools/perf/bench/Build b/tools/perf/bench/Build index 0f158dc8139b..07bbc449329e 100644 --- a/tools/perf/bench/Build +++ b/tools/perf/bench/Build @@ -1,5 +1,6 @@ perf-y += sched-messaging.o perf-y += sched-pipe.o +perf-y += sched-seccomp-notify.o perf-y += syscall.o perf-y += mem-functions.o perf-y += futex-hash.o diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h index 0d2b65976212..a0625c77bea3 100644 --- a/tools/perf/bench/bench.h +++ b/tools/perf/bench/bench.h @@ -21,6 +21,7 @@ extern struct timeval bench__start, bench__end, bench__runtime; int bench_numa(int argc, const char **argv); int bench_sched_messaging(int argc, const char **argv); int bench_sched_pipe(int argc, const char **argv); +int bench_sched_seccomp_notify(int argc, const char **argv); int bench_syscall_basic(int argc, const char **argv); int bench_syscall_getpgid(int argc, const char **argv); int bench_syscall_fork(int argc, const char **argv); diff --git a/tools/perf/bench/sched-seccomp-notify.c b/tools/perf/bench/sched-seccomp-notify.c new file mode 100644 index 000000000000..b04ebcde4036 --- /dev/null +++ b/tools/perf/bench/sched-seccomp-notify.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "bench.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOOPS_DEFAULT 1000000UL +static uint64_t loops = LOOPS_DEFAULT; +static bool sync_mode; + +static const struct option options[] = { + OPT_U64('l', "loop", &loops, "Specify number of loops"), + OPT_BOOLEAN('s', "sync-mode", &sync_mode, + "Enable the synchronious mode for seccomp notifications"), + OPT_END() +}; + +static const char * const bench_seccomp_usage[] = { + "perf bench sched secccomp-notify ", + NULL +}; + +static int seccomp(unsigned int op, unsigned int flags, void *args) +{ + return syscall(__NR_seccomp, op, flags, args); +} + +static int user_notif_syscall(int nr, unsigned int flags) +{ + struct sock_filter filter[] = { + BPF_STMT(BPF_LD|BPF_W|BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP|BPF_JEQ|BPF_K, nr, 0, 1), + BPF_STMT(BPF_RET|BPF_K, SECCOMP_RET_USER_NOTIF), + BPF_STMT(BPF_RET|BPF_K, SECCOMP_RET_ALLOW), + }; + + struct sock_fprog prog = { + .len = (unsigned short)ARRAY_SIZE(filter), + .filter = filter, + }; + + return seccomp(SECCOMP_SET_MODE_FILTER, flags, &prog); +} + +#define USER_NOTIF_MAGIC INT_MAX +static void user_notification_sync_loop(int listener) +{ + struct seccomp_notif_resp resp; + struct seccomp_notif req; + uint64_t nr; + + for (nr = 0; nr < loops; nr++) { + memset(&req, 0, sizeof(req)); + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req)) + err(EXIT_FAILURE, "SECCOMP_IOCTL_NOTIF_RECV failed"); + + if (req.data.nr != __NR_gettid) + errx(EXIT_FAILURE, "unexpected syscall: %d", req.data.nr); + + resp.id = req.id; + resp.error = 0; + resp.val = USER_NOTIF_MAGIC; + resp.flags = 0; + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &resp)) + err(EXIT_FAILURE, "SECCOMP_IOCTL_NOTIF_SEND failed"); + } +} + +#ifndef SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP +#define SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP (1UL << 0) +#define SECCOMP_IOCTL_NOTIF_SET_FLAGS SECCOMP_IOW(4, __u64) +#endif +int bench_sched_seccomp_notify(int argc, const char **argv) +{ + struct timeval start, stop, diff; + unsigned long long result_usec = 0; + int status, listener; + pid_t pid; + long ret; + + argc = parse_options(argc, argv, options, bench_seccomp_usage, 0); + + gettimeofday(&start, NULL); + + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + listener = user_notif_syscall(__NR_gettid, + SECCOMP_FILTER_FLAG_NEW_LISTENER); + if (listener < 0) + err(EXIT_FAILURE, "can't create a notification descriptor"); + + pid = fork(); + if (pid < 0) + err(EXIT_FAILURE, "fork"); + if (pid == 0) { + if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0)) + err(EXIT_FAILURE, "can't set the parent death signal"); + while (1) { + ret = syscall(__NR_gettid); + if (ret == USER_NOTIF_MAGIC) + continue; + break; + } + _exit(1); + } + + if (sync_mode) { + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SET_FLAGS, + SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP, 0)) + err(EXIT_FAILURE, + "can't set SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP"); + } + user_notification_sync_loop(listener); + + kill(pid, SIGKILL); + if (waitpid(pid, &status, 0) != pid) + err(EXIT_FAILURE, "waitpid(%d) failed", pid); + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGKILL) + errx(EXIT_FAILURE, "unexpected exit code: %d", status); + + gettimeofday(&stop, NULL); + timersub(&stop, &start, &diff); + + switch (bench_format) { + case BENCH_FORMAT_DEFAULT: + printf("# Executed %" PRIu64 " system calls\n\n", + loops); + + result_usec = diff.tv_sec * USEC_PER_SEC; + result_usec += diff.tv_usec; + + printf(" %14s: %lu.%03lu [sec]\n\n", "Total time", + (unsigned long) diff.tv_sec, + (unsigned long) (diff.tv_usec / USEC_PER_MSEC)); + + printf(" %14lf usecs/op\n", + (double)result_usec / (double)loops); + printf(" %14d ops/sec\n", + (int)((double)loops / + ((double)result_usec / (double)USEC_PER_SEC))); + break; + + case BENCH_FORMAT_SIMPLE: + printf("%lu.%03lu\n", + (unsigned long) diff.tv_sec, + (unsigned long) (diff.tv_usec / USEC_PER_MSEC)); + break; + + default: + /* reaching here is something disaster */ + fprintf(stderr, "Unknown format:%d\n", bench_format); + exit(1); + break; + } + + return 0; +} diff --git a/tools/perf/builtin-bench.c b/tools/perf/builtin-bench.c index db435b791a09..5033e8bab276 100644 --- a/tools/perf/builtin-bench.c +++ b/tools/perf/builtin-bench.c @@ -47,6 +47,7 @@ static struct bench numa_benchmarks[] = { static struct bench sched_benchmarks[] = { { "messaging", "Benchmark for scheduling and IPC", bench_sched_messaging }, { "pipe", "Benchmark for pipe() between two processes", bench_sched_pipe }, + { "seccomp-notify", "Benchmark for seccomp user notify", bench_sched_seccomp_notify}, { "all", "Run all scheduler benchmarks", NULL }, { NULL, NULL, NULL } }; From 6716f4d39c17febf7aa4fa5f5923da67a8d10e85 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 03:41:37 -0700 Subject: [PATCH 058/130] rcu: Update synchronize_rcu_mult() comment for call_rcu_hurry() Those who have worked with RCU for some time will naturally think in terms of the long-standing call_rcu() API rather than the much newer call_rcu_hurry() API. But it is call_rcu_hurry() that you should normally pass to synchronize_rcu_mult(). This commit therefore updates the header comment to point this out. Signed-off-by: Paul E. McKenney Reviewed-by: Joel Fernandes (Google) --- include/linux/rcupdate_wait.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 699b938358bf..5e0f74f2f8ca 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -42,6 +42,11 @@ do { \ * call_srcu() function, with this wrapper supplying the pointer to the * corresponding srcu_struct. * + * Note that call_rcu_hurry() should be used instead of call_rcu() + * because in kernels built with CONFIG_RCU_LAZY=y the delay between the + * invocation of call_rcu() and that of the corresponding RCU callback + * can be multiple seconds. + * * The first argument tells Tiny RCU's _wait_rcu_gp() not to * bother waiting for RCU. The reason for this is because anywhere * synchronize_rcu_mult() can be called is automatically already a full From c924bf5a43e47d5591d527d39b474f8ca9e63c0e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 23 May 2023 02:22:10 -0700 Subject: [PATCH 059/130] rcu: Clarify rcu_is_watching() kernel-doc comment Make it clear that this function always returns either true or false without other planned failure modes. Reported-by: Masami Hiramatsu Signed-off-by: Paul E. McKenney Reviewed-by: Joel Fernandes (Google) --- kernel/rcu/tree.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 1449cb69a0e0..f788981272f8 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -677,12 +677,16 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) } /** - * rcu_is_watching - see if RCU thinks that the current CPU is not idle + * rcu_is_watching - RCU read-side critical sections permitted on current CPU? * - * Return true if RCU is watching the running CPU, which means that this - * CPU can safely enter RCU read-side critical sections. In other words, - * if the current CPU is not in its idle loop or is in an interrupt or - * NMI handler, return true. + * Return @true if RCU is watching the running CPU and @false otherwise. + * An @true return means that this CPU can safely enter RCU read-side + * critical sections. + * + * Although calls to rcu_is_watching() from most parts of the kernel + * will return @true, there are important exceptions. For example, if the + * current CPU is deep within its idle loop, in kernel entry/exit code, + * or offline, rcu_is_watching() will return @false. * * Make notrace because it can be called by the internal functions of * ftrace, and making this notrace removes unnecessary recursion calls. From db5dc3502dfac007088ab823a267d9cd7e2c5604 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 10 Jul 2023 17:34:45 -0700 Subject: [PATCH 060/130] torture: Add srcu_lockdep.sh to torture.sh This commit adds srcu_lockdep.sh to torture.sh, thus exercizing the extended SRCU-aware lockdep-RCU functionality on a regular basis. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/torture.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index d36b839bf586..bc63344ce4cd 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -56,6 +56,7 @@ do_kcsan=no do_clocksourcewd=yes do_rt=yes do_rcutasksflavors=yes +do_srcu_lockdep=yes # doyesno - Helper function for yes/no arguments function doyesno () { @@ -87,6 +88,7 @@ usage () { echo " --do-refscale / --do-no-refscale / --no-refscale" echo " --do-rt / --do-no-rt / --no-rt" echo " --do-scftorture / --do-no-scftorture / --no-scftorture" + echo " --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep" echo " --duration [ | h | d ]" echo " --kcsan-kmake-arg kernel-make-arguments" exit 1 @@ -128,6 +130,7 @@ do do_kasan=yes do_kcsan=yes do_clocksourcewd=yes + do_srcu_lockdep=yes ;; --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig) do_allmodconfig=`doyesno "$1" --do-allmodconfig` @@ -160,6 +163,7 @@ do do_kasan=no do_kcsan=no do_clocksourcewd=no + do_srcu_lockdep=no ;; --do-rcuscale|--do-no-rcuscale|--no-rcuscale) do_rcuscale=`doyesno "$1" --do-rcuscale` @@ -179,6 +183,9 @@ do --do-scftorture|--do-no-scftorture|--no-scftorture) do_scftorture=`doyesno "$1" --do-scftorture` ;; + --do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep) + do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep` + ;; --duration) checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' mult=1 @@ -432,6 +439,23 @@ then torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --trust-make fi +if test "$do_srcu_lockdep" = "yes" +then + echo " --- do-srcu-lockdep:" Start `date` | tee -a $T/log + tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh --datestamp "$ds/results-srcu-lockdep" > $T/srcu_lockdep.sh.out 2>&1 + retcode=$? + cp $T/srcu_lockdep.sh.out "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + if test "$retcode" -eq 0 + then + echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/successes + echo Success >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + else + echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/failures + echo " --- srcu_lockdep Test Summary:" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + echo " --- Summary: Exit code $retcode from srcu_lockdep.sh, see ds/results-srcu-lockdep" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" + fi +fi + if test "$do_refscale" = yes then primlist="`grep '\.name[ ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" From 10f84c2cfb5045e37d78cb5d4c8e8321e06ae18f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 11 Jul 2023 11:02:09 -0700 Subject: [PATCH 061/130] torture: Avoid torture-test reboot loops Currently, the various torture tests sometimes react to an early-boot bug by rebooting. This is almost always counterproductive, needlessly consuming CPU time and bloating the console log. This commit therefore adds the "-no-reboot" argument to qemu so that reboot requests will cause qemu to exit. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index 1e7566997e51..3bc0140438ef 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh @@ -9,9 +9,10 @@ # # Usage: kvm-test-1-run.sh config resdir seconds qemu-args boot_args_in # -# qemu-args defaults to "-enable-kvm -nographic", along with arguments -# specifying the number of CPUs and other options -# generated from the underlying CPU architecture. +# qemu-args defaults to "-enable-kvm -nographic -no-reboot", along with +# arguments specifying the number of CPUs and +# other options generated from the underlying +# CPU architecture. # boot_args_in defaults to value returned by the per_version_boot_params # shell function. # @@ -139,7 +140,7 @@ then fi # Generate -smp qemu argument. -qemu_args="-enable-kvm -nographic $qemu_args" +qemu_args="-enable-kvm -nographic -no-reboot $qemu_args" cpu_count=`configNR_CPUS.sh $resdir/ConfigFragment` cpu_count=`configfrag_boot_cpus "$boot_args_in" "$config_template" "$cpu_count"` if test "$cpu_count" -gt "$TORTURE_ALLOTED_CPUS" From ecf671cf76b026dc27e6febb31a87b6b15d2a7f7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 13 Jul 2023 16:38:14 -0700 Subject: [PATCH 062/130] torture: Add init-program support for loongarch This commit adds the __loongarch__, __loongarch_lp64, and __loongarch_double_float targets to rcutorture's mkinitrd.sh script in order to allow nolibc init programs for loongarch. [ paulmck: Apply feedback from Feiyang Chen. ] Cc: Feiyang Chen Cc: Huacai Chen Cc: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/mkinitrd.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh index 3af3a86a0091..435ae6db5cc5 100755 --- a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh +++ b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh @@ -63,7 +63,7 @@ ___EOF___ # build using nolibc on supported archs (smaller executable) and fall # back to regular glibc on other ones. if echo -e "#if __x86_64__||__i386__||__i486__||__i586__||__i686__" \ - "||__ARM_EABI__||__aarch64__||__s390x__\nyes\n#endif" \ + "||__ARM_EABI__||__aarch64__||__s390x__||__loongarch__\nyes\n#endif" \ | ${CROSS_COMPILE}gcc -E -nostdlib -xc - \ | grep -q '^yes'; then # architecture supported by nolibc From 93a556b8b3c8cae25230fac8f1376284a697c489 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 13 Jul 2023 17:39:02 -0700 Subject: [PATCH 063/130] torture: Switch qemu from -nographic to -display none This commit switches the qemu argument "-nographic" to "-display none", aligning with the nolibc tests. Cc: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index 3bc0140438ef..b33cd8753689 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh @@ -9,9 +9,9 @@ # # Usage: kvm-test-1-run.sh config resdir seconds qemu-args boot_args_in # -# qemu-args defaults to "-enable-kvm -nographic -no-reboot", along with -# arguments specifying the number of CPUs and -# other options generated from the underlying +# qemu-args defaults to "-enable-kvm -display none -no-reboot", along +# with arguments specifying the number of CPUs +# and other options generated from the underlying # CPU architecture. # boot_args_in defaults to value returned by the per_version_boot_params # shell function. @@ -140,7 +140,7 @@ then fi # Generate -smp qemu argument. -qemu_args="-enable-kvm -nographic -no-reboot $qemu_args" +qemu_args="-enable-kvm -display none -no-reboot $qemu_args" cpu_count=`configNR_CPUS.sh $resdir/ConfigFragment` cpu_count=`configfrag_boot_cpus "$boot_args_in" "$config_template" "$cpu_count"` if test "$cpu_count" -gt "$TORTURE_ALLOTED_CPUS" From 451d2a52f652446f002f289431c171c2064a06fb Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 13 Jul 2023 18:22:07 -0700 Subject: [PATCH 064/130] torture: Make init program dump command-line arguments This commit causes the init program generated by mkinitrd.sh dump out its parameters. Although this is in some sense redundant given that the kernel already dumps them out, confirmation can be a good thing. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/mkinitrd.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh index 435ae6db5cc5..558f5a669dbd 100755 --- a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh +++ b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh @@ -33,12 +33,16 @@ cat > init.c << '___EOF___' volatile unsigned long delaycount; -int main(int argc, int argv[]) +int main(int argc, char *argv[]) { int i; struct timeval tv; struct timeval tvb; + printf("Torture-test rudimentary init program started, command line:\n"); + for (i = 0; i < argc; i++) + printf(" %s", argv[i]); + printf("\n"); for (;;) { sleep(1); /* Need some userspace time. */ From 7c25ee819f39a6c84d5fbe87fee4a7f78dd204a1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 13 Jul 2023 18:29:36 -0700 Subject: [PATCH 065/130] torture: Cause mkinitrd.sh to indicate failure on compile errors Currently, if the C program created by mkinitrd.sh has compile errors, the errors are printed, but kvm.sh soldiers on, building kernels that have init-less initrd setups. The kernels then fail on boot when they attempt to mount non-existent root filesystems. This commit therefore improves user friendliness by making mkinitrd.sh return non-zero exit status on compile errors, which in turn causes kvm.sh to take an early exit, with the compile errors still clearly visible. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/mkinitrd.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh index 558f5a669dbd..212c52ca90b5 100755 --- a/tools/testing/selftests/rcutorture/bin/mkinitrd.sh +++ b/tools/testing/selftests/rcutorture/bin/mkinitrd.sh @@ -74,8 +74,16 @@ if echo -e "#if __x86_64__||__i386__||__i486__||__i586__||__i686__" \ ${CROSS_COMPILE}gcc -fno-asynchronous-unwind-tables -fno-ident \ -nostdlib -include ../../../../include/nolibc/nolibc.h \ -s -static -Os -o init init.c -lgcc + ret=$? else ${CROSS_COMPILE}gcc -s -static -Os -o init init.c + ret=$? +fi + +if [ "$ret" -ne 0 ] +then + echo "Failed to create a statically linked C-language initrd" + exit "$ret" fi rm init.c From 84dd7f19e72656fac5faf67f1b1809e57e9788cc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 6 Jul 2023 11:48:07 -0700 Subject: [PATCH 066/130] checkpatch: Complain about unexpected uses of RCU Tasks Trace RCU Tasks Trace is quite specialized, having been created specifically for sleepable BPF programs. Because it allows general blocking within readers, any new use of RCU Tasks Trace must take current use cases into account. Therefore, update checkpatch.pl to complain about use of any of the RCU Tasks Trace API members outside of BPF and outside of RCU itself. [ paulmck: Apply Joe Perches feedback. ] Cc: Andy Whitcroft (maintainer:CHECKPATCH) Cc: Joe Perches (maintainer:CHECKPATCH) Cc: Dwaipayan Ray (reviewer:CHECKPATCH) Cc: Lukas Bulwahn Cc: Alexei Starovoitov Cc: Daniel Borkmann Cc: John Fastabend Cc: Signed-off-by: Paul E. McKenney --- scripts/checkpatch.pl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 880fde13d9b8..a9841148cde2 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -7457,6 +7457,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { From 630fdd592912614a72d00026fdadad72d9ef62eb Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 26 Jul 2023 14:59:57 -0700 Subject: [PATCH 067/130] seq_file: seq_show_option_n() is used for precise sizes When seq_show_option_n() is used, it is for non-string memory that happens to be printable bytes. As such, we must use memcpy() to copy the bytes and then explicitly NUL-terminate the result. Cc: Andy Shevchenko Cc: Andrew Morton Cc: Al Viro Cc: Muchun Song Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20230726215957.never.619-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/seq_file.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index bd023dd38ae6..386ab580b839 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -249,18 +249,19 @@ static inline void seq_show_option(struct seq_file *m, const char *name, /** * seq_show_option_n - display mount options with appropriate escapes - * where @value must be a specific length. + * where @value must be a specific length (i.e. + * not NUL-terminated). * @m: the seq_file handle * @name: the mount option name * @value: the mount option name's value, cannot be NULL - * @length: the length of @value to display + * @length: the exact length of @value to display, must be constant expression * * This is a macro since this uses "length" to define the size of the * stack buffer. */ #define seq_show_option_n(m, name, value, length) { \ char val_buf[length + 1]; \ - strncpy(val_buf, value, length); \ + memcpy(val_buf, value, length); \ val_buf[length] = '\0'; \ seq_show_option(m, name, val_buf); \ } From 61ce78f29a694772c3b2c5c749589682dbdfec2d Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Mon, 3 Jul 2023 16:06:41 +0000 Subject: [PATCH 068/130] um: Remove strlcpy declaration strlcpy() reads the entire source buffer first. This read may exceed the destination size limit. This is both inefficient and can lead to linear read overflows if a source string is not NUL-terminated [1]. In an effort to remove strlcpy() completely [2], replace strlcpy() here with strscpy(). No return values were used, so direct replacement is safe. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#strlcpy [2] https://github.com/KSPP/linux/issues/89 Signed-off-by: Azeem Shaikh Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230703160641.1790935-1-azeemshaikh38@gmail.com Signed-off-by: Kees Cook --- arch/um/include/shared/user.h | 1 - arch/um/os-Linux/umid.c | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/um/include/shared/user.h b/arch/um/include/shared/user.h index 0347a190429c..981e11d8e025 100644 --- a/arch/um/include/shared/user.h +++ b/arch/um/include/shared/user.h @@ -50,7 +50,6 @@ static inline int printk(const char *fmt, ...) #endif extern int in_aton(char *str); -extern size_t strlcpy(char *, const char *, size_t); extern size_t strlcat(char *, const char *, size_t); extern size_t strscpy(char *, const char *, size_t); diff --git a/arch/um/os-Linux/umid.c b/arch/um/os-Linux/umid.c index 7a1abb829930..288c422bfa96 100644 --- a/arch/um/os-Linux/umid.c +++ b/arch/um/os-Linux/umid.c @@ -40,7 +40,7 @@ static int __init make_uml_dir(void) __func__); goto err; } - strlcpy(dir, home, sizeof(dir)); + strscpy(dir, home, sizeof(dir)); uml_dir++; } strlcat(dir, uml_dir, sizeof(dir)); @@ -243,7 +243,7 @@ int __init set_umid(char *name) if (strlen(name) > UMID_LEN - 1) return -E2BIG; - strlcpy(umid, name, sizeof(umid)); + strscpy(umid, name, sizeof(umid)); return 0; } @@ -262,7 +262,7 @@ static int __init make_umid(void) make_uml_dir(); if (*umid == '\0') { - strlcpy(tmp, uml_dir, sizeof(tmp)); + strscpy(tmp, uml_dir, sizeof(tmp)); strlcat(tmp, "XXXXXX", sizeof(tmp)); fd = mkstemp(tmp); if (fd < 0) { From c9732f1461f947429f8ee9289792c5ffea793350 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Mon, 3 Jul 2023 16:58:16 +0000 Subject: [PATCH 069/130] perf: Replace strlcpy with strscpy strlcpy() reads the entire source buffer first. This read may exceed the destination size limit. This is both inefficient and can lead to linear read overflows if a source string is not NUL-terminated [1]. In an effort to remove strlcpy() completely [2], replace strlcpy() here with strscpy(). No return values were used, so direct replacement is safe. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#strlcpy [2] https://github.com/KSPP/linux/issues/89 Signed-off-by: Azeem Shaikh Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230703165817.2840457-1-azeemshaikh38@gmail.com Signed-off-by: Kees Cook --- kernel/events/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 78ae7b6f90fd..2554f5fc70dc 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8249,7 +8249,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) unsigned int size; memset(comm, 0, sizeof(comm)); - strlcpy(comm, comm_event->task->comm, sizeof(comm)); + strscpy(comm, comm_event->task->comm, sizeof(comm)); size = ALIGN(strlen(comm)+1, sizeof(u64)); comm_event->comm = comm; @@ -8704,7 +8704,7 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) } cpy_name: - strlcpy(tmp, name, sizeof(tmp)); + strscpy(tmp, name, sizeof(tmp)); name = tmp; got_name: /* @@ -9128,7 +9128,7 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN) goto err; - strlcpy(name, sym, KSYM_NAME_LEN); + strscpy(name, sym, KSYM_NAME_LEN); name_len = strlen(name) + 1; while (!IS_ALIGNED(name_len, sizeof(u64))) name[name_len++] = '\0'; From b6ed2f7758a558485ff889b4a3912fcb6abcf689 Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Thu, 6 Jul 2023 17:58:04 +0000 Subject: [PATCH 070/130] EISA: Replace all non-returning strlcpy with strscpy strlcpy() reads the entire source buffer first. This read may exceed the destination size limit. This is both inefficient and can lead to linear read overflows if a source string is not NUL-terminated [1]. In an effort to remove strlcpy() completely [2], replace strlcpy() here with strscpy(). No return values were used, so direct replacement is safe. [1] https://www.kernel.org/doc/html/latest/process/deprecated.html#strlcpy [2] https://github.com/KSPP/linux/issues/89 Signed-off-by: Azeem Shaikh Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230706175804.2249018-1-azeemshaikh38@gmail.com Signed-off-by: Kees Cook # diff --git a/drivers/eisa/eisa-bus.c b/drivers/eisa/eisa-bus.c # index 713582cc27d1..33f0ba11c6ad 100644 # --- a/drivers/eisa/eisa-bus.c # +++ b/drivers/eisa/eisa-bus.c # @@ -60,7 +60,7 @@ static void __init eisa_name_device(struct eisa_device *edev) # int i; # for (i = 0; i < EISA_INFOS; i++) { # if (!strcmp(edev->id.sig, eisa_table[i].id.sig)) { # - strlcpy(edev->pretty_name, # + strscpy(edev->pretty_name, # eisa_table[i].name, # sizeof(edev->pretty_name)); # return; --- drivers/eisa/eisa-bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/eisa/eisa-bus.c b/drivers/eisa/eisa-bus.c index 713582cc27d1..33f0ba11c6ad 100644 --- a/drivers/eisa/eisa-bus.c +++ b/drivers/eisa/eisa-bus.c @@ -60,7 +60,7 @@ static void __init eisa_name_device(struct eisa_device *edev) int i; for (i = 0; i < EISA_INFOS; i++) { if (!strcmp(edev->id.sig, eisa_table[i].id.sig)) { - strlcpy(edev->pretty_name, + strscpy(edev->pretty_name, eisa_table[i].name, sizeof(edev->pretty_name)); return; From 7a4dcb4a5de1214c4a59448a759e2e264c2c4473 Mon Sep 17 00:00:00 2001 From: Laurent Dufour Date: Wed, 5 Jul 2023 16:51:34 +0200 Subject: [PATCH 071/130] cpu/hotplug: Remove dependancy against cpu_primary_thread_mask The commit 18415f33e2ac ("cpu/hotplug: Allow "parallel" bringup up to CPUHP_BP_KICK_AP_STATE") introduce a dependancy against a global variable cpu_primary_thread_mask exported by the X86 code. This variable is only used when CONFIG_HOTPLUG_PARALLEL is set. Since cpuhp_get_primary_thread_mask() and cpuhp_smt_aware() are only used when CONFIG_HOTPLUG_PARALLEL is set, don't define them when it is not set. No functional change. Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-2-ldufour@linux.ibm.com --- kernel/cpu.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/kernel/cpu.c b/kernel/cpu.c index 88a7ede322bd..03309f2f35a4 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -650,22 +650,8 @@ bool cpu_smt_possible(void) } EXPORT_SYMBOL_GPL(cpu_smt_possible); -static inline bool cpuhp_smt_aware(void) -{ - return topology_smt_supported(); -} - -static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) -{ - return cpu_primary_thread_mask; -} #else static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } -static inline bool cpuhp_smt_aware(void) { return false; } -static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) -{ - return cpu_present_mask; -} #endif static inline enum cpuhp_state @@ -1793,6 +1779,16 @@ static int __init parallel_bringup_parse_param(char *arg) } early_param("cpuhp.parallel", parallel_bringup_parse_param); +static inline bool cpuhp_smt_aware(void) +{ + return topology_smt_supported(); +} + +static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) +{ + return cpu_primary_thread_mask; +} + /* * On architectures which have enabled parallel bringup this invokes all BP * prepare states for each of the to be onlined APs first. The last state From 3f9169196be55590a794b52f49637561ddd1ba4f Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 5 Jul 2023 16:51:35 +0200 Subject: [PATCH 072/130] cpu/SMT: Move SMT prototypes into cpu_smt.h In order to export the cpuhp_smt_control enum as part of the interface between generic and architecture code, the architecture code needs to include asm/topology.h. But that leads to circular header dependencies. So split the enum and related declarations into a separate header. [ ldufour: Reworded the commit's description ] Signed-off-by: Michael Ellerman Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-3-ldufour@linux.ibm.com --- arch/x86/include/asm/topology.h | 2 ++ include/linux/cpu.h | 25 +------------------------ include/linux/cpu_smt.h | 29 +++++++++++++++++++++++++++++ kernel/cpu.c | 1 + 4 files changed, 33 insertions(+), 24 deletions(-) create mode 100644 include/linux/cpu_smt.h diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index caf41c4869a0..ae49ed4417d0 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -136,6 +136,8 @@ static inline int topology_max_smt_threads(void) return __max_smt_threads; } +#include + int topology_update_package_map(unsigned int apicid, unsigned int cpu); int topology_update_die_map(unsigned int dieid, unsigned int cpu); int topology_phys_to_logical_pkg(unsigned int pkg); diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 6e6e57ec69e8..6b326a9e8191 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -18,6 +18,7 @@ #include #include #include +#include struct device; struct device_node; @@ -204,30 +205,6 @@ void cpuhp_report_idle_dead(void); static inline void cpuhp_report_idle_dead(void) { } #endif /* #ifdef CONFIG_HOTPLUG_CPU */ -enum cpuhp_smt_control { - CPU_SMT_ENABLED, - CPU_SMT_DISABLED, - CPU_SMT_FORCE_DISABLED, - CPU_SMT_NOT_SUPPORTED, - CPU_SMT_NOT_IMPLEMENTED, -}; - -#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) -extern enum cpuhp_smt_control cpu_smt_control; -extern void cpu_smt_disable(bool force); -extern void cpu_smt_check_topology(void); -extern bool cpu_smt_possible(void); -extern int cpuhp_smt_enable(void); -extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); -#else -# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) -static inline void cpu_smt_disable(bool force) { } -static inline void cpu_smt_check_topology(void) { } -static inline bool cpu_smt_possible(void) { return false; } -static inline int cpuhp_smt_enable(void) { return 0; } -static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } -#endif - extern bool cpu_mitigations_off(void); extern bool cpu_mitigations_auto_nosmt(void); diff --git a/include/linux/cpu_smt.h b/include/linux/cpu_smt.h new file mode 100644 index 000000000000..722c2e306fef --- /dev/null +++ b/include/linux/cpu_smt.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CPU_SMT_H_ +#define _LINUX_CPU_SMT_H_ + +enum cpuhp_smt_control { + CPU_SMT_ENABLED, + CPU_SMT_DISABLED, + CPU_SMT_FORCE_DISABLED, + CPU_SMT_NOT_SUPPORTED, + CPU_SMT_NOT_IMPLEMENTED, +}; + +#if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) +extern enum cpuhp_smt_control cpu_smt_control; +extern void cpu_smt_disable(bool force); +extern void cpu_smt_check_topology(void); +extern bool cpu_smt_possible(void); +extern int cpuhp_smt_enable(void); +extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); +#else +# define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) +static inline void cpu_smt_disable(bool force) { } +static inline void cpu_smt_check_topology(void) { } +static inline bool cpu_smt_possible(void) { return false; } +static inline int cpuhp_smt_enable(void) { return 0; } +static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } +#endif + +#endif /* _LINUX_CPU_SMT_H_ */ diff --git a/kernel/cpu.c b/kernel/cpu.c index 03309f2f35a4..e02204c4675a 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -592,6 +592,7 @@ static void lockdep_release_cpus_lock(void) void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMT + enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; void __init cpu_smt_disable(bool force) From c53361ce7d8771129c517dca529d2f2dc5bf04d1 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 5 Jul 2023 16:51:36 +0200 Subject: [PATCH 073/130] cpu/SMT: Move smt/control simple exit cases earlier Move the simple exit cases, i.e. those which don't depend on the value written, earlier in the function. That makes it clearer that regardless of the input those states cannot be transitioned out of. That does have a user-visible effect, in that the error returned will now always be EPERM/ENODEV for those states, regardless of the value written. Previously writing an invalid value would return EINVAL even when in those states. Signed-off-by: Michael Ellerman Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-4-ldufour@linux.ibm.com --- kernel/cpu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/cpu.c b/kernel/cpu.c index e02204c4675a..b6fe170c93e9 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2841,6 +2841,12 @@ __store_smt_control(struct device *dev, struct device_attribute *attr, { int ctrlval, ret; + if (cpu_smt_control == CPU_SMT_FORCE_DISABLED) + return -EPERM; + + if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED) + return -ENODEV; + if (sysfs_streq(buf, "on")) ctrlval = CPU_SMT_ENABLED; else if (sysfs_streq(buf, "off")) @@ -2850,12 +2856,6 @@ __store_smt_control(struct device *dev, struct device_attribute *attr, else return -EINVAL; - if (cpu_smt_control == CPU_SMT_FORCE_DISABLED) - return -EPERM; - - if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED) - return -ENODEV; - ret = lock_device_hotplug_sysfs(); if (ret) return ret; From 447ae4ac41130a7f127c2581a5e816bb0800b560 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 5 Jul 2023 16:51:37 +0200 Subject: [PATCH 074/130] cpu/SMT: Store the current/max number of threads Some architectures allow partial SMT states at boot time, ie. when not all SMT threads are brought online. To support that the SMT code needs to know the maximum number of SMT threads, and also the currently configured number. The architecture code knows the max number of threads, so have the architecture code pass that value to cpu_smt_set_num_threads(). Note that although topology_max_smt_threads() exists, it is not configured early enough to be used here. As architecture, like PowerPC, allows the threads number to be set through the kernel command line, also pass that value. [ ldufour: Slightly reword the commit message ] [ ldufour: Rename cpu_smt_check_topology and add a num_threads argument ] Signed-off-by: Michael Ellerman Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-5-ldufour@linux.ibm.com --- arch/x86/kernel/cpu/common.c | 2 +- include/linux/cpu_smt.h | 8 ++++++-- kernel/cpu.c | 21 ++++++++++++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 52683fddafaf..12a48a85da3d 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2317,7 +2317,7 @@ void __init arch_cpu_finalize_init(void) * identify_boot_cpu() initialized SMT support information, let the * core code know. */ - cpu_smt_check_topology(); + cpu_smt_set_num_threads(smp_num_siblings, smp_num_siblings); if (!IS_ENABLED(CONFIG_SMP)) { pr_info("CPU: "); diff --git a/include/linux/cpu_smt.h b/include/linux/cpu_smt.h index 722c2e306fef..0c1664294b57 100644 --- a/include/linux/cpu_smt.h +++ b/include/linux/cpu_smt.h @@ -12,15 +12,19 @@ enum cpuhp_smt_control { #if defined(CONFIG_SMP) && defined(CONFIG_HOTPLUG_SMT) extern enum cpuhp_smt_control cpu_smt_control; +extern unsigned int cpu_smt_num_threads; extern void cpu_smt_disable(bool force); -extern void cpu_smt_check_topology(void); +extern void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads); extern bool cpu_smt_possible(void); extern int cpuhp_smt_enable(void); extern int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval); #else # define cpu_smt_control (CPU_SMT_NOT_IMPLEMENTED) +# define cpu_smt_num_threads 1 static inline void cpu_smt_disable(bool force) { } -static inline void cpu_smt_check_topology(void) { } +static inline void cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { } static inline bool cpu_smt_possible(void) { return false; } static inline int cpuhp_smt_enable(void) { return 0; } static inline int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { return 0; } diff --git a/kernel/cpu.c b/kernel/cpu.c index b6fe170c93e9..d7dd535cb5b5 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -594,6 +594,8 @@ void __weak arch_smt_update(void) { } #ifdef CONFIG_HOTPLUG_SMT enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED; +static unsigned int cpu_smt_max_threads __ro_after_init; +unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX; void __init cpu_smt_disable(bool force) { @@ -607,16 +609,33 @@ void __init cpu_smt_disable(bool force) pr_info("SMT: disabled\n"); cpu_smt_control = CPU_SMT_DISABLED; } + cpu_smt_num_threads = 1; } /* * The decision whether SMT is supported can only be done after the full * CPU identification. Called from architecture code. */ -void __init cpu_smt_check_topology(void) +void __init cpu_smt_set_num_threads(unsigned int num_threads, + unsigned int max_threads) { + WARN_ON(!num_threads || (num_threads > max_threads)); + if (!topology_smt_supported()) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; + + cpu_smt_max_threads = max_threads; + + /* + * If SMT has been disabled via the kernel command line or SMT is + * not supported, set cpu_smt_num_threads to 1 for consistency. + * If enabled, take the architecture requested number of threads + * to bring up into account. + */ + if (cpu_smt_control != CPU_SMT_ENABLED) + cpu_smt_num_threads = 1; + else if (num_threads < cpu_smt_num_threads) + cpu_smt_num_threads = num_threads; } static int __init smt_cmdline_disable(char *str) From 91b4a7dbfe05ddb6fd3cf78cc11fb5ed64d3af90 Mon Sep 17 00:00:00 2001 From: Laurent Dufour Date: Wed, 5 Jul 2023 16:51:38 +0200 Subject: [PATCH 075/130] cpu/SMT: Remove topology_smt_supported() Since the maximum number of threads is now passed to cpu_smt_set_num_threads(), checking that value is enough to know whether SMT is supported. Suggested-by: Thomas Gleixner Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-6-ldufour@linux.ibm.com --- arch/x86/include/asm/topology.h | 2 -- arch/x86/kernel/smpboot.c | 8 -------- kernel/cpu.c | 4 ++-- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index ae49ed4417d0..3235ba1e5b06 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -141,7 +141,6 @@ static inline int topology_max_smt_threads(void) int topology_update_package_map(unsigned int apicid, unsigned int cpu); int topology_update_die_map(unsigned int dieid, unsigned int cpu); int topology_phys_to_logical_pkg(unsigned int pkg); -bool topology_smt_supported(void); extern struct cpumask __cpu_primary_thread_mask; #define cpu_primary_thread_mask ((const struct cpumask *)&__cpu_primary_thread_mask) @@ -164,7 +163,6 @@ static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; } static inline int topology_max_die_per_package(void) { return 1; } static inline int topology_max_smt_threads(void) { return 1; } static inline bool topology_is_primary_thread(unsigned int cpu) { return true; } -static inline bool topology_smt_supported(void) { return false; } #endif /* !CONFIG_SMP */ static inline void arch_fix_phys_package_id(int num, u32 slot) diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index e1aa2cd7734b..d4e897b820c4 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -326,14 +326,6 @@ static void notrace start_secondary(void *unused) cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } -/** - * topology_smt_supported - Check whether SMT is supported by the CPUs - */ -bool topology_smt_supported(void) -{ - return smp_num_siblings > 1; -} - /** * topology_phys_to_logical_pkg - Map a physical package id to a logical * @phys_pkg: The physical package id to map diff --git a/kernel/cpu.c b/kernel/cpu.c index d7dd535cb5b5..70add058e77b 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -621,7 +621,7 @@ void __init cpu_smt_set_num_threads(unsigned int num_threads, { WARN_ON(!num_threads || (num_threads > max_threads)); - if (!topology_smt_supported()) + if (max_threads == 1) cpu_smt_control = CPU_SMT_NOT_SUPPORTED; cpu_smt_max_threads = max_threads; @@ -1801,7 +1801,7 @@ early_param("cpuhp.parallel", parallel_bringup_parse_param); static inline bool cpuhp_smt_aware(void) { - return topology_smt_supported(); + return cpu_smt_max_threads > 1; } static inline const struct cpumask *cpuhp_get_primary_thread_mask(void) From 38253464bc821d6de6bba81bb1412ebb36f6cbd1 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 5 Jul 2023 16:51:39 +0200 Subject: [PATCH 076/130] cpu/SMT: Create topology_smt_thread_allowed() Some architectures allows partial SMT states, i.e. when not all SMT threads are brought online. To support that, add an architecture helper which checks whether a given CPU is allowed to be brought online depending on how many SMT threads are currently enabled. Since this is only applicable to architecture supporting partial SMT, only these architectures should select the new configuration variable CONFIG_SMT_NUM_THREADS_DYNAMIC. For the other architectures, not supporting the partial SMT states, there is no need to define topology_cpu_smt_allowed(), the generic code assumed that all the threads are allowed or only the primary ones. Call the helper from cpu_smt_enable(), and cpu_smt_allowed() when SMT is enabled, to check if the particular thread should be onlined. Notably, also call it from cpu_smt_disable() if CPU_SMT_ENABLED, to allow offlining some threads to move from a higher to lower number of threads online. [ ldufour: Slightly reword the commit's description ] [ ldufour: Introduce CONFIG_SMT_NUM_THREADS_DYNAMIC ] Suggested-by: Thomas Gleixner Signed-off-by: Michael Ellerman Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-7-ldufour@linux.ibm.com --- arch/Kconfig | 3 +++ kernel/cpu.c | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/Kconfig b/arch/Kconfig index aff2746c8af2..63c5d6a2022b 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -34,6 +34,9 @@ config ARCH_HAS_SUBPAGE_FAULTS config HOTPLUG_SMT bool +config SMT_NUM_THREADS_DYNAMIC + bool + # Selected by HOTPLUG_CORE_SYNC_DEAD or HOTPLUG_CORE_SYNC_FULL config HOTPLUG_CORE_SYNC bool diff --git a/kernel/cpu.c b/kernel/cpu.c index 70add058e77b..9a8d0685e055 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -645,9 +645,23 @@ static int __init smt_cmdline_disable(char *str) } early_param("nosmt", smt_cmdline_disable); +/* + * For Archicture supporting partial SMT states check if the thread is allowed. + * Otherwise this has already been checked through cpu_smt_max_threads when + * setting the SMT level. + */ +static inline bool cpu_smt_thread_allowed(unsigned int cpu) +{ +#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC + return topology_smt_thread_allowed(cpu); +#else + return true; +#endif +} + static inline bool cpu_smt_allowed(unsigned int cpu) { - if (cpu_smt_control == CPU_SMT_ENABLED) + if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu)) return true; if (topology_is_primary_thread(cpu)) @@ -2642,6 +2656,12 @@ int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) for_each_online_cpu(cpu) { if (topology_is_primary_thread(cpu)) continue; + /* + * Disable can be called with CPU_SMT_ENABLED when changing + * from a higher to lower number of SMT threads per core. + */ + if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu)) + continue; ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE); if (ret) break; @@ -2676,6 +2696,8 @@ int cpuhp_smt_enable(void) /* Skip online CPUs and CPUs on offline nodes */ if (cpu_online(cpu) || !node_online(cpu_to_node(cpu))) continue; + if (!cpu_smt_thread_allowed(cpu)) + continue; ret = _cpu_up(cpu, 0, CPUHP_ONLINE); if (ret) break; From 7f48405c3c3437d578d0f1e4287aa0a3bfa5623f Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 5 Jul 2023 16:51:40 +0200 Subject: [PATCH 077/130] cpu/SMT: Allow enabling partial SMT states via sysfs Add support to the /sys/devices/system/cpu/smt/control interface for enabling a specified number of SMT threads per core, including partial SMT states where not all threads are brought online. The current interface accepts "on" and "off", to enable either 1 or all SMT threads per core. This commit allows writing an integer, between 1 and the number of SMT threads supported by the machine. Writing 1 is a synonym for "off", 2 or more enables SMT with the specified number of threads. When reading the file, if all threads are online "on" is returned, to avoid changing behaviour for existing users. If some other number of threads is online then the integer value is returned. Architectures like x86 only supporting 1 thread or all threads, should not define CONFIG_SMT_NUM_THREADS_DYNAMIC. Architecture supporting partial SMT states, like PowerPC, should define it. [ ldufour: Slightly reword the commit's description ] [ ldufour: Remove switch() in __store_smt_control() ] [ ldufour: Rix build issue in control_show() ] Reported-by: kernel test robot Signed-off-by: Michael Ellerman Signed-off-by: Laurent Dufour Signed-off-by: Thomas Gleixner Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230705145143.40545-8-ldufour@linux.ibm.com --- .../ABI/testing/sysfs-devices-system-cpu | 1 + kernel/cpu.c | 60 ++++++++++++++----- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index ecd585ca2d50..6dba65fb1956 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -555,6 +555,7 @@ Description: Control Symmetric Multi Threading (SMT) ================ ========================================= "on" SMT is enabled "off" SMT is disabled + "" SMT is enabled with N threads per core. "forceoff" SMT is force disabled. Cannot be changed. "notsupported" SMT is not supported by the CPU "notimplemented" SMT runtime toggling is not diff --git a/kernel/cpu.c b/kernel/cpu.c index 9a8d0685e055..7e8f1b044772 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2876,11 +2876,19 @@ static const struct attribute_group cpuhp_cpu_root_attr_group = { #ifdef CONFIG_HOTPLUG_SMT +static bool cpu_smt_num_threads_valid(unsigned int threads) +{ + if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC)) + return threads >= 1 && threads <= cpu_smt_max_threads; + return threads == 1 || threads == cpu_smt_max_threads; +} + static ssize_t __store_smt_control(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - int ctrlval, ret; + int ctrlval, ret, num_threads, orig_threads; + bool force_off; if (cpu_smt_control == CPU_SMT_FORCE_DISABLED) return -EPERM; @@ -2888,30 +2896,39 @@ __store_smt_control(struct device *dev, struct device_attribute *attr, if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED) return -ENODEV; - if (sysfs_streq(buf, "on")) + if (sysfs_streq(buf, "on")) { ctrlval = CPU_SMT_ENABLED; - else if (sysfs_streq(buf, "off")) + num_threads = cpu_smt_max_threads; + } else if (sysfs_streq(buf, "off")) { ctrlval = CPU_SMT_DISABLED; - else if (sysfs_streq(buf, "forceoff")) + num_threads = 1; + } else if (sysfs_streq(buf, "forceoff")) { ctrlval = CPU_SMT_FORCE_DISABLED; - else + num_threads = 1; + } else if (kstrtoint(buf, 10, &num_threads) == 0) { + if (num_threads == 1) + ctrlval = CPU_SMT_DISABLED; + else if (cpu_smt_num_threads_valid(num_threads)) + ctrlval = CPU_SMT_ENABLED; + else + return -EINVAL; + } else { return -EINVAL; + } ret = lock_device_hotplug_sysfs(); if (ret) return ret; - if (ctrlval != cpu_smt_control) { - switch (ctrlval) { - case CPU_SMT_ENABLED: - ret = cpuhp_smt_enable(); - break; - case CPU_SMT_DISABLED: - case CPU_SMT_FORCE_DISABLED: - ret = cpuhp_smt_disable(ctrlval); - break; - } - } + orig_threads = cpu_smt_num_threads; + cpu_smt_num_threads = num_threads; + + force_off = ctrlval != cpu_smt_control && ctrlval == CPU_SMT_FORCE_DISABLED; + + if (num_threads > orig_threads) + ret = cpuhp_smt_enable(); + else if (num_threads < orig_threads || force_off) + ret = cpuhp_smt_disable(ctrlval); unlock_device_hotplug(); return ret ? ret : count; @@ -2939,6 +2956,17 @@ static ssize_t control_show(struct device *dev, { const char *state = smt_states[cpu_smt_control]; +#ifdef CONFIG_HOTPLUG_SMT + /* + * If SMT is enabled but not all threads are enabled then show the + * number of threads. If all threads are enabled show "on". Otherwise + * show the state name. + */ + if (cpu_smt_control == CPU_SMT_ENABLED && + cpu_smt_num_threads != cpu_smt_max_threads) + return sysfs_emit(buf, "%d\n", cpu_smt_num_threads); +#endif + return snprintf(buf, PAGE_SIZE - 2, "%s\n", state); } From e2c12739ccf76aae12e9e949526a475b9d17adaa Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Thu, 6 Jul 2023 16:22:25 +0200 Subject: [PATCH 078/130] genirq: Prevent nested thread vs synchronize_hardirq() deadlock There is a possibility of deadlock if synchronize_hardirq() is called when the nested threaded interrupt is active. The following scenario was observed on a uniprocessor PREEMPT_NONE system: Thread 1 Thread 2 handle_nested_thread() Set INPROGRESS Call ->thread_fn() thread_fn goes to sleep free_irq() __synchronize_hardirq() Busy-loop forever waiting for INPROGRESS to be cleared The INPROGRESS flag is only supposed to be used for hard interrupt handlers. Remove the incorrect usage in the nested threaded interrupt case and instead re-use the threads_active / wait_for_threads mechanism to wait for nested threaded interrupts to complete. Signed-off-by: Vincent Whitchurch Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230613-genirq-nested-v3-1-ae58221143eb@axis.com --- kernel/irq/chip.c | 11 ++++------- kernel/irq/internals.h | 2 ++ kernel/irq/manage.c | 26 ++++++++++++++------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index ee8c0acf39df..dc94e0bf2c94 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -473,11 +473,12 @@ void handle_nested_irq(unsigned int irq) action = desc->action; if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) { desc->istate |= IRQS_PENDING; - goto out_unlock; + raw_spin_unlock_irq(&desc->lock); + return; } kstat_incr_irqs_this_cpu(desc); - irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS); + atomic_inc(&desc->threads_active); raw_spin_unlock_irq(&desc->lock); action_ret = IRQ_NONE; @@ -487,11 +488,7 @@ void handle_nested_irq(unsigned int irq) if (!irq_settings_no_debug(desc)) note_interrupt(desc, action_ret); - raw_spin_lock_irq(&desc->lock); - irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); - -out_unlock: - raw_spin_unlock_irq(&desc->lock); + wake_threads_waitq(desc); } EXPORT_SYMBOL_GPL(handle_nested_irq); diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index bdd35bb9c735..831f38308466 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -121,6 +121,8 @@ void irq_resend_init(struct irq_desc *desc); bool irq_wait_for_poll(struct irq_desc *desc); void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action); +void wake_threads_waitq(struct irq_desc *desc); + #ifdef CONFIG_PROC_FS extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index d2742af0f0fd..d309ba84e08a 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -108,6 +108,16 @@ bool synchronize_hardirq(unsigned int irq) } EXPORT_SYMBOL(synchronize_hardirq); +static void __synchronize_irq(struct irq_desc *desc) +{ + __synchronize_hardirq(desc, true); + /* + * We made sure that no hardirq handler is running. Now verify that no + * threaded handlers are active. + */ + wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active)); +} + /** * synchronize_irq - wait for pending IRQ handlers (on other CPUs) * @irq: interrupt number to wait for @@ -127,16 +137,8 @@ void synchronize_irq(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); - if (desc) { - __synchronize_hardirq(desc, true); - /* - * We made sure that no hardirq handler is - * running. Now verify that no threaded handlers are - * active. - */ - wait_event(desc->wait_for_threads, - !atomic_read(&desc->threads_active)); - } + if (desc) + __synchronize_irq(desc); } EXPORT_SYMBOL(synchronize_irq); @@ -1216,7 +1218,7 @@ static irqreturn_t irq_thread_fn(struct irq_desc *desc, return ret; } -static void wake_threads_waitq(struct irq_desc *desc) +void wake_threads_waitq(struct irq_desc *desc) { if (atomic_dec_and_test(&desc->threads_active)) wake_up(&desc->wait_for_threads); @@ -1944,7 +1946,7 @@ static struct irqaction *__free_irq(struct irq_desc *desc, void *dev_id) * supports it also make sure that there is no (not yet serviced) * interrupt in flight at the hardware level. */ - __synchronize_hardirq(desc, true); + __synchronize_irq(desc); #ifdef CONFIG_DEBUG_SHIRQ /* From 51a5acce7162185a9b9d695b314a12fc067562b4 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Thu, 20 Jul 2023 22:36:25 +0800 Subject: [PATCH 079/130] genirq: Remove unused extern declaration commit 3795de236d67 ("genirq: Distangle kernel/irq/handle.c") left behind this. Signed-off-by: YueHaibing Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230720143625.29176-1-yuehaibing@huawei.com --- kernel/irq/internals.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 831f38308466..bcc7f21db9ee 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -108,8 +108,6 @@ extern int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which, bool *state); -extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr); - irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc); irqreturn_t handle_irq_event_percpu(struct irq_desc *desc); irqreturn_t handle_irq_event(struct irq_desc *desc); From 52b38b7ad589c89a8c69272497bed20034f8e322 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Fri, 28 Jul 2023 23:53:13 +0800 Subject: [PATCH 080/130] cpu/SMT: Fix cpu_smt_possible() comment Commit e1572f1d08be ("cpu/SMT: create and export cpu_smt_possible()") introduces cpu_smt_possible() to represent if SMT is theoretically possible. It returns true when SMT is supported and not forcefully disabled ('nosmt=force'). But the comment of it says "Returns true if SMT is not supported of forcefully (irreversibly) disabled", which is wrong. Fix that comment accordingly. Signed-off-by: Zhang Rui Signed-off-by: Thomas Gleixner Reviewed-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20230728155313.44170-1-rui.zhang@intel.com --- kernel/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/cpu.c b/kernel/cpu.c index 7e8f1b044772..f6811c857102 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -676,7 +676,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu) return !cpumask_test_cpu(cpu, &cpus_booted_once_mask); } -/* Returns true if SMT is not supported of forcefully (irreversibly) disabled */ +/* Returns true if SMT is supported and not forcefully (irreversibly) disabled */ bool cpu_smt_possible(void) { return cpu_smt_control != CPU_SMT_FORCE_DISABLED && From cb88f7f51bc6f351a529ff61d0a706c6eae1417a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 28 Jul 2023 13:33:02 -0700 Subject: [PATCH 081/130] rcu-tasks: Permit use of debug-objects with RCU Tasks flavors Currently, cblist_init_generic() holds a raw spinlock when invoking INIT_WORK(). This fails in kernels built with CONFIG_DEBUG_OBJECTS=y due to memory allocation being forbidden while holding a raw spinlock. But the only reason for holding the raw spinlock is to synchronize with early boot calls to call_rcu_tasks(), call_rcu_tasks_rude, and, last but not least, call_rcu_tasks_trace(). These calls also invoke cblist_init_generic() in order to support early boot queueing of callbacks. Except that there are no early boot calls to either of these three functions, and the BPF guys confirm that they have no plans to add any such calls. This commit therefore removes the synchronization and adds a WARN_ON_ONCE() to catch the case of now-prohibited early boot RCU Tasks callback queueing. If early boot queueing is needed, an "initialized" flag may be added to the rcu_tasks structure. Then queueing a callback before this flag is set would initialize the callback list (if needed) and queue the callback. The decision as to where to queue the callback given the possibility of non-zero boot CPUs is left as an exercise for the reader. Reported-by: Jakub Kicinski Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 291d536b1789..a4cd4ea2402c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -236,7 +236,7 @@ static const char *tasks_gp_state_getname(struct rcu_tasks *rtp) #endif /* #ifndef CONFIG_TINY_RCU */ // Initialize per-CPU callback lists for the specified flavor of -// Tasks RCU. +// Tasks RCU. Do not enqueue callbacks before this function is invoked. static void cblist_init_generic(struct rcu_tasks *rtp) { int cpu; @@ -244,7 +244,6 @@ static void cblist_init_generic(struct rcu_tasks *rtp) int lim; int shift; - raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rcu_task_enqueue_lim < 0) { rcu_task_enqueue_lim = 1; rcu_task_cb_adjust = true; @@ -267,17 +266,16 @@ static void cblist_init_generic(struct rcu_tasks *rtp) WARN_ON_ONCE(!rtpcp); if (cpu) raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock)); - raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. + local_irq_save(flags); // serialize initialization if (rcu_segcblist_empty(&rtpcp->cblist)) rcu_segcblist_init(&rtpcp->cblist); + local_irq_restore(flags); INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; if (!rtpcp->rtp_blkd_tasks.next) INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); - raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. } - raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d.\n", rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust); @@ -351,11 +349,9 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, READ_ONCE(rtp->percpu_enqueue_lim) != nr_cpu_ids) needadjust = true; // Defer adjustment to avoid deadlock. } - if (!rcu_segcblist_is_enabled(&rtpcp->cblist)) { - raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. - cblist_init_generic(rtp); - raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. - } + // Queuing callbacks before initialization not yet supported. + if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) + rcu_segcblist_init(&rtpcp->cblist); needwake = (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { From fcce1c6cb156d79debaf84f63233f146540031b1 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 26 Jul 2023 16:11:43 -0700 Subject: [PATCH 082/130] x86/paravirt: Fix tlb_remove_table function callback prototype warning Under W=1, this warning is visible in Clang 16 and newer: arch/x86/kernel/paravirt.c:337:4: warning: cast from 'void (*)(struct mmu_gather *, struct page *)' to 'void (*)(struct mmu_gather *, void *)' converts to incompatible function type [-Wcast-function-type-strict] (void (*)(struct mmu_gather *, void *))tlb_remove_page, ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Add a direct wrapper instead, which will make this warning (and potential KCFI failures) go away. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202307260332.pJntWR6o-lkp@intel.com/ Cc: Juergen Gross Cc: Peter Zijlstra Cc: Sami Tolvanen Cc: Nathan Chancellor Cc: Ajay Kaher Cc: Alexey Makhalov Cc: VMware PV-Drivers Reviewers Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: virtualization@lists.linux-foundation.org Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20230726231139.never.601-kees@kernel.org Signed-off-by: Kees Cook --- arch/x86/kernel/paravirt.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index ac10b46c5832..23d4d7114473 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -79,6 +79,11 @@ void __init native_pv_lock_init(void) static_branch_disable(&virt_spin_lock_key); } +static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) +{ + tlb_remove_page(tlb, table); +} + unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr, unsigned int len) { @@ -295,8 +300,7 @@ struct paravirt_patch_template pv_ops = { .mmu.flush_tlb_kernel = native_flush_tlb_global, .mmu.flush_tlb_one_user = native_flush_tlb_one_user, .mmu.flush_tlb_multi = native_flush_tlb_multi, - .mmu.tlb_remove_table = - (void (*)(struct mmu_gather *, void *))tlb_remove_page, + .mmu.tlb_remove_table = native_tlb_remove_table, .mmu.exit_mmap = paravirt_nop, .mmu.notify_page_enc_status_changed = paravirt_nop, From 33af38d85b5c9e8ffe26f9712319ed28c5ded64f Mon Sep 17 00:00:00 2001 From: Yue Haibing Date: Sat, 5 Aug 2023 19:04:06 +0800 Subject: [PATCH 083/130] cpu/hotplug: Remove unused function declaration cpu_set_state_online() Commit 5356297d12d9 ("cpu/hotplug: Remove cpu_report_state() and related unused cruft") removed function but leave the declaration. Signed-off-by: Yue Haibing Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230805110406.45900-1-yuehaibing@huawei.com --- include/linux/cpu.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 6b326a9e8191..ed56b2534500 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -191,7 +191,6 @@ void arch_cpu_finalize_init(void); static inline void arch_cpu_finalize_init(void) { } #endif -void cpu_set_state_online(int cpu); void play_idle_precise(u64 duration_ns, u64 latency_ns); static inline void play_idle(unsigned long duration_us) From e0a99a839f04c90bf9f16919997c4b34f9c8f1f0 Mon Sep 17 00:00:00 2001 From: Anna-Maria Behnsen Date: Mon, 15 May 2023 18:20:38 +0200 Subject: [PATCH 084/130] Documentation: core-api/cpuhotplug: Fix state names Dynamic allocated hotplug states in documentation and the comment above cpuhp_state enum do not match the code. To not get confused by wrong documentation, change to proper state names. Signed-off-by: Anna-Maria Behnsen Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230515162038.62703-1-anna-maria@linutronix.de --- Documentation/core-api/cpu_hotplug.rst | 10 +++++----- include/linux/cpuhotplug.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst index e6f5bc39cf5c..b9ae591d0b18 100644 --- a/Documentation/core-api/cpu_hotplug.rst +++ b/Documentation/core-api/cpu_hotplug.rst @@ -395,8 +395,8 @@ multi-instance state the following function is available: * cpuhp_setup_state_multi(state, name, startup, teardown) The @state argument is either a statically allocated state or one of the -constants for dynamically allocated states - CPUHP_PREPARE_DYN, -CPUHP_ONLINE_DYN - depending on the state section (PREPARE, ONLINE) for +constants for dynamically allocated states - CPUHP_BP_PREPARE_DYN, +CPUHP_AP_ONLINE_DYN - depending on the state section (PREPARE, ONLINE) for which a dynamic state should be allocated. The @name argument is used for sysfs output and for instrumentation. The @@ -588,7 +588,7 @@ notifications on online and offline operations:: Setup and teardown a dynamically allocated state in the ONLINE section for notifications on offline operations:: - state = cpuhp_setup_state(CPUHP_ONLINE_DYN, "subsys:offline", NULL, subsys_cpu_offline); + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "subsys:offline", NULL, subsys_cpu_offline); if (state < 0) return state; .... @@ -597,7 +597,7 @@ for notifications on offline operations:: Setup and teardown a dynamically allocated state in the ONLINE section for notifications on online operations without invoking the callbacks:: - state = cpuhp_setup_state_nocalls(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, NULL); + state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "subsys:online", subsys_cpu_online, NULL); if (state < 0) return state; .... @@ -606,7 +606,7 @@ for notifications on online operations without invoking the callbacks:: Setup, use and teardown a dynamically allocated multi-instance state in the ONLINE section for notifications on online and offline operation:: - state = cpuhp_setup_state_multi(CPUHP_ONLINE_DYN, "subsys:online", subsys_cpu_online, subsys_cpu_offline); + state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "subsys:online", subsys_cpu_online, subsys_cpu_offline); if (state < 0) return state; .... diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 25b6e6e6ba6b..06dda85f0424 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -48,7 +48,7 @@ * same section. * * If neither #1 nor #2 apply, please use the dynamic state space when - * setting up a state by using CPUHP_PREPARE_DYN or CPUHP_PREPARE_ONLINE + * setting up a state by using CPUHP_BP_PREPARE_DYN or CPUHP_AP_ONLINE_DYN * for the @state argument of the setup function. * * See Documentation/core-api/cpu_hotplug.rst for further information and From fbc5d382407eb5d6a2eeef245cc2ca278d590645 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 2 Aug 2023 21:04:32 -0700 Subject: [PATCH 085/130] selftests/seccomp: Handle arm32 corner cases better It turns out arm32 doesn't handle syscall -1 gracefully, so skip testing for that. Additionally skip tests that depend on clone3 when it is not available (for example when building the seccomp selftests on an old arm image without clone3 headers). And improve error reporting for when nanosleep fails, as seen on arm32 since v5.15. Cc: Lecopzer Chen Signed-off-by: Kees Cook --- tools/testing/selftests/seccomp/seccomp_bpf.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index f6a04d88e02f..38f651469968 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -2184,6 +2184,9 @@ FIXTURE_TEARDOWN(TRACE_syscall) TEST(negative_ENOSYS) { +#if defined(__arm__) + SKIP(return, "arm32 does not support calling syscall -1"); +#endif /* * There should be no difference between an "internal" skip * and userspace asking for syscall "-1". @@ -3072,7 +3075,8 @@ TEST(syscall_restart) timeout.tv_sec = 1; errno = 0; EXPECT_EQ(0, nanosleep(&timeout, NULL)) { - TH_LOG("Call to nanosleep() failed (errno %d)", errno); + TH_LOG("Call to nanosleep() failed (errno %d: %s)", + errno, strerror(errno)); } /* Read final sync from parent. */ @@ -3908,6 +3912,9 @@ TEST(user_notification_filter_empty) TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); } + if (__NR_clone3 < 0) + SKIP(return, "Test not built with clone3 support"); + pid = sys_clone3(&args, sizeof(args)); ASSERT_GE(pid, 0); @@ -3962,6 +3969,9 @@ TEST(user_notification_filter_empty_threaded) TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); } + if (__NR_clone3 < 0) + SKIP(return, "Test not built with clone3 support"); + pid = sys_clone3(&args, sizeof(args)); ASSERT_GE(pid, 0); From b3d46e11fec0c5a8972e5061bb1462119ae5736d Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 7 Aug 2023 10:43:58 -0700 Subject: [PATCH 086/130] selftests/harness: Actually report SKIP for signal tests Tests that were expecting a signal were not correctly checking for a SKIP condition. Move the check before the signal checking when processing test result. Cc: Shuah Khan Cc: Andy Lutomirski Cc: Will Drewry Cc: linux-kselftest@vger.kernel.org Fixes: 9847d24af95c ("selftests/harness: Refactor XFAIL into SKIP") Signed-off-by: Kees Cook --- tools/testing/selftests/kselftest_harness.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 5fd49ad0c696..e05ac8261046 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -938,7 +938,11 @@ void __wait_for_test(struct __test_metadata *t) fprintf(TH_LOG_STREAM, "# %s: Test terminated by timeout\n", t->name); } else if (WIFEXITED(status)) { - if (t->termsig != -1) { + if (WEXITSTATUS(status) == 255) { + /* SKIP */ + t->passed = 1; + t->skip = 1; + } else if (t->termsig != -1) { t->passed = 0; fprintf(TH_LOG_STREAM, "# %s: Test exited normally instead of by signal (code: %d)\n", @@ -950,11 +954,6 @@ void __wait_for_test(struct __test_metadata *t) case 0: t->passed = 1; break; - /* SKIP */ - case 255: - t->passed = 1; - t->skip = 1; - break; /* Other failure, assume step report. */ default: t->passed = 0; From 2e3f65ccfe6b0778b261ad69c9603ae85f210334 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 7 Aug 2023 09:41:19 -0700 Subject: [PATCH 087/130] gcc-plugins: Rename last_stmt() for GCC 14+ In GCC 14, last_stmt() was renamed to last_nondebug_stmt(). Add a helper macro to handle the renaming. Cc: linux-hardening@vger.kernel.org Signed-off-by: Kees Cook --- scripts/gcc-plugins/gcc-common.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 84c730da36dd..1ae39b9f4a95 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -440,4 +440,8 @@ static inline void debug_gimple_stmt(const_gimple s) #define SET_DECL_MODE(decl, mode) DECL_MODE(decl) = (mode) #endif +#if BUILDING_GCC_VERSION >= 14000 +#define last_stmt(x) last_nondebug_stmt(x) +#endif + #endif From 9d0cce2bc3874dd03f7471ec00ae4acb5a77e43c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 1 Aug 2023 12:11:18 -0700 Subject: [PATCH 088/130] rcu-tasks: Fix boot-time RCU tasks debug-only deadlock In kernels built with CONFIG_PROVE_RCU=y (for example, lockdep kernels), the following sequence of events can occur: o rcu_init_tasks_generic() is invoked just before init is spawned. It invokes rcu_spawn_tasks_kthread() and friends. o rcu_spawn_tasks_kthread() invokes rcu_spawn_tasks_kthread_generic(), which uses kthread_run() to create the needed kthread. o Control returns to rcu_init_tasks_generic(), which, because this is a CONFIG_PROVE_RCU=y kernel, invokes the version of the rcu_tasks_initiate_self_tests() function that actually does something, including invoking synchronize_rcu_tasks(), which in turn invokes synchronize_rcu_tasks_generic(). o synchronize_rcu_tasks_generic() sees that the ->kthread_ptr is still NULL, because the newly spawned kthread has not yet started. o The new kthread starts, preempting synchronize_rcu_tasks_generic() just after its check. This kthread invokes rcu_tasks_one_gp(), which acquires ->tasks_gp_mutex, and, seeing no work, blocks in rcuwait_wait_event(). Note that this step requires either a preemptible kernel or a fault-injection-style sleep at the beginning of mutex_lock(). o synchronize_rcu_tasks_generic() resumes and invokes rcu_tasks_one_gp(). o rcu_tasks_one_gp() attempts to acquire ->tasks_gp_mutex, which is still held by the newly spawned kthread's rcu_tasks_one_gp() function. Deadlock. Because the only reason for ->tasks_gp_mutex is to handle pre-kthread synchronous grace periods, this commit avoids this deadlock by having rcu_tasks_one_gp() momentarily release ->tasks_gp_mutex while invoking rcuwait_wait_event(). This allows the call to rcu_tasks_one_gp() from synchronize_rcu_tasks_generic() proceed. Note that it is not necessary to release the mutex anywhere else in rcu_tasks_one_gp() because rcuwait_wait_event() is the only function that can block indefinitely. Reported-by: Guenter Roeck Reported-by: Roy Hopkins Reported-by: Peter Zijlstra Signed-off-by: Paul E. McKenney Tested-by: Roy Hopkins --- kernel/rcu/tasks.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index a4cd4ea2402c..5572f4a22996 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -570,10 +570,12 @@ static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot) if (unlikely(midboot)) { needgpcb = 0x2; } else { + mutex_unlock(&rtp->tasks_gp_mutex); set_tasks_gp_state(rtp, RTGS_WAIT_CBS); rcuwait_wait_event(&rtp->cbs_wait, (needgpcb = rcu_tasks_need_gpcb(rtp)), TASK_IDLE); + mutex_lock(&rtp->tasks_gp_mutex); } if (needgpcb & 0x2) { From 67d5404d274376890d6d095a10e6565854918f8e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 19 Jul 2023 15:50:07 -0700 Subject: [PATCH 089/130] torture: Add a kthread-creation callback to _torture_create_kthread() This commit adds a kthread-creation callback to the _torture_create_kthread() function, which allows callers of a new torture_create_kthread_cb() macro to specify a function to be invoked after the kthread is created but before it is awakened for the first time. Signed-off-by: Paul E. McKenney Cc: Dietmar Eggemann Cc: Josh Triplett Cc: Juri Lelli Cc: Valentin Schneider Cc: Dietmar Eggemann Cc: kernel-team@android.com Reviewed-by: Joel Fernandes (Google) Acked-by: John Stultz --- include/linux/torture.h | 7 +++++-- kernel/torture.c | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/linux/torture.h b/include/linux/torture.h index 7038104463e4..bb466eec01e4 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -108,12 +108,15 @@ bool torture_must_stop(void); bool torture_must_stop_irq(void); void torture_kthread_stopping(char *title); int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, - char *f, struct task_struct **tp); + char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)); void _torture_stop_kthread(char *m, struct task_struct **tp); #define torture_create_kthread(n, arg, tp) \ _torture_create_kthread(n, (arg), #n, "Creating " #n " task", \ - "Failed to create " #n, &(tp)) + "Failed to create " #n, &(tp), NULL) +#define torture_create_kthread_cb(n, arg, tp, cbf) \ + _torture_create_kthread(n, (arg), #n, "Creating " #n " task", \ + "Failed to create " #n, &(tp), cbf) #define torture_stop_kthread(n, tp) \ _torture_stop_kthread("Stopping " #n " task", &(tp)) diff --git a/kernel/torture.c b/kernel/torture.c index 8be83fdc6be1..b88a1a86d9da 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -932,7 +932,7 @@ EXPORT_SYMBOL_GPL(torture_kthread_stopping); * it starts, you will need to open-code your own. */ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, - char *f, struct task_struct **tp) + char *f, struct task_struct **tp, void (*cbf)(struct task_struct *tp)) { int ret = 0; @@ -944,6 +944,10 @@ int _torture_create_kthread(int (*fn)(void *arg), void *arg, char *s, char *m, *tp = NULL; return ret; } + + if (cbf) + cbf(*tp); + wake_up_process(*tp); // Process is sleeping, so ordering provided. torture_shuffle_task_register(*tp); return ret; From 5d248bb39fe1388943acb6510f8f48fa5570e0ec Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 2 Jun 2023 22:02:10 +0000 Subject: [PATCH 090/130] torture: Add lock_torture writer_fifo module parameter This commit adds a module parameter that causes the locktorture writer to run at real-time priority. To use it: insmod /lib/modules/torture.ko random_shuffle=1 insmod /lib/modules/locktorture.ko torture_type=mutex_lock rt_boost=1 rt_boost_factor=50 nested_locks=3 writer_fifo=1 ^^^^^^^^^^^^^ A predecessor to this patch has been helpful to uncover issues with the proxy-execution series. [ paulmck: Remove locktorture-specific code from kernel/torture.c. ] Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: kernel-team@android.com Signed-off-by: Dietmar Eggemann [jstultz: Include header change to build, reword commit message] Signed-off-by: John Stultz Acked-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 4 ++++ kernel/locking/locktorture.c | 12 +++++++----- kernel/torture.c | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..7b94455e9ae2 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2918,6 +2918,10 @@ locktorture.torture_type= [KNL] Specify the locking implementation to test. + locktorture.writer_fifo= [KNL] + Run the write-side locktorture kthreads at + sched_set_fifo() real-time priority. + locktorture.verbose= [KNL] Enable additional printk() statements. diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 949d3deae506..270c7f80ce84 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -45,6 +45,7 @@ torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); torture_param(int, rt_boost, 2, "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); +torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority"); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ @@ -809,7 +810,8 @@ static int lock_torture_writer(void *arg) bool skip_main_lock; VERBOSE_TOROUT_STRING("lock_torture_writer task started"); - set_user_nice(current, MAX_NICE); + if (!rt_task(current)) + set_user_nice(current, MAX_NICE); do { if ((torture_random(&rand) & 0xfffff) == 0) @@ -1015,8 +1017,7 @@ static void lock_torture_cleanup(void) if (writer_tasks) { for (i = 0; i < cxt.nrealwriters_stress; i++) - torture_stop_kthread(lock_torture_writer, - writer_tasks[i]); + torture_stop_kthread(lock_torture_writer, writer_tasks[i]); kfree(writer_tasks); writer_tasks = NULL; } @@ -1244,8 +1245,9 @@ static int __init lock_torture_init(void) goto create_reader; /* Create writer. */ - firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], - writer_tasks[i]); + firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i], + writer_tasks[i], + writer_fifo ? sched_set_fifo : NULL); if (torture_init_error(firsterr)) goto unwind; diff --git a/kernel/torture.c b/kernel/torture.c index b88a1a86d9da..a1ac493488e2 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "rcu/rcu.h" MODULE_LICENSE("GPL"); @@ -734,7 +735,7 @@ bool stutter_wait(const char *title) cond_resched_tasks_rcu_qs(); spt = READ_ONCE(stutter_pause_test); for (; spt; spt = READ_ONCE(stutter_pause_test)) { - if (!ret) { + if (!ret && !rt_task(current)) { sched_set_normal(current, MAX_NICE); ret = true; } From 872948c665f50a1446e8a34b1ed57bb0b3a9ca4a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 14:59:29 -0700 Subject: [PATCH 091/130] torture: Make torture_hrtimeout_*() use TASK_IDLE Given that it is expected that more code will use torture_hrtimeout_*(), including for longer timeouts, make it use TASK_IDLE instead of TASK_UNINTERRUPTIBLE. Signed-off-by: Paul E. McKenney --- kernel/torture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index a1ac493488e2..68c831a43a58 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -93,7 +93,7 @@ int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_s if (trsp) hto += (torture_random(trsp) >> 3) % fuzzt_ns; - set_current_state(TASK_UNINTERRUPTIBLE); + set_current_state(TASK_IDLE); return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); } EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); From 3f0c06e1cba6207703733474bc7b55292bc86c6a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 15:19:56 -0700 Subject: [PATCH 092/130] torture: Move torture_onoff() timeouts to hrtimers In order to gain better race coverage, move the CPU-hotplug-related timed waits in torture_onoff() to torture_hrtimeout_jiffies(). Signed-off-by: Paul E. McKenney --- kernel/torture.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/torture.c b/kernel/torture.c index 68c831a43a58..2c441d5a1bdd 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -354,12 +354,12 @@ torture_onoff(void *arg) if (onoff_holdoff > 0) { VERBOSE_TOROUT_STRING("torture_onoff begin holdoff"); - schedule_timeout_interruptible(onoff_holdoff); + torture_hrtimeout_jiffies(onoff_holdoff, &rand); VERBOSE_TOROUT_STRING("torture_onoff end holdoff"); } while (!torture_must_stop()) { if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) { - schedule_timeout_interruptible(HZ / 10); + torture_hrtimeout_jiffies(HZ / 10, &rand); continue; } cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); @@ -369,7 +369,7 @@ torture_onoff(void *arg) torture_online(cpu, &n_online_attempts, &n_online_successes, &sum_online, &min_online, &max_online); - schedule_timeout_interruptible(onoff_interval); + torture_hrtimeout_jiffies(onoff_interval, &rand); } stop: From dea81dcfd3497e75eb23e7543434f88c34289d31 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 16:27:02 -0700 Subject: [PATCH 093/130] torture: Move torture_shuffle() timeouts to hrtimers In order to gain better race coverage, move the CPU-migration timed waits in torture_shuffle() to torture_hrtimeout_jiffies(). Signed-off-by: Paul E. McKenney --- kernel/torture.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index 2c441d5a1bdd..c5311154bc4d 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -557,9 +557,11 @@ static void torture_shuffle_tasks(void) */ static int torture_shuffle(void *arg) { + DEFINE_TORTURE_RANDOM(rand); + VERBOSE_TOROUT_STRING("torture_shuffle task started"); do { - schedule_timeout_interruptible(shuffle_interval); + torture_hrtimeout_jiffies(shuffle_interval, &rand); torture_shuffle_tasks(); torture_shutdown_absorb("torture_shuffle"); } while (!torture_must_stop()); From 10af43671e8bf4ac153c4991a17cdf57bc6d2cfe Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 16:52:35 -0700 Subject: [PATCH 094/130] torture: Move stutter_wait() timeouts to hrtimers In order to gain better race coverage, move the test start/stop waits in stutter_wait() to torture_hrtimeout_jiffies(). Signed-off-by: Paul E. McKenney --- kernel/torture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/torture.c b/kernel/torture.c index c5311154bc4d..984651de4b55 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -742,7 +742,7 @@ bool stutter_wait(const char *title) ret = true; } if (spt == 1) { - schedule_timeout_interruptible(1); + torture_hrtimeout_jiffies(1, NULL); } else if (spt == 2) { while (READ_ONCE(stutter_pause_test)) { if (!(i++ & 0xffff)) @@ -750,7 +750,7 @@ bool stutter_wait(const char *title) cond_resched(); } } else { - schedule_timeout_interruptible(round_jiffies_relative(HZ)); + torture_hrtimeout_jiffies(round_jiffies_relative(HZ), NULL); } torture_shutdown_absorb(title); } From 6cab60ceb1d32b795ae22fea5719fb6150e4cda9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 12 Jul 2023 14:56:51 -0700 Subject: [PATCH 095/130] torture: Stop right-shifting torture_random() return values Now that torture_random() uses swahw32(), its callers no longer see not-so-random low-order bits, as these are now swapped up into the upper 16 bits of the torture_random() function's return value. This commit therefore removes the right-shifting of torture_random() return values. Signed-off-by: Paul E. McKenney --- kernel/torture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/torture.c b/kernel/torture.c index 984651de4b55..b28b05bbef02 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -92,7 +92,7 @@ int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_s ktime_t hto = baset_ns; if (trsp) - hto += (torture_random(trsp) >> 3) % fuzzt_ns; + hto += torture_random(trsp) % fuzzt_ns; set_current_state(TASK_IDLE); return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); } @@ -362,7 +362,7 @@ torture_onoff(void *arg) torture_hrtimeout_jiffies(HZ / 10, &rand); continue; } - cpu = (torture_random(&rand) >> 4) % (maxcpu + 1); + cpu = torture_random(&rand) % (maxcpu + 1); if (!torture_offline(cpu, &n_offline_attempts, &n_offline_successes, &sum_offline, &min_offline, &max_offline)) From bc19e86e285f679d39e1671e72e2f02ac04de999 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 12 Jul 2023 16:16:37 -0700 Subject: [PATCH 096/130] rcutorture: Stop right-shifting torture_random() return values Now that torture_random() uses swahw32(), its callers no longer see not-so-random low-order bits, as these are now swapped up into the upper 16 bits of the torture_random() function's return value. This commit therefore removes the right-shifting of torture_random() return values. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index d291a1438c30..ade42d6a9d9b 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1877,7 +1877,7 @@ static int rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) { int mask = rcutorture_extend_mask_max(); - unsigned long randmask1 = torture_random(trsp) >> 8; + unsigned long randmask1 = torture_random(trsp); unsigned long randmask2 = randmask1 >> 3; unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; @@ -1936,7 +1936,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, if (!((mask - 1) & mask)) return rtrsp; /* Current RCU reader not extendable. */ /* Bias towards larger numbers of loops. */ - i = (torture_random(trsp) >> 3); + i = torture_random(trsp); i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; for (j = 0; j < i; j++) { mask = rcutorture_extend_mask(*readstate, trsp); @@ -2137,7 +2137,7 @@ static int rcu_nocb_toggle(void *arg) toggle_fuzz = NSEC_PER_USEC; do { r = torture_random(&rand); - cpu = (r >> 4) % (maxcpu + 1); + cpu = (r >> 1) % (maxcpu + 1); if (r & 0x1) { rcu_nocb_cpu_offload(cpu); atomic_long_inc(&n_nocb_offload); From 7a0fd5e1678505534573b3c14c6ff69ed8592596 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 11 Aug 2023 17:18:38 +0200 Subject: [PATCH 097/130] compiler_types: Introduce the Clang __preserve_most function attribute [1]: "On X86-64 and AArch64 targets, this attribute changes the calling convention of a function. The preserve_most calling convention attempts to make the code in the caller as unintrusive as possible. This convention behaves identically to the C calling convention on how arguments and return values are passed, but it uses a different set of caller/callee-saved registers. This alleviates the burden of saving and recovering a large register set before and after the call in the caller. If the arguments are passed in callee-saved registers, then they will be preserved by the callee across the call. This doesn't apply for values returned in callee-saved registers. * On X86-64 the callee preserves all general purpose registers, except for R11. R11 can be used as a scratch register. Floating-point registers (XMMs/YMMs) are not preserved and need to be saved by the caller. * On AArch64 the callee preserve all general purpose registers, except x0-X8 and X16-X18." [1] https://clang.llvm.org/docs/AttributeReference.html#preserve-most Introduce the attribute to compiler_types.h as __preserve_most. Use of this attribute results in better code generation for calls to very rarely called functions, such as error-reporting functions, or rarely executed slow paths. Beware that the attribute conflicts with instrumentation calls inserted on function entry which do not use __preserve_most themselves. Notably, function tracing which assumes the normal C calling convention for the given architecture. Where the attribute is supported, __preserve_most will imply notrace. It is recommended to restrict use of the attribute to functions that should or already disable tracing. Note: The additional preprocessor check against architecture should not be necessary if __has_attribute() only returns true where supported; also see https://github.com/ClangBuiltLinux/linux/issues/1908. But until __has_attribute() does the right thing, we also guard by known-supported architectures to avoid build warnings on other architectures. The attribute may be supported by a future GCC version (see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110899). Signed-off-by: Marco Elver Reviewed-by: Miguel Ojeda Reviewed-by: Nick Desaulniers Acked-by: "Steven Rostedt (Google)" Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20230811151847.1594958-1-elver@google.com Signed-off-by: Kees Cook --- include/linux/compiler_types.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 547ea1ff806e..c523c6683789 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -106,6 +106,34 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } #define __cold #endif +/* + * On x86-64 and arm64 targets, __preserve_most changes the calling convention + * of a function to make the code in the caller as unintrusive as possible. This + * convention behaves identically to the C calling convention on how arguments + * and return values are passed, but uses a different set of caller- and callee- + * saved registers. + * + * The purpose is to alleviates the burden of saving and recovering a large + * register set before and after the call in the caller. This is beneficial for + * rarely taken slow paths, such as error-reporting functions that may be called + * from hot paths. + * + * Note: This may conflict with instrumentation inserted on function entry which + * does not use __preserve_most or equivalent convention (if in assembly). Since + * function tracing assumes the normal C calling convention, where the attribute + * is supported, __preserve_most implies notrace. It is recommended to restrict + * use of the attribute to functions that should or already disable tracing. + * + * Optional: not supported by gcc. + * + * clang: https://clang.llvm.org/docs/AttributeReference.html#preserve-most + */ +#if __has_attribute(__preserve_most__) && (defined(CONFIG_X86_64) || defined(CONFIG_ARM64)) +# define __preserve_most notrace __attribute__((__preserve_most__)) +#else +# define __preserve_most +#endif + /* Builtins */ /* From b16c42c8fde808b4f047d94f1f2aeda93487670d Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 11 Aug 2023 17:18:39 +0200 Subject: [PATCH 098/130] list_debug: Introduce inline wrappers for debug checks Turn the list debug checking functions __list_*_valid() into inline functions that wrap the out-of-line functions. Care is taken to ensure the inline wrappers are always inlined, so that additional compiler instrumentation (such as sanitizers) does not result in redundant outlining. This change is preparation for performing checks in the inline wrappers. No functional change intended. Signed-off-by: Marco Elver Link: https://lore.kernel.org/r/20230811151847.1594958-2-elver@google.com Signed-off-by: Kees Cook --- arch/arm64/kvm/hyp/nvhe/list_debug.c | 6 ++--- include/linux/list.h | 37 +++++++++++++++++++++++++--- lib/list_debug.c | 11 ++++----- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/list_debug.c b/arch/arm64/kvm/hyp/nvhe/list_debug.c index d68abd7ea124..16266a939a4c 100644 --- a/arch/arm64/kvm/hyp/nvhe/list_debug.c +++ b/arch/arm64/kvm/hyp/nvhe/list_debug.c @@ -26,8 +26,8 @@ static inline __must_check bool nvhe_check_data_corruption(bool v) /* The predicates checked here are taken from lib/list_debug.c. */ -bool __list_add_valid(struct list_head *new, struct list_head *prev, - struct list_head *next) +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) { if (NVHE_CHECK_DATA_CORRUPTION(next->prev != prev) || NVHE_CHECK_DATA_CORRUPTION(prev->next != next) || @@ -37,7 +37,7 @@ bool __list_add_valid(struct list_head *new, struct list_head *prev, return true; } -bool __list_del_entry_valid(struct list_head *entry) +bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; diff --git a/include/linux/list.h b/include/linux/list.h index f10344dbad4d..130c6a1bb45c 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -39,10 +39,39 @@ static inline void INIT_LIST_HEAD(struct list_head *list) } #ifdef CONFIG_DEBUG_LIST -extern bool __list_add_valid(struct list_head *new, - struct list_head *prev, - struct list_head *next); -extern bool __list_del_entry_valid(struct list_head *entry); +/* + * Performs the full set of list corruption checks before __list_add(). + * On list corruption reports a warning, and returns false. + */ +extern bool __list_add_valid_or_report(struct list_head *new, + struct list_head *prev, + struct list_head *next); + +/* + * Performs list corruption checks before __list_add(). Returns false if a + * corruption is detected, true otherwise. + */ +static __always_inline bool __list_add_valid(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + return __list_add_valid_or_report(new, prev, next); +} + +/* + * Performs the full set of list corruption checks before __list_del_entry(). + * On list corruption reports a warning, and returns false. + */ +extern bool __list_del_entry_valid_or_report(struct list_head *entry); + +/* + * Performs list corruption checks before __list_del_entry(). Returns false if a + * corruption is detected, true otherwise. + */ +static __always_inline bool __list_del_entry_valid(struct list_head *entry) +{ + return __list_del_entry_valid_or_report(entry); +} #else static inline bool __list_add_valid(struct list_head *new, struct list_head *prev, diff --git a/lib/list_debug.c b/lib/list_debug.c index d98d43f80958..2def33b1491f 100644 --- a/lib/list_debug.c +++ b/lib/list_debug.c @@ -17,8 +17,8 @@ * attempt). */ -bool __list_add_valid(struct list_head *new, struct list_head *prev, - struct list_head *next) +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) { if (CHECK_DATA_CORRUPTION(prev == NULL, "list_add corruption. prev is NULL.\n") || @@ -37,9 +37,9 @@ bool __list_add_valid(struct list_head *new, struct list_head *prev, return true; } -EXPORT_SYMBOL(__list_add_valid); +EXPORT_SYMBOL(__list_add_valid_or_report); -bool __list_del_entry_valid(struct list_head *entry) +bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; @@ -65,6 +65,5 @@ bool __list_del_entry_valid(struct list_head *entry) return false; return true; - } -EXPORT_SYMBOL(__list_del_entry_valid); +EXPORT_SYMBOL(__list_del_entry_valid_or_report); From aebc7b0d8d91bbc69e976909963046bc48bca4fd Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 11 Aug 2023 17:18:40 +0200 Subject: [PATCH 099/130] list: Introduce CONFIG_LIST_HARDENED Numerous production kernel configs (see [1, 2]) are choosing to enable CONFIG_DEBUG_LIST, which is also being recommended by KSPP for hardened configs [3]. The motivation behind this is that the option can be used as a security hardening feature (e.g. CVE-2019-2215 and CVE-2019-2025 are mitigated by the option [4]). The feature has never been designed with performance in mind, yet common list manipulation is happening across hot paths all over the kernel. Introduce CONFIG_LIST_HARDENED, which performs list pointer checking inline, and only upon list corruption calls the reporting slow path. To generate optimal machine code with CONFIG_LIST_HARDENED: 1. Elide checking for pointer values which upon dereference would result in an immediate access fault (i.e. minimal hardening checks). The trade-off is lower-quality error reports. 2. Use the __preserve_most function attribute (available with Clang, but not yet with GCC) to minimize the code footprint for calling the reporting slow path. As a result, function size of callers is reduced by avoiding saving registers before calling the rarely called reporting slow path. Note that all TUs in lib/Makefile already disable function tracing, including list_debug.c, and __preserve_most's implied notrace has no effect in this case. 3. Because the inline checks are a subset of the full set of checks in __list_*_valid_or_report(), always return false if the inline checks failed. This avoids redundant compare and conditional branch right after return from the slow path. As a side-effect of the checks being inline, if the compiler can prove some condition to always be true, it can completely elide some checks. Since DEBUG_LIST is functionally a superset of LIST_HARDENED, the Kconfig variables are changed to reflect that: DEBUG_LIST selects LIST_HARDENED, whereas LIST_HARDENED itself has no dependency on DEBUG_LIST. Running netperf with CONFIG_LIST_HARDENED (using a Clang compiler with "preserve_most") shows throughput improvements, in my case of ~7% on average (up to 20-30% on some test cases). Link: https://r.android.com/1266735 [1] Link: https://gitlab.archlinux.org/archlinux/packaging/packages/linux/-/blob/main/config [2] Link: https://kernsec.org/wiki/index.php/Kernel_Self_Protection_Project/Recommended_Settings [3] Link: https://googleprojectzero.blogspot.com/2019/11/bad-binder-android-in-wild-exploit.html [4] Signed-off-by: Marco Elver Link: https://lore.kernel.org/r/20230811151847.1594958-3-elver@google.com Signed-off-by: Kees Cook --- arch/arm64/kvm/hyp/nvhe/Makefile | 2 +- arch/arm64/kvm/hyp/nvhe/list_debug.c | 2 + drivers/misc/lkdtm/bugs.c | 4 +- include/linux/list.h | 64 +++++++++++++++++++++++++--- lib/Kconfig.debug | 9 +++- lib/Makefile | 2 +- lib/list_debug.c | 5 ++- security/Kconfig.hardening | 13 ++++++ 8 files changed, 88 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 9ddc025e4b86..2250253a6429 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -25,7 +25,7 @@ hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o -hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o +hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o hyp-obj-y += $(lib-objs) ## diff --git a/arch/arm64/kvm/hyp/nvhe/list_debug.c b/arch/arm64/kvm/hyp/nvhe/list_debug.c index 16266a939a4c..46a2d4f2b3c6 100644 --- a/arch/arm64/kvm/hyp/nvhe/list_debug.c +++ b/arch/arm64/kvm/hyp/nvhe/list_debug.c @@ -26,6 +26,7 @@ static inline __must_check bool nvhe_check_data_corruption(bool v) /* The predicates checked here are taken from lib/list_debug.c. */ +__list_valid_slowpath bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, struct list_head *next) { @@ -37,6 +38,7 @@ bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, return true; } +__list_valid_slowpath bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 3c95600ab2f7..963b4dee6a7d 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -393,7 +393,7 @@ static void lkdtm_CORRUPT_LIST_ADD(void) pr_err("Overwrite did not happen, but no BUG?!\n"); else { pr_err("list_add() corruption not detected!\n"); - pr_expected_config(CONFIG_DEBUG_LIST); + pr_expected_config(CONFIG_LIST_HARDENED); } } @@ -420,7 +420,7 @@ static void lkdtm_CORRUPT_LIST_DEL(void) pr_err("Overwrite did not happen, but no BUG?!\n"); else { pr_err("list_del() corruption not detected!\n"); - pr_expected_config(CONFIG_DEBUG_LIST); + pr_expected_config(CONFIG_LIST_HARDENED); } } diff --git a/include/linux/list.h b/include/linux/list.h index 130c6a1bb45c..164b4d0e9d2a 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -38,39 +38,91 @@ static inline void INIT_LIST_HEAD(struct list_head *list) WRITE_ONCE(list->prev, list); } +#ifdef CONFIG_LIST_HARDENED + #ifdef CONFIG_DEBUG_LIST +# define __list_valid_slowpath +#else +# define __list_valid_slowpath __cold __preserve_most +#endif + /* * Performs the full set of list corruption checks before __list_add(). * On list corruption reports a warning, and returns false. */ -extern bool __list_add_valid_or_report(struct list_head *new, - struct list_head *prev, - struct list_head *next); +extern bool __list_valid_slowpath __list_add_valid_or_report(struct list_head *new, + struct list_head *prev, + struct list_head *next); /* * Performs list corruption checks before __list_add(). Returns false if a * corruption is detected, true otherwise. + * + * With CONFIG_LIST_HARDENED only, performs minimal list integrity checking + * inline to catch non-faulting corruptions, and only if a corruption is + * detected calls the reporting function __list_add_valid_or_report(). */ static __always_inline bool __list_add_valid(struct list_head *new, struct list_head *prev, struct list_head *next) { - return __list_add_valid_or_report(new, prev, next); + bool ret = true; + + if (!IS_ENABLED(CONFIG_DEBUG_LIST)) { + /* + * With the hardening version, elide checking if next and prev + * are NULL, since the immediate dereference of them below would + * result in a fault if NULL. + * + * With the reduced set of checks, we can afford to inline the + * checks, which also gives the compiler a chance to elide some + * of them completely if they can be proven at compile-time. If + * one of the pre-conditions does not hold, the slow-path will + * show a report which pre-condition failed. + */ + if (likely(next->prev == prev && prev->next == next && new != prev && new != next)) + return true; + ret = false; + } + + ret &= __list_add_valid_or_report(new, prev, next); + return ret; } /* * Performs the full set of list corruption checks before __list_del_entry(). * On list corruption reports a warning, and returns false. */ -extern bool __list_del_entry_valid_or_report(struct list_head *entry); +extern bool __list_valid_slowpath __list_del_entry_valid_or_report(struct list_head *entry); /* * Performs list corruption checks before __list_del_entry(). Returns false if a * corruption is detected, true otherwise. + * + * With CONFIG_LIST_HARDENED only, performs minimal list integrity checking + * inline to catch non-faulting corruptions, and only if a corruption is + * detected calls the reporting function __list_del_entry_valid_or_report(). */ static __always_inline bool __list_del_entry_valid(struct list_head *entry) { - return __list_del_entry_valid_or_report(entry); + bool ret = true; + + if (!IS_ENABLED(CONFIG_DEBUG_LIST)) { + struct list_head *prev = entry->prev; + struct list_head *next = entry->next; + + /* + * With the hardening version, elide checking if next and prev + * are NULL, LIST_POISON1 or LIST_POISON2, since the immediate + * dereference of them below would result in a fault. + */ + if (likely(prev->next == entry && next->prev == entry)) + return true; + ret = false; + } + + ret &= __list_del_entry_valid_or_report(entry); + return ret; } #else static inline bool __list_add_valid(struct list_head *new, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index fbc89baf7de6..c38745ad46eb 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1674,9 +1674,14 @@ menu "Debug kernel data structures" config DEBUG_LIST bool "Debug linked list manipulation" depends on DEBUG_KERNEL || BUG_ON_DATA_CORRUPTION + select LIST_HARDENED help - Enable this to turn on extended checks in the linked-list - walking routines. + Enable this to turn on extended checks in the linked-list walking + routines. + + This option trades better quality error reports for performance, and + is more suitable for kernel debugging. If you care about performance, + you should only enable CONFIG_LIST_HARDENED instead. If unsure, say N. diff --git a/lib/Makefile b/lib/Makefile index 42d307ade225..110936c9a68b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -161,7 +161,7 @@ obj-$(CONFIG_BTREE) += btree.o obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o -obj-$(CONFIG_DEBUG_LIST) += list_debug.o +obj-$(CONFIG_LIST_HARDENED) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o obj-$(CONFIG_BITREVERSE) += bitrev.o diff --git a/lib/list_debug.c b/lib/list_debug.c index 2def33b1491f..db602417febf 100644 --- a/lib/list_debug.c +++ b/lib/list_debug.c @@ -2,7 +2,8 @@ * Copyright 2006, Red Hat, Inc., Dave Jones * Released under the General Public License (GPL). * - * This file contains the linked list validation for DEBUG_LIST. + * This file contains the linked list validation and error reporting for + * LIST_HARDENED and DEBUG_LIST. */ #include @@ -17,6 +18,7 @@ * attempt). */ +__list_valid_slowpath bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, struct list_head *next) { @@ -39,6 +41,7 @@ bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, } EXPORT_SYMBOL(__list_add_valid_or_report); +__list_valid_slowpath bool __list_del_entry_valid_or_report(struct list_head *entry) { struct list_head *prev, *next; diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index 0f295961e773..ffc3c702b461 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -279,6 +279,19 @@ config ZERO_CALL_USED_REGS endmenu +menu "Hardening of kernel data structures" + +config LIST_HARDENED + bool "Check integrity of linked list manipulation" + help + Minimal integrity checking in the linked-list manipulation routines + to catch memory corruptions that are not guaranteed to result in an + immediate access fault. + + If unsure, say N. + +endmenu + config CC_HAS_RANDSTRUCT def_bool $(cc-option,-frandomize-layout-seed-file=/dev/null) # Randstruct was first added in Clang 15, but it isn't safe to use until From aa9f10d57056cea51d41283d3785bccbbb9f459e Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 11 Aug 2023 17:18:41 +0200 Subject: [PATCH 100/130] hardening: Move BUG_ON_DATA_CORRUPTION to hardening options BUG_ON_DATA_CORRUPTION is turning detected corruptions of list data structures from WARNings into BUGs. This can be useful to stop further corruptions or even exploitation attempts. However, the option has less to do with debugging than with hardening. With the introduction of LIST_HARDENED, it makes more sense to move it to the hardening options, where it selects LIST_HARDENED instead. Without this change, combining BUG_ON_DATA_CORRUPTION with LIST_HARDENED alone wouldn't be possible, because DEBUG_LIST would always be selected by BUG_ON_DATA_CORRUPTION. Signed-off-by: Marco Elver Link: https://lore.kernel.org/r/20230811151847.1594958-4-elver@google.com Signed-off-by: Kees Cook --- lib/Kconfig.debug | 12 +----------- security/Kconfig.hardening | 10 ++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index c38745ad46eb..c7348d1fabe5 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1673,7 +1673,7 @@ menu "Debug kernel data structures" config DEBUG_LIST bool "Debug linked list manipulation" - depends on DEBUG_KERNEL || BUG_ON_DATA_CORRUPTION + depends on DEBUG_KERNEL select LIST_HARDENED help Enable this to turn on extended checks in the linked-list walking @@ -1715,16 +1715,6 @@ config DEBUG_NOTIFIERS This is a relatively cheap check but if you care about maximum performance, say N. -config BUG_ON_DATA_CORRUPTION - bool "Trigger a BUG when data corruption is detected" - select DEBUG_LIST - help - Select this option if the kernel should BUG when it encounters - data corruption in kernel memory structures when they get checked - for validity. - - If unsure, say N. - config DEBUG_MAPLE_TREE bool "Debug maple trees" depends on DEBUG_KERNEL diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index ffc3c702b461..2cff851ebfd7 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -290,6 +290,16 @@ config LIST_HARDENED If unsure, say N. +config BUG_ON_DATA_CORRUPTION + bool "Trigger a BUG when data corruption is detected" + select LIST_HARDENED + help + Select this option if the kernel should BUG when it encounters + data corruption in kernel memory structures when they get checked + for validity. + + If unsure, say N. + endmenu config CC_HAS_RANDSTRUCT From 967afdf808cf66908a55c55b8ec5937cc20676ce Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 2 Aug 2023 07:25:56 -0600 Subject: [PATCH 101/130] alpha: Replace one-element array with flexible-array member One-element and zero-length arrays are deprecated. So, replace one-element array in struct osf_dirent with flexible-array member. This results in no differences in binary output. Signed-off-by: "Gustavo A. R. Silva" Link: https://lore.kernel.org/r/ZMpZZBShlLqyD3ax@work Signed-off-by: Kees Cook --- arch/alpha/kernel/osf_sys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index d98701ee36c6..5db88b627439 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -97,7 +97,7 @@ struct osf_dirent { unsigned int d_ino; unsigned short d_reclen; unsigned short d_namlen; - char d_name[1]; + char d_name[]; }; struct osf_dirent_callback { From 30bed99e0c6335b711119b9fda806da7b4031dfb Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 7 Aug 2023 18:22:30 +0000 Subject: [PATCH 102/130] um: vector: refactor deprecated strncpy `strncpy` is deprecated for use on NUL-terminated destination strings [1]. A suitable replacement is `strscpy` [2] due to the fact that it guarantees NUL-termination on its destination buffer argument which is _not_ the case for `strncpy`! In this case, we are able to drop the now superfluous `... - 1` instances because `strscpy` will automatically truncate the last byte by setting it to a NUL byte if the source size exceeds the destination size or if the source string is not NUL-terminated. I've also opted to remove the seemingly useless char* casts. I'm not sure why they're present at all since (after expanding the `ifr_name` macro) `ifr.ifr_ifrn.ifrn_name` is a char* already. All in all, `strscpy` is a more robust and less ambiguous interface while also letting us remove some `... -1`'s which cleans things up a bit. [1]: www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [2]: https://manpages.debian.org/testing/linux-manual-4.8/strscpy.9.en.html Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Acked-by: Anton Ivanov Link: https://lore.kernel.org/r/20230807-arch-um-drivers-v1-1-10d602c5577a@google.com Signed-off-by: Kees Cook --- arch/um/drivers/vector_user.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/um/drivers/vector_user.c b/arch/um/drivers/vector_user.c index c650e428432b..c719e1ec4645 100644 --- a/arch/um/drivers/vector_user.c +++ b/arch/um/drivers/vector_user.c @@ -141,7 +141,7 @@ static int create_tap_fd(char *iface) } memset(&ifr, 0, sizeof(ifr)); ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR; - strncpy((char *)&ifr.ifr_name, iface, sizeof(ifr.ifr_name) - 1); + strscpy(ifr.ifr_name, iface, sizeof(ifr.ifr_name)); err = ioctl(fd, TUNSETIFF, (void *) &ifr); if (err != 0) { @@ -171,7 +171,7 @@ static int create_raw_fd(char *iface, int flags, int proto) goto raw_fd_cleanup; } memset(&ifr, 0, sizeof(ifr)); - strncpy((char *)&ifr.ifr_name, iface, sizeof(ifr.ifr_name) - 1); + strscpy(ifr.ifr_name, iface, sizeof(ifr.ifr_name)); if (ioctl(fd, SIOCGIFINDEX, (void *) &ifr) < 0) { err = -errno; goto raw_fd_cleanup; From be8dffa04de3894bad0bf31d1531a2f06353b70e Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Wed, 9 Aug 2023 18:19:32 +0000 Subject: [PATCH 103/130] um: refactor deprecated strncpy to memcpy Use `memcpy` since `console_buf` is not expected to be NUL-terminated and it more accurately describes what is happening with the buffers `console_buf` and `string` as per Kees' analysis [1]. Also mark char buffer as `__nonstring` as per Kees' suggestion [2]. This change now makes it more clear what this code does and that `console_buf` is not expected to be NUL-terminated. Link: https://lore.kernel.org/all/202308081708.D5ADC80F@keescook/ [1] Link: https://github.com/KSPP/linux/issues/90 [2] Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings Cc: linux-hardening@vger.kernel.org Suggested-by: Kees Cook Signed-off-by: Justin Stitt Link: https://lore.kernel.org/r/20230809-arch-um-v3-1-f63e1122d77e@google.com Signed-off-by: Kees Cook --- arch/um/drivers/mconsole_kern.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c index 5026e7b9adfe..ff4bda95b9c7 100644 --- a/arch/um/drivers/mconsole_kern.c +++ b/arch/um/drivers/mconsole_kern.c @@ -554,7 +554,7 @@ struct mconsole_output { static DEFINE_SPINLOCK(client_lock); static LIST_HEAD(clients); -static char console_buf[MCONSOLE_MAX_DATA]; +static char console_buf[MCONSOLE_MAX_DATA] __nonstring; static void console_write(struct console *console, const char *string, unsigned int len) @@ -567,7 +567,7 @@ static void console_write(struct console *console, const char *string, while (len > 0) { n = min((size_t) len, ARRAY_SIZE(console_buf)); - strncpy(console_buf, string, n); + memcpy(console_buf, string, n); string += n; len -= n; From cf007647475b5090819c5fe8da771073145c7334 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 10 Aug 2023 12:54:18 -0700 Subject: [PATCH 104/130] ARM: ptrace: Restore syscall restart tracing Since commit 4e57a4ddf6b0 ("ARM: 9107/1: syscall: always store thread_info->abi_syscall"), the seccomp selftests "syscall_restart" has been broken. This was caused by the restart syscall not being stored to "abi_syscall" during restart setup before branching to the "local_restart" label. Tracers would see the wrong syscall, and scno would get overwritten while returning from the TIF_WORK path. Add the missing store. Cc: Russell King Cc: Arnd Bergmann Cc: Lecopzer Chen Cc: Oleg Nesterov Cc: linux-arm-kernel@lists.infradead.org Fixes: 4e57a4ddf6b0 ("ARM: 9107/1: syscall: always store thread_info->abi_syscall") Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/r/20230810195422.2304827-1-keescook@chromium.org Signed-off-by: Kees Cook --- arch/arm/kernel/entry-common.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index bcc4c9ec3aa4..5c31e9de7a60 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -90,6 +90,7 @@ slow_work_pending: cmp r0, #0 beq no_work_pending movlt scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE) + str scno, [tsk, #TI_ABI_SYSCALL] @ make sure tracers see update ldmia sp, {r0 - r6} @ have to reload r0 - r6 b local_restart @ ... and off we go ENDPROC(ret_fast_syscall) From 4697b5848bd933f68ebd04836362c8de0cacaf71 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 10 Aug 2023 12:54:19 -0700 Subject: [PATCH 105/130] ARM: ptrace: Restore syscall skipping for tracers Since commit 4e57a4ddf6b0 ("ARM: 9107/1: syscall: always store thread_info->abi_syscall"), the seccomp selftests "syscall_errno" and "syscall_faked" have been broken. Both seccomp and PTRACE depend on using the special value of "-1" for skipping syscalls. This value wasn't working because it was getting masked by __NR_SYSCALL_MASK in both PTRACE_SET_SYSCALL and get_syscall_nr(). Explicitly test for -1 in PTRACE_SET_SYSCALL and get_syscall_nr(), leaving it exposed when present, allowing tracers to skip syscalls again. Cc: Russell King Cc: Arnd Bergmann Cc: Lecopzer Chen Cc: Oleg Nesterov Cc: linux-arm-kernel@lists.infradead.org Fixes: 4e57a4ddf6b0 ("ARM: 9107/1: syscall: always store thread_info->abi_syscall") Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/r/20230810195422.2304827-2-keescook@chromium.org Signed-off-by: Kees Cook --- arch/arm/include/asm/syscall.h | 3 +++ arch/arm/kernel/ptrace.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/asm/syscall.h b/arch/arm/include/asm/syscall.h index dfeed440254a..fe4326d938c1 100644 --- a/arch/arm/include/asm/syscall.h +++ b/arch/arm/include/asm/syscall.h @@ -25,6 +25,9 @@ static inline int syscall_get_nr(struct task_struct *task, if (IS_ENABLED(CONFIG_AEABI) && !IS_ENABLED(CONFIG_OABI_COMPAT)) return task_thread_info(task)->abi_syscall; + if (task_thread_info(task)->abi_syscall == -1) + return -1; + return task_thread_info(task)->abi_syscall & __NR_SYSCALL_MASK; } diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 2d8e2516906b..fef32d73f912 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -783,8 +783,9 @@ long arch_ptrace(struct task_struct *child, long request, break; case PTRACE_SET_SYSCALL: - task_thread_info(child)->abi_syscall = data & - __NR_SYSCALL_MASK; + if (data != -1) + data &= __NR_SYSCALL_MASK; + task_thread_info(child)->abi_syscall = data; ret = 0; break; From ed2b9e1b6d82c729b2757b449097f28f108b023b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 16 Jun 2023 16:07:24 -0700 Subject: [PATCH 106/130] srcu,notifier: Remove #ifdefs in favor of SRCU Tiny srcu_usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit removes two #ifdef directives from include/linux/notifier.h by causing SRCU Tiny to provide a dummy srcu_usage structure and a dummy __SRCU_USAGE_INIT() macro. Suggested-by: Linus Torvalds Cc: Matthias Brugger Cc: "Rafael J. Wysocki" Cc: "Michał Mirosław" Cc: Dmitry Osipenko Cc: Sachin Sant Cc: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- include/linux/notifier.h | 11 ----------- include/linux/srcutiny.h | 4 ++++ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 86544707236a..45702bdcbceb 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -73,9 +73,7 @@ struct raw_notifier_head { struct srcu_notifier_head { struct mutex mutex; -#ifdef CONFIG_TREE_SRCU struct srcu_usage srcuu; -#endif struct srcu_struct srcu; struct notifier_block __rcu *head; }; @@ -106,7 +104,6 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); #define RAW_NOTIFIER_INIT(name) { \ .head = NULL } -#ifdef CONFIG_TREE_SRCU #define SRCU_NOTIFIER_INIT(name, pcpu) \ { \ .mutex = __MUTEX_INITIALIZER(name.mutex), \ @@ -114,14 +111,6 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); .srcuu = __SRCU_USAGE_INIT(name.srcuu), \ .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ } -#else -#define SRCU_NOTIFIER_INIT(name, pcpu) \ - { \ - .mutex = __MUTEX_INITIALIZER(name.mutex), \ - .head = NULL, \ - .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \ - } -#endif #define ATOMIC_NOTIFIER_HEAD(name) \ struct atomic_notifier_head name = \ diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index ebd72491af99..447133171d95 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -48,6 +48,10 @@ void srcu_drive_gp(struct work_struct *wp); #define DEFINE_STATIC_SRCU(name) \ static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name) +// Dummy structure for srcu_notifier_head. +struct srcu_usage { }; +#define __SRCU_USAGE_INIT(name) { } + void synchronize_srcu(struct srcu_struct *ssp); /* From 343640cb5b4ed349b3656ee4b100b36e2ae7e2da Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 18:37:05 -0700 Subject: [PATCH 107/130] rcu: Mark __rcu_irq_enter_check_tick() ->rcu_urgent_qs load The rcu_request_urgent_qs_task() function does a cross-CPU store to ->rcu_urgent_qs, so this commit therefore marks the load in __rcu_irq_enter_check_tick() with READ_ONCE(). Signed-off-by: Paul E. McKenney Reviewed-by: Joel Fernandes (Google) --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f788981272f8..cb1caefa8bd0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -632,7 +632,7 @@ void __rcu_irq_enter_check_tick(void) // prevents self-deadlock. So we can safely recheck under the lock. // Note that the nohz_full state currently cannot change. raw_spin_lock_rcu_node(rdp->mynode); - if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { + if (READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) { // A nohz_full CPU is in the kernel and RCU needs a // quiescent state. Turn on the tick! WRITE_ONCE(rdp->rcu_forced_tick, true); From 3292ba0229dbe5f3e79b78b230354ada2888dc29 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 5 Jul 2023 16:20:58 -0700 Subject: [PATCH 108/130] rcu: Make the rcu_nocb_poll boot parameter usable via boot config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rcu_nocb_poll kernel boot parameter is defined via early_param(), whose parsing functions are invoked from parse_early_param() which is in turn invoked by setup_arch(), which is very early indeed.  It is invoked so early that the console output timestamps read 0.000000, in other words, before time begins. This use of early_param() means that the rcu_nocb_poll kernel boot parameter cannot usefully be embedded into the kernel image. Yes, you can embed it, but setup_boot_config() is invoked from start_kernel() too late for it to be parsed. But it makes no sense to parse this parameter so early. After all, it cannot do anything until the rcuog kthreads are created, which is long after rcu_init() time, let alone setup_boot_config() time. This commit therefore switches the rcu_nocb_poll kernel boot parameter from early_param() to __setup(), which allows boot-config parsing of this parameter, in turn allowing it to be embedded into the kernel image. Signed-off-by: Paul E. McKenney Reviewed-by: Joel Fernandes (Google) --- kernel/rcu/tree_nocb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 43229d2b0c44..5598212d1f27 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -77,9 +77,9 @@ __setup("rcu_nocbs", rcu_nocb_setup); static int __init parse_rcu_nocb_poll(char *arg) { rcu_nocb_poll = true; - return 0; + return 1; } -early_param("rcu_nocb_poll", parse_rcu_nocb_poll); +__setup("rcu_nocb_poll", parse_rcu_nocb_poll); /* * Don't bother bypassing ->cblist if the call_rcu() rate is low. From efd04f8a8b45b8b98704b5860e363bab239b8bae Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Thu, 6 Jul 2023 10:28:48 +0000 Subject: [PATCH 109/130] rcu: Use WRITE_ONCE() for assignments to ->next for rculist_nulls When the objects managed by rculist_nulls are allocated with SLAB_TYPESAFE_BY_RCU, old readers may still hold references to an object even though it is just now being added, which means the modification of ->next is visible to readers. This patch therefore uses WRITE_ONCE() for assignments to ->next. Signed-off-by: Alan Huang Signed-off-by: Paul E. McKenney Reviewed-by: Joel Fernandes (Google) --- include/linux/rculist_nulls.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index ba4c00dd8005..89186c499dd4 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h @@ -101,7 +101,7 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, { struct hlist_nulls_node *first = h->first; - n->next = first; + WRITE_ONCE(n->next, first); WRITE_ONCE(n->pprev, &h->first); rcu_assign_pointer(hlist_nulls_first_rcu(h), n); if (!is_a_nulls(first)) @@ -137,7 +137,7 @@ static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, last = i; if (last) { - n->next = last->next; + WRITE_ONCE(n->next, last->next); n->pprev = &last->next; rcu_assign_pointer(hlist_nulls_next_rcu(last), n); } else { From 46822860a5a9a5a558475d323a55c8aab0b54012 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 17 Aug 2023 12:32:15 -0700 Subject: [PATCH 110/130] seccomp: Add missing kerndoc notations The kerndoc for some struct member and function arguments were missing. Add them. Cc: Andy Lutomirski Cc: Will Drewry Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202308171742.AncabIG1-lkp@intel.com/ Signed-off-by: Kees Cook --- kernel/seccomp.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kernel/seccomp.c b/kernel/seccomp.c index d3fdc0086168..255999ba9190 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -110,11 +110,13 @@ struct seccomp_knotif { * @flags: The flags for the new file descriptor. At the moment, only O_CLOEXEC * is allowed. * @ioctl_flags: The flags used for the seccomp_addfd ioctl. + * @setfd: whether or not SECCOMP_ADDFD_FLAG_SETFD was set during notify_addfd * @ret: The return value of the installing process. It is set to the fd num * upon success (>= 0). * @completion: Indicates that the installing process has completed fd * installation, or gone away (either due to successful * reply, or signal) + * @list: list_head for chaining seccomp_kaddfd together. * */ struct seccomp_kaddfd { @@ -138,12 +140,12 @@ struct seccomp_kaddfd { * structure is fairly large, we store the notification-specific stuff in a * separate structure. * - * @request: A semaphore that users of this notification can wait on for - * changes. Actual reads and writes are still controlled with - * filter->notify_lock. + * @requests: A semaphore that users of this notification can wait on for + * changes. Actual reads and writes are still controlled with + * filter->notify_lock. + * @flags: A set of SECCOMP_USER_NOTIF_FD_* flags. * @next_id: The id of the next request. * @notifications: A list of struct seccomp_knotif elements. - * @flags: A set of SECCOMP_USER_NOTIF_FD_* flags. */ struct notification { @@ -558,6 +560,8 @@ static void __seccomp_filter_release(struct seccomp_filter *orig) * drop its reference count, and notify * about unused filters * + * @tsk: task the filter should be released from. + * * This function should only be called when the task is exiting as * it detaches it from its filter tree. As such, READ_ONCE() and * barriers are not needed here, as would normally be needed. @@ -577,6 +581,8 @@ void seccomp_filter_release(struct task_struct *tsk) /** * seccomp_sync_threads: sets all threads to use current's filter * + * @flags: SECCOMP_FILTER_FLAG_* flags to set during sync. + * * Expects sighand and cred_guard_mutex locks to be held, and for * seccomp_can_sync_threads() to have returned success already * without dropping the locks. From c8248faf3ca276ebdf60f003b3e04bf764daba91 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 17 Aug 2023 13:06:03 -0700 Subject: [PATCH 111/130] Compiler Attributes: counted_by: Adjust name and identifier expansion GCC and Clang's current RFCs name this attribute "counted_by", and have moved away from using a string for the member name. Update the kernel's macros to match. Additionally provide a UAPI no-op macro for UAPI structs that will gain annotations. Cc: Miguel Ojeda Cc: Nick Desaulniers Fixes: dd06e72e68bc ("Compiler Attributes: Add __counted_by macro") Acked-by: Miguel Ojeda Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20230817200558.never.077-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/compiler_attributes.h | 26 +++++++++++++------------- include/uapi/linux/stddef.h | 4 ++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 00efa35c350f..28566624f008 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -94,6 +94,19 @@ # define __copy(symbol) #endif +/* + * Optional: only supported since gcc >= 14 + * Optional: only supported since clang >= 18 + * + * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 + * clang: https://reviews.llvm.org/D148381 + */ +#if __has_attribute(__counted_by__) +# define __counted_by(member) __attribute__((__counted_by__(member))) +#else +# define __counted_by(member) +#endif + /* * Optional: not supported by gcc * Optional: only supported since clang >= 14.0 @@ -129,19 +142,6 @@ # define __designated_init #endif -/* - * Optional: only supported since gcc >= 14 - * Optional: only supported since clang >= 17 - * - * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 - * clang: https://reviews.llvm.org/D148381 - */ -#if __has_attribute(__element_count__) -# define __counted_by(member) __attribute__((__element_count__(#member))) -#else -# define __counted_by(member) -#endif - /* * Optional: only supported since clang >= 14.0 * diff --git a/include/uapi/linux/stddef.h b/include/uapi/linux/stddef.h index 7837ba4fe728..7c3fc3980881 100644 --- a/include/uapi/linux/stddef.h +++ b/include/uapi/linux/stddef.h @@ -45,3 +45,7 @@ TYPE NAME[]; \ } #endif + +#ifndef __counted_by +#define __counted_by(m) +#endif From 5d207e83ca41206e75c2cd414d40b451ef04c259 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 16 Aug 2023 21:27:35 -0700 Subject: [PATCH 112/130] lkdtm: Add FAM_BOUNDS test for __counted_by Add new CONFIG_UBSAN_BOUNDS test for __counted_by attribute. Cc: Greg Kroah-Hartman Cc: Arnd Bergmann Signed-off-by: Kees Cook --- drivers/misc/lkdtm/bugs.c | 47 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 963b4dee6a7d..c66cc05a68c4 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -273,8 +273,8 @@ static void lkdtm_HUNG_TASK(void) schedule(); } -volatile unsigned int huge = INT_MAX - 2; -volatile unsigned int ignored; +static volatile unsigned int huge = INT_MAX - 2; +static volatile unsigned int ignored; static void lkdtm_OVERFLOW_SIGNED(void) { @@ -305,7 +305,7 @@ static void lkdtm_OVERFLOW_UNSIGNED(void) ignored = value; } -/* Intentionally using old-style flex array definition of 1 byte. */ +/* Intentionally using unannotated flex array definition. */ struct array_bounds_flex_array { int one; int two; @@ -357,6 +357,46 @@ static void lkdtm_ARRAY_BOUNDS(void) pr_expected_config(CONFIG_UBSAN_BOUNDS); } +struct lkdtm_annotated { + unsigned long flags; + int count; + int array[] __counted_by(count); +}; + +static volatile int fam_count = 4; + +static void lkdtm_FAM_BOUNDS(void) +{ + struct lkdtm_annotated *inst; + + inst = kzalloc(struct_size(inst, array, fam_count + 1), GFP_KERNEL); + if (!inst) { + pr_err("FAIL: could not allocate test struct!\n"); + return; + } + + inst->count = fam_count; + pr_info("Array access within bounds ...\n"); + inst->array[1] = fam_count; + ignored = inst->array[1]; + + pr_info("Array access beyond bounds ...\n"); + inst->array[fam_count] = fam_count; + ignored = inst->array[fam_count]; + + kfree(inst); + + pr_err("FAIL: survived access of invalid flexible array member index!\n"); + + if (!__has_attribute(__counted_by__)) + pr_warn("This is expected since this %s was built a compiler supporting __counted_by\n", + lkdtm_kernel_info); + else if (IS_ENABLED(CONFIG_UBSAN_BOUNDS)) + pr_expected_config(CONFIG_UBSAN_TRAP); + else + pr_expected_config(CONFIG_UBSAN_BOUNDS); +} + static void lkdtm_CORRUPT_LIST_ADD(void) { /* @@ -616,6 +656,7 @@ static struct crashtype crashtypes[] = { CRASHTYPE(OVERFLOW_SIGNED), CRASHTYPE(OVERFLOW_UNSIGNED), CRASHTYPE(ARRAY_BOUNDS), + CRASHTYPE(FAM_BOUNDS), CRASHTYPE(CORRUPT_LIST_ADD), CRASHTYPE(CORRUPT_LIST_DEL), CRASHTYPE(STACK_GUARD_PAGE_LEADING), From a4b35d4d05b9f2e84c1dd6301d3dca65719334ef Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 17 Aug 2023 14:03:28 -0700 Subject: [PATCH 113/130] integrity: Annotate struct ima_rule_opt_list with __counted_by Prepare for the coming implementation by GCC and Clang of the __counted_by attribute. Flexible array members annotated with __counted_by can have their accesses bounds-checked at run-time checking via CONFIG_UBSAN_BOUNDS (for array indexing) and CONFIG_FORTIFY_SOURCE (for strcpy/memcpy-family functions). As found with Coccinelle[1], add __counted_by for struct ima_rule_opt_list. Additionally, since the element count member must be set before accessing the annotated flexible array member, move its initialization earlier. [1] https://github.com/kees/kernel-tools/blob/trunk/coccinelle/examples/counted_by.cocci Cc: Mimi Zohar Cc: Dmitry Kasatkin Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-integrity@vger.kernel.org Cc: linux-security-module@vger.kernel.org Acked-by: Mimi Zohar Reviewed-by: "Gustavo A. R. Silva" Acked-by: Jarkko Sakkinen Link: https://lore.kernel.org/r/20230817210327.never.598-kees@kernel.org Signed-off-by: Kees Cook --- security/integrity/ima/ima_policy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c index c9b3bd8f1bb9..7a0420cf1a6a 100644 --- a/security/integrity/ima/ima_policy.c +++ b/security/integrity/ima/ima_policy.c @@ -68,7 +68,7 @@ enum policy_rule_list { IMA_DEFAULT_POLICY = 1, IMA_CUSTOM_POLICY }; struct ima_rule_opt_list { size_t count; - char *items[]; + char *items[] __counted_by(count); }; /* @@ -342,6 +342,7 @@ static struct ima_rule_opt_list *ima_alloc_rule_opt_list(const substring_t *src) kfree(src_copy); return ERR_PTR(-ENOMEM); } + opt_list->count = count; /* * strsep() has already replaced all instances of '|' with '\0', @@ -357,7 +358,6 @@ static struct ima_rule_opt_list *ima_alloc_rule_opt_list(const substring_t *src) opt_list->items[i] = cur; cur = strchr(cur, '\0') + 1; } - opt_list->count = count; return opt_list; } From 2e99b73afde18853754c5fae8e8d1a66fe5e3f64 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Fri, 11 Aug 2023 17:58:04 +0800 Subject: [PATCH 114/130] irqchip/loongson-eiointc: Fix return value checking of eiointc_index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return value of function eiointc_index is int, however it is converted into uint32_t and then compared smaller than zero, this will cause logic problem. Fixes: dd281e1a1a93 ("irqchip: Add Loongson Extended I/O interrupt controller support") Signed-off-by: Bibo Mao Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230811095805.2974722-2-maobibo@loongson.cn --- drivers/irqchip/irq-loongson-eiointc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index 92d8aa28bdf5..1623cd779175 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -144,7 +144,7 @@ static int eiointc_router_init(unsigned int cpu) int i, bit; uint32_t data; uint32_t node = cpu_to_eio_node(cpu); - uint32_t index = eiointc_index(node); + int index = eiointc_index(node); if (index < 0) { pr_err("Error: invalid nodemap!\n"); From 9bdd26b6477f798e292d0c196b9a8e5b77ce5e1d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 10 Aug 2023 14:33:54 +0200 Subject: [PATCH 115/130] irqchip/xtensa-pic: Include header for xtensa_pic_init_legacy() The declaration for this function is not included, which leads to a harmless warning: drivers/irqchip/irq-xtensa-pic.c:91:12: error: no previous prototype for 'xtensa_pic_init_legacy' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Max Filippov Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230810123404.1222332-1-arnd@kernel.org --- drivers/irqchip/irq-xtensa-pic.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/irqchip/irq-xtensa-pic.c b/drivers/irqchip/irq-xtensa-pic.c index ab12328be5ee..0c18d1f1e264 100644 --- a/drivers/irqchip/irq-xtensa-pic.c +++ b/drivers/irqchip/irq-xtensa-pic.c @@ -16,6 +16,7 @@ #include #include #include +#include #include unsigned int cached_irq_mask; From 90e921d768677172cbb4be1d7597d556ae018020 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 10 Aug 2023 14:33:55 +0200 Subject: [PATCH 116/130] irqchip/mips-gic: Mark gic_irq_domain_free() static MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function is only used locally and should be static to avoid a warning: drivers/irqchip/irq-mips-gic.c:560:6: error: no previous prototype for 'gic_irq_domain_free' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Serge Semin Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230810123404.1222332-2-arnd@kernel.org --- drivers/irqchip/irq-mips-gic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index 6d5ecc10a870..76253e864f23 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -557,7 +557,7 @@ static int gic_irq_domain_alloc(struct irq_domain *d, unsigned int virq, return gic_irq_domain_map(d, virq, hwirq); } -void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, +static void gic_irq_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) { } From 424a4aec7a2392d248e2f260ba571a01e24b8ec2 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 10 Aug 2023 14:33:56 +0200 Subject: [PATCH 117/130] irqchipr/i8259: Mark i8259_of_init() static i8259_of_init() is only used as an initcall and does not need to be global, so mark it static to avoid: drivers/irqchip/irq-i8259.c:343:12: warning: no previous prototype for 'i8259_of_init' [-Wmissing-prototypes] Signed-off-by: Arnd Bergmann Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230810123404.1222332-3-arnd@kernel.org --- drivers/irqchip/irq-i8259.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c index b70ce0d3c092..115bdcffab24 100644 --- a/drivers/irqchip/irq-i8259.c +++ b/drivers/irqchip/irq-i8259.c @@ -340,7 +340,7 @@ static void i8259_irq_dispatch(struct irq_desc *desc) generic_handle_domain_irq(domain, hwirq); } -int __init i8259_of_init(struct device_node *node, struct device_node *parent) +static int __init i8259_of_init(struct device_node *node, struct device_node *parent) { struct irq_domain *domain; unsigned int parent_irq; From 2f54db449b6185024ef884fd3083802e1eae6921 Mon Sep 17 00:00:00 2001 From: Ruan Jinjie Date: Thu, 3 Aug 2023 16:34:51 +0800 Subject: [PATCH 118/130] irqchip/imx-mu-msi: Do not check for 0 return after calling platform_get_irq() It is not possible for platform_get_irq() to return 0. Use the return value from platform_get_irq(). Signed-off-by: Ruan Jinjie Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230803083452.4085650-2-ruanjinjie@huawei.com --- drivers/irqchip/irq-imx-mu-msi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c index 229039eda1b1..90d41c1407ac 100644 --- a/drivers/irqchip/irq-imx-mu-msi.c +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -339,8 +339,8 @@ static int __init imx_mu_of_init(struct device_node *dn, msi_data->msiir_addr = res->start + msi_data->cfg->xTR; irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return -ENODEV; + if (irq < 0) + return irq; platform_set_drvdata(pdev, msi_data); From d0afed833970808f141ed52fcc591998087f3a26 Mon Sep 17 00:00:00 2001 From: Ruan Jinjie Date: Thu, 3 Aug 2023 16:34:52 +0800 Subject: [PATCH 119/130] irqchip/irq-pruss-intc: Do not check for 0 return after calling platform_get_irq() It is not possible for platform_get_irq() to return 0. Use the return value from platform_get_irq(). Signed-off-by: Ruan Jinjie Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230803083452.4085650-3-ruanjinjie@huawei.com --- drivers/irqchip/irq-pruss-intc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c index fa8d89b02ec0..ee69e8d83b9a 100644 --- a/drivers/irqchip/irq-pruss-intc.c +++ b/drivers/irqchip/irq-pruss-intc.c @@ -565,8 +565,8 @@ static int pruss_intc_probe(struct platform_device *pdev) continue; irq = platform_get_irq_byname(pdev, irq_names[i]); - if (irq <= 0) { - ret = (irq == 0) ? -EINVAL : irq; + if (irq < 0) { + ret = irq; goto fail_irq; } From 70befedaf5b020a08d397e2d6a9c306f0f611cfb Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 26 Jul 2023 11:07:41 +0800 Subject: [PATCH 120/130] irqchip/orion: Use of_address_count() helper After commit 16988c742968 ("of/address: introduce of_address_count() helper"), Use of_address_count() to instead of open-coding it, it's no functional change. Signed-off-by: Yang Yingliang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230726030741.1136244-1-yangyingliang@huawei.com --- drivers/irqchip/irq-orion.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-orion.c b/drivers/irqchip/irq-orion.c index 17c2c7a07f10..4e4e874e09a8 100644 --- a/drivers/irqchip/irq-orion.c +++ b/drivers/irqchip/irq-orion.c @@ -57,8 +57,7 @@ static int __init orion_irq_init(struct device_node *np, struct resource r; /* count number of irq chips by valid reg addresses */ - while (of_address_to_resource(np, num_chips, &r) == 0) - num_chips++; + num_chips = of_address_count(np); orion_irq_domain = irq_domain_add_linear(np, num_chips * ORION_IRQS_PER_CHIP, From ee07675046389046d501319b08158b20a8902a5e Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 14 Jul 2023 11:46:43 -0600 Subject: [PATCH 121/130] irqchip: Explicitly include correct DT includes The DT of_device.h and of_platform.h date back to the separate of_platform_bus_type before it as merged into the regular platform bus. As part of that merge prepping Arm DT support 13 years ago, they "temporarily" include each other. They also include platform_device.h and of.h. As a result, there's a pretty much random mix of those include files used throughout the tree. In order to detangle these headers and replace the implicit includes with struct declarations, users need to explicitly include the correct includes. Signed-off-by: Rob Herring Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230714174645.4058547-1-robh@kernel.org --- drivers/irqchip/irq-bcm6345-l1.c | 1 - drivers/irqchip/irq-bcm7038-l1.c | 1 - drivers/irqchip/irq-brcmstb-l2.c | 1 - drivers/irqchip/irq-gic-pm.c | 2 +- drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c | 2 -- drivers/irqchip/irq-imx-intmux.c | 3 ++- drivers/irqchip/irq-imx-irqsteer.c | 3 ++- drivers/irqchip/irq-keystone.c | 2 +- drivers/irqchip/irq-loongson-htvec.c | 1 - drivers/irqchip/irq-loongson-pch-pic.c | 2 +- drivers/irqchip/irq-madera.c | 4 +--- drivers/irqchip/irq-pruss-intc.c | 2 +- drivers/irqchip/irq-qcom-mpm.c | 2 +- drivers/irqchip/irq-renesas-intc-irqpin.c | 1 - drivers/irqchip/irq-st.c | 2 +- drivers/irqchip/irq-stm32-exti.c | 3 ++- drivers/irqchip/irq-sunxi-nmi.c | 1 - drivers/irqchip/irq-tb10x.c | 1 - drivers/irqchip/irq-ti-sci-inta.c | 4 ++-- drivers/irqchip/irq-ti-sci-intr.c | 4 ++-- drivers/irqchip/irq-uniphier-aidet.c | 1 - drivers/irqchip/irqchip.c | 2 +- drivers/irqchip/qcom-pdc.c | 1 - 23 files changed, 18 insertions(+), 28 deletions(-) diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c index fa113cb2529a..b521897277a8 100644 --- a/drivers/irqchip/irq-bcm6345-l1.c +++ b/drivers/irqchip/irq-bcm6345-l1.c @@ -60,7 +60,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index a62b96237b82..24ca1d656adc 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 091b0fe7e324..5559c943f03f 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-gic-pm.c b/drivers/irqchip/irq-gic-pm.c index 3989d16f997b..a275a8071a25 100644 --- a/drivers/irqchip/irq-gic-pm.c +++ b/drivers/irqchip/irq-gic-pm.c @@ -4,7 +4,7 @@ */ #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c index 634263dfd7b5..8e87fc35f8aa 100644 --- a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c +++ b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c @@ -9,8 +9,6 @@ #include #include -#include -#include #include #include #include diff --git a/drivers/irqchip/irq-imx-intmux.c b/drivers/irqchip/irq-imx-intmux.c index 80aaea82468a..6d9a08238c9d 100644 --- a/drivers/irqchip/irq-imx-intmux.c +++ b/drivers/irqchip/irq-imx-intmux.c @@ -50,8 +50,9 @@ #include #include #include +#include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-imx-irqsteer.c b/drivers/irqchip/irq-imx-irqsteer.c index 96230a04ec23..bd9543314539 100644 --- a/drivers/irqchip/irq-imx-irqsteer.c +++ b/drivers/irqchip/irq-imx-irqsteer.c @@ -10,8 +10,9 @@ #include #include #include +#include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-keystone.c b/drivers/irqchip/irq-keystone.c index ba9792e60329..a36396db4b08 100644 --- a/drivers/irqchip/irq-keystone.c +++ b/drivers/irqchip/irq-keystone.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index fc8bf1f5d41b..0bff728b25e3 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -15,7 +15,6 @@ #include #include #include -#include #include /* Registers */ diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index 93a71f66efeb..63db8e2172e0 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -12,9 +12,9 @@ #include #include #include +#include #include #include -#include #include /* Registers */ diff --git a/drivers/irqchip/irq-madera.c b/drivers/irqchip/irq-madera.c index 8b81271c823c..3eb1f8cdf674 100644 --- a/drivers/irqchip/irq-madera.c +++ b/drivers/irqchip/irq-madera.c @@ -10,12 +10,10 @@ #include #include #include +#include #include #include #include -#include -#include -#include #include #include #include diff --git a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c index ee69e8d83b9a..0f64ecb9b1f4 100644 --- a/drivers/irqchip/irq-pruss-intc.c +++ b/drivers/irqchip/irq-pruss-intc.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include /* diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index d30614661eea..7124565234a5 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c index 26e4c17a7bf2..fa19585f3dee 100644 --- a/drivers/irqchip/irq-renesas-intc-irqpin.c +++ b/drivers/irqchip/irq-renesas-intc-irqpin.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #define INTC_IRQPIN_MAX 8 /* maximum 8 interrupts per driver instance */ diff --git a/drivers/irqchip/irq-st.c b/drivers/irqchip/irq-st.c index 819a12297b58..de71bb350d57 100644 --- a/drivers/irqchip/irq-st.c +++ b/drivers/irqchip/irq-st.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c index b5fa76ce5046..d8ba5fba7450 100644 --- a/drivers/irqchip/irq-stm32-exti.c +++ b/drivers/irqchip/irq-stm32-exti.c @@ -14,10 +14,11 @@ #include #include #include +#include #include #include #include -#include +#include #include #include diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c index 21d49791f855..e760b1278143 100644 --- a/drivers/irqchip/irq-sunxi-nmi.c +++ b/drivers/irqchip/irq-sunxi-nmi.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include diff --git a/drivers/irqchip/irq-tb10x.c b/drivers/irqchip/irq-tb10x.c index 8a0e69298e83..680586354d12 100644 --- a/drivers/irqchip/irq-tb10x.c +++ b/drivers/irqchip/irq-tb10x.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c index 7133f9fa6fd9..b83f5cbab123 100644 --- a/drivers/irqchip/irq-ti-sci-inta.c +++ b/drivers/irqchip/irq-ti-sci-inta.c @@ -15,9 +15,9 @@ #include #include #include -#include +#include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c index 1186f1e431a3..c027cd9e4a69 100644 --- a/drivers/irqchip/irq-ti-sci-intr.c +++ b/drivers/irqchip/irq-ti-sci-intr.c @@ -12,9 +12,9 @@ #include #include #include -#include -#include +#include #include +#include #include /** diff --git a/drivers/irqchip/irq-uniphier-aidet.c b/drivers/irqchip/irq-uniphier-aidet.c index 716b1bb88bf2..601f9343d5b3 100644 --- a/drivers/irqchip/irq-uniphier-aidet.c +++ b/drivers/irqchip/irq-uniphier-aidet.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 7899607fbee8..1eeb0d0156ce 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index d96916cf6a41..a32c0d28d038 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include From 32b5f8a6f5de384291ea4083541ad10725c7a054 Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Tue, 4 Jul 2023 18:02:52 +0800 Subject: [PATCH 122/130] irqchip/ls-scfg-msi: Use devm_platform_get_and_ioremap_resource() Convert platform_get_resource(), devm_ioremap_resource() to a single call to devm_platform_get_and_ioremap_resource(), as this is exactly what this function does. Signed-off-by: Yangtao Li Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230704100253.36045-1-frank.li@vivo.com --- drivers/irqchip/irq-ls-scfg-msi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c index f5ba3f9f8415..f31a262fe438 100644 --- a/drivers/irqchip/irq-ls-scfg-msi.c +++ b/drivers/irqchip/irq-ls-scfg-msi.c @@ -349,8 +349,7 @@ static int ls_scfg_msi_probe(struct platform_device *pdev) msi_data->cfg = (struct ls_scfg_msi_cfg *) match->data; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - msi_data->regs = devm_ioremap_resource(&pdev->dev, res); + msi_data->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(msi_data->regs)) { dev_err(&pdev->dev, "failed to initialize 'regs'\n"); return PTR_ERR(msi_data->regs); From 69da32b5438a9f3dc5b26eee0a94372b232a603c Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Tue, 4 Jul 2023 18:02:53 +0800 Subject: [PATCH 123/130] irqchip/irq-mvebu-sei: Use devm_platform_get_and_ioremap_resource() Convert platform_get_resource(), devm_ioremap_resource() to a single call to devm_platform_get_and_ioremap_resource(), as this is exactly what this function does. Signed-off-by: Yangtao Li Reviewed-by: Andrew Lunn Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230704100253.36045-2-frank.li@vivo.com --- drivers/irqchip/irq-mvebu-sei.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-mvebu-sei.c b/drivers/irqchip/irq-mvebu-sei.c index 4ecef6d83777..a48dbe91b036 100644 --- a/drivers/irqchip/irq-mvebu-sei.c +++ b/drivers/irqchip/irq-mvebu-sei.c @@ -377,8 +377,7 @@ static int mvebu_sei_probe(struct platform_device *pdev) mutex_init(&sei->cp_msi_lock); raw_spin_lock_init(&sei->mask_lock); - sei->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - sei->base = devm_ioremap_resource(sei->dev, sei->res); + sei->base = devm_platform_get_and_ioremap_resource(pdev, 0, &sei->res); if (IS_ERR(sei->base)) return PTR_ERR(sei->base); From 2b709c6760ecf6d953ce15b2301ae0c24793cbe1 Mon Sep 17 00:00:00 2001 From: Huqiang Qin Date: Wed, 28 Jun 2023 17:15:32 +0800 Subject: [PATCH 124/130] dt-bindings: interrupt-controller: Add support for Amlogic-C3 SoCs Update dt-binding document for GPIO interrupt controller of Amlogic-C3 SoCs Signed-off-by: Huqiang Qin Acked-by: Conor Dooley Acked-by: Martin Blumenstingl Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230628091533.3884385-2-huqiang.qin@amlogic.com --- .../bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml index e84e4f33b358..3d06db98e978 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml @@ -35,6 +35,7 @@ properties: - amlogic,meson-sm1-gpio-intc - amlogic,meson-a1-gpio-intc - amlogic,meson-s4-gpio-intc + - amlogic,c3-gpio-intc - const: amlogic,meson-gpio-intc reg: From 19b5a44bee16518104e8a159ab9a60788292fbd4 Mon Sep 17 00:00:00 2001 From: Huqiang Qin Date: Wed, 28 Jun 2023 17:15:33 +0800 Subject: [PATCH 125/130] irqchip: Add support for Amlogic-C3 SoCs The Amlogic-C3 SoCs support 12 GPIO IRQ lines compared with previous serial chips and have something different, details are as below. IRQ Number: - 54 1 pins on bank TESTN - 53:40 14 pins on bank X - 39:33 7 pins on bank D - 32:27 6 pins on bank A - 26:22 5 pins on bank E - 21:15 7 pins on bank C - 14:0 15 pins on bank B Signed-off-by: Huqiang Qin Acked-by: Martin Blumenstingl Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230628091533.3884385-3-huqiang.qin@amlogic.com --- drivers/irqchip/irq-meson-gpio.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index 7da18ef95211..f88df39f4129 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -150,6 +150,10 @@ static const struct meson_gpio_irq_params s4_params = { INIT_MESON_S4_COMMON_DATA(82) }; +static const struct meson_gpio_irq_params c3_params = { + INIT_MESON_S4_COMMON_DATA(55) +}; + static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson8-gpio-intc", .data = &meson8_params }, { .compatible = "amlogic,meson8b-gpio-intc", .data = &meson8b_params }, @@ -160,6 +164,7 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson-sm1-gpio-intc", .data = &sm1_params }, { .compatible = "amlogic,meson-a1-gpio-intc", .data = &a1_params }, { .compatible = "amlogic,meson-s4-gpio-intc", .data = &s4_params }, + { .compatible = "amlogic,c3-gpio-intc", .data = &c3_params }, { } }; From 2ddd3cac1fa988323684cee567356e970e6750bd Mon Sep 17 00:00:00 2001 From: Elena Reshetova Date: Thu, 17 Aug 2023 21:13:32 -0700 Subject: [PATCH 126/130] nsproxy: Convert nsproxy.count to refcount_t atomic_t variables are currently used to implement reference counters with the following properties: - counter is initialized to 1 using atomic_set() - a resource is freed upon counter reaching zero - once counter reaches zero, its further increments aren't allowed - counter schema uses basic atomic operations (set, inc, inc_not_zero, dec_and_test, etc.) Such atomic variables should be converted to a newly provided refcount_t type and API that prevents accidental counter overflows and underflows. This is important since overflows and underflows can lead to use-after-free situation and be exploitable. The variable nsproxy.count is used as pure reference counter. Convert it to refcount_t and fix up the operations. **Important note for maintainers: Some functions from refcount_t API defined in refcount.h have different memory ordering guarantees than their atomic counterparts. Please check Documentation/core-api/refcount-vs-atomic.rst for more information. Normally the differences should not matter since refcount_t provides enough guarantees to satisfy the refcounting use cases, but in some rare cases it might matter. Please double check that you don't have some undocumented memory guarantees for this variable usage. For the nsproxy.count it might make a difference in following places: - put_nsproxy() and switch_task_namespaces(): decrement in refcount_dec_and_test() only provides RELEASE ordering and ACQUIRE ordering on success vs. fully ordered atomic counterpart Suggested-by: Kees Cook Signed-off-by: Elena Reshetova Reviewed-by: David Windsor Reviewed-by: Hans Liljestrand Reviewed-by: Christian Brauner Link: https://lore.kernel.org/r/20230818041327.gonna.210-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/nsproxy.h | 7 +++---- kernel/nsproxy.c | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h index fee881cded01..771cb0285872 100644 --- a/include/linux/nsproxy.h +++ b/include/linux/nsproxy.h @@ -29,7 +29,7 @@ struct fs_struct; * nsproxy is copied. */ struct nsproxy { - atomic_t count; + refcount_t count; struct uts_namespace *uts_ns; struct ipc_namespace *ipc_ns; struct mnt_namespace *mnt_ns; @@ -102,14 +102,13 @@ int __init nsproxy_cache_init(void); static inline void put_nsproxy(struct nsproxy *ns) { - if (atomic_dec_and_test(&ns->count)) { + if (refcount_dec_and_test(&ns->count)) free_nsproxy(ns); - } } static inline void get_nsproxy(struct nsproxy *ns) { - atomic_inc(&ns->count); + refcount_inc(&ns->count); } #endif diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index 80d9c6d77a45..15781acaac1c 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -30,7 +30,7 @@ static struct kmem_cache *nsproxy_cachep; struct nsproxy init_nsproxy = { - .count = ATOMIC_INIT(1), + .count = REFCOUNT_INIT(1), .uts_ns = &init_uts_ns, #if defined(CONFIG_POSIX_MQUEUE) || defined(CONFIG_SYSVIPC) .ipc_ns = &init_ipc_ns, @@ -55,7 +55,7 @@ static inline struct nsproxy *create_nsproxy(void) nsproxy = kmem_cache_alloc(nsproxy_cachep, GFP_KERNEL); if (nsproxy) - atomic_set(&nsproxy->count, 1); + refcount_set(&nsproxy->count, 1); return nsproxy; } From 1dfe3a5a7cefbe2162cecb759f3933baea22c393 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 21 Aug 2023 17:35:26 +0100 Subject: [PATCH 127/130] entry: Remove empty addr_limit_user_check() Back when set_fs() was a generic API for altering the address limit, addr_limit_user_check() was a safety measure to prevent userspace being able to issue syscalls with an unbound limit. With the the removal of set_fs() as a generic API, the last user of addr_limit_user_check() was removed in commit: b5a5a01d8e9a44ec ("arm64: uaccess: remove addr_limit_user_check()") ... as since that commit, no architecture defines TIF_FSCHECK, and hence addr_limit_user_check() always expands to nothing. Remove addr_limit_user_check(), updating the comment in exit_to_user_mode_prepare() to no longer refer to it. At the same time, the comment is reworded to be a little more generic so as to cover kmap_assert_nomap() in addition to lockdep_sys_exit(). No functional change. Signed-off-by: Mark Rutland Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230821163526.2319443-1-mark.rutland@arm.com --- include/linux/syscalls.h | 16 ---------------- kernel/entry/common.c | 3 +-- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 03e3d0121d5e..c4b9b66b0d05 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -283,22 +283,6 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event) #define SYSCALL32_DEFINE6 SYSCALL_DEFINE6 #endif -/* - * Called before coming back to user-mode. Returning to user-mode with an - * address limit different than USER_DS can allow to overwrite kernel memory. - */ -static inline void addr_limit_user_check(void) -{ -#ifdef TIF_FSCHECK - if (!test_thread_flag(TIF_FSCHECK)) - return; -#endif - -#ifdef TIF_FSCHECK - clear_thread_flag(TIF_FSCHECK); -#endif -} - /* * These syscall function prototypes are kept in the same order as * include/uapi/asm-generic/unistd.h. Architecture specific entries go below, diff --git a/kernel/entry/common.c b/kernel/entry/common.c index be61332c66b5..d7ee4bc3f2ba 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -205,8 +205,7 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) arch_exit_to_user_mode_prepare(regs, ti_work); - /* Ensure that the address limit is intact and no locks are held */ - addr_limit_user_check(); + /* Ensure that kernel state is sane for a return to userspace */ kmap_assert_nomap(); lockdep_assert_irqs_disabled(); lockdep_sys_exit(); From 33f0467fe06934d5e4ea6e24ce2b9c65ce618e26 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Thu, 24 Aug 2023 20:46:59 -0700 Subject: [PATCH 128/130] kallsyms: Fix kallsyms_selftest failure Kernel test robot reported a kallsyms_test failure when clang lto is enabled (thin or full) and CONFIG_KALLSYMS_SELFTEST is also enabled. I can reproduce in my local environment with the following error message with thin lto: [ 1.877897] kallsyms_selftest: Test for 1750th symbol failed: (tsc_cs_mark_unstable) addr=ffffffff81038090 [ 1.877901] kallsyms_selftest: abort It appears that commit 8cc32a9bbf29 ("kallsyms: strip LTO-only suffixes from promoted global functions") caused the failure. Commit 8cc32a9bbf29 changed cleanup_symbol_name() based on ".llvm." instead of '.' where ".llvm." is appended to a before-lto-optimization local symbol name. We need to propagate such knowledge in kallsyms_selftest.c as well. Further more, compare_symbol_name() in kallsyms.c needs change as well. In scripts/kallsyms.c, kallsyms_names and kallsyms_seqs_of_names are used to record symbol names themselves and index to symbol names respectively. For example: kallsyms_names: ... __amd_smn_rw._entry <== seq 1000 __amd_smn_rw._entry.5 <== seq 1001 __amd_smn_rw.llvm. <== seq 1002 ... kallsyms_seqs_of_names are sorted based on cleanup_symbol_name() through, so the order in kallsyms_seqs_of_names actually has index 1000: seq 1002 <== __amd_smn_rw.llvm. (actual symbol comparison using '__amd_smn_rw') index 1001: seq 1000 <== __amd_smn_rw._entry index 1002: seq 1001 <== __amd_smn_rw._entry.5 Let us say at a particular point, at index 1000, symbol '__amd_smn_rw.llvm.' is comparing to '__amd_smn_rw._entry' where '__amd_smn_rw._entry' is the one to search e.g., with function kallsyms_on_each_match_symbol(). The current implementation will find out '__amd_smn_rw._entry' is less than '__amd_smn_rw.llvm.' and then continue to search e.g., index 999 and never found a match although the actual index 1001 is a match. To fix this issue, let us do cleanup_symbol_name() first and then do comparison. In the above case, comparing '__amd_smn_rw' vs '__amd_smn_rw._entry' and '__amd_smn_rw._entry' being greater than '__amd_smn_rw', the next comparison will be > index 1000 and eventually index 1001 will be hit an a match is found. For any symbols not having '.llvm.' substr, there is no functionality change for compare_symbol_name(). Fixes: 8cc32a9bbf29 ("kallsyms: strip LTO-only suffixes from promoted global functions") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-lkp/202308232200.1c932a90-oliver.sang@intel.com Signed-off-by: Yonghong Song Reviewed-by: Song Liu Reviewed-by: Zhen Lei Link: https://lore.kernel.org/r/20230825034659.1037627-1-yonghong.song@linux.dev Cc: stable@vger.kernel.org Signed-off-by: Kees Cook --- kernel/kallsyms.c | 17 +++++++---------- kernel/kallsyms_selftest.c | 23 +---------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 016d997131d4..e12d26c10dba 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -188,16 +188,13 @@ static bool cleanup_symbol_name(char *s) static int compare_symbol_name(const char *name, char *namebuf) { - int ret; - - ret = strcmp(name, namebuf); - if (!ret) - return ret; - - if (cleanup_symbol_name(namebuf) && !strcmp(name, namebuf)) - return 0; - - return ret; + /* The kallsyms_seqs_of_names is sorted based on names after + * cleanup_symbol_name() (see scripts/kallsyms.c) if clang lto is enabled. + * To ensure correct bisection in kallsyms_lookup_names(), do + * cleanup_symbol_name(namebuf) before comparing name and namebuf. + */ + cleanup_symbol_name(namebuf); + return strcmp(name, namebuf); } static unsigned int get_symbol_seq(int index) diff --git a/kernel/kallsyms_selftest.c b/kernel/kallsyms_selftest.c index a2e3745d15c4..e05ddc33a752 100644 --- a/kernel/kallsyms_selftest.c +++ b/kernel/kallsyms_selftest.c @@ -196,7 +196,7 @@ static bool match_cleanup_name(const char *s, const char *name) if (!IS_ENABLED(CONFIG_LTO_CLANG)) return false; - p = strchr(s, '.'); + p = strstr(s, ".llvm."); if (!p) return false; @@ -344,27 +344,6 @@ static int test_kallsyms_basic_function(void) goto failed; } - /* - * The first '.' may be the initial letter, in which case the - * entire symbol name will be truncated to an empty string in - * cleanup_symbol_name(). Do not test these symbols. - * - * For example: - * cat /proc/kallsyms | awk '{print $3}' | grep -E "^\." | head - * .E_read_words - * .E_leading_bytes - * .E_trailing_bytes - * .E_write_words - * .E_copy - * .str.292.llvm.12122243386960820698 - * .str.24.llvm.12122243386960820698 - * .str.29.llvm.12122243386960820698 - * .str.75.llvm.12122243386960820698 - * .str.99.llvm.12122243386960820698 - */ - if (IS_ENABLED(CONFIG_LTO_CLANG) && !namebuf[0]) - continue; - lookup_addr = kallsyms_lookup_name(namebuf); memset(stat, 0, sizeof(*stat)); From 76903a9648744c081547c91f31ec3917204b74e5 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 25 Aug 2023 13:20:36 -0700 Subject: [PATCH 129/130] kallsyms: Change func signature for cleanup_symbol_name() All users of cleanup_symbol_name() do not use the return value. So let us change the return value of cleanup_symbol_name() to 'void' to reflect its usage pattern. Suggested-by: Nick Desaulniers Signed-off-by: Yonghong Song Reviewed-by: Nick Desaulniers Reviewed-by: Song Liu Link: https://lore.kernel.org/r/20230825202036.441212-1-yonghong.song@linux.dev Signed-off-by: Kees Cook --- kernel/kallsyms.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index e12d26c10dba..18edd57b5fe8 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -163,12 +163,12 @@ unsigned long kallsyms_sym_address(int idx) return kallsyms_relative_base - 1 - kallsyms_offsets[idx]; } -static bool cleanup_symbol_name(char *s) +static void cleanup_symbol_name(char *s) { char *res; if (!IS_ENABLED(CONFIG_LTO_CLANG)) - return false; + return; /* * LLVM appends various suffixes for local functions and variables that @@ -178,12 +178,10 @@ static bool cleanup_symbol_name(char *s) * - foo.llvm.[0-9a-f]+ */ res = strstr(s, ".llvm."); - if (res) { + if (res) *res = '\0'; - return true; - } - return false; + return; } static int compare_symbol_name(const char *name, char *namebuf) From 5f536ac6a5a7b67351e4e5ae4f9e1e57d31268e6 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 17 Aug 2023 16:59:56 -0700 Subject: [PATCH 130/130] LoadPin: Annotate struct dm_verity_loadpin_trusted_root_digest with __counted_by Prepare for the coming implementation by GCC and Clang of the __counted_by attribute. Flexible array members annotated with __counted_by can have their accesses bounds-checked at run-time checking via CONFIG_UBSAN_BOUNDS (for array indexing) and CONFIG_FORTIFY_SOURCE (for strcpy/memcpy-family functions). As found with Coccinelle[1], add __counted_by for struct dm_verity_loadpin_trusted_root_digest. Additionally, since the element count member must be set before accessing the annotated flexible array member, move its initialization earlier. [1] https://github.com/kees/kernel-tools/blob/trunk/coccinelle/examples/counted_by.cocci Cc: Alasdair Kergon Cc: Mike Snitzer Cc: dm-devel@redhat.com Cc: Paul Moore Cc: James Morris Cc: "Serge E. Hallyn" Cc: linux-security-module@vger.kernel.org Link: https://lore.kernel.org/r/20230817235955.never.762-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/dm-verity-loadpin.h | 2 +- security/loadpin/loadpin.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/linux/dm-verity-loadpin.h b/include/linux/dm-verity-loadpin.h index 552b817ab102..3ac6dbaeaa37 100644 --- a/include/linux/dm-verity-loadpin.h +++ b/include/linux/dm-verity-loadpin.h @@ -12,7 +12,7 @@ extern struct list_head dm_verity_loadpin_trusted_root_digests; struct dm_verity_loadpin_trusted_root_digest { struct list_head node; unsigned int len; - u8 data[]; + u8 data[] __counted_by(len); }; #if IS_ENABLED(CONFIG_SECURITY_LOADPIN_VERITY) diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index ebae964f7cc9..a9d40456a064 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -336,6 +336,7 @@ static int read_trusted_verity_root_digests(unsigned int fd) rc = -ENOMEM; goto err; } + trd->len = len; if (hex2bin(trd->data, d, len)) { kfree(trd); @@ -343,8 +344,6 @@ static int read_trusted_verity_root_digests(unsigned int fd) goto err; } - trd->len = len; - list_add_tail(&trd->node, &dm_verity_loadpin_trusted_root_digests); }