From 11e46f0804c406ff5cd67f5ed8387ee8a2f30b8a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 15 Jun 2021 11:14:19 -0700 Subject: [PATCH 001/134] torture: Apply CONFIG_KCSAN_STRICT to kvm.sh --kcsan argument Currently, the --kcsan argument to kvm.sh applies a laundry list of Kconfig options. Now that KCSAN provides the CONFIG_KCSAN_STRICT Kconfig option, this commit reduces the laundry list to this one option. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index f442d84fb2a3..6cf9ec6a3d1c 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -184,7 +184,7 @@ do TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG ;; --kcsan) - TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_INTERRUPT_WATCHER=y CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG + TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_STRICT=y CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y"; export TORTURE_KCONFIG_KCSAN_ARG ;; --kmake-arg|--kmake-args) checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' From 57d4374be94aa27712fbcb2f0fa2a30802c43556 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 3 Sep 2021 15:48:22 +0000 Subject: [PATCH 002/134] audit: rename struct node to struct audit_node to prevent future name collisions Future work in the powerpc code results in a name collision with the identified "node" as struct node defined in kernel/audit_tree.c conflicts with struct node defined in include/linux/node.h (below). This patch takes the proactive route and renames the audit code's struct node to audit_node. CC kernel/audit_tree.o kernel/audit_tree.c:33:9: error: redefinition of 'struct node' 33 | struct node { | ^~~~ In file included from ./include/linux/cpu.h:17, from ./include/linux/static_call.h:102, from ./arch/powerpc/include/asm/machdep.h:10, from ./arch/powerpc/include/asm/archrandom.h:7, from ./include/linux/random.h:121, from ./include/linux/net.h:18, from ./include/linux/skbuff.h:26, from kernel/audit.h:11, from kernel/audit_tree.c:2: ./include/linux/node.h:84:8: note: originally defined here 84 | struct node { | ^~~~ make[2]: *** [kernel/audit_tree.o] Error 1 Signed-off-by: Christophe Leroy Reviewed-by: Richard Guy Briggs [PM: rewrite subj/desc as the build failure is just a RFC patch] Signed-off-by: Paul Moore --- kernel/audit_tree.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 2cd7b5694422..6ea71fe4aeff 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -30,7 +30,7 @@ struct audit_chunk { int count; atomic_long_t refs; struct rcu_head head; - struct node { + struct audit_node { struct list_head list; struct audit_tree *owner; unsigned index; /* index; upper bit indicates 'will prune' */ @@ -269,7 +269,7 @@ bool audit_tree_match(struct audit_chunk *chunk, struct audit_tree *tree) /* tagging and untagging inodes with trees */ -static struct audit_chunk *find_chunk(struct node *p) +static struct audit_chunk *find_chunk(struct audit_node *p) { int index = p->index & ~(1U<<31); p -= index; @@ -322,7 +322,7 @@ static void replace_chunk(struct audit_chunk *new, struct audit_chunk *old) list_replace_rcu(&old->hash, &new->hash); } -static void remove_chunk_node(struct audit_chunk *chunk, struct node *p) +static void remove_chunk_node(struct audit_chunk *chunk, struct audit_node *p) { struct audit_tree *owner = p->owner; @@ -459,7 +459,7 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree) { struct fsnotify_mark *mark; struct audit_chunk *chunk, *old; - struct node *p; + struct audit_node *p; int n; mutex_lock(&audit_tree_group->mark_mutex); @@ -570,11 +570,11 @@ static void prune_tree_chunks(struct audit_tree *victim, bool tagged) { spin_lock(&hash_lock); while (!list_empty(&victim->chunks)) { - struct node *p; + struct audit_node *p; struct audit_chunk *chunk; struct fsnotify_mark *mark; - p = list_first_entry(&victim->chunks, struct node, list); + p = list_first_entry(&victim->chunks, struct audit_node, list); /* have we run out of marked? */ if (tagged && !(p->index & (1U<<31))) break; @@ -616,7 +616,7 @@ static void trim_marked(struct audit_tree *tree) } /* reorder */ for (p = tree->chunks.next; p != &tree->chunks; p = q) { - struct node *node = list_entry(p, struct node, list); + struct audit_node *node = list_entry(p, struct audit_node, list); q = p->next; if (node->index & (1U<<31)) { list_del_init(p); @@ -684,7 +684,7 @@ void audit_trim_trees(void) struct audit_tree *tree; struct path path; struct vfsmount *root_mnt; - struct node *node; + struct audit_node *node; int err; tree = container_of(cursor.next, struct audit_tree, list); @@ -839,7 +839,7 @@ int audit_add_tree_rule(struct audit_krule *rule) drop_collected_mounts(mnt); if (!err) { - struct node *node; + struct audit_node *node; spin_lock(&hash_lock); list_for_each_entry(node, &tree->chunks, list) node->index &= ~(1U<<31); @@ -938,7 +938,7 @@ int audit_tag_tree(char *old, char *new) mutex_unlock(&audit_filter_mutex); if (!failed) { - struct node *node; + struct audit_node *node; spin_lock(&hash_lock); list_for_each_entry(node, &tree->chunks, list) node->index &= ~(1U<<31); From 2431774f04d1050292054c763070021bade7b151 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 20 Jul 2021 06:16:27 -0700 Subject: [PATCH 003/134] rcu: Mark accesses to rcu_state.n_force_qs This commit marks accesses to the rcu_state.n_force_qs. These data races are hard to make happen, but syzkaller was equal to the task. Reported-by: syzbot+e08a83a1940ec3846cd5@syzkaller.appspotmail.com Acked-by: Marco Elver Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index bce848e50512..c89f5e6c4154 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1907,7 +1907,7 @@ static void rcu_gp_fqs(bool first_time) struct rcu_node *rnp = rcu_get_root(); WRITE_ONCE(rcu_state.gp_activity, jiffies); - rcu_state.n_force_qs++; + WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1); if (first_time) { /* Collect dyntick-idle snapshots. */ force_qs_rnp(dyntick_save_progress_counter); @@ -2550,7 +2550,7 @@ static void rcu_do_batch(struct rcu_data *rdp) /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */ if (count == 0 && rdp->qlen_last_fqs_check != 0) { rdp->qlen_last_fqs_check = 0; - rdp->n_force_qs_snap = rcu_state.n_force_qs; + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); } else if (count < rdp->qlen_last_fqs_check - qhimark) rdp->qlen_last_fqs_check = count; @@ -2898,10 +2898,10 @@ static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, } else { /* Give the grace period a kick. */ rdp->blimit = DEFAULT_MAX_RCU_BLIMIT; - if (rcu_state.n_force_qs == rdp->n_force_qs_snap && + if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap && rcu_segcblist_first_pend_cb(&rdp->cblist) != head) rcu_force_quiescent_state(); - rdp->n_force_qs_snap = rcu_state.n_force_qs; + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); } } @@ -4128,7 +4128,7 @@ int rcutree_prepare_cpu(unsigned int cpu) /* Set up local state, ensuring consistent view of global state. */ raw_spin_lock_irqsave_rcu_node(rnp, flags); rdp->qlen_last_fqs_check = 0; - rdp->n_force_qs_snap = rcu_state.n_force_qs; + rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); rdp->blimit = blimit; rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ rcu_dynticks_eqs_online(); From 52b030aa278642194f5d25872c33360013b0167e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 21 Jul 2021 13:24:35 -0700 Subject: [PATCH 004/134] rcu-nocb: Fix a couple of tree_nocb code-style nits This commit removes a non-value-returning "return" statement at the end of __call_rcu_nocb_wake() and adds a blank line following declarations in nocb_cb_can_run(). Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_nocb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 8fdf44f8523f..368ef7b9af4f 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -549,7 +549,6 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, rcu_nocb_unlock_irqrestore(rdp, flags); trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); } - return; } /* @@ -767,6 +766,7 @@ static int rcu_nocb_gp_kthread(void *arg) static inline bool nocb_cb_can_run(struct rcu_data *rdp) { u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_CB; + return rcu_segcblist_test_flags(&rdp->cblist, flags); } From 88ee23ef1c129e40309f4612f80dd74be4590c03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 22 Jul 2021 15:49:05 -0700 Subject: [PATCH 005/134] rcu: Eliminate rcu_implicit_dynticks_qs() local variable rnhqp The rcu_implicit_dynticks_qs() function's local variable rnhqp references the ->rcu_need_heavy_qs field in the rcu_data structure referenced by the function parameter rdp, with a rather odd method for computing the pointer to this field. This commit therefore simplifies things and saves a few lines of code by replacing each instance of rnhqp with &rdp->need_heavy_qs. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c89f5e6c4154..18d2f35d1450 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1219,7 +1219,6 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp) static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) { unsigned long jtsq; - bool *rnhqp; bool *ruqp; struct rcu_node *rnp = rdp->mynode; @@ -1286,12 +1285,11 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) */ jtsq = READ_ONCE(jiffies_to_sched_qs); ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu); - rnhqp = per_cpu_ptr(&rcu_data.rcu_need_heavy_qs, rdp->cpu); - if (!READ_ONCE(*rnhqp) && + if (!READ_ONCE(rdp->rcu_need_heavy_qs) && (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || time_after(jiffies, rcu_state.jiffies_resched) || rcu_state.cbovld)) { - WRITE_ONCE(*rnhqp, true); + WRITE_ONCE(rdp->rcu_need_heavy_qs, true); /* Store rcu_need_heavy_qs before rcu_urgent_qs. */ smp_store_release(ruqp, true); } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { From 9424b867a759febc2b67b6777bfa27f0f830d437 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 22 Jul 2021 16:47:42 -0700 Subject: [PATCH 006/134] rcu: Eliminate rcu_implicit_dynticks_qs() local variable ruqp The rcu_implicit_dynticks_qs() function's local variable ruqp references the ->rcu_urgent_qs field in the rcu_data structure referenced by the function parameter rdp, with a rather odd method for computing the pointer to this field. This commit therefore simplifies things and saves a couple of lines of code by replacing each instance of ruqp with &rdp->need_heavy_qs. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 18d2f35d1450..0bfebec94277 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1219,7 +1219,6 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp) static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) { unsigned long jtsq; - bool *ruqp; struct rcu_node *rnp = rdp->mynode; /* @@ -1284,16 +1283,15 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) * is set way high. */ jtsq = READ_ONCE(jiffies_to_sched_qs); - ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu); if (!READ_ONCE(rdp->rcu_need_heavy_qs) && (time_after(jiffies, rcu_state.gp_start + jtsq * 2) || time_after(jiffies, rcu_state.jiffies_resched) || rcu_state.cbovld)) { WRITE_ONCE(rdp->rcu_need_heavy_qs, true); /* Store rcu_need_heavy_qs before rcu_urgent_qs. */ - smp_store_release(ruqp, true); + smp_store_release(&rdp->rcu_urgent_qs, true); } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) { - WRITE_ONCE(*ruqp, true); + WRITE_ONCE(rdp->rcu_urgent_qs, true); } /* @@ -1307,7 +1305,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) if (tick_nohz_full_cpu(rdp->cpu) && (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) || rcu_state.cbovld)) { - WRITE_ONCE(*ruqp, true); + WRITE_ONCE(rdp->rcu_urgent_qs, true); resched_cpu(rdp->cpu); WRITE_ONCE(rdp->last_fqs_resched, jiffies); } From 13bc8fa8057a064007d4e69c12799450123ef731 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 22 Jul 2021 21:41:48 -0700 Subject: [PATCH 007/134] doc: Add another stall-warning root cause in stallwarn.rst This commit adds a bullet item noting that both deficiencies and surpluses of calls to rcu_*_enter() and rcu_*_exit() can result in RCU CPU stall warnings. Signed-off-by: Paul E. McKenney --- Documentation/RCU/stallwarn.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index 5036df24ae61..28f8ad16db25 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -96,6 +96,16 @@ warnings: the ``rcu_.*timer wakeup didn't happen for`` console-log message, which will include additional debugging information. +- A low-level kernel issue that either fails to invoke one of the + variants of rcu_user_enter(), rcu_user_exit(), rcu_idle_enter(), + rcu_idle_exit(), rcu_irq_enter(), or rcu_irq_exit() on the one + hand, or that invokes one of them too many times on the other. + Historically, the most frequent issue has been an omission + of either irq_enter() or irq_exit(), which in turn invoke + rcu_irq_enter() or rcu_irq_exit(), respectively. Building your + kernel with CONFIG_RCU_EQS_DEBUG=y can help track down these types + of issues, which sometimes arise in architecture-specific code. + - A bug in the RCU implementation. - A hardware failure. This is quite unlikely, but has occurred From 3ac858785231e2573d2964950738aac087ebf30c Mon Sep 17 00:00:00 2001 From: Zhouyi Zhou Date: Mon, 26 Jul 2021 05:43:33 +0800 Subject: [PATCH 008/134] rcu: Fix undefined Kconfig macros Invoking scripts/checkkconfigsymbols.py in the Linux-kernel source tree located the following issues: 1. TREE_PREEMPT_RCU Referencing files: arch/sh/configs/sdk7786_defconfig It should now be CONFIG_PREEMPT_RCU. Except that the CONFIG_PREEMPT=y in that same file implies CONFIG_PREEMPT_RCU=y. Therefore, delete the CONFIG_TREE_PREEMPT_RCU=y line. The reason is as follows: In kernel/rcu/Kconfig, we have config PREEMPT_RCU bool default y if PREEMPTION https://www.kernel.org/doc/Documentation/kbuild/kconfig-language.txt says, "The default value is only assigned to the config symbol if no other value was set by the user (via the input prompt above)." there is no prompt in config PREEMPT_RCU entry, so we are guaranteed to get CONFIG_PREEMPT_RCU=y when CONFIG_PREEMPT is present. 2. RCU_CPU_STALL_INFO Referencing files: arch/xtensa/configs/nommu_kc705_defconfig The old Kconfig option RCU_CPU_STALL_INFO was removed by commit 75c27f119b64 ("rcu: Remove CONFIG_RCU_CPU_STALL_INFO"), and the kernel now acts as if this Kconfig option was unconditionally enabled. 3. RCU_NOCB_CPU_ALL Referencing files: Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst This is an old snapshot of the code. I update this from the real rcu_prepare_for_idle() function in kernel/rcu/tree_plugin.h. This change was tested by invoking "make htmldocs". 4. RCU_TORTURE_TESTS Referencing files: kernel/rcu/rcutorture.c Forward-progress checking conflicts with CPU-stall testing, so we should complain at "modprobe rcutorture" when both are enabled. Signed-off-by: Zhouyi Zhou Signed-off-by: Paul E. McKenney --- .../Tree-RCU-Memory-Ordering.rst | 69 +++++++++---------- arch/sh/configs/sdk7786_defconfig | 1 - arch/xtensa/configs/nommu_kc705_defconfig | 1 - kernel/rcu/rcutorture.c | 2 +- 4 files changed, 33 insertions(+), 40 deletions(-) diff --git a/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst b/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst index eeb351296df1..7fdf151a8680 100644 --- a/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst +++ b/Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst @@ -202,49 +202,44 @@ newly arrived RCU callbacks against future grace periods: 1 static void rcu_prepare_for_idle(void) 2 { 3 bool needwake; - 4 struct rcu_data *rdp; - 5 struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); - 6 struct rcu_node *rnp; - 7 struct rcu_state *rsp; - 8 int tne; - 9 - 10 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_ALL) || - 11 rcu_is_nocb_cpu(smp_processor_id())) - 12 return; + 4 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + 5 struct rcu_node *rnp; + 6 int tne; + 7 + 8 lockdep_assert_irqs_disabled(); + 9 if (rcu_rdp_is_offloaded(rdp)) + 10 return; + 11 + 12 /* Handle nohz enablement switches conservatively. */ 13 tne = READ_ONCE(tick_nohz_active); - 14 if (tne != rdtp->tick_nohz_enabled_snap) { - 15 if (rcu_cpu_has_callbacks(NULL)) - 16 invoke_rcu_core(); - 17 rdtp->tick_nohz_enabled_snap = tne; + 14 if (tne != rdp->tick_nohz_enabled_snap) { + 15 if (!rcu_segcblist_empty(&rdp->cblist)) + 16 invoke_rcu_core(); /* force nohz to see update. */ + 17 rdp->tick_nohz_enabled_snap = tne; 18 return; - 19 } + 19 } 20 if (!tne) 21 return; - 22 if (rdtp->all_lazy && - 23 rdtp->nonlazy_posted != rdtp->nonlazy_posted_snap) { - 24 rdtp->all_lazy = false; - 25 rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted; - 26 invoke_rcu_core(); - 27 return; - 28 } - 29 if (rdtp->last_accelerate == jiffies) - 30 return; - 31 rdtp->last_accelerate = jiffies; - 32 for_each_rcu_flavor(rsp) { - 33 rdp = this_cpu_ptr(rsp->rda); - 34 if (rcu_segcblist_pend_cbs(&rdp->cblist)) - 35 continue; - 36 rnp = rdp->mynode; - 37 raw_spin_lock_rcu_node(rnp); - 38 needwake = rcu_accelerate_cbs(rsp, rnp, rdp); - 39 raw_spin_unlock_rcu_node(rnp); - 40 if (needwake) - 41 rcu_gp_kthread_wake(rsp); - 42 } - 43 } + 22 + 23 /* + 24 * If we have not yet accelerated this jiffy, accelerate all + 25 * callbacks on this CPU. + 26 */ + 27 if (rdp->last_accelerate == jiffies) + 28 return; + 29 rdp->last_accelerate = jiffies; + 30 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { + 31 rnp = rdp->mynode; + 32 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ + 33 needwake = rcu_accelerate_cbs(rnp, rdp); + 34 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ + 35 if (needwake) + 36 rcu_gp_kthread_wake(); + 37 } + 38 } But the only part of ``rcu_prepare_for_idle()`` that really matters for -this discussion are lines 37–39. We will therefore abbreviate this +this discussion are lines 32–34. We will therefore abbreviate this function as follows: .. kernel-figure:: rcu_node-lock.svg diff --git a/arch/sh/configs/sdk7786_defconfig b/arch/sh/configs/sdk7786_defconfig index f776a1d0d277..a8662b6927ec 100644 --- a/arch/sh/configs/sdk7786_defconfig +++ b/arch/sh/configs/sdk7786_defconfig @@ -5,7 +5,6 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y CONFIG_AUDIT=y CONFIG_AUDITSYSCALL=y -CONFIG_TREE_PREEMPT_RCU=y CONFIG_RCU_TRACE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y diff --git a/arch/xtensa/configs/nommu_kc705_defconfig b/arch/xtensa/configs/nommu_kc705_defconfig index 88b2e222d4bf..fcb620ef3799 100644 --- a/arch/xtensa/configs/nommu_kc705_defconfig +++ b/arch/xtensa/configs/nommu_kc705_defconfig @@ -119,7 +119,6 @@ CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_STACKTRACE=y -# CONFIG_RCU_CPU_STALL_INFO is not set CONFIG_RCU_TRACE=y # CONFIG_FTRACE is not set # CONFIG_LD_NO_RELAX is not set diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ab4215266ebe..f640cbd9262c 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2449,7 +2449,7 @@ static int __init rcu_torture_fwd_prog_init(void) } if (stall_cpu > 0) { VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); - if (IS_MODULE(CONFIG_RCU_TORTURE_TESTS)) + if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) return -EINVAL; /* In module, can fail back to user. */ WARN_ON(1); /* Make sure rcutorture notices conflict. */ return 0; From ebc88ad491362e6a4fae5bfb1c23c06c876f70be Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jul 2021 11:57:39 -0700 Subject: [PATCH 009/134] rcu: Comment rcu_gp_init() code waiting for CPU-hotplug operations Near the beginning of rcu_gp_init() is a per-rcu_node loop that waits for CPU-hotplug operations that might have started before the new grace period did. This commit adds a comment explaining that this wait does not exclude CPU-hotplug operations. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0bfebec94277..e6e1b9281530 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1775,6 +1775,8 @@ static noinline_for_stack bool rcu_gp_init(void) */ WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF); rcu_for_each_leaf_node(rnp) { + // Wait for CPU-hotplug operations that might have + // started before this grace period did. smp_mb(); // Pair with barriers used when updating ->ofl_seq to odd values. firstseq = READ_ONCE(rnp->ofl_seq); if (firstseq & 0x1) From 2caebefb00f03b5ba13d44aa6cc3723759b43822 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 28 Jul 2021 12:38:42 -0700 Subject: [PATCH 010/134] rcu: Move rcu_dynticks_eqs_online() to rcu_cpu_starting() The purpose of rcu_dynticks_eqs_online() is to adjust the ->dynticks counter of an incoming CPU when required. It is currently invoked from rcutree_prepare_cpu(), which runs before the incoming CPU is running, and thus on some other CPU. This makes the per-CPU accesses in rcu_dynticks_eqs_online() iffy at best, and it all "works" only because the running CPU cannot possibly be in dyntick-idle mode, which means that rcu_dynticks_eqs_online() never has any effect. It is currently OK for rcu_dynticks_eqs_online() to have no effect, but only because the CPU-offline process just happens to leave ->dynticks in the correct state. After all, if ->dynticks were in the wrong state on a just-onlined CPU, rcutorture would complain bitterly the next time that CPU went idle, at least in kernels built with CONFIG_RCU_EQS_DEBUG=y, for example, those built by rcutorture scenario TREE04. One could argue that this means that rcu_dynticks_eqs_online() is unnecessary, however, removing it would make the CPU-online process vulnerable to slight changes in the CPU-offline process. One could also ask why it is safe to move the rcu_dynticks_eqs_online() call so late in the CPU-online process. Indeed, there was a time when it would not have been safe, which does much to explain its current location. However, the marking of a CPU as online from an RCU perspective has long since moved from rcutree_prepare_cpu() to rcu_cpu_starting(), and all that is required is that ->dynticks be set correctly by the time that the CPU is marked as online from an RCU perspective. After all, the RCU grace-period kthread does not check to see if offline CPUs are also idle. (In case you were curious, this is one reason why there is quiescent-state reporting as part of the offlining process.) This commit therefore moves the call to rcu_dynticks_eqs_online() from rcutree_prepare_cpu() to rcu_cpu_starting(), this latter being guaranteed to be running on the incoming CPU. The call to this function must of course be placed before this rcu_cpu_starting() announces this CPU's presence to RCU. Reported-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index e6e1b9281530..801075e36515 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4129,7 +4129,6 @@ int rcutree_prepare_cpu(unsigned int cpu) rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs); rdp->blimit = blimit; rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */ - rcu_dynticks_eqs_online(); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ /* @@ -4249,6 +4248,7 @@ void rcu_cpu_starting(unsigned int cpu) mask = rdp->grpmask; WRITE_ONCE(rnp->ofl_seq, rnp->ofl_seq + 1); WARN_ON_ONCE(!(rnp->ofl_seq & 0x1)); + rcu_dynticks_eqs_online(); smp_mb(); // Pair with rcu_gp_cleanup()'s ->ofl_seq barrier(). raw_spin_lock_irqsave_rcu_node(rnp, flags); WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext | mask); From 768f5d50e6ad88363291f96a2e230442b8d633bc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 29 Jul 2021 15:35:21 -0700 Subject: [PATCH 011/134] rcu: Simplify rcu_report_dead() call to rcu_report_exp_rdp() Currently, rcu_report_dead() disables preemption across its call to rcu_report_exp_rdp(), but this is pointless because interrupts are already disabled by the caller. In addition, rcu_report_dead() computes the address of the outgoing CPU's rcu_data structure, which is also pointless because this address is already present in local variable rdp. This commit therefore drops the preemption disabling and passes rdp to rcu_report_exp_rdp(). Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 801075e36515..dc2968473593 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4294,9 +4294,7 @@ void rcu_report_dead(unsigned int cpu) do_nocb_deferred_wakeup(rdp); /* QS for any half-done expedited grace period. */ - preempt_disable(); - rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); - preempt_enable(); + rcu_report_exp_rdp(rdp); rcu_preempt_deferred_qs(current); /* Remove outgoing CPU from mask in the leaf rcu_node structure. */ From 4aa846f97c0c0d9740d120f9ac3e2fba1522ac0c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 29 Jul 2021 20:30:32 -0700 Subject: [PATCH 012/134] rcu: Make rcutree_dying_cpu() use its "cpu" parameter The CPU-hotplug functions take a "cpu" parameter, but rcutree_dying_cpu() ignores it in favor of this_cpu_ptr(). This works at the moment, but it would be better to be consistent. This might also work better given some possible future changes. This commit therefore uses per_cpu_ptr() to avoid ignoring the rcutree_dying_cpu() function's argument. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index dc2968473593..6a1e9d3374db 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2356,7 +2356,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp) int rcutree_dying_cpu(unsigned int cpu) { bool blkd; - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); struct rcu_node *rnp = rdp->mynode; if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) From ebb6d30d9ed1fe7137486e1be2ae9d621e918c4a Mon Sep 17 00:00:00 2001 From: Juri Lelli Date: Tue, 10 Aug 2021 10:48:15 +0200 Subject: [PATCH 013/134] rcu: Make rcu_normal_after_boot writable again Certain configurations (e.g., systems that make heavy use of netns) need to use synchronize_rcu_expedited() to service RCU grace periods even after boot. Even though synchronize_rcu_expedited() has been traditionally considered harmful for RT for the heavy use of IPIs, it is perfectly usable under certain conditions (e.g. nohz_full). Make rcupdate.rcu_normal_after_boot= again writeable on RT (if NO_HZ_ FULL is defined), but keep its default value to 1 (enabled) to avoid regressions. Users who need synchronize_rcu_expedited() will boot with rcupdate.rcu_normal_after_ boot=0 in the kernel cmdline. Reflect the change in synchronize_rcu_expedited_wait() by removing the WARN related to CONFIG_PREEMPT_RT. Signed-off-by: Juri Lelli Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 1 - kernel/rcu/update.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 2796084ef85a..d9e4f8eb9ae2 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -512,7 +512,6 @@ static void synchronize_rcu_expedited_wait(void) j = READ_ONCE(jiffies_till_first_fqs); if (synchronize_rcu_expedited_wait_once(j + HZ)) return; - WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT)); } for (;;) { diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index c21b38cc25e9..bd551134e2f4 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -57,7 +57,7 @@ module_param(rcu_expedited, int, 0); module_param(rcu_normal, int, 0); static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT); -#ifndef CONFIG_PREEMPT_RT +#if !defined(CONFIG_PREEMPT_RT) || defined(CONFIG_NO_HZ_FULL) module_param(rcu_normal_after_boot, int, 0); #endif #endif /* #ifndef CONFIG_TINY_RCU */ From 1eac0075ebeecbf5c972f575ac448a0ea92e4f3a Mon Sep 17 00:00:00 2001 From: Juri Lelli Date: Tue, 10 Aug 2021 10:48:16 +0200 Subject: [PATCH 014/134] rcu: Make rcu update module parameters world-readable rcu update module parameters currently don't appear in sysfs and this is a serviceability issue as it might be needed to access their default values at runtime. Fix this issue by changing rcu update module parameters permissions to world-readable. Suggested-by: Paul E. McKenney Signed-off-by: Juri Lelli Signed-off-by: Paul E. McKenney --- kernel/rcu/update.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index bd551134e2f4..94282dc12bab 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -54,11 +54,11 @@ #define MODULE_PARAM_PREFIX "rcupdate." #ifndef CONFIG_TINY_RCU -module_param(rcu_expedited, int, 0); -module_param(rcu_normal, int, 0); +module_param(rcu_expedited, int, 0444); +module_param(rcu_normal, int, 0444); static int rcu_normal_after_boot = IS_ENABLED(CONFIG_PREEMPT_RT); #if !defined(CONFIG_PREEMPT_RT) || defined(CONFIG_NO_HZ_FULL) -module_param(rcu_normal_after_boot, int, 0); +module_param(rcu_normal_after_boot, int, 0444); #endif #endif /* #ifndef CONFIG_TINY_RCU */ From f0b2b2df5423fb369ac762c77900bc7765496d58 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 18 Aug 2021 13:34:00 +0530 Subject: [PATCH 015/134] rcu: Fix existing exp request check in sync_sched_exp_online_cleanup() The sync_sched_exp_online_cleanup() checks to see if RCU needs an expedited quiescent state from the incoming CPU, sending it an IPI if so. Before sending IPI, it checks whether expedited qs need has been already requested for the incoming CPU, by checking rcu_data.cpu_no_qs.b.exp for the current cpu, on which sync_sched_exp_online_cleanup() is running. This works for the case where incoming CPU is same as self. However, for the case where incoming CPU is different from self, expedited request won't get marked, which can potentially delay reporting of expedited quiescent state for the incoming CPU. Fixes: e015a3411220 ("rcu: Avoid self-IPI in sync_sched_exp_online_cleanup()") Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index d9e4f8eb9ae2..f3947c49eee7 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -759,7 +759,7 @@ static void sync_sched_exp_online_cleanup(int cpu) my_cpu = get_cpu(); /* Quiescent state either not needed or already requested, leave. */ if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) { + rdp->cpu_no_qs.b.exp) { put_cpu(); return; } From cbe0d8d91415c9692fe88191940d98952b6855d9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 30 Jul 2021 12:17:59 -0700 Subject: [PATCH 016/134] rcu-tasks: Wait for trc_read_check_handler() IPIs Currently, RCU Tasks Trace initializes the trc_n_readers_need_end counter to the value one, increments it before each trc_read_check_handler() IPI, then decrements it within trc_read_check_handler() if the target task was in a quiescent state (or if the target task moved to some other CPU while the IPI was in flight), complaining if the new value was zero. The rationale for complaining is that the initial value of one must be decremented away before zero can be reached, and this decrement has not yet happened. Except that trc_read_check_handler() is initiated with an asynchronous smp_call_function_single(), which might be significantly delayed. This can result in false-positive complaints about the counter reaching zero. This commit therefore waits for in-flight IPI handlers to complete before decrementing away the initial value of one from the trc_n_readers_need_end counter. Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 806160c44b17..3b2f8038064a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1150,14 +1150,28 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, } } +static void rcu_tasks_trace_empty_fn(void *unused) +{ +} + /* Wait for grace period to complete and provide ordering. */ static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp) { + int cpu; bool firstreport; struct task_struct *g, *t; LIST_HEAD(holdouts); long ret; + // Wait for any lingering IPI handlers to complete. Note that + // if a CPU has gone offline or transitioned to userspace in the + // meantime, all IPI handlers should have been drained beforehand. + // Yes, this assumes that CPUs process IPIs in order. If that ever + // changes, there will need to be a recheck and/or timed wait. + for_each_online_cpu(cpu) + if (smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu))) + smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1); + // Remove the safety count. smp_mb__before_atomic(); // Order vs. earlier atomics atomic_dec(&trc_n_readers_need_end); From fda84866b1e68ab409074e7fcf1a7db800615445 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 3 Aug 2021 17:42:25 -0700 Subject: [PATCH 017/134] rcutorture: Suppressing read-exit testing is not an error Currently, specifying the rcutorture.read_exit_burst=0 kernel boot parameter will result in a -EINVAL exit code that will stop the rcutorture test run before it has fully initialized. This commit therefore uses a zero exit code in that case, thus allowing rcutorture.read_exit_burst=0 to complete normally. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ab4215266ebe..59254fa15cc6 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2741,7 +2741,7 @@ static int rcu_torture_read_exit(void *unused) static int rcu_torture_read_exit_init(void) { if (read_exit_burst <= 0) - return -EINVAL; + return 0; init_waitqueue_head(&read_exit_wq); read_exit_child_stop = false; read_exit_child_stopped = false; From efeff6b39b9de4480572c7b0c5eb77204795cb57 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 13:28:24 -0700 Subject: [PATCH 018/134] rcutorture: Warn on individual rcu_torture_init() error conditions When running rcutorture as a module, any rcu_torture_init() issues will be reflected in the error code from modprobe or insmod, as the case may be. However, these error codes are not available when running rcutorture built-in, for example, when using the kvm.sh script. This commit therefore adds WARN_ON_ONCE() to allow distinguishing rcu_torture_init() errors when running rcutorture built-in. Signed-off-by: Paul E. McKenney --- include/linux/torture.h | 8 ++++++++ kernel/rcu/rcutorture.c | 30 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/include/linux/torture.h b/include/linux/torture.h index 0910c5803f35..24f58e50a94b 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -47,6 +47,14 @@ do { \ } while (0) void verbose_torout_sleep(void); +#define torture_init_error(firsterr) \ +({ \ + int ___firsterr = (firsterr); \ + \ + WARN_ONCE(!IS_MODULE(CONFIG_RCU_TORTURE_TEST) && ___firsterr < 0, "Torture-test initialization failed with error code %d\n", ___firsterr); \ + ___firsterr < 0; \ +}) + /* Definitions for online/offline exerciser. */ #ifdef CONFIG_HOTPLUG_CPU int torture_num_online_cpus(void); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 59254fa15cc6..b90cd4d98a20 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3037,7 +3037,7 @@ rcu_torture_init(void) rcu_torture_write_types(); firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; if (nfakewriters > 0) { fakewriter_tasks = kcalloc(nfakewriters, @@ -3052,7 +3052,7 @@ rcu_torture_init(void) for (i = 0; i < nfakewriters; i++) { firsterr = torture_create_kthread(rcu_torture_fakewriter, NULL, fakewriter_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), @@ -3068,7 +3068,7 @@ rcu_torture_init(void) rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, reader_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } nrealnocbers = nocbs_nthreads; @@ -3088,18 +3088,18 @@ rcu_torture_init(void) } for (i = 0; i < nrealnocbers; i++) { firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stat_interval > 0) { firsterr = torture_create_kthread(rcu_torture_stats, NULL, stats_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (test_no_idle_hz && shuffle_interval > 0) { firsterr = torture_shuffle_init(shuffle_interval * HZ); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stutter < 0) @@ -3109,7 +3109,7 @@ rcu_torture_init(void) t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; firsterr = torture_stutter_init(stutter * HZ, t); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (fqs_duration < 0) @@ -3118,7 +3118,7 @@ rcu_torture_init(void) /* Create the fqs thread */ firsterr = torture_create_kthread(rcu_torture_fqs, NULL, fqs_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (test_boost_interval < 1) @@ -3132,7 +3132,7 @@ rcu_torture_init(void) firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", rcutorture_booster_init, rcutorture_booster_cleanup); - if (firsterr < 0) + if (torture_init_error(firsterr)) goto unwind; rcutor_hp = firsterr; @@ -3153,23 +3153,23 @@ rcu_torture_init(void) } shutdown_jiffies = jiffies + shutdown_secs * HZ; firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, rcutorture_sync); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; firsterr = rcu_torture_stall_init(); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; firsterr = rcu_torture_fwd_prog_init(); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; firsterr = rcu_torture_barrier_init(); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; firsterr = rcu_torture_read_exit_init(); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; if (object_debug) rcu_test_debug_objects(); From b3b3cc618ee07f5f4c409e8ca86ac7fbac085ebd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 15:53:10 -0700 Subject: [PATCH 019/134] locktorture: Warn on individual lock_torture_init() error conditions When running locktorture as a module, any lock_torture_init() issues will be reflected in the error code from modprobe or insmod, as the case may be. However, these error codes are not available when running locktorture built-in, for example, when using the kvm.sh script. This commit therefore adds WARN_ON_ONCE() to allow distinguishing lock_torture_init() errors when running locktorture built-in. Signed-off-by: Paul E. McKenney --- kernel/locking/locktorture.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/locking/locktorture.c b/kernel/locking/locktorture.c index 7c5a4a087cc7..397ac13d2ef7 100644 --- a/kernel/locking/locktorture.c +++ b/kernel/locking/locktorture.c @@ -1022,23 +1022,23 @@ static int __init lock_torture_init(void) if (onoff_interval > 0) { firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ, NULL); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (shuffle_interval > 0) { firsterr = torture_shuffle_init(shuffle_interval); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (shutdown_secs > 0) { firsterr = torture_shutdown_init(shutdown_secs, lock_torture_cleanup); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stutter > 0) { firsterr = torture_stutter_init(stutter, stutter); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } @@ -1082,7 +1082,7 @@ static int __init lock_torture_init(void) /* Create writer. */ firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i], writer_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; create_reader: @@ -1091,13 +1091,13 @@ static int __init lock_torture_init(void) /* Create reader. */ firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], reader_tasks[j]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stat_interval > 0) { firsterr = torture_create_kthread(lock_torture_stats, NULL, stats_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } torture_init_end(); From ed60ad733aa49b70720c9d8dded1b18374ec5022 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 15:57:12 -0700 Subject: [PATCH 020/134] refscale: Warn on individual ref_scale_init() error conditions When running refscale as a module, any ref_scale_init() issues will be reflected in the error code from modprobe or insmod, as the case may be. However, these error codes are not available when running refscale built-in, for example, when using the kvm.sh script. This commit therefore adds WARN_ON_ONCE() to allow distinguishing ref_scale_init() errors when running refscale built-in. Signed-off-by: Paul E. McKenney --- kernel/rcu/refscale.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index 66dc14cf5687..1631ef8a138d 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -824,7 +824,7 @@ ref_scale_init(void) init_waitqueue_head(&shutdown_wq); firsterr = torture_create_kthread(ref_scale_shutdown, NULL, shutdown_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; schedule_timeout_uninterruptible(1); } @@ -851,7 +851,7 @@ ref_scale_init(void) for (i = 0; i < nreaders; i++) { firsterr = torture_create_kthread(ref_scale_reader, (void *)i, reader_tasks[i].task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; init_waitqueue_head(&(reader_tasks[i].wq)); @@ -860,7 +860,7 @@ ref_scale_init(void) // Main Task init_waitqueue_head(&main_wq); firsterr = torture_create_kthread(main_func, NULL, main_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; torture_init_end(); From eb77abfdeed29dd032c923e16fe8d91fa95cd316 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 15:58:53 -0700 Subject: [PATCH 021/134] rcuscale: Warn on individual rcu_scale_init() error conditions When running rcuscale as a module, any rcu_scale_init() issues will be reflected in the error code from modprobe or insmod, as the case may be. However, these error codes are not available when running rcuscale built-in, for example, when using the kvm.sh script. This commit therefore adds WARN_ON_ONCE() to allow distinguishing rcu_scale_init() errors when running rcuscale built-in. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index 2cc34a22a506..228f143bf935 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -758,7 +758,7 @@ kfree_scale_init(void) init_waitqueue_head(&shutdown_wq); firsterr = torture_create_kthread(kfree_scale_shutdown, NULL, shutdown_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; schedule_timeout_uninterruptible(1); } @@ -775,7 +775,7 @@ kfree_scale_init(void) for (i = 0; i < kfree_nrealthreads; i++) { firsterr = torture_create_kthread(kfree_scale_thread, (void *)i, kfree_reader_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } @@ -838,7 +838,7 @@ rcu_scale_init(void) init_waitqueue_head(&shutdown_wq); firsterr = torture_create_kthread(rcu_scale_shutdown, NULL, shutdown_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; schedule_timeout_uninterruptible(1); } @@ -852,7 +852,7 @@ rcu_scale_init(void) for (i = 0; i < nrealreaders; i++) { firsterr = torture_create_kthread(rcu_scale_reader, (void *)i, reader_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) @@ -879,7 +879,7 @@ rcu_scale_init(void) } firsterr = torture_create_kthread(rcu_scale_writer, (void *)i, writer_tasks[i]); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } torture_init_end(); From fd13fe16db0d82612b260640f4e26f6d9d1e11fd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 6 Aug 2021 08:57:26 -0700 Subject: [PATCH 022/134] rcutorture: Don't cpuhp_remove_state() if cpuhp_setup_state() failed Currently, in CONFIG_RCU_BOOST kernels, if the rcu_torture_init() function's call to cpuhp_setup_state() fails, rcu_torture_cleanup() gamely passes nonsense to cpuhp_remove_state(). This results in strange and misleading splats. This commit therefore ensures that if the rcu_torture_init() function's call to cpuhp_setup_state() fails, rcu_torture_cleanup() avoids invoking cpuhp_remove_state(). Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index b90cd4d98a20..424184764ef0 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2819,7 +2819,7 @@ rcu_torture_cleanup(void) rcutorture_seq_diff(gp_seq, start_gp_seq)); torture_stop_kthread(rcu_torture_stats, stats_task); torture_stop_kthread(rcu_torture_fqs, fqs_task); - if (rcu_torture_can_boost()) + if (rcu_torture_can_boost() && rcutor_hp >= 0) cpuhp_remove_state(rcutor_hp); /* @@ -3132,9 +3132,9 @@ rcu_torture_init(void) firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", rcutorture_booster_init, rcutorture_booster_cleanup); + rcutor_hp = firsterr; if (torture_init_error(firsterr)) goto unwind; - rcutor_hp = firsterr; // Testing RCU priority boosting requires rcutorture do // some serious abuse. Counter this by running ksoftirqd From 71921a9606ddbcc1d98c00eca7ae82c373d1fecd Mon Sep 17 00:00:00 2001 From: Scott Wood Date: Fri, 20 Aug 2021 09:42:36 +0200 Subject: [PATCH 023/134] rcutorture: Avoid problematic critical section nesting on PREEMPT_RT rcutorture is generating some nesting scenarios that are not compatible on PREEMPT_RT. For example: preempt_disable(); rcu_read_lock_bh(); preempt_enable(); rcu_read_unlock_bh(); The problem here is that on PREEMPT_RT the bottom halves have to be disabled and enabled in preemptible context. Reorder locking: start with BH locking and continue with then with disabling preemption or interrupts. In the unlocking do it reverse by first enabling interrupts and preemption and BH at the very end. Ensure that on PREEMPT_RT BH locking remains unchanged if in non-preemptible context. Link: https://lkml.kernel.org/r/20190911165729.11178-6-swood@redhat.com Link: https://lkml.kernel.org/r/20210819182035.GF4126399@paulmck-ThinkPad-P17-Gen-1 Signed-off-by: Scott Wood [bigeasy: Drop ATOM_BH, make it only about changing BH in atomic context. Allow enabling RCU in IRQ-off section. Reword commit message.] Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 48 ++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 424184764ef0..fb079b78c232 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1432,28 +1432,34 @@ static void rcutorture_one_extend(int *readstate, int newstate, /* First, put new protection in place to avoid critical-section gap. */ if (statesnew & RCUTORTURE_RDR_BH) local_bh_disable(); + if (statesnew & RCUTORTURE_RDR_RBH) + rcu_read_lock_bh(); if (statesnew & RCUTORTURE_RDR_IRQ) local_irq_disable(); if (statesnew & RCUTORTURE_RDR_PREEMPT) preempt_disable(); - if (statesnew & RCUTORTURE_RDR_RBH) - rcu_read_lock_bh(); if (statesnew & RCUTORTURE_RDR_SCHED) rcu_read_lock_sched(); if (statesnew & RCUTORTURE_RDR_RCU) idxnew = cur_ops->readlock() << RCUTORTURE_RDR_SHIFT; - /* Next, remove old protection, irq first due to bh conflict. */ + /* + * Next, remove old protection, in decreasing order of strength + * to avoid unlock paths that aren't safe in the stronger + * context. Namely: BH can not be enabled with disabled interrupts. + * Additionally PREEMPT_RT requires that BH is enabled in preemptible + * context. + */ if (statesold & RCUTORTURE_RDR_IRQ) local_irq_enable(); - if (statesold & RCUTORTURE_RDR_BH) - local_bh_enable(); if (statesold & RCUTORTURE_RDR_PREEMPT) preempt_enable(); - if (statesold & RCUTORTURE_RDR_RBH) - rcu_read_unlock_bh(); if (statesold & RCUTORTURE_RDR_SCHED) rcu_read_unlock_sched(); + if (statesold & RCUTORTURE_RDR_BH) + local_bh_enable(); + if (statesold & RCUTORTURE_RDR_RBH) + rcu_read_unlock_bh(); if (statesold & RCUTORTURE_RDR_RCU) { bool lockit = !statesnew && !(torture_random(trsp) & 0xffff); @@ -1496,6 +1502,9 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) int mask = rcutorture_extend_mask_max(); unsigned long randmask1 = torture_random(trsp) >> 8; unsigned long randmask2 = randmask1 >> 3; + unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; + unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; + unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT); /* Mostly only one bit (need preemption!), sometimes lots of bits. */ @@ -1503,11 +1512,26 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) mask = mask & randmask2; else mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); - /* Can't enable bh w/irq disabled. */ - if ((mask & RCUTORTURE_RDR_IRQ) && - ((!(mask & RCUTORTURE_RDR_BH) && (oldmask & RCUTORTURE_RDR_BH)) || - (!(mask & RCUTORTURE_RDR_RBH) && (oldmask & RCUTORTURE_RDR_RBH)))) - mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; + + /* + * Can't enable bh w/irq disabled. + */ + if (mask & RCUTORTURE_RDR_IRQ) + mask |= oldmask & bhs; + + /* + * Ideally these sequences would be detected in debug builds + * (regardless of RT), but until then don't stop testing + * them on non-RT. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + /* Can't modify BH in atomic context */ + if (oldmask & preempts_irq) + mask &= ~bhs; + if ((oldmask | mask) & preempts_irq) + mask |= oldmask & bhs; + } + return mask ?: RCUTORTURE_RDR_RCU; } From b380b10b84c3b8334aba51a5a10eabb30f37589f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 15 Jul 2021 15:14:56 -0700 Subject: [PATCH 024/134] torture: Make torture.sh print the number of files to be compressed Compressing gigabyte vmlinux files can take some time, and it can be a bit annoying to not know many more batches of compression there will be. This commit therefore makes torture.sh print the number of files to be compressed just before starting compression and just after compression completes. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/torture.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index 363f56081eff..8e882346d2a6 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -434,7 +434,12 @@ then batchno=1 if test -s $T/xz-todo then - echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log + for i in `cat $T/xz-todo` + do + find $i -name 'vmlinux*' -print + done | wc -l | awk '{ print $1 }' > $T/xz-todo-count + n2compress="`cat $T/xz-todo-count`" + echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log for i in `cat $T/xz-todo` do echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1 @@ -456,7 +461,7 @@ then echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log fi wait - echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log + echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log else echo No compression needed: `date` >> "$tdir/log-xz" 2>&1 From d680c6b49c5edb532e3e5a134d9f48f000a691e1 Mon Sep 17 00:00:00 2001 From: Cai Huoqing Date: Tue, 14 Sep 2021 11:33:38 +0800 Subject: [PATCH 025/134] audit: Convert to SPDX identifier Use SPDX-License-Identifier instead of a verbose license text. Signed-off-by: Cai Huoqing Signed-off-by: Paul Moore --- kernel/auditsc.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 8dd73a64f921..1af4e7d0c16e 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* auditsc.c -- System-call auditing support * Handles all system-call specific auditing features. * @@ -6,20 +7,6 @@ * Copyright (C) 2005, 2006 IBM Corporation * All Rights Reserved. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * Written by Rickard E. (Rik) Faith * * Many of the ideas implemented here are from Stephen C. Tweedie, From 96017bf9039763a2e02dcc6adaa18592cd73a39d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 28 Jul 2021 10:53:41 -0700 Subject: [PATCH 026/134] rcu-tasks: Simplify trc_read_check_handler() atomic operations Currently, trc_wait_for_one_reader() atomically increments the trc_n_readers_need_end counter before sending the IPI invoking trc_read_check_handler(). All failure paths out of trc_read_check_handler() and also from the smp_call_function_single() within trc_wait_for_one_reader() must carefully atomically decrement this counter. This is more complex than it needs to be. This commit therefore simplifies things and saves a few lines of code by dispensing with the atomic decrements in favor of having trc_read_check_handler() do the atomic increment only in the success case. In theory, this represents no change in functionality. Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 3b2f8038064a..c9d8583ffe59 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -890,32 +890,24 @@ static void trc_read_check_handler(void *t_in) // If the task is no longer running on this CPU, leave. if (unlikely(texp != t)) { - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) - wake_up(&trc_wait); goto reset_ipi; // Already on holdout list, so will check later. } // If the task is not in a read-side critical section, and // if this is the last reader, awaken the grace-period kthread. if (likely(!READ_ONCE(t->trc_reader_nesting))) { - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) - wake_up(&trc_wait); - // Mark as checked after decrement to avoid false - // positives on the above WARN_ON_ONCE(). WRITE_ONCE(t->trc_reader_checked, true); goto reset_ipi; } // If we are racing with an rcu_read_unlock_trace(), try again later. - if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) { - if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end))) - wake_up(&trc_wait); + if (unlikely(READ_ONCE(t->trc_reader_nesting) < 0)) goto reset_ipi; - } WRITE_ONCE(t->trc_reader_checked, true); // Get here if the task is in a read-side critical section. Set // its state so that it will awaken the grace-period kthread upon // exit from that critical section. + atomic_inc(&trc_n_readers_need_end); // One more to wait on. WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs)); WRITE_ONCE(t->trc_reader_special.b.need_qs, true); @@ -1015,21 +1007,15 @@ static void trc_wait_for_one_reader(struct task_struct *t, if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0) return; - atomic_inc(&trc_n_readers_need_end); per_cpu(trc_ipi_to_cpu, cpu) = true; t->trc_ipi_to_cpu = cpu; rcu_tasks_trace.n_ipis++; - if (smp_call_function_single(cpu, - trc_read_check_handler, t, 0)) { + if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) { // Just in case there is some other reason for // failure than the target CPU being offline. rcu_tasks_trace.n_ipis_fails++; per_cpu(trc_ipi_to_cpu, cpu) = false; t->trc_ipi_to_cpu = cpu; - if (atomic_dec_and_test(&trc_n_readers_need_end)) { - WARN_ON_ONCE(1); - wake_up(&trc_wait); - } } } } From 18f08e758f34e6dfe0668bee51bd2af7adacf381 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 28 Jul 2021 11:32:28 -0700 Subject: [PATCH 027/134] rcu-tasks: Add trc_inspect_reader() checks for exiting critical section Currently, trc_inspect_reader() treats a task exiting its RCU Tasks Trace read-side critical section the same as being within that critical section. However, this can fail because that task might have already checked its .need_qs field, which means that it might never decrement the all-important trc_n_readers_need_end counter. Of course, for that to happen, the task would need to never again execute an RCU Tasks Trace read-side critical section, but this really could happen if the system's last trampoline was removed. Note that exit from such a critical section cannot be treated as a quiescent state due to the possibility of nested critical sections. This means that if trc_inspect_reader() sees a negative nesting value, it must set up to try again later. This commit therefore ignores tasks that are exiting their RCU Tasks Trace read-side critical sections so that they will be rechecked later. [ paulmck: Apply feedback from Neeraj Upadhyay and Boqun Feng. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index c9d8583ffe59..8387e70e6b00 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -923,7 +923,7 @@ reset_ipi: static bool trc_inspect_reader(struct task_struct *t, void *arg) { int cpu = task_cpu(t); - bool in_qs = false; + int nesting; bool ofl = cpu_is_offline(cpu); if (task_curr(t)) { @@ -943,18 +943,18 @@ static bool trc_inspect_reader(struct task_struct *t, void *arg) n_heavy_reader_updates++; if (ofl) n_heavy_reader_ofl_updates++; - in_qs = true; + nesting = 0; } else { // The task is not running, so C-language access is safe. - in_qs = likely(!t->trc_reader_nesting); + nesting = t->trc_reader_nesting; } - // Mark as checked so that the grace-period kthread will - // remove it from the holdout list. - t->trc_reader_checked = true; - - if (in_qs) - return true; // Already in quiescent state, done!!! + // If not exiting a read-side critical section, mark as checked + // so that the grace-period kthread will remove it from the + // holdout list. + t->trc_reader_checked = nesting >= 0; + if (nesting <= 0) + return !nesting; // If in QS, done, otherwise try again later. // The task is in a read-side critical section, so set up its // state so that it will awaken the grace-period kthread upon exit From a5c071ccfa1728508f31e61213ee795e4529d0d4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 28 Jul 2021 12:28:27 -0700 Subject: [PATCH 028/134] rcu-tasks: Remove second argument of rcu_read_unlock_trace_special() The second argument of rcu_read_unlock_trace_special() is always zero. When called from exit_tasks_rcu_finish_trace(), it is the constant zero, and rcu_read_unlock_trace_special() doesn't get called from rcu_read_unlock_trace() unless the value of local variable "nesting" is zero because in that case the early return is taken instead. This commit therefore removes the "nesting" argument from the rcu_read_unlock_trace_special() function, substituting the constant zero within that function. This commit also adds a WARN_ON_ONCE() to rcu_read_lock_trace_held() in case non-zeroness some day appears. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 5 +++-- kernel/rcu/tasks.h | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index 86c8f6c98412..6f9c35817398 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -31,7 +31,7 @@ static inline int rcu_read_lock_trace_held(void) #ifdef CONFIG_TASKS_TRACE_RCU -void rcu_read_unlock_trace_special(struct task_struct *t, int nesting); +void rcu_read_unlock_trace_special(struct task_struct *t); /** * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section @@ -80,7 +80,8 @@ static inline void rcu_read_unlock_trace(void) WRITE_ONCE(t->trc_reader_nesting, nesting); return; // We assume shallow reader nesting. } - rcu_read_unlock_trace_special(t, nesting); + WARN_ON_ONCE(nesting != 0); + rcu_read_unlock_trace_special(t); } void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func); diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 8387e70e6b00..a3f4f9bd8c67 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -848,7 +848,7 @@ static void rcu_read_unlock_iw(struct irq_work *iwp) static DEFINE_IRQ_WORK(rcu_tasks_trace_iw, rcu_read_unlock_iw); /* If we are the last reader, wake up the grace-period kthread. */ -void rcu_read_unlock_trace_special(struct task_struct *t, int nesting) +void rcu_read_unlock_trace_special(struct task_struct *t) { int nq = READ_ONCE(t->trc_reader_special.b.need_qs); @@ -858,7 +858,7 @@ void rcu_read_unlock_trace_special(struct task_struct *t, int nesting) // Update .need_qs before ->trc_reader_nesting for irq/NMI handlers. if (nq) WRITE_ONCE(t->trc_reader_special.b.need_qs, false); - WRITE_ONCE(t->trc_reader_nesting, nesting); + WRITE_ONCE(t->trc_reader_nesting, 0); if (nq && atomic_dec_and_test(&trc_n_readers_need_end)) irq_work_queue(&rcu_tasks_trace_iw); } @@ -1200,7 +1200,7 @@ static void exit_tasks_rcu_finish_trace(struct task_struct *t) WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting)); WRITE_ONCE(t->trc_reader_nesting, 0); if (WARN_ON_ONCE(READ_ONCE(t->trc_reader_special.b.need_qs))) - rcu_read_unlock_trace_special(t, 0); + rcu_read_unlock_trace_special(t); } /** From c4f113ac450afc9c7c4d2ce84a08f516dbec69b1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 09:54:45 -0700 Subject: [PATCH 029/134] rcu-tasks: Fix s/instruction/instructions/ typo in comment Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index a3f4f9bd8c67..43c0f715ac63 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -758,7 +758,7 @@ EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread); // 2. Protects code in the idle loop, exception entry/exit, and // CPU-hotplug code paths, similar to the capabilities of SRCU. // -// 3. Avoids expensive read-side instruction, having overhead similar +// 3. Avoids expensive read-side instructions, having overhead similar // to that of Preemptible RCU. // // There are of course downsides. The grace-period code can send IPIs to From 0db7c32ad3160ae06f497d48a74bd46a2a35e6bf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 11 Aug 2021 09:07:44 -0700 Subject: [PATCH 030/134] rcu-tasks: Move RTGS_WAIT_CBS to beginning of rcu_tasks_kthread() loop Early in debugging, it made some sense to differentiate the first iteration from subsequent iterations, but now this just causes confusion. This commit therefore moves the "set_tasks_gp_state(rtp, RTGS_WAIT_CBS)" statement to the beginning of the "for" loop in rcu_tasks_kthread(). Reported-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 43c0f715ac63..7e2641783e43 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -197,6 +197,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) * This loop is terminated by the system going down. ;-) */ for (;;) { + set_tasks_gp_state(rtp, RTGS_WAIT_CBS); /* Pick up any new callbacks. */ raw_spin_lock_irqsave(&rtp->cbs_lock, flags); @@ -236,8 +237,6 @@ static int __noreturn rcu_tasks_kthread(void *arg) } /* Paranoid sleep to keep this from entering a tight loop */ schedule_timeout_idle(rtp->gp_sleep); - - set_tasks_gp_state(rtp, RTGS_WAIT_CBS); } } From d0a85858569ead8d39ba5b41501cd99bc7d7e7bd Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 18 Aug 2021 12:58:39 +0530 Subject: [PATCH 031/134] rcu-tasks: Fix s/rcu_add_holdout/trc_add_holdout/ typo in comment Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 7e2641783e43..75e7888b3fc9 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -991,7 +991,7 @@ static void trc_wait_for_one_reader(struct task_struct *t, // If this task is not yet on the holdout list, then we are in // an RCU read-side critical section. Otherwise, the invocation of - // rcu_add_holdout() that added it to the list did the necessary + // trc_add_holdout() that added it to the list did the necessary // get_task_struct(). Either way, the task cannot be freed out // from under this code. From 89401176daf0a44ab517d9a0e296adb85af246df Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 18 Aug 2021 12:58:40 +0530 Subject: [PATCH 032/134] rcu-tasks: Correct firstreport usage in check_all_holdout_tasks_trace In check_all_holdout_tasks_trace(), firstreport is a pointer argument; so, check the dereferenced value, instead of checking the pointer. Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 75e7888b3fc9..e2ec548fc0c4 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1129,7 +1129,7 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, cpus_read_unlock(); if (needreport) { - if (firstreport) + if (*firstreport) pr_err("INFO: rcu_tasks_trace detected stalls? (Late IPI?)\n"); show_stalled_ipi_trace(); } From d39ec8f3c12abe3710f7031ce3d5564bda12b19e Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 18 Aug 2021 12:58:41 +0530 Subject: [PATCH 033/134] rcu-tasks: Correct comparisons for CPU numbers in show_stalled_task_trace Valid CPU numbers can be zero or greater, but the checks for ->trc_ipi_to_cpu and tick_nohz_full_cpu()'s argument are for strictly greater than. This commit therefore corrects the check for no_hz_full cpu in show_stalled_task_trace() so as to include cpu 0. Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index e2ec548fc0c4..af7388849bed 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1084,9 +1084,9 @@ static void show_stalled_task_trace(struct task_struct *t, bool *firstreport) cpu = task_cpu(t); pr_alert("P%d: %c%c%c nesting: %d%c cpu: %d\n", t->pid, - ".I"[READ_ONCE(t->trc_ipi_to_cpu) > 0], + ".I"[READ_ONCE(t->trc_ipi_to_cpu) >= 0], ".i"[is_idle_task(t)], - ".N"[cpu > 0 && tick_nohz_full_cpu(cpu)], + ".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)], READ_ONCE(t->trc_reader_nesting), " N"[!!READ_ONCE(t->trc_reader_special.b.need_qs)], cpu); From a6517e9ce0115e33617062c9e73b4c5e6f787525 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 18 Aug 2021 12:58:43 +0530 Subject: [PATCH 034/134] rcu-tasks: Clarify read side section info for rcu_tasks_rude GP primitives RCU tasks rude variant does not check whether the current running context on a CPU is usermode. Read side critical section ends on transition to usermode execution, by the virtue of usermode execution being schedulable. Clarify this in comments for call_rcu_tasks_rude() and synchronize_rcu_tasks_rude(). Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index af7388849bed..8c63b4d23829 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -677,11 +677,11 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * period elapses, in other words after all currently executing RCU * read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, - * cond_resched_rcu_qs(), or transition to usermode execution. As such, - * there are no read-side primitives analogous to rcu_read_lock() and - * rcu_read_unlock() because this primitive is intended to determine - * that all tasks have passed through a safe state, not so much for - * data-structure synchronization. + * cond_resched_rcu_qs(), or transition to usermode execution (as + * usermode execution is schedulable). As such, there are no read-side + * primitives analogous to rcu_read_lock() and rcu_read_unlock() because + * this primitive is intended to determine that all tasks have passed + * through a safe state, not so much for data-structure synchronization. * * See the description of call_rcu() for more detailed information on * memory ordering guarantees. @@ -699,8 +699,8 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks_rude); * grace period has elapsed, in other words after all currently * executing rcu-tasks read-side critical sections have elapsed. These * read-side critical sections are delimited by calls to schedule(), - * cond_resched_tasks_rcu_qs(), userspace execution, and (in theory, - * anyway) cond_resched(). + * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable + * context), and (in theory, anyway) cond_resched(). * * This is a very specialized primitive, intended only for a few uses in * tracing and other situations requiring manipulation of function preambles From ed42c38067129c85ab1bda39f2fd91924a432dc0 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 25 Aug 2021 12:40:50 +0530 Subject: [PATCH 035/134] rcu-tasks: Fix read-side primitives comment for call_rcu_tasks_trace call_rcu_tasks_trace() does have read-side primitives - rcu_read_lock_trace() and rcu_read_unlock_trace(). Fix this information in the comments. Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 8c63b4d23829..47a29e411217 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1207,15 +1207,11 @@ static void exit_tasks_rcu_finish_trace(struct task_struct *t) * @rhp: structure to be used for queueing the RCU updates. * @func: actual callback function to be invoked after the grace period * - * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_tasks_trace() - * assumes that the read-side critical sections end at context switch, - * cond_resched_rcu_qs(), or transition to usermode execution. As such, - * there are no read-side primitives analogous to rcu_read_lock() and - * rcu_read_unlock() because this primitive is intended to determine - * that all tasks have passed through a safe state, not so much for - * data-structure synchronization. + * The callback function will be invoked some time after a trace rcu-tasks + * grace period elapses, in other words after all currently executing + * trace rcu-tasks read-side critical sections have completed. These + * read-side critical sections are delimited by calls to rcu_read_lock_trace() + * and rcu_read_unlock_trace(). * * See the description of call_rcu() for more detailed information on * memory ordering guarantees. @@ -1231,7 +1227,7 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks_trace); * * Control will return to the caller some time after a trace rcu-tasks * grace period has elapsed, in other words after all currently executing - * rcu-tasks read-side critical sections have elapsed. These read-side + * trace rcu-tasks read-side critical sections have elapsed. These read-side * critical sections are delimited by calls to rcu_read_lock_trace() * and rcu_read_unlock_trace(). * From 46aa886c483f57ef13cd5ea0a85e70b93eb1d381 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Fri, 27 Aug 2021 13:43:35 +0530 Subject: [PATCH 036/134] rcu-tasks: Fix IPI failure handling in trc_wait_for_one_reader The trc_wait_for_one_reader() function is called at multiple stages of trace rcu-tasks GP function, rcu_tasks_wait_gp(): - First, it is called as part of per task function - rcu_tasks_trace_pertask(), for all non-idle tasks. As part of per task processing, this function add the task in the holdout list and if the task is currently running on a CPU, it sends IPI to the task's CPU. The IPI handler takes action depending on whether task is in trace rcu-tasks read side critical section or not: - a. If the task is in trace rcu-tasks read side critical section (t->trc_reader_nesting != 0), the IPI handler sets the task's ->trc_reader_special.b.need_qs, so that this task notifies exit from its outermost read side critical section (by decrementing trc_n_readers_need_end) to the GP handling function. trc_wait_for_one_reader() also increments trc_n_readers_need_end, so that the trace rcu-tasks GP handler function waits for this task's read side exit notification. The IPI handler also sets t->trc_reader_checked to true, and no further IPIs are sent for this task, for this trace rcu-tasks grace period and this task can be removed from holdout list. - b. If the task is in the process of exiting its trace rcu-tasks read side critical section, (t->trc_reader_nesting < 0), defer this task's processing to future calls to trc_wait_for_one_reader(). - c. If task is not in rcu-task read side critical section, t->trc_reader_nesting == 0, ->trc_reader_checked is set for this task, so that this task is removed from holdout list. - Second, trc_wait_for_one_reader() is called as part of post scan, in function rcu_tasks_trace_postscan(), for all idle tasks. - Third, in function check_all_holdout_tasks_trace(), this function is called for each task in the holdout list, but only if there isn't a pending IPI for the task (->trc_ipi_to_cpu == -1). This function removed the task from holdout list, if IPI handler has completed the required work, to ensure that the current trace rcu-tasks grace period either waits for this task, or this task is not in a trace rcu-tasks read side critical section. Now, considering the scenario where smp_call_function_single() fails in first case, inside rcu_tasks_trace_pertask(). In this case, ->trc_ipi_to_cpu is set to the current CPU for that task. This will result in trc_wait_for_one_reader() getting skipped in third case, inside check_all_holdout_tasks_trace(), for this task. This further results in ->trc_reader_checked never getting set for this task, and the task not getting removed from holdout list. This can cause the current trace rcu-tasks grace period to stall. Fix the above problem, by resetting ->trc_ipi_to_cpu to -1, on smp_call_function_single() failure, so that future IPI calls can be send for this task. Note that all three of the trc_wait_for_one_reader() function's callers (rcu_tasks_trace_pertask(), rcu_tasks_trace_postscan(), check_all_holdout_tasks_trace()) hold cpu_read_lock(). This means that smp_call_function_single() cannot race with CPU hotplug, and thus should never fail. Therefore, also add a warning in order to report any such failure in case smp_call_function_single() grows some other reason for failure. Signed-off-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 47a29e411217..0c10c8407dca 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1012,9 +1012,11 @@ static void trc_wait_for_one_reader(struct task_struct *t, if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) { // Just in case there is some other reason for // failure than the target CPU being offline. + WARN_ONCE(1, "%s(): smp_call_function_single() failed for CPU: %d\n", + __func__, cpu); rcu_tasks_trace.n_ipis_fails++; per_cpu(trc_ipi_to_cpu, cpu) = false; - t->trc_ipi_to_cpu = cpu; + t->trc_ipi_to_cpu = -1; } } } From 8af9e2c7826a67a26c2c7a0cd3ce09a5acaf8035 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 15 Sep 2021 09:24:18 -0700 Subject: [PATCH 037/134] rcu-tasks: Update comments to cond_resched_tasks_rcu_qs() The cond_resched_rcu_qs() function no longer exists, despite being mentioned several times in kernel/rcu/tasks.h. This commit therefore updates it to the current cond_resched_tasks_rcu_qs(). Reported-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 0c10c8407dca..66e7586a33e9 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -368,7 +368,7 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) //////////////////////////////////////////////////////////////////////// // // Simple variant of RCU whose quiescent states are voluntary context -// switch, cond_resched_rcu_qs(), user-space execution, and idle. +// switch, cond_resched_tasks_rcu_qs(), user-space execution, and idle. // As such, grace periods can take one good long time. There are no // read-side primitives similar to rcu_read_lock() and rcu_read_unlock() // because this implementation is intended to get the system into a safe @@ -539,7 +539,7 @@ DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks"); * period elapses, in other words after all currently executing RCU * read-side critical sections have completed. call_rcu_tasks() assumes * that the read-side critical sections end at a voluntary context - * switch (not a preemption!), cond_resched_rcu_qs(), entry into idle, + * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, * or transition to usermode execution. As such, there are no read-side * primitives analogous to rcu_read_lock() and rcu_read_unlock() because * this primitive is intended to determine that all tasks have passed @@ -677,7 +677,7 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * period elapses, in other words after all currently executing RCU * read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, - * cond_resched_rcu_qs(), or transition to usermode execution (as + * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as * usermode execution is schedulable). As such, there are no read-side * primitives analogous to rcu_read_lock() and rcu_read_unlock() because * this primitive is intended to determine that all tasks have passed From 925da92ba5cb0c82d07cdd5049a07e40f54e9c44 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 26 Aug 2021 22:21:22 -0400 Subject: [PATCH 038/134] rcu: Avoid unneeded function call in rcu_read_unlock() Since commit aa40c138cc8f3 ("rcu: Report QS for outermost PREEMPT=n rcu_read_unlock() for strict GPs") the function rcu_read_unlock_strict() is invoked by the inlined rcu_read_unlock() function. However, rcu_read_unlock_strict() is an empty function in production kernels, which are built with CONFIG_RCU_STRICT_GRACE_PERIOD=n. There is a mention of rcu_read_unlock_strict() in the BPF verifier, but this is in a deny-list, meaning that BPF does not care whether rcu_read_unlock_strict() is ever called. This commit therefore provides a slight performance improvement by hoisting the check of CONFIG_RCU_STRICT_GRACE_PERIOD from rcu_read_unlock_strict() into rcu_read_unlock(), thus avoiding the pointless call to an empty function. Cc: Alexei Starovoitov Acked-by: Andrii Nakryiko Signed-off-by: Waiman Long Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 3 ++- kernel/rcu/tree_plugin.h | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 434d12fe2d4f..5e0beb5c5659 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -71,7 +71,8 @@ static inline void __rcu_read_lock(void) static inline void __rcu_read_unlock(void) { preempt_enable(); - rcu_read_unlock_strict(); + if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD)) + rcu_read_unlock_strict(); } static inline int rcu_preempt_depth(void) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index d070059163d7..1a6fdb03d0a5 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -814,8 +814,7 @@ void rcu_read_unlock_strict(void) { struct rcu_data *rdp; - if (!IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) || - irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) + if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) return; rdp = this_cpu_ptr(&rcu_data); rcu_report_qs_rdp(rdp); From 2f611d044b8dcab245b6bbe5f691b6dce173ff56 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jul 2021 14:12:54 -0700 Subject: [PATCH 039/134] scftorture: Allow zero weight to exclude an smp_call_function*() category This commit reworks the weighting calculations to allow zero to be specified to disable a given weight. For example, specifying the scftorture.weight_resched=0 kernel boot parameter without specifying a non-zero value for any of the other scftorture.weight_* parameters would provide the default weights for the others, but would refrain from doing any resched-based IPIs. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 64a08288b1a6..bc3f8e26345c 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -553,18 +553,18 @@ static int __init scf_torture_init(void) scftorture_print_module_parms("Start of test"); - if (weight_resched == -1 && - weight_single == -1 && weight_single_rpc == -1 && weight_single_wait == -1 && - weight_many == -1 && weight_many_wait == -1 && - weight_all == -1 && weight_all_wait == -1) { - weight_resched1 = 2 * nr_cpu_ids; - weight_single1 = 2 * nr_cpu_ids; - weight_single_rpc1 = 2 * nr_cpu_ids; - weight_single_wait1 = 2 * nr_cpu_ids; - weight_many1 = 2; - weight_many_wait1 = 2; - weight_all1 = 1; - weight_all_wait1 = 1; + if (weight_resched <= 0 && + weight_single <= 0 && weight_single_rpc <= 0 && weight_single_wait <= 0 && + weight_many <= 0 && weight_many_wait <= 0 && + weight_all <= 0 && weight_all_wait <= 0) { + weight_resched1 = weight_resched == 0 ? 0 : 2 * nr_cpu_ids; + weight_single1 = weight_single == 0 ? 0 : 2 * nr_cpu_ids; + weight_single_rpc1 = weight_single_rpc == 0 ? 0 : 2 * nr_cpu_ids; + weight_single_wait1 = weight_single_wait == 0 ? 0 : 2 * nr_cpu_ids; + weight_many1 = weight_many == 0 ? 0 : 2; + weight_many_wait1 = weight_many_wait == 0 ? 0 : 2; + weight_all1 = weight_all == 0 ? 0 : 1; + weight_all_wait1 = weight_all_wait == 0 ? 0 : 1; } else { if (weight_resched == -1) weight_resched1 = 0; From 2b1388f8a408e68fda6443ec166f42ae4ffca87c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jul 2021 14:20:35 -0700 Subject: [PATCH 040/134] scftorture: Shut down if nonsensical arguments given If (say) a 10-hour scftorture run is started, but the module parameters are so nonsensical that the run doesn't even start, then scftorture will wait the full ten hours when run built into a guest OS. This commit therefore shuts down the system in this case so that the error is reported immediately instead of ten hours hence. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index bc3f8e26345c..31b458b3b113 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -651,6 +651,10 @@ static int __init scf_torture_init(void) unwind: torture_init_end(); scf_torture_cleanup(); + if (shutdown_secs) { + WARN_ON(!IS_MODULE(CONFIG_SCF_TORTURE_TEST)); + kernel_power_off(); + } return firsterr; } From da9366c627ef459a1ceb6e5535648683e2adbdb2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 13 Jul 2021 15:13:56 -0700 Subject: [PATCH 041/134] scftorture: Account for weight_resched when checking for all zeroes The "all zero weights makes no sense" error is emitted even when scftorture.weight_resched is non-zero because it was left out of the enclosing "if" condition. This commit adds it in. Fixes: 1ac78b49d61d4 ("scftorture: Add an alternative IPI vector") Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 31b458b3b113..74348abc792b 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -583,8 +583,8 @@ static int __init scf_torture_init(void) if (weight_all_wait == -1) weight_all_wait1 = 0; } - if (weight_single1 == 0 && weight_single_rpc1 == 0 && weight_single_wait1 == 0 && - weight_many1 == 0 && weight_many_wait1 == 0 && + if (weight_resched1 == 0 && weight_single1 == 0 && weight_single_rpc1 == 0 && + weight_single_wait1 == 0 && weight_many1 == 0 && weight_many_wait1 == 0 && weight_all1 == 0 && weight_all_wait1 == 0) { VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense"); firsterr = -EINVAL; From c3d0258d5af2a50529e8928fe458344e38653d25 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 14 Jul 2021 06:53:51 -0700 Subject: [PATCH 042/134] scftorture: Count reschedule IPIs Currently, only those IPIs that invoke scftorture's scf_handler() IPI handler function are counted. This means that runs exercising only scftorture.weight_resched will look like they have made no forward progress, resulting in "GP HANG" complaints from the rcutorture scripting. This commit therefore increments the scf_invoked_count per-CPU counter immediately after calling resched_cpu(). Fixes: 1ac78b49d61d4 ("scftorture: Add an alternative IPI vector") Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 74348abc792b..00bba2bdd4d1 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -341,6 +341,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra cpu = torture_random(trsp) % nr_cpu_ids; scfp->n_resched++; resched_cpu(cpu); + this_cpu_inc(scf_invoked_count); } break; case SCF_PRIM_SINGLE: From f2bdf7dc0da234d78636994367e6ea4af055c689 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 16:01:51 -0700 Subject: [PATCH 043/134] scftorture: Warn on individual scf_torture_init() error conditions When running scftorture as a module, any scf_torture_init() issues will be reflected in the error code from modprobe or insmod, as the case may be. However, these error codes are not available when running scftorture built-in, for example, when using the kvm.sh script. This commit therefore adds WARN_ON_ONCE() to allow distinguishing scf_torture_init() errors when running scftorture built-in. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 00bba2bdd4d1..5d42f44e3e1a 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -606,17 +606,17 @@ static int __init scf_torture_init(void) if (onoff_interval > 0) { firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (shutdown_secs > 0) { firsterr = torture_shutdown_init(shutdown_secs, scf_torture_cleanup); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stutter > 0) { firsterr = torture_stutter_init(stutter, stutter); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } @@ -637,12 +637,12 @@ static int __init scf_torture_init(void) scf_stats_p[i].cpu = i; firsterr = torture_create_kthread(scftorture_invoker, (void *)&scf_stats_p[i], scf_stats_p[i].task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } if (stat_interval > 0) { firsterr = torture_create_kthread(scf_torture_stats, NULL, scf_torture_stats_task); - if (firsterr) + if (torture_init_error(firsterr)) goto unwind; } From 2010776f8ccb68b85efbade3f19a11b17fb33d74 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 19 Jul 2021 10:08:12 -0700 Subject: [PATCH 044/134] tools/rcu: Add an extract-stall script This commit adds a script that extracts RCU CPU stall warnings from console output. The user can optionally specify the number of lines preceding the stall to output, and also the number of lines of stall-warning text. Signed-off-by: Paul E. McKenney --- tools/rcu/extract-stall.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tools/rcu/extract-stall.sh diff --git a/tools/rcu/extract-stall.sh b/tools/rcu/extract-stall.sh new file mode 100644 index 000000000000..e565697c9f90 --- /dev/null +++ b/tools/rcu/extract-stall.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# +# Extract any RCU CPU stall warnings present in specified file. +# Filter out clocksource lines. Note that preceding-lines excludes the +# initial line of the stall warning but trailing-lines includes it. +# +# Usage: extract-stall.sh dmesg-file [ preceding-lines [ trailing-lines ] ] + +echo $1 +preceding_lines="${2-3}" +trailing_lines="${3-10}" + +awk -v preceding_lines="$preceding_lines" -v trailing_lines="$trailing_lines" ' +suffix <= 0 { + for (i = preceding_lines; i > 0; i--) + last[i] = last[i - 1]; + last[0] = $0; +} + +suffix > 0 { + print $0; + suffix--; + if (suffix <= 0) + print ""; +} + +suffix <= 0 && /detected stall/ { + for (i = preceding_lines; i >= 0; i--) + if (last[i] != "") + print last[i]; + suffix = trailing_lines; +}' < "$1" | tr -d '\015' | grep -v clocksource + From ae3357ac11273fe1aad9c790febf179ef05b930a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 21 Jul 2021 16:28:43 -0700 Subject: [PATCH 045/134] torture: Allot 1G of memory for scftorture runs By default, torture.sh allots 512M of memory for each guest OS. However, when running scftorture with KASAN, 1G is needed. This commit therefore causes torture.sh to provide the required 1G. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/torture.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index 8e882346d2a6..eae88aacca2a 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -351,7 +351,7 @@ fi if test "$do_scftorture" = "yes" then torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot" - torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make + torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --memory 1G --trust-make fi if test "$do_refscale" = yes From faaaf2ac03a81ad6f9dece28d9cb0b65b515a5cb Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 5 Aug 2021 10:51:11 -0700 Subject: [PATCH 046/134] torture: Make kvm-remote.sh print size of downloaded tarball This commit causes kvm-remote.sh to print the size of the tarball that is downloaded to each of the remote systems. This size can help with performance projections and analysis. Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-remote.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh index 03126eb6ec5a..c7d42ef80c53 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh @@ -149,6 +149,7 @@ do done # Download and expand the tarball on all systems. +echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log" for i in $systems do echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log" From 04cb788ecee8369c11edb7104775f30ed6247a26 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 24 Aug 2021 14:05:13 -0700 Subject: [PATCH 047/134] crypto: jitter - drop kernel-doc notation Drop "begin kernel-doc (/**)" entries in jitterentropy.c since they are not in kernel-doc format and they cause many complaints (warnings) from scripts/kernel-doc. Signed-off-by: Randy Dunlap Cc: Stephan Mueller Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Signed-off-by: Herbert Xu --- crypto/jitterentropy.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index a11b3208760f..4dc2261cdeef 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -125,7 +125,7 @@ struct rand_data { * This test complies with SP800-90B section 4.4.2. ***************************************************************************/ -/** +/* * Reset the APT counter * * @ec [in] Reference to entropy collector @@ -138,7 +138,7 @@ static void jent_apt_reset(struct rand_data *ec, unsigned int delta_masked) ec->apt_observations = 0; } -/** +/* * Insert a new entropy event into APT * * @ec [in] Reference to entropy collector @@ -182,7 +182,7 @@ static void jent_apt_insert(struct rand_data *ec, unsigned int delta_masked) * the end. The caller of the Jitter RNG is informed with an error code. ***************************************************************************/ -/** +/* * Repetition Count Test as defined in SP800-90B section 4.4.1 * * @ec [in] Reference to entropy collector @@ -223,7 +223,7 @@ static void jent_rct_insert(struct rand_data *ec, int stuck) } } -/** +/* * Is there an RCT health test failure? * * @ec [in] Reference to entropy collector @@ -246,7 +246,7 @@ static inline __u64 jent_delta(__u64 prev, __u64 next) (JENT_UINT64_MAX - prev + 1 + next); } -/** +/* * Stuck test by checking the: * 1st derivative of the jitter measurement (time delta) * 2nd derivative of the jitter measurement (delta of time deltas) @@ -288,7 +288,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta) return 0; } -/** +/* * Report any health test failures * * @ec [in] Reference to entropy collector @@ -310,7 +310,7 @@ static int jent_health_failure(struct rand_data *ec) * Noise sources ***************************************************************************/ -/** +/* * Update of the loop count used for the next round of * an entropy collection. * @@ -353,7 +353,7 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, return (shuffle + (1<data = new; } -/** +/* * Memory Access noise source -- this is a noise source based on variations in * memory access times * @@ -500,7 +500,7 @@ static void jent_memaccess(struct rand_data *ec, __u64 loop_cnt) /*************************************************************************** * Start of entropy processing logic ***************************************************************************/ -/** +/* * This is the heart of the entropy generation: calculate time deltas and * use the CPU jitter in the time deltas. The jitter is injected into the * entropy pool. @@ -539,7 +539,7 @@ static int jent_measure_jitter(struct rand_data *ec) return stuck; } -/** +/* * Generator of one 64 bit random number * Function fills rand_data->data * @@ -566,7 +566,7 @@ static void jent_gen_entropy(struct rand_data *ec) } } -/** +/* * Entry function: Obtain entropy for the caller. * * This function invokes the entropy gathering logic as often to generate From d5e93b3374e465700fb8c11c955a4433cad5b17e Mon Sep 17 00:00:00 2001 From: Cai Huoqing Date: Wed, 25 Aug 2021 12:05:52 +0800 Subject: [PATCH 048/134] hwrng: Kconfig - Add helper dependency on COMPILE_TEST it's helpful to do a complie test in other platform(e.g.X86) Signed-off-by: Cai Huoqing Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 239eca4d6805..814b3d0ca7b7 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -63,7 +63,7 @@ config HW_RANDOM_AMD config HW_RANDOM_ATMEL tristate "Atmel Random Number Generator support" - depends on ARCH_AT91 && HAVE_CLK && OF + depends on (ARCH_AT91 || COMPILE_TEST) && HAVE_CLK && OF default HW_RANDOM help This driver provides kernel-side support for the Random Number @@ -87,7 +87,7 @@ config HW_RANDOM_BA431 config HW_RANDOM_BCM2835 tristate "Broadcom BCM2835/BCM63xx Random Number Generator support" depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X || \ - ARCH_BCM_63XX || BCM63XX || BMIPS_GENERIC + ARCH_BCM_63XX || BCM63XX || BMIPS_GENERIC || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the Random Number @@ -100,7 +100,7 @@ config HW_RANDOM_BCM2835 config HW_RANDOM_IPROC_RNG200 tristate "Broadcom iProc/STB RNG200 support" - depends on ARCH_BCM_IPROC || ARCH_BCM2835 || ARCH_BRCMSTB + depends on ARCH_BCM_IPROC || ARCH_BCM2835 || ARCH_BRCMSTB || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the RNG200 @@ -165,7 +165,7 @@ config HW_RANDOM_IXP4XX config HW_RANDOM_OMAP tristate "OMAP Random Number Generator support" - depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3 + depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3 || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the Random Number @@ -179,7 +179,7 @@ config HW_RANDOM_OMAP config HW_RANDOM_OMAP3_ROM tristate "OMAP3 ROM Random Number Generator support" - depends on ARCH_OMAP3 + depends on ARCH_OMAP3 || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the Random Number @@ -298,7 +298,7 @@ config HW_RANDOM_INGENIC_TRNG config HW_RANDOM_NOMADIK tristate "ST-Ericsson Nomadik Random Number Generator support" - depends on ARCH_NOMADIK + depends on ARCH_NOMADIK || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the Random Number From 4a7e1e5fc294687a8941fa3eeb4a7e8539ca5e2f Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 25 Aug 2021 13:38:59 -0700 Subject: [PATCH 049/134] crypto: sm4 - Do not change section of ck and sbox When building with clang and GNU as, there is a warning about ignored changed section attributes: /tmp/sm4-c916c8.s: Assembler messages: /tmp/sm4-c916c8.s:677: Warning: ignoring changed section attributes for .data..cacheline_aligned "static const" places the data in .rodata but __cacheline_aligned has the section attribute to place it in .data..cacheline_aligned, in addition to the aligned attribute. To keep the alignment but avoid attempting to change sections, use the ____cacheline_aligned attribute, which is just the aligned attribute. Fixes: 2b31277af577 ("crypto: sm4 - create SM4 library based on sm4 generic code") Link: https://github.com/ClangBuiltLinux/linux/issues/1441 Signed-off-by: Nathan Chancellor Reviewed-by: Tianjia Zhang Signed-off-by: Herbert Xu --- lib/crypto/sm4.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c index 633b59fed9db..284e62576d0c 100644 --- a/lib/crypto/sm4.c +++ b/lib/crypto/sm4.c @@ -15,7 +15,7 @@ static const u32 fk[4] = { 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc }; -static const u32 __cacheline_aligned ck[32] = { +static const u32 ____cacheline_aligned ck[32] = { 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, @@ -26,7 +26,7 @@ static const u32 __cacheline_aligned ck[32] = { 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 }; -static const u8 __cacheline_aligned sbox[256] = { +static const u8 ____cacheline_aligned sbox[256] = { 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, From b9e699f91236cb1cc97a3d02ccb8063af5e4cc46 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:36 +0200 Subject: [PATCH 050/134] crypto: arm64/gcm-aes-ce - remove non-SIMD fallback path Now that kernel mode SIMD is guaranteed to be available when executing in task or softirq context, we no longer need scalar fallbacks to use when the NEON is unavailable. So get rid of them. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-glue.c | 217 ++++++++---------------------- 1 file changed, 55 insertions(+), 162 deletions(-) diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 720cd3a58da3..15794fe21a0b 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -362,84 +362,36 @@ static int gcm_encrypt(struct aead_request *req) err = skcipher_walk_aead_encrypt(&walk, req, false); - if (likely(crypto_simd_usable())) { - do { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - - tag = (u8 *)&lengths; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { - src = dst = memcpy(buf + sizeof(buf) - nbytes, - src, nbytes); - } else if (nbytes < walk.total) { - nbytes &= ~(AES_BLOCK_SIZE - 1); - tag = NULL; - } - - kernel_neon_begin(); - pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, nrounds, - tag); - kernel_neon_end(); - - if (unlikely(!nbytes)) - break; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) - memcpy(walk.dst.virt.addr, - buf + sizeof(buf) - nbytes, nbytes); - - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } while (walk.nbytes); - } else { - while (walk.nbytes >= AES_BLOCK_SIZE) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int remaining = blocks; - - do { - aes_encrypt(&ctx->aes_key, buf, iv); - crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); - crypto_inc(iv, AES_BLOCK_SIZE); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - } while (--remaining > 0); - - ghash_do_update(blocks, dg, walk.dst.virt.addr, - &ctx->ghash_key, NULL); - - err = skcipher_walk_done(&walk, - walk.nbytes % AES_BLOCK_SIZE); - } - - /* handle the tail */ - if (walk.nbytes) { - aes_encrypt(&ctx->aes_key, buf, iv); - - crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, - buf, walk.nbytes); - - memcpy(buf, walk.dst.virt.addr, walk.nbytes); - memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes); - } + do { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + int nbytes = walk.nbytes; tag = (u8 *)&lengths; - ghash_do_update(1, dg, tag, &ctx->ghash_key, - walk.nbytes ? buf : NULL); - if (walk.nbytes) - err = skcipher_walk_done(&walk, 0); + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); + } else if (nbytes < walk.total) { + nbytes &= ~(AES_BLOCK_SIZE - 1); + tag = NULL; + } - put_unaligned_be64(dg[1], tag); - put_unaligned_be64(dg[0], tag + 8); - put_unaligned_be32(1, iv + GCM_IV_SIZE); - aes_encrypt(&ctx->aes_key, iv, iv); - crypto_xor(tag, iv, AES_BLOCK_SIZE); - } + kernel_neon_begin(); + pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, nrounds, + tag); + kernel_neon_end(); + + if (unlikely(!nbytes)) + break; + + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } while (walk.nbytes); if (err) return err; @@ -464,6 +416,7 @@ static int gcm_decrypt(struct aead_request *req) u64 dg[2] = {}; be128 lengths; u8 *tag; + int ret; int err; lengths.a = cpu_to_be64(req->assoclen * 8); @@ -481,101 +434,41 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_aead_decrypt(&walk, req, false); - if (likely(crypto_simd_usable())) { - int ret; - - do { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - int nbytes = walk.nbytes; - - tag = (u8 *)&lengths; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { - src = dst = memcpy(buf + sizeof(buf) - nbytes, - src, nbytes); - } else if (nbytes < walk.total) { - nbytes &= ~(AES_BLOCK_SIZE - 1); - tag = NULL; - } - - kernel_neon_begin(); - ret = pmull_gcm_decrypt(nbytes, dst, src, - ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, - nrounds, tag, otag, authsize); - kernel_neon_end(); - - if (unlikely(!nbytes)) - break; - - if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) - memcpy(walk.dst.virt.addr, - buf + sizeof(buf) - nbytes, nbytes); - - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } while (walk.nbytes); - - if (err) - return err; - if (ret) - return -EBADMSG; - } else { - while (walk.nbytes >= AES_BLOCK_SIZE) { - int blocks = walk.nbytes / AES_BLOCK_SIZE; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - - ghash_do_update(blocks, dg, walk.src.virt.addr, - &ctx->ghash_key, NULL); - - do { - aes_encrypt(&ctx->aes_key, buf, iv); - crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE); - crypto_inc(iv, AES_BLOCK_SIZE); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - } while (--blocks > 0); - - err = skcipher_walk_done(&walk, - walk.nbytes % AES_BLOCK_SIZE); - } - - /* handle the tail */ - if (walk.nbytes) { - memcpy(buf, walk.src.virt.addr, walk.nbytes); - memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes); - } + do { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + int nbytes = walk.nbytes; tag = (u8 *)&lengths; - ghash_do_update(1, dg, tag, &ctx->ghash_key, - walk.nbytes ? buf : NULL); - if (walk.nbytes) { - aes_encrypt(&ctx->aes_key, buf, iv); - - crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, - buf, walk.nbytes); - - err = skcipher_walk_done(&walk, 0); + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) { + src = dst = memcpy(buf + sizeof(buf) - nbytes, + src, nbytes); + } else if (nbytes < walk.total) { + nbytes &= ~(AES_BLOCK_SIZE - 1); + tag = NULL; } - if (err) - return err; + kernel_neon_begin(); + ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, + nrounds, tag, otag, authsize); + kernel_neon_end(); - put_unaligned_be64(dg[1], tag); - put_unaligned_be64(dg[0], tag + 8); - put_unaligned_be32(1, iv + GCM_IV_SIZE); - aes_encrypt(&ctx->aes_key, iv, iv); - crypto_xor(tag, iv, AES_BLOCK_SIZE); + if (unlikely(!nbytes)) + break; - if (crypto_memneq(tag, otag, authsize)) { - memzero_explicit(tag, AES_BLOCK_SIZE); - return -EBADMSG; - } - } - return 0; + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, + buf + sizeof(buf) - nbytes, nbytes); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + } while (walk.nbytes); + + if (err) + return err; + + return ret ? -EBADMSG : 0; } static struct aead_alg gcm_aes_alg = { From 96c34e143689bc498508c27ea35c4446aed9f42e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:37 +0200 Subject: [PATCH 051/134] crypto: arm64/aes-neonbs - stop using SIMD helper for skciphers Calls into the skcipher API can only occur from contexts where the SIMD unit is available, so there is no need for the SIMD helper. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 2 - arch/arm64/crypto/aes-neonbs-glue.c | 122 ++-------------------------- 2 files changed, 9 insertions(+), 115 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 55f19450091b..4b70aaab0f35 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -122,8 +122,6 @@ config CRYPTO_AES_ARM64_BS depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_NEON_BLK - select CRYPTO_AES_ARM64 select CRYPTO_LIB_AES - select CRYPTO_SIMD endif diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index fb507d569922..8df6ad8cb09d 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -63,11 +63,6 @@ struct aesbs_cbc_ctx { u32 enc[AES_MAX_KEYLENGTH_U32]; }; -struct aesbs_ctr_ctx { - struct aesbs_ctx key; /* must be first member */ - struct crypto_aes_ctx fallback; -}; - struct aesbs_xts_ctx { struct aesbs_ctx key; u32 twkey[AES_MAX_KEYLENGTH_U32]; @@ -207,25 +202,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = aes_expandkey(&ctx->fallback, in_key, key_len); - if (err) - return err; - - ctx->key.rounds = 6 + key_len / 4; - - kernel_neon_begin(); - aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); - kernel_neon_end(); - - return 0; -} - static int ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -292,29 +268,6 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key, return aesbs_setkey(tfm, in_key, key_len); } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(&ctx->fallback, dst, src); - local_irq_restore(flags); -} - -static int ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int __xts_crypt(struct skcipher_request *req, bool encrypt, void (*fn)(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[])) @@ -431,13 +384,12 @@ static int xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { - .base.cra_name = "__ecb(aes)", - .base.cra_driver_name = "__ecb-aes-neonbs", + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -446,13 +398,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, { - .base.cra_name = "__cbc(aes)", - .base.cra_driver_name = "__cbc-aes-neonbs", + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -462,13 +413,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, { - .base.cra_name = "__ctr(aes)", - .base.cra_driver_name = "__ctr-aes-neonbs", + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct aesbs_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -479,29 +429,12 @@ static struct skcipher_alg aes_algs[] = { { .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, { - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-neonbs", - .base.cra_priority = 250 - 1, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct aesbs_ctr_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .chunksize = AES_BLOCK_SIZE, - .walksize = 8 * AES_BLOCK_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = aesbs_ctr_setkey_sync, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base.cra_name = "__xts(aes)", - .base.cra_driver_name = "__xts-aes-neonbs", + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "xts-aes-neonbs", .base.cra_priority = 250, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -512,54 +445,17 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = xts_decrypt, } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; - int err; - int i; - if (!cpu_have_named_feature(ASIMD)) return -ENODEV; - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; - -unregister_simds: - aes_exit(); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } module_init(aes_init); From 676e508122d96aec199ab7181fd62c592c2cb8d5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:38 +0200 Subject: [PATCH 052/134] crypto: arm64/aes-ce - stop using SIMD helper for skciphers Calls into the skcipher API can only occur from contexts where the SIMD unit is available, so there is no need for the SIMD helper. Reviewed-by: Eric Biggers Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/Kconfig | 4 -- arch/arm64/crypto/aes-glue.c | 102 +++++------------------------------ 2 files changed, 13 insertions(+), 93 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 4b70aaab0f35..addfa413650b 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -88,16 +88,12 @@ config CRYPTO_AES_ARM64_CE_BLK depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_CE - select CRYPTO_AES_ARM64 - select CRYPTO_SIMD config CRYPTO_AES_ARM64_NEON_BLK tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER - select CRYPTO_AES_ARM64 select CRYPTO_LIB_AES - select CRYPTO_SIMD config CRYPTO_CHACHA20_NEON tristate "ChaCha20, XChaCha20, and XChaCha12 stream ciphers using NEON instructions" diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 17e735931a0c..30b7cc6a7079 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -444,7 +444,7 @@ static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req) return err ?: cbc_decrypt_walk(req, &walk); } -static int ctr_encrypt(struct skcipher_request *req) +static int __maybe_unused ctr_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -485,29 +485,6 @@ static int ctr_encrypt(struct skcipher_request *req) return err; } -static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) -{ - const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned long flags; - - /* - * Temporarily disable interrupts to avoid races where - * cachelines are evicted when the CPU is interrupted - * to do something else. - */ - local_irq_save(flags); - aes_encrypt(ctx, dst, src); - local_irq_restore(flags); -} - -static int __maybe_unused ctr_encrypt_sync(struct skcipher_request *req) -{ - if (!crypto_simd_usable()) - return crypto_ctr_encrypt_walk(req, ctr_encrypt_one); - - return ctr_encrypt(req); -} - static int __maybe_unused xts_encrypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -656,10 +633,9 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) static struct skcipher_alg aes_algs[] = { { #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) .base = { - .cra_name = "__ecb(aes)", - .cra_driver_name = "__ecb-aes-" MODE, + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -671,10 +647,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = ecb_decrypt, }, { .base = { - .cra_name = "__cbc(aes)", - .cra_driver_name = "__cbc-aes-" MODE, + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -687,10 +662,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__ctr(aes)", - .cra_driver_name = "__ctr-aes-" MODE, + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -704,26 +678,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = ctr_encrypt, }, { .base = { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-" MODE, - .cra_priority = PRIO - 1, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto_aes_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .chunksize = AES_BLOCK_SIZE, - .setkey = skcipher_aes_setkey, - .encrypt = ctr_encrypt_sync, - .decrypt = ctr_encrypt_sync, -}, { - .base = { - .cra_name = "__xts(aes)", - .cra_driver_name = "__xts-aes-" MODE, + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .cra_module = THIS_MODULE, @@ -738,10 +695,9 @@ static struct skcipher_alg aes_algs[] = { { }, { #endif .base = { - .cra_name = "__cts(cbc(aes))", - .cra_driver_name = "__cts-cbc-aes-" MODE, + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "cts-cbc-aes-" MODE, .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_module = THIS_MODULE, @@ -755,10 +711,9 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = cts_cbc_decrypt, }, { .base = { - .cra_name = "__essiv(cbc(aes),sha256)", - .cra_driver_name = "__essiv-cbc-aes-sha256-" MODE, + .cra_name = "essiv(cbc(aes),sha256)", + .cra_driver_name = "essiv-cbc-aes-sha256-" MODE, .cra_priority = PRIO + 1, - .cra_flags = CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx), .cra_module = THIS_MODULE, @@ -997,28 +952,15 @@ static struct shash_alg mac_algs[] = { { .descsize = sizeof(struct mac_desc_ctx), } }; -static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)]; - static void aes_exit(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++) - if (aes_simd_algs[i]) - simd_skcipher_free(aes_simd_algs[i]); - crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - struct simd_skcipher_alg *simd; - const char *basename; - const char *algname; - const char *drvname; int err; - int i; err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); if (err) @@ -1028,26 +970,8 @@ static int __init aes_init(void) if (err) goto unregister_ciphers; - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL)) - continue; - - algname = aes_algs[i].base.cra_name + 2; - drvname = aes_algs[i].base.cra_driver_name + 2; - basename = aes_algs[i].base.cra_driver_name; - simd = simd_skcipher_create_compat(algname, drvname, basename); - err = PTR_ERR(simd); - if (IS_ERR(simd)) - goto unregister_simds; - - aes_simd_algs[i] = simd; - } - return 0; -unregister_simds: - aes_exit(); - return err; unregister_ciphers: crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); return err; From 36a916af641dc71ef7d4b98417bf4019ddeb4ebe Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:39 +0200 Subject: [PATCH 053/134] crypto: arm64/aes-ccm - yield NEON when processing auth-only data In SIMD accelerated crypto drivers, we typically yield the SIMD unit after processing 4 KiB of input, to avoid scheduling blackouts caused by the fact that claiming the SIMD unit disables preemption as well as softirq processing. The arm64 CCM driver does this implicitly for the ciphertext, due to the fact that the skcipher API never processes more than a single page at a time. However, the scatterwalk performed by this driver when processing the authenticate-only data will keep the SIMD unit occupied until it completes. So cap the scatterwalk steps to 4 KiB. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-glue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index f6d19b0dc893..fe9c837ac4b9 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -161,6 +161,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } + n = min_t(u32, n, SZ_4K); /* yield NEON at least every 4k */ p = scatterwalk_map(&walk); ccm_update_mac(ctx, mac, p, n, &macp); len -= n; From b3482635e5d69c8a40288bd025f61a994b3b1126 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:40 +0200 Subject: [PATCH 054/134] crypto: arm64/aes-ccm - remove non-SIMD fallback path AES/CCM on arm64 is implemented as a synchronous AEAD, and so it is guaranteed by the API that it is only invoked in task or softirq context. Since softirqs are now only handled when the SIMD is not being used in the task context that was interrupted to service the softirq, we no longer need a fallback path. Let's remove it. Signed-off-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-glue.c | 153 ++++++---------------------- 1 file changed, 32 insertions(+), 121 deletions(-) diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index fe9c837ac4b9..c1f221a181a5 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -6,12 +6,10 @@ */ #include -#include #include #include #include #include -#include #include #include @@ -99,36 +97,10 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], u32 abytes, u32 *macp) { - if (crypto_simd_usable()) { - kernel_neon_begin(); - ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, - num_rounds(key)); - kernel_neon_end(); - } else { - if (*macp > 0 && *macp < AES_BLOCK_SIZE) { - int added = min(abytes, AES_BLOCK_SIZE - *macp); - - crypto_xor(&mac[*macp], in, added); - - *macp += added; - in += added; - abytes -= added; - } - - while (abytes >= AES_BLOCK_SIZE) { - aes_encrypt(key, mac, mac); - crypto_xor(mac, in, AES_BLOCK_SIZE); - - in += AES_BLOCK_SIZE; - abytes -= AES_BLOCK_SIZE; - } - - if (abytes > 0) { - aes_encrypt(key, mac, mac); - crypto_xor(mac, in, abytes); - *macp = abytes; - } - } + kernel_neon_begin(); + ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, + num_rounds(key)); + kernel_neon_end(); } static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) @@ -172,54 +144,6 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) } while (len); } -static int ccm_crypt_fallback(struct skcipher_walk *walk, u8 mac[], u8 iv0[], - struct crypto_aes_ctx *ctx, bool enc) -{ - u8 buf[AES_BLOCK_SIZE]; - int err = 0; - - while (walk->nbytes) { - int blocks = walk->nbytes / AES_BLOCK_SIZE; - u32 tail = walk->nbytes % AES_BLOCK_SIZE; - u8 *dst = walk->dst.virt.addr; - u8 *src = walk->src.virt.addr; - u32 nbytes = walk->nbytes; - - if (nbytes == walk->total && tail > 0) { - blocks++; - tail = 0; - } - - do { - u32 bsize = AES_BLOCK_SIZE; - - if (nbytes < AES_BLOCK_SIZE) - bsize = nbytes; - - crypto_inc(walk->iv, AES_BLOCK_SIZE); - aes_encrypt(ctx, buf, walk->iv); - aes_encrypt(ctx, mac, mac); - if (enc) - crypto_xor(mac, src, bsize); - crypto_xor_cpy(dst, src, buf, bsize); - if (!enc) - crypto_xor(mac, dst, bsize); - dst += bsize; - src += bsize; - nbytes -= bsize; - } while (--blocks); - - err = skcipher_walk_done(walk, tail); - } - - if (!err) { - aes_encrypt(ctx, buf, iv0); - aes_encrypt(ctx, mac, mac); - crypto_xor(mac, buf, AES_BLOCK_SIZE); - } - return err; -} - static int ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -242,30 +166,24 @@ static int ccm_encrypt(struct aead_request *req) err = skcipher_walk_aead_encrypt(&walk, req, false); - if (crypto_simd_usable()) { - while (walk.nbytes) { - u32 tail = walk.nbytes % AES_BLOCK_SIZE; + while (walk.nbytes) { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; - if (walk.nbytes == walk.total) - tail = 0; + if (walk.nbytes == walk.total) + tail = 0; - kernel_neon_begin(); - ce_aes_ccm_encrypt(walk.dst.virt.addr, - walk.src.virt.addr, - walk.nbytes - tail, ctx->key_enc, - num_rounds(ctx), mac, walk.iv); - kernel_neon_end(); + kernel_neon_begin(); + ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes - tail, ctx->key_enc, + num_rounds(ctx), mac, walk.iv); + kernel_neon_end(); - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, - num_rounds(ctx)); - kernel_neon_end(); - } - } else { - err = ccm_crypt_fallback(&walk, mac, buf, ctx, true); + err = skcipher_walk_done(&walk, tail); + } + if (!err) { + kernel_neon_begin(); + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + kernel_neon_end(); } if (err) return err; @@ -300,32 +218,25 @@ static int ccm_decrypt(struct aead_request *req) err = skcipher_walk_aead_decrypt(&walk, req, false); - if (crypto_simd_usable()) { - while (walk.nbytes) { - u32 tail = walk.nbytes % AES_BLOCK_SIZE; + while (walk.nbytes) { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; - if (walk.nbytes == walk.total) - tail = 0; + if (walk.nbytes == walk.total) + tail = 0; - kernel_neon_begin(); - ce_aes_ccm_decrypt(walk.dst.virt.addr, - walk.src.virt.addr, + kernel_neon_begin(); + ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); - kernel_neon_end(); + kernel_neon_end(); - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, - num_rounds(ctx)); - kernel_neon_end(); - } - } else { - err = ccm_crypt_fallback(&walk, mac, buf, ctx, false); + err = skcipher_walk_done(&walk, tail); + } + if (!err) { + kernel_neon_begin(); + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + kernel_neon_end(); } - if (err) return err; From 741691c44606b1903e674d12f3e4a4b68ade69ad Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:41 +0200 Subject: [PATCH 055/134] crypto: arm64/aes-ccm - reduce NEON begin/end calls for common case AES-CCM (as used in WPA2 CCMP, for instance) typically involves authenticate-only data, and operates on a single network packet, and so the common case is for the authenticate, en/decrypt and finalize SIMD helpers to all be called exactly once in sequence. Since kernel_neon_end() now involves manipulation of the preemption state as well as the softirq mask state, let's reduce the number of times we are forced to call it to only once if we are handling this common case. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-core.S | 1 + arch/arm64/crypto/aes-ce-ccm-glue.c | 80 +++++++++++++++++------------ 2 files changed, 49 insertions(+), 32 deletions(-) diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 99a028e298ed..8adff299fcd3 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -124,6 +124,7 @@ SYM_FUNC_START(ce_aes_ccm_final) SYM_FUNC_END(ce_aes_ccm_final) .macro aes_ccm_do_crypt,enc + cbz x2, 5f ldr x8, [x6, #8] /* load lower ctr */ ld1 {v0.16b}, [x5] /* load mac */ CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index c1f221a181a5..d973655fab7e 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -97,10 +97,8 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], u32 abytes, u32 *macp) { - kernel_neon_begin(); ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, num_rounds(key)); - kernel_neon_end(); } static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) @@ -136,6 +134,12 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) n = min_t(u32, n, SZ_4K); /* yield NEON at least every 4k */ p = scatterwalk_map(&walk); ccm_update_mac(ctx, mac, p, n, &macp); + + if (len / SZ_4K > (len - n) / SZ_4K) { + kernel_neon_end(); + kernel_neon_begin(); + } + len -= n; scatterwalk_unmap(p); @@ -158,35 +162,41 @@ static int ccm_encrypt(struct aead_request *req) if (err) return err; - if (req->assoclen) - ccm_calculate_auth_mac(req, mac); - /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); err = skcipher_walk_aead_encrypt(&walk, req, false); + if (unlikely(err)) + return err; - while (walk.nbytes) { + kernel_neon_begin(); + + if (req->assoclen) + ccm_calculate_auth_mac(req, mac); + + do { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == walk.total) tail = 0; - kernel_neon_begin(); ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr, walk.nbytes - tail, ctx->key_enc, num_rounds(ctx), mac, walk.iv); + + if (walk.nbytes == walk.total) + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + kernel_neon_end(); - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); - kernel_neon_end(); - } - if (err) - return err; + if (walk.nbytes) { + err = skcipher_walk_done(&walk, tail); + if (unlikely(err)) + return err; + if (unlikely(walk.nbytes)) + kernel_neon_begin(); + } + } while (walk.nbytes); /* copy authtag to end of dst */ scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen, @@ -210,35 +220,41 @@ static int ccm_decrypt(struct aead_request *req) if (err) return err; - if (req->assoclen) - ccm_calculate_auth_mac(req, mac); - /* preserve the original iv for the final round */ memcpy(buf, req->iv, AES_BLOCK_SIZE); err = skcipher_walk_aead_decrypt(&walk, req, false); + if (unlikely(err)) + return err; - while (walk.nbytes) { + kernel_neon_begin(); + + if (req->assoclen) + ccm_calculate_auth_mac(req, mac); + + do { u32 tail = walk.nbytes % AES_BLOCK_SIZE; if (walk.nbytes == walk.total) tail = 0; - kernel_neon_begin(); ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr, - walk.nbytes - tail, ctx->key_enc, - num_rounds(ctx), mac, walk.iv); + walk.nbytes - tail, ctx->key_enc, + num_rounds(ctx), mac, walk.iv); + + if (walk.nbytes == walk.total) + ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); + kernel_neon_end(); - err = skcipher_walk_done(&walk, tail); - } - if (!err) { - kernel_neon_begin(); - ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx)); - kernel_neon_end(); - } - if (err) - return err; + if (walk.nbytes) { + err = skcipher_walk_done(&walk, tail); + if (unlikely(err)) + return err; + if (unlikely(walk.nbytes)) + kernel_neon_begin(); + } + } while (walk.nbytes); /* compare calculated auth tag with the stored one */ scatterwalk_map_and_copy(buf, req->src, From 898387e40cf538b7d1605e05d456699fe418a77f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 27 Aug 2021 09:03:42 +0200 Subject: [PATCH 056/134] crypto: arm64/aes-ccm - avoid by-ref argument for ce_aes_ccm_auth_data With the SIMD code path removed, we can clean up the CCM auth-only path a bit further, by passing the 'macp' input buffer pointer by value, rather than by reference, and taking the output value from the function's return value. This way, the compiler is no longer forced to allocate macp on the stack. This is not expected to make any difference in practice, it just makes for slightly cleaner code. Signed-off-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm64/crypto/aes-ce-ccm-core.S | 23 +++++++++++------------ arch/arm64/crypto/aes-ce-ccm-glue.c | 19 +++++++------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 8adff299fcd3..b03f7f71f893 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -12,22 +12,21 @@ .arch armv8-a+crypto /* - * void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, - * u32 *macp, u8 const rk[], u32 rounds); + * u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, + * u32 macp, u8 const rk[], u32 rounds); */ SYM_FUNC_START(ce_aes_ccm_auth_data) - ldr w8, [x3] /* leftover from prev round? */ ld1 {v0.16b}, [x0] /* load mac */ - cbz w8, 1f - sub w8, w8, #16 + cbz w3, 1f + sub w3, w3, #16 eor v1.16b, v1.16b, v1.16b 0: ldrb w7, [x1], #1 /* get 1 byte of input */ subs w2, w2, #1 - add w8, w8, #1 + add w3, w3, #1 ins v1.b[0], w7 ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */ beq 8f /* out of input? */ - cbnz w8, 0b + cbnz w3, 0b eor v0.16b, v0.16b, v1.16b 1: ld1 {v3.4s}, [x4] /* load first round key */ prfm pldl1strm, [x1] @@ -62,7 +61,7 @@ SYM_FUNC_START(ce_aes_ccm_auth_data) beq 10f adds w2, w2, #16 beq 10f - mov w8, w2 + mov w3, w2 7: ldrb w7, [x1], #1 umov w6, v0.b[0] eor w6, w6, w7 @@ -71,15 +70,15 @@ SYM_FUNC_START(ce_aes_ccm_auth_data) beq 10f ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ b 7b -8: cbz w8, 91f - mov w7, w8 - add w8, w8, #16 +8: cbz w3, 91f + mov w7, w3 + add w3, w3, #16 9: ext v1.16b, v1.16b, v1.16b, #1 adds w7, w7, #1 bne 9b 91: eor v0.16b, v0.16b, v1.16b st1 {v0.16b}, [x0] -10: str w8, [x3] +10: mov w0, w3 ret SYM_FUNC_END(ce_aes_ccm_auth_data) diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index d973655fab7e..c4f14415f5f0 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -27,8 +27,8 @@ static int num_rounds(struct crypto_aes_ctx *ctx) return 6 + ctx->key_length / 4; } -asmlinkage void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, - u32 *macp, u32 const rk[], u32 rounds); +asmlinkage u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, + u32 macp, u32 const rk[], u32 rounds); asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, u32 const rk[], u32 rounds, u8 mac[], @@ -94,13 +94,6 @@ static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen) return 0; } -static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[], - u32 abytes, u32 *macp) -{ - ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc, - num_rounds(key)); -} - static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) { struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -120,7 +113,8 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) ltag.len = 6; } - ccm_update_mac(ctx, mac, (u8 *)<ag, ltag.len, &macp); + macp = ce_aes_ccm_auth_data(mac, (u8 *)<ag, ltag.len, macp, + ctx->key_enc, num_rounds(ctx)); scatterwalk_start(&walk, req->src); do { @@ -133,13 +127,14 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) } n = min_t(u32, n, SZ_4K); /* yield NEON at least every 4k */ p = scatterwalk_map(&walk); - ccm_update_mac(ctx, mac, p, n, &macp); + + macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc, + num_rounds(ctx)); if (len / SZ_4K > (len - n) / SZ_4K) { kernel_neon_end(); kernel_neon_begin(); } - len -= n; scatterwalk_unmap(p); From cbbb5f07ab737f868f90d429255d5d644280f6a9 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 27 Aug 2021 16:43:10 +0800 Subject: [PATCH 057/134] crypto: hisilicon - Fix sscanf format signedness The function qm_qos_value_init expects an unsigned integer but is incorrectly supplying a signed format to sscanf. This patch fixes it. Reported-by: kernel test robot Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 369562d34d66..cf9ec5625a1e 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -4185,7 +4185,7 @@ static ssize_t qm_qos_value_init(const char *buf, unsigned long *val) return -EINVAL; } - ret = sscanf(buf, "%ld", val); + ret = sscanf(buf, "%lu", val); if (ret != QM_QOS_VAL_NUM) return -EINVAL; From 8de8d4fe7d5a38e1ded234269f808898e72e9fb2 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 28 Aug 2021 11:39:38 +0800 Subject: [PATCH 058/134] crypto: hisilicon/qm - fix the uacce mmap failed Allocate an extra memory page for qp. This extra page is used to set the device or qp status. But this page not be used currently. Meanwhile it leads to dus size not equal to mmap size as using uacce sva mode, and cause the app task broken. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index cf9ec5625a1e..88ec66b34766 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3122,8 +3122,10 @@ static int qm_alloc_uacce(struct hisi_qm *qm) else mmio_page_nr = qm->db_interval / PAGE_SIZE; + /* Add one more page for device or qp status */ dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT; + sizeof(struct qm_cqe) * QM_Q_DEPTH + PAGE_SIZE) >> + PAGE_SHIFT; uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; From 8bb765271aded24ca724a39701c6e686234c7020 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 28 Aug 2021 11:39:39 +0800 Subject: [PATCH 059/134] crypto: hisilicon/qm - support the userspace task resetting Allocate an extra memory page for qp in the qp memory initialization. Set a qp error flag in the extra page addr when device resetting. This error flag can be seen in the userspace. This flag can helps users to stop tasks when device resetting. After resetting, this error flag will be reset when this qp is created again. So app should release the old qp and request a new one, and do the task on the new queue again. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 46 ++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 88ec66b34766..6c57149e28f0 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -233,6 +233,8 @@ #define QM_DBG_WRITE_LEN 1024 #define QM_DBG_TMP_BUF_LEN 22 #define QM_PCI_COMMAND_INVALID ~0 +#define QM_RESET_STOP_TX_OFFSET 1 +#define QM_RESET_STOP_RX_OFFSET 2 #define WAIT_PERIOD 20 #define REMOVE_WAIT_DELAY 10 @@ -883,6 +885,20 @@ static irqreturn_t qm_mb_cmd_irq(int irq, void *data) return IRQ_HANDLED; } +static void qm_set_qp_disable(struct hisi_qp *qp, int offset) +{ + u32 *addr; + + if (qp->is_in_kernel) + return; + + addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset; + *addr = 1; + + /* make sure setup is completed */ + mb(); +} + static irqreturn_t qm_aeq_irq(int irq, void *data) { struct hisi_qm *qm = data; @@ -2467,6 +2483,15 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) return qp->sqe + sq_tail * qp->qm->sqe_size; } +static void hisi_qm_unset_hw_reset(struct hisi_qp *qp) +{ + u64 *addr; + + /* Use last 64 bits of DUS to reset status. */ + addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET; + *addr = 0; +} + static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) { struct device *dev = &qm->pdev->dev; @@ -2492,7 +2517,7 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) } qp = &qm->qp_array[qp_id]; - + hisi_qm_unset_hw_reset(qp); memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH); qp->event_cb = NULL; @@ -2912,6 +2937,14 @@ static int hisi_qm_get_available_instances(struct uacce_device *uacce) return hisi_qm_get_free_qp_num(uacce->priv); } +static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) +{ + int i; + + for (i = 0; i < qm->qp_num; i++) + qm_set_qp_disable(&qm->qp_array[i], offset); +} + static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, unsigned long arg, struct uacce_queue *q) @@ -3684,11 +3717,13 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) if (qm->status.stop_reason == QM_SOFT_RESET || qm->status.stop_reason == QM_FLR) { + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); ret = qm_stop_started_qp(qm); if (ret < 0) { dev_err(dev, "Failed to stop started qp!\n"); goto err_unlock; } + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); } /* Mask eq and aeq irq */ @@ -5047,6 +5082,8 @@ static int qm_controller_reset(struct hisi_qm *qm) ret = qm_controller_reset_prepare(qm); if (ret) { + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); clear_bit(QM_RST_SCHED, &qm->misc_ctl); return ret; } @@ -5133,6 +5170,8 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) ret = hisi_qm_stop(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); return; } @@ -5316,9 +5355,14 @@ static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, atomic_set(&qm->status.flags, QM_STOP); cmd = QM_VF_PREPARE_FAIL; goto err_prepare; + } else { + goto out; } err_prepare: + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); +out: pci_save_state(pdev); ret = qm->ops->ping_pf(qm, cmd); if (ret) From ba79a32acfde1ffdaefc05b02420c4124b60dbd3 Mon Sep 17 00:00:00 2001 From: Wojciech Ziemba Date: Wed, 1 Sep 2021 18:36:05 +0100 Subject: [PATCH 060/134] crypto: qat - replace deprecated MSI API Replace deprecated MSI enable and disable respectively and update handling of return values. Signed-off-by: Wojciech Ziemba Co-developed-by: Gupta Shashank Signed-off-by: Gupta Shashank Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 1 - drivers/crypto/qat/qat_common/adf_isr.c | 102 +++++++++--------- drivers/crypto/qat/qat_common/adf_vf_isr.c | 12 +-- 3 files changed, 55 insertions(+), 60 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 38c0af6d4e43..87de40d6c9a5 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -45,7 +45,6 @@ struct adf_bar { } __packed; struct adf_accel_msix { - struct msix_entry *entries; char **names; u32 num_entries; } __packed; diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index c678d5c531aa..a2ab16651a56 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -27,35 +27,26 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 msix_num_entries = 1; + u32 msix_num_entries = hw_data->num_banks + 1; + int ret; if (hw_data->set_msix_rttable) hw_data->set_msix_rttable(accel_dev); - /* If SR-IOV is disabled, add entries for each bank */ - if (!accel_dev->pf.vf_info) { - int i; - - msix_num_entries += hw_data->num_banks; - for (i = 0; i < msix_num_entries; i++) - pci_dev_info->msix_entries.entries[i].entry = i; - } else { - pci_dev_info->msix_entries.entries[0].entry = - hw_data->num_banks; - } - - if (pci_enable_msix_exact(pci_dev_info->pci_dev, - pci_dev_info->msix_entries.entries, - msix_num_entries)) { - dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); - return -EFAULT; + ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries, + msix_num_entries, PCI_IRQ_MSIX); + if (unlikely(ret < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to allocate %d MSI-X vectors\n", + msix_num_entries); + return ret; } return 0; } static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) { - pci_disable_msix(pci_dev_info->pci_dev); + pci_free_irq_vectors(pci_dev_info->pci_dev); } static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) @@ -139,9 +130,9 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int ret, i = 0; + int clust_irq = hw_data->num_banks; + int ret, irq, i = 0; char *name; /* Request msix irq for all banks unless SR-IOV enabled */ @@ -153,19 +144,25 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) name = *(pci_dev_info->msix_entries.names + i); snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, "qat%d-bundle%d", accel_dev->accel_id, i); - ret = request_irq(msixe[i].vector, - adf_msix_isr_bundle, 0, name, bank); + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + if (unlikely(irq < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get IRQ number of device vector %d - %s\n", + i, name); + return irq; + } + ret = request_irq(irq, adf_msix_isr_bundle, 0, + &name[0], bank); if (ret) { dev_err(&GET_DEV(accel_dev), - "failed to enable irq %d for %s\n", - msixe[i].vector, name); + "Failed to allocate IRQ %d for %s\n", + irq, name); return ret; } cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; - irq_set_affinity_hint(msixe[i].vector, - get_cpu_mask(cpu)); + irq_set_affinity_hint(irq, get_cpu_mask(cpu)); } } @@ -173,11 +170,17 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) name = *(pci_dev_info->msix_entries.names + i); snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, "qat%d-ae-cluster", accel_dev->accel_id); - ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev); + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + if (unlikely(irq < 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to get IRQ number of device vector %d - %s\n", + i, name); + return irq; + } + ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev); if (ret) { dev_err(&GET_DEV(accel_dev), - "failed to enable irq %d, for %s\n", - msixe[i].vector, name); + "Failed to allocate IRQ %d for %s\n", irq, name); return ret; } return ret; @@ -187,25 +190,27 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int i = 0; + int clust_irq = hw_data->num_banks; + int irq, i = 0; if (pci_dev_info->msix_entries.num_entries > 1) { for (i = 0; i < hw_data->num_banks; i++) { - irq_set_affinity_hint(msixe[i].vector, NULL); - free_irq(msixe[i].vector, &etr_data->banks[i]); + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + irq_set_affinity_hint(irq, NULL); + free_irq(irq, &etr_data->banks[i]); } } - irq_set_affinity_hint(msixe[i].vector, NULL); - free_irq(msixe[i].vector, accel_dev); + + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + irq_set_affinity_hint(irq, NULL); + free_irq(irq, accel_dev); } -static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) +static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) { int i; char **names; - struct msix_entry *entries; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msix_num_entries = 1; @@ -213,39 +218,30 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) if (!accel_dev->pf.vf_info) msix_num_entries += hw_data->num_banks; - entries = kcalloc_node(msix_num_entries, sizeof(*entries), - GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); - if (!entries) + names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); + if (!names) return -ENOMEM; - names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); - if (!names) { - kfree(entries); - return -ENOMEM; - } for (i = 0; i < msix_num_entries; i++) { *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); if (!(*(names + i))) goto err; } accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; - accel_dev->accel_pci_dev.msix_entries.entries = entries; accel_dev->accel_pci_dev.msix_entries.names = names; return 0; err: for (i = 0; i < msix_num_entries; i++) kfree(*(names + i)); - kfree(entries); kfree(names); return -ENOMEM; } -static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) +static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev) { char **names = accel_dev->accel_pci_dev.msix_entries.names; int i; - kfree(accel_dev->accel_pci_dev.msix_entries.entries); for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) kfree(*(names + i)); kfree(names); @@ -287,7 +283,7 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev) adf_free_irqs(accel_dev); adf_cleanup_bh(accel_dev); adf_disable_msix(&accel_dev->accel_pci_dev); - adf_isr_free_msix_entry_table(accel_dev); + adf_isr_free_msix_vectors_data(accel_dev); } EXPORT_SYMBOL_GPL(adf_isr_resource_free); @@ -303,7 +299,7 @@ int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_isr_alloc_msix_entry_table(accel_dev); + ret = adf_isr_alloc_msix_vectors_data(accel_dev); if (ret) goto err_out; @@ -328,7 +324,7 @@ err_disable_msix: adf_disable_msix(&accel_dev->accel_pci_dev); err_free_msix_table: - adf_isr_free_msix_entry_table(accel_dev); + adf_isr_free_msix_vectors_data(accel_dev); err_out: return ret; diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index 7828a6573f3e..695c5050b6f3 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -53,11 +53,11 @@ EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); static int adf_enable_msi(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - int stat = pci_enable_msi(pci_dev_info->pci_dev); - - if (stat) { + int stat = pci_alloc_irq_vectors(pci_dev_info->pci_dev, 1, 1, + PCI_IRQ_MSI); + if (unlikely(stat < 0)) { dev_err(&GET_DEV(accel_dev), - "Failed to enable MSI interrupts\n"); + "Failed to enable MSI interrupt: %d\n", stat); return stat; } @@ -65,7 +65,7 @@ static int adf_enable_msi(struct adf_accel_dev *accel_dev) if (!accel_dev->vf.irq_name) return -ENOMEM; - return stat; + return 0; } static void adf_disable_msi(struct adf_accel_dev *accel_dev) @@ -73,7 +73,7 @@ static void adf_disable_msi(struct adf_accel_dev *accel_dev) struct pci_dev *pdev = accel_to_pci_dev(accel_dev); kfree(accel_dev->vf.irq_name); - pci_disable_msi(pdev); + pci_free_irq_vectors(pdev); } static void adf_dev_stop_async(struct work_struct *work) From 0e64dcd7c94b94f90b820bfbe57bbcea8bf21545 Mon Sep 17 00:00:00 2001 From: Wojciech Ziemba Date: Wed, 1 Sep 2021 18:36:06 +0100 Subject: [PATCH 061/134] crypto: qat - remove unmatched CPU affinity to cluster IRQ This patch removes redundant CPU affinity to AE cluster IRQ. Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_isr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index a2ab16651a56..26b546d3f821 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -203,7 +203,6 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) } irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); - irq_set_affinity_hint(irq, NULL); free_irq(irq, accel_dev); } From 9832fdc917de0f28772558688d7fa6c097c9c6cc Mon Sep 17 00:00:00 2001 From: Maksim Lukoshkov Date: Wed, 1 Sep 2021 18:36:07 +0100 Subject: [PATCH 062/134] crypto: qat - free irqs only if allocated Change the irq allocation logic so that it is possible to free only the allocated irqs in case of error. A new flag is introduced for every PF/VF interrupt. This flag is set to "true" only when the interrupt is requested. During clean up, devm_free_irq() is only called if this flag is set. Signed-off-by: Maksim Lukoshkov Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 10 +++- drivers/crypto/qat/qat_common/adf_isr.c | 50 ++++++++----------- drivers/crypto/qat/qat_common/adf_vf_isr.c | 12 ++--- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 87de40d6c9a5..e391ca0662bc 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -44,8 +44,13 @@ struct adf_bar { resource_size_t size; } __packed; +struct adf_irq { + bool enabled; + char name[ADF_MAX_MSIX_VECTOR_NAME]; +}; + struct adf_accel_msix { - char **names; + struct adf_irq *irqs; u32 num_entries; } __packed; @@ -250,7 +255,8 @@ struct adf_accel_dev { struct adf_accel_vf_info *vf_info; } pf; struct { - char *irq_name; + bool irq_enabled; + char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; struct tasklet_struct pf2vf_bh_tasklet; struct mutex vf2pf_lock; /* protect CSR access */ struct completion iov_msg_completion; diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 26b546d3f821..861a9368b9db 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -130,6 +130,7 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; struct adf_etr_data *etr_data = accel_dev->transport; int clust_irq = hw_data->num_banks; int ret, irq, i = 0; @@ -141,7 +142,7 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) struct adf_etr_bank_data *bank = &etr_data->banks[i]; unsigned int cpu, cpus = num_online_cpus(); - name = *(pci_dev_info->msix_entries.names + i); + name = irqs[i].name; snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, "qat%d-bundle%d", accel_dev->accel_id, i); irq = pci_irq_vector(pci_dev_info->pci_dev, i); @@ -163,11 +164,12 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; irq_set_affinity_hint(irq, get_cpu_mask(cpu)); + irqs[i].enabled = true; } } /* Request msix irq for AE */ - name = *(pci_dev_info->msix_entries.names + i); + name = irqs[i].name; snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, "qat%d-ae-cluster", accel_dev->accel_id); irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); @@ -183,6 +185,7 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) "Failed to allocate IRQ %d for %s\n", irq, name); return ret; } + irqs[i].enabled = true; return ret; } @@ -190,60 +193,51 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; struct adf_etr_data *etr_data = accel_dev->transport; int clust_irq = hw_data->num_banks; int irq, i = 0; if (pci_dev_info->msix_entries.num_entries > 1) { for (i = 0; i < hw_data->num_banks; i++) { - irq = pci_irq_vector(pci_dev_info->pci_dev, i); - irq_set_affinity_hint(irq, NULL); - free_irq(irq, &etr_data->banks[i]); + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + irq_set_affinity_hint(irq, NULL); + free_irq(irq, &etr_data->banks[i]); + } } } - irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); - free_irq(irq, accel_dev); + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + free_irq(irq, accel_dev); + } } static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) { - int i; - char **names; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msix_num_entries = 1; + struct adf_irq *irqs; /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ if (!accel_dev->pf.vf_info) msix_num_entries += hw_data->num_banks; - names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); - if (!names) + irqs = kzalloc_node(msix_num_entries * sizeof(*irqs), + GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); + if (!irqs) return -ENOMEM; - for (i = 0; i < msix_num_entries; i++) { - *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); - if (!(*(names + i))) - goto err; - } accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; - accel_dev->accel_pci_dev.msix_entries.names = names; + accel_dev->accel_pci_dev.msix_entries.irqs = irqs; return 0; -err: - for (i = 0; i < msix_num_entries; i++) - kfree(*(names + i)); - kfree(names); - return -ENOMEM; } static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev) { - char **names = accel_dev->accel_pci_dev.msix_entries.names; - int i; - - for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) - kfree(*(names + i)); - kfree(names); + kfree(accel_dev->accel_pci_dev.msix_entries.irqs); + accel_dev->accel_pci_dev.msix_entries.irqs = NULL; } static int adf_setup_bh(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index 695c5050b6f3..07f81682c19b 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -61,10 +61,6 @@ static int adf_enable_msi(struct adf_accel_dev *accel_dev) return stat; } - accel_dev->vf.irq_name = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); - if (!accel_dev->vf.irq_name) - return -ENOMEM; - return 0; } @@ -72,7 +68,6 @@ static void adf_disable_msi(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - kfree(accel_dev->vf.irq_name); pci_free_irq_vectors(pdev); } @@ -240,6 +235,7 @@ static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) } cpu = accel_dev->accel_id % num_online_cpus(); irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); + accel_dev->vf.irq_enabled = true; return ret; } @@ -271,8 +267,10 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); - irq_set_affinity_hint(pdev->irq, NULL); - free_irq(pdev->irq, (void *)accel_dev); + if (accel_dev->vf.irq_enabled) { + irq_set_affinity_hint(pdev->irq, NULL); + free_irq(pdev->irq, accel_dev); + } adf_cleanup_bh(accel_dev); adf_cleanup_pf2vf_bh(accel_dev); adf_disable_msi(accel_dev); From 70fead3adb4eea70cf6f9dba681394653b1387e3 Mon Sep 17 00:00:00 2001 From: Wojciech Ziemba Date: Wed, 1 Sep 2021 18:36:08 +0100 Subject: [PATCH 063/134] crypto: qat - free irq in case of failure If devm_request_irq() fails inside adf_request_irqs(), unwind properly by freeing the allocated irqs. Signed-off-by: Wojciech Ziemba Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_isr.c | 63 +++++++++++++------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 861a9368b9db..c55a9f14b0d2 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -126,6 +126,31 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) return IRQ_NONE; } +static void adf_free_irqs(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; + struct adf_etr_data *etr_data = accel_dev->transport; + int clust_irq = hw_data->num_banks; + int irq, i = 0; + + if (pci_dev_info->msix_entries.num_entries > 1) { + for (i = 0; i < hw_data->num_banks; i++) { + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, i); + irq_set_affinity_hint(irq, NULL); + free_irq(irq, &etr_data->banks[i]); + } + } + } + + if (irqs[i].enabled) { + irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); + free_irq(irq, accel_dev); + } +} + static int adf_request_irqs(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; @@ -150,7 +175,8 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to get IRQ number of device vector %d - %s\n", i, name); - return irq; + ret = irq; + goto err; } ret = request_irq(irq, adf_msix_isr_bundle, 0, &name[0], bank); @@ -158,7 +184,7 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to allocate IRQ %d for %s\n", irq, name); - return ret; + goto err; } cpu = ((accel_dev->accel_id * hw_data->num_banks) + @@ -177,41 +203,20 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to get IRQ number of device vector %d - %s\n", i, name); - return irq; + ret = irq; + goto err; } ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to allocate IRQ %d for %s\n", irq, name); - return ret; + goto err; } irqs[i].enabled = true; return ret; -} - -static void adf_free_irqs(struct adf_accel_dev *accel_dev) -{ - struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_irq *irqs = pci_dev_info->msix_entries.irqs; - struct adf_etr_data *etr_data = accel_dev->transport; - int clust_irq = hw_data->num_banks; - int irq, i = 0; - - if (pci_dev_info->msix_entries.num_entries > 1) { - for (i = 0; i < hw_data->num_banks; i++) { - if (irqs[i].enabled) { - irq = pci_irq_vector(pci_dev_info->pci_dev, i); - irq_set_affinity_hint(irq, NULL); - free_irq(irq, &etr_data->banks[i]); - } - } - } - - if (irqs[i].enabled) { - irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq); - free_irq(irq, accel_dev); - } +err: + adf_free_irqs(accel_dev); + return ret; } static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev) From 40da865381ad061ab75a7a9da469ed4e623bdfeb Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 2 Sep 2021 09:34:59 +0100 Subject: [PATCH 064/134] crypto: qat - remove unneeded packed attribute Remove packed attribute from structures that do not need to be packed. These are just used internally and not shared with firmware. This also fixes a series of warning when compiling the driver with the flag -Waddress-of-packed-member, similar to the following: drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c:102:28: warning: taking address of packed member 'csr_ops' of class or structure 'adf_hw_device_data' may result in an unaligned pointer value Signed-off-by: Giovanni Cabiddu Reported-by: kernel test robot Reviewed-by: Wojciech Ziemba Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index e391ca0662bc..b62b34fccc90 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -42,7 +42,7 @@ struct adf_bar { resource_size_t base_addr; void __iomem *virt_addr; resource_size_t size; -} __packed; +}; struct adf_irq { bool enabled; @@ -52,7 +52,7 @@ struct adf_irq { struct adf_accel_msix { struct adf_irq *irqs; u32 num_entries; -} __packed; +}; struct adf_accel_pci { struct pci_dev *pci_dev; @@ -60,7 +60,7 @@ struct adf_accel_pci { struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; u8 revid; u8 sku; -} __packed; +}; enum dev_state { DEV_DOWN = 0, @@ -100,7 +100,7 @@ struct adf_hw_device_class { const char *name; const enum adf_device_type type; u32 instances; -} __packed; +}; struct arb_info { u32 arb_cfg; @@ -199,7 +199,7 @@ struct adf_hw_device_data { u8 num_logical_accel; u8 num_engines; u8 min_iov_compat_ver; -} __packed; +}; /* CSR write macro */ #define ADF_CSR_WR(csr_base, csr_offset, val) \ @@ -266,5 +266,5 @@ struct adf_accel_dev { }; bool is_vf; u32 accel_id; -} __packed; +}; #endif From 29601c8159c8089782fb5da25acadd3c146f2944 Mon Sep 17 00:00:00 2001 From: Cai Huoqing Date: Tue, 7 Sep 2021 15:40:41 +0800 Subject: [PATCH 065/134] hwrng: ixp4xx - Make use of the helper function devm_platform_ioremap_resource() Use the devm_platform_ioremap_resource() helper instead of calling platform_get_resource() and devm_ioremap_resource() separately Signed-off-by: Cai Huoqing Signed-off-by: Herbert Xu --- drivers/char/hw_random/ixp4xx-rng.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/char/hw_random/ixp4xx-rng.c b/drivers/char/hw_random/ixp4xx-rng.c index 188854dd16a9..7df5e9f7519d 100644 --- a/drivers/char/hw_random/ixp4xx-rng.c +++ b/drivers/char/hw_random/ixp4xx-rng.c @@ -42,13 +42,11 @@ static int ixp4xx_rng_probe(struct platform_device *pdev) { void __iomem * rng_base; struct device *dev = &pdev->dev; - struct resource *res; if (!cpu_is_ixp46x()) /* includes IXP455 */ return -ENOSYS; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - rng_base = devm_ioremap_resource(dev, res); + rng_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rng_base)) return PTR_ERR(rng_base); From 5e91f56a0bb32fd14096e20224bf4f93f1b174b1 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 10 Sep 2021 13:15:32 +0100 Subject: [PATCH 066/134] crypto: img-hash - remove need for error return variable ret The assignment to error return variable ret and then the jump to an error exit path can be simplified by just returning the error return at the failure point. This allows variable ret and the error return path to be removed. This cleans up a static analysis warninng that variable ret is being assigned (value never being used) and being re-assigned later. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/img-hash.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index aa4c7b2af3e2..d8e82d69745d 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -674,14 +674,12 @@ static int img_hash_digest(struct ahash_request *req) static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) { struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm); - int err = -ENOMEM; ctx->fallback = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback)) { pr_err("img_hash: Could not load fallback driver.\n"); - err = PTR_ERR(ctx->fallback); - goto err; + return PTR_ERR(ctx->fallback); } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct img_hash_request_ctx) + @@ -689,9 +687,6 @@ static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) IMG_HASH_DMA_THRESHOLD); return 0; - -err: - return err; } static int img_hash_cra_md5_init(struct crypto_tfm *tfm) From 81f53028dfbc79844f727a7c13d337ba827a471c Mon Sep 17 00:00:00 2001 From: Tim Gardner Date: Fri, 10 Sep 2021 13:01:17 -0600 Subject: [PATCH 067/134] crypto: drbg - Fix unused value warning in drbg_healthcheck_sanity() Coverity warns uf an unused value: CID 44865 (#2 of 2): Unused value (UNUSED_VALUE) assigned_value: Assigning value -14 to ret here, but that stored value is overwritten before it can be used. 2006 int ret = -EFAULT; ... value_overwrite: Overwriting previous write to ret with value from drbg_seed(drbg, &addtl, false). 2052 ret = drbg_seed(drbg, &addtl, false); Fix this by removing the variable initializer. Cc: Herbert Xu Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Tim Gardner Signed-off-by: Herbert Xu --- crypto/drbg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/drbg.c b/crypto/drbg.c index ea85d4a0fe9e..f72f340a1321 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -2003,7 +2003,7 @@ static inline int __init drbg_healthcheck_sanity(void) #define OUTBUFLEN 16 unsigned char buf[OUTBUFLEN]; struct drbg_state *drbg = NULL; - int ret = -EFAULT; + int ret; int rc = -EFAULT; bool pr = false; int coreref = 0; From a2d3cbc80d2527b435154ff0f89b56ef4b84370f Mon Sep 17 00:00:00 2001 From: Shreyansh Chouhan Date: Sat, 11 Sep 2021 16:37:59 +0530 Subject: [PATCH 068/134] crypto: aesni - check walk.nbytes instead of err In the code for xts_crypt(), we check for the err value returned by skcipher_walk_virt() and return from the function if it is non zero. However, skcipher_walk_virt() can set walk.nbytes to 0, which would cause us to call kernel_fpu_begin(), and then skip the kernel_fpu_end() call. This patch checks for the walk.nbytes value instead, and returns if walk.nbytes is 0. This prevents us from calling kernel_fpu_begin() in the first place and also covers the case of having a non zero err value returned from skcipher_walk_virt(). Reported-by: Dan Carpenter Signed-off-by: Shreyansh Chouhan Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 0fc961bef299..e09f4672dd38 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -866,7 +866,7 @@ static int xts_crypt(struct skcipher_request *req, bool encrypt) req = &subreq; err = skcipher_walk_virt(&walk, req, false); - if (err) + if (!walk.nbytes) return err; } else { tail = 0; From 12c5e81d3fd0a690c49dfe1c3a99bf80a24075c7 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 16 Feb 2021 19:46:48 -0500 Subject: [PATCH 069/134] audit: prepare audit_context for use in calling contexts beyond syscalls This patch cleans up some of our audit_context handling by abstracting out the reset and return code fixup handling to dedicated functions. Not only does this help make things easier to read and inspect, it allows for easier reuse by future patches. We also convert the simple audit_context->in_syscall flag into an enum which can be used to by future patches to indicate a calling context other than the syscall context. Thanks to Richard Guy Briggs for review and feedback. Acked-by: Richard Guy Briggs Signed-off-by: Paul Moore --- kernel/audit.h | 5 +- kernel/auditsc.c | 256 ++++++++++++++++++++++++++++++----------------- 2 files changed, 167 insertions(+), 94 deletions(-) diff --git a/kernel/audit.h b/kernel/audit.h index d6a2c899a8db..13abc48de0bd 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -100,7 +100,10 @@ struct audit_proctitle { /* The per-task audit context. */ struct audit_context { int dummy; /* must be the first element */ - int in_syscall; /* 1 if task is in a syscall */ + enum { + AUDIT_CTX_UNUSED, /* audit_context is currently unused */ + AUDIT_CTX_SYSCALL, /* in use by syscall */ + } context; enum audit_state state, current_state; unsigned int serial; /* serial number for record */ int major; /* syscall number */ diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 8dd73a64f921..f3d309b05c2d 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -915,10 +915,80 @@ static inline void audit_free_aux(struct audit_context *context) context->aux = aux->next; kfree(aux); } + context->aux = NULL; while ((aux = context->aux_pids)) { context->aux_pids = aux->next; kfree(aux); } + context->aux_pids = NULL; +} + +/** + * audit_reset_context - reset a audit_context structure + * @ctx: the audit_context to reset + * + * All fields in the audit_context will be reset to an initial state, all + * references held by fields will be dropped, and private memory will be + * released. When this function returns the audit_context will be suitable + * for reuse, so long as the passed context is not NULL or a dummy context. + */ +static void audit_reset_context(struct audit_context *ctx) +{ + if (!ctx) + return; + + /* if ctx is non-null, reset the "ctx->state" regardless */ + ctx->context = AUDIT_CTX_UNUSED; + if (ctx->dummy) + return; + + /* + * NOTE: It shouldn't matter in what order we release the fields, so + * release them in the order in which they appear in the struct; + * this gives us some hope of quickly making sure we are + * resetting the audit_context properly. + * + * Other things worth mentioning: + * - we don't reset "dummy" + * - we don't reset "state", we do reset "current_state" + * - we preserve "filterkey" if "state" is AUDIT_STATE_RECORD + * - much of this is likely overkill, but play it safe for now + * - we really need to work on improving the audit_context struct + */ + + ctx->current_state = ctx->state; + ctx->serial = 0; + ctx->major = 0; + ctx->ctime = (struct timespec64){ .tv_sec = 0, .tv_nsec = 0 }; + memset(ctx->argv, 0, sizeof(ctx->argv)); + ctx->return_code = 0; + ctx->prio = (ctx->state == AUDIT_STATE_RECORD ? ~0ULL : 0); + ctx->return_valid = AUDITSC_INVALID; + audit_free_names(ctx); + if (ctx->state != AUDIT_STATE_RECORD) { + kfree(ctx->filterkey); + ctx->filterkey = NULL; + } + audit_free_aux(ctx); + kfree(ctx->sockaddr); + ctx->sockaddr = NULL; + ctx->sockaddr_len = 0; + ctx->pid = ctx->ppid = 0; + ctx->uid = ctx->euid = ctx->suid = ctx->fsuid = KUIDT_INIT(0); + ctx->gid = ctx->egid = ctx->sgid = ctx->fsgid = KGIDT_INIT(0); + ctx->personality = 0; + ctx->arch = 0; + ctx->target_pid = 0; + ctx->target_auid = ctx->target_uid = KUIDT_INIT(0); + ctx->target_sessionid = 0; + ctx->target_sid = 0; + ctx->target_comm[0] = '\0'; + unroll_tree_refs(ctx, NULL, 0); + WARN_ON(!list_empty(&ctx->killed_trees)); + ctx->type = 0; + audit_free_module(ctx); + ctx->fds[0] = -1; + audit_proctitle_free(ctx); } static inline struct audit_context *audit_alloc_context(enum audit_state state) @@ -928,6 +998,7 @@ static inline struct audit_context *audit_alloc_context(enum audit_state state) context = kzalloc(sizeof(*context), GFP_KERNEL); if (!context) return NULL; + context->context = AUDIT_CTX_UNUSED; context->state = state; context->prio = state == AUDIT_STATE_RECORD ? ~0ULL : 0; INIT_LIST_HEAD(&context->killed_trees); @@ -953,7 +1024,7 @@ int audit_alloc(struct task_struct *tsk) char *key = NULL; if (likely(!audit_ever_enabled)) - return 0; /* Return if not auditing. */ + return 0; state = audit_filter_task(tsk, &key); if (state == AUDIT_STATE_DISABLED) { @@ -975,14 +1046,10 @@ int audit_alloc(struct task_struct *tsk) static inline void audit_free_context(struct audit_context *context) { - audit_free_module(context); - audit_free_names(context); - unroll_tree_refs(context, NULL, 0); + /* resetting is extra work, but it is likely just noise */ + audit_reset_context(context); free_tree_refs(context); - audit_free_aux(context); kfree(context->filterkey); - kfree(context->sockaddr); - audit_proctitle_free(context); kfree(context); } @@ -1489,29 +1556,35 @@ static void audit_log_exit(void) context->personality = current->personality; - ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL); - if (!ab) - return; /* audit_panic has been called */ - audit_log_format(ab, "arch=%x syscall=%d", - context->arch, context->major); - if (context->personality != PER_LINUX) - audit_log_format(ab, " per=%lx", context->personality); - if (context->return_valid != AUDITSC_INVALID) - audit_log_format(ab, " success=%s exit=%ld", - (context->return_valid==AUDITSC_SUCCESS)?"yes":"no", - context->return_code); - - audit_log_format(ab, - " a0=%lx a1=%lx a2=%lx a3=%lx items=%d", - context->argv[0], - context->argv[1], - context->argv[2], - context->argv[3], - context->name_count); - - audit_log_task_info(ab); - audit_log_key(ab, context->filterkey); - audit_log_end(ab); + switch (context->context) { + case AUDIT_CTX_SYSCALL: + ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL); + if (!ab) + return; + audit_log_format(ab, "arch=%x syscall=%d", + context->arch, context->major); + if (context->personality != PER_LINUX) + audit_log_format(ab, " per=%lx", context->personality); + if (context->return_valid != AUDITSC_INVALID) + audit_log_format(ab, " success=%s exit=%ld", + (context->return_valid == AUDITSC_SUCCESS ? + "yes" : "no"), + context->return_code); + audit_log_format(ab, + " a0=%lx a1=%lx a2=%lx a3=%lx items=%d", + context->argv[0], + context->argv[1], + context->argv[2], + context->argv[3], + context->name_count); + audit_log_task_info(ab); + audit_log_key(ab, context->filterkey); + audit_log_end(ab); + break; + default: + BUG(); + break; + } for (aux = context->aux; aux; aux = aux->next) { @@ -1602,14 +1675,15 @@ static void audit_log_exit(void) audit_log_name(context, n, NULL, i++, &call_panic); } - audit_log_proctitle(); + if (context->context == AUDIT_CTX_SYSCALL) + audit_log_proctitle(); /* Send end of event record to help user space know we are finished */ ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE); if (ab) audit_log_end(ab); if (call_panic) - audit_panic("error converting sid to string"); + audit_panic("error in audit_log_exit()"); } /** @@ -1625,6 +1699,7 @@ void __audit_free(struct task_struct *tsk) if (!context) return; + /* this may generate CONFIG_CHANGE records */ if (!list_empty(&context->killed_trees)) audit_kill_trees(context); @@ -1633,7 +1708,8 @@ void __audit_free(struct task_struct *tsk) * random task_struct that doesn't doesn't have any meaningful data we * need to log via audit_log_exit(). */ - if (tsk == current && !context->dummy && context->in_syscall) { + if (tsk == current && !context->dummy && + context->context == AUDIT_CTX_SYSCALL) { context->return_valid = AUDITSC_INVALID; context->return_code = 0; @@ -1647,6 +1723,34 @@ void __audit_free(struct task_struct *tsk) audit_free_context(context); } +/** + * audit_return_fixup - fixup the return codes in the audit_context + * @ctx: the audit_context + * @success: true/false value to indicate if the operation succeeded or not + * @code: operation return code + * + * We need to fixup the return code in the audit logs if the actual return + * codes are later going to be fixed by the arch specific signal handlers. + */ +static void audit_return_fixup(struct audit_context *ctx, + int success, long code) +{ + /* + * This is actually a test for: + * (rc == ERESTARTSYS ) || (rc == ERESTARTNOINTR) || + * (rc == ERESTARTNOHAND) || (rc == ERESTART_RESTARTBLOCK) + * + * but is faster than a bunch of || + */ + if (unlikely(code <= -ERESTARTSYS) && + (code >= -ERESTART_RESTARTBLOCK) && + (code != -ENOIOCTLCMD)) + ctx->return_code = -EINTR; + else + ctx->return_code = code; + ctx->return_valid = (success ? AUDITSC_SUCCESS : AUDITSC_FAILURE); +} + /** * __audit_syscall_entry - fill in an audit record at syscall entry * @major: major syscall type (function) @@ -1672,7 +1776,12 @@ void __audit_syscall_entry(int major, unsigned long a1, unsigned long a2, if (!audit_enabled || !context) return; - BUG_ON(context->in_syscall || context->name_count); + WARN_ON(context->context != AUDIT_CTX_UNUSED); + WARN_ON(context->name_count); + if (context->context != AUDIT_CTX_UNUSED || context->name_count) { + audit_panic("unrecoverable error in audit_syscall_entry()"); + return; + } state = context->state; if (state == AUDIT_STATE_DISABLED) @@ -1691,10 +1800,8 @@ void __audit_syscall_entry(int major, unsigned long a1, unsigned long a2, context->argv[1] = a2; context->argv[2] = a3; context->argv[3] = a4; - context->serial = 0; - context->in_syscall = 1; + context->context = AUDIT_CTX_SYSCALL; context->current_state = state; - context->ppid = 0; ktime_get_coarse_real_ts64(&context->ctime); } @@ -1711,63 +1818,27 @@ void __audit_syscall_entry(int major, unsigned long a1, unsigned long a2, */ void __audit_syscall_exit(int success, long return_code) { - struct audit_context *context; + struct audit_context *context = audit_context(); - context = audit_context(); - if (!context) - return; + if (!context || context->dummy || + context->context != AUDIT_CTX_SYSCALL) + goto out; + /* this may generate CONFIG_CHANGE records */ if (!list_empty(&context->killed_trees)) audit_kill_trees(context); - if (!context->dummy && context->in_syscall) { - if (success) - context->return_valid = AUDITSC_SUCCESS; - else - context->return_valid = AUDITSC_FAILURE; + /* run through both filters to ensure we set the filterkey properly */ + audit_filter_syscall(current, context); + audit_filter_inodes(current, context); + if (context->current_state < AUDIT_STATE_RECORD) + goto out; - /* - * we need to fix up the return code in the audit logs if the - * actual return codes are later going to be fixed up by the - * arch specific signal handlers - * - * This is actually a test for: - * (rc == ERESTARTSYS ) || (rc == ERESTARTNOINTR) || - * (rc == ERESTARTNOHAND) || (rc == ERESTART_RESTARTBLOCK) - * - * but is faster than a bunch of || - */ - if (unlikely(return_code <= -ERESTARTSYS) && - (return_code >= -ERESTART_RESTARTBLOCK) && - (return_code != -ENOIOCTLCMD)) - context->return_code = -EINTR; - else - context->return_code = return_code; + audit_return_fixup(context, success, return_code); + audit_log_exit(); - audit_filter_syscall(current, context); - audit_filter_inodes(current, context); - if (context->current_state == AUDIT_STATE_RECORD) - audit_log_exit(); - } - - context->in_syscall = 0; - context->prio = context->state == AUDIT_STATE_RECORD ? ~0ULL : 0; - - audit_free_module(context); - audit_free_names(context); - unroll_tree_refs(context, NULL, 0); - audit_free_aux(context); - context->aux = NULL; - context->aux_pids = NULL; - context->target_pid = 0; - context->target_sid = 0; - context->sockaddr_len = 0; - context->type = 0; - context->fds[0] = -1; - if (context->state != AUDIT_STATE_RECORD) { - kfree(context->filterkey); - context->filterkey = NULL; - } +out: + audit_reset_context(context); } static inline void handle_one(const struct inode *inode) @@ -1919,7 +1990,7 @@ void __audit_getname(struct filename *name) struct audit_context *context = audit_context(); struct audit_names *n; - if (!context->in_syscall) + if (context->context == AUDIT_CTX_UNUSED) return; n = audit_alloc_name(context, AUDIT_TYPE_UNKNOWN); @@ -1991,7 +2062,7 @@ void __audit_inode(struct filename *name, const struct dentry *dentry, struct list_head *list = &audit_filter_list[AUDIT_FILTER_FS]; int i; - if (!context->in_syscall) + if (context->context == AUDIT_CTX_UNUSED) return; rcu_read_lock(); @@ -2109,7 +2180,7 @@ void __audit_inode_child(struct inode *parent, struct list_head *list = &audit_filter_list[AUDIT_FILTER_FS]; int i; - if (!context->in_syscall) + if (context->context == AUDIT_CTX_UNUSED) return; rcu_read_lock(); @@ -2208,7 +2279,7 @@ EXPORT_SYMBOL_GPL(__audit_inode_child); int auditsc_get_stamp(struct audit_context *ctx, struct timespec64 *t, unsigned int *serial) { - if (!ctx->in_syscall) + if (ctx->context == AUDIT_CTX_UNUSED) return 0; if (!ctx->serial) ctx->serial = audit_serial(); @@ -2706,8 +2777,7 @@ void audit_seccomp_actions_logged(const char *names, const char *old_names, struct list_head *audit_killed_trees(void) { struct audit_context *ctx = audit_context(); - - if (likely(!ctx || !ctx->in_syscall)) + if (likely(!ctx || ctx->context == AUDIT_CTX_UNUSED)) return NULL; return &ctx->killed_trees; } From 5bd2182d58e9d9c6279b7a8a2f9b41add0e7f9cb Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 16 Feb 2021 19:46:48 -0500 Subject: [PATCH 070/134] audit,io_uring,io-wq: add some basic audit support to io_uring This patch adds basic auditing to io_uring operations, regardless of their context. This is accomplished by allocating audit_context structures for the io-wq worker and io_uring SQPOLL kernel threads as well as explicitly auditing the io_uring operations in io_issue_sqe(). Individual io_uring operations can bypass auditing through the "audit_skip" field in the struct io_op_def definition for the operation; although great care must be taken so that security relevant io_uring operations do not bypass auditing; please contact the audit mailing list (see the MAINTAINERS file) with any questions. The io_uring operations are audited using a new AUDIT_URINGOP record, an example is shown below: type=UNKNOWN[1336] msg=audit(1631800225.981:37289): uring_op=19 success=yes exit=0 items=0 ppid=15454 pid=15681 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 key=(null) Thanks to Richard Guy Briggs for review and feedback. Signed-off-by: Paul Moore --- fs/io-wq.c | 4 + fs/io_uring.c | 55 ++++++++++-- include/linux/audit.h | 26 ++++++ include/uapi/linux/audit.h | 1 + kernel/audit.h | 2 + kernel/auditsc.c | 166 +++++++++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 6 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 6c55362c1f99..dac5c5961c9d 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "io-wq.h" @@ -562,6 +563,8 @@ static int io_wqe_worker(void *data) snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid); set_task_comm(current, buf); + audit_alloc_kernel(current); + while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { long ret; @@ -601,6 +604,7 @@ loop: io_worker_handle_work(worker); } + audit_free(current); io_worker_exit(worker); return 0; } diff --git a/fs/io_uring.c b/fs/io_uring.c index 16fb7436043c..388754b24785 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -79,6 +79,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -917,6 +918,8 @@ struct io_op_def { unsigned needs_async_setup : 1; /* should block plug */ unsigned plug : 1; + /* skip auditing */ + unsigned audit_skip : 1; /* size of async data needed, if any */ unsigned short async_size; }; @@ -930,6 +933,7 @@ static const struct io_op_def io_op_defs[] = { .buffer_select = 1, .needs_async_setup = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_WRITEV] = { @@ -939,16 +943,19 @@ static const struct io_op_def io_op_defs[] = { .pollout = 1, .needs_async_setup = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_FSYNC] = { .needs_file = 1, + .audit_skip = 1, }, [IORING_OP_READ_FIXED] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_WRITE_FIXED] = { @@ -957,15 +964,20 @@ static const struct io_op_def io_op_defs[] = { .unbound_nonreg_file = 1, .pollout = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_POLL_ADD] = { .needs_file = 1, .unbound_nonreg_file = 1, + .audit_skip = 1, + }, + [IORING_OP_POLL_REMOVE] = { + .audit_skip = 1, }, - [IORING_OP_POLL_REMOVE] = {}, [IORING_OP_SYNC_FILE_RANGE] = { .needs_file = 1, + .audit_skip = 1, }, [IORING_OP_SENDMSG] = { .needs_file = 1, @@ -983,18 +995,23 @@ static const struct io_op_def io_op_defs[] = { .async_size = sizeof(struct io_async_msghdr), }, [IORING_OP_TIMEOUT] = { + .audit_skip = 1, .async_size = sizeof(struct io_timeout_data), }, [IORING_OP_TIMEOUT_REMOVE] = { /* used by timeout updates' prep() */ + .audit_skip = 1, }, [IORING_OP_ACCEPT] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, }, - [IORING_OP_ASYNC_CANCEL] = {}, + [IORING_OP_ASYNC_CANCEL] = { + .audit_skip = 1, + }, [IORING_OP_LINK_TIMEOUT] = { + .audit_skip = 1, .async_size = sizeof(struct io_timeout_data), }, [IORING_OP_CONNECT] = { @@ -1009,14 +1026,19 @@ static const struct io_op_def io_op_defs[] = { }, [IORING_OP_OPENAT] = {}, [IORING_OP_CLOSE] = {}, - [IORING_OP_FILES_UPDATE] = {}, - [IORING_OP_STATX] = {}, + [IORING_OP_FILES_UPDATE] = { + .audit_skip = 1, + }, + [IORING_OP_STATX] = { + .audit_skip = 1, + }, [IORING_OP_READ] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, .buffer_select = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_WRITE] = { @@ -1025,39 +1047,50 @@ static const struct io_op_def io_op_defs[] = { .unbound_nonreg_file = 1, .pollout = 1, .plug = 1, + .audit_skip = 1, .async_size = sizeof(struct io_async_rw), }, [IORING_OP_FADVISE] = { .needs_file = 1, + .audit_skip = 1, }, [IORING_OP_MADVISE] = {}, [IORING_OP_SEND] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, + .audit_skip = 1, }, [IORING_OP_RECV] = { .needs_file = 1, .unbound_nonreg_file = 1, .pollin = 1, .buffer_select = 1, + .audit_skip = 1, }, [IORING_OP_OPENAT2] = { }, [IORING_OP_EPOLL_CTL] = { .unbound_nonreg_file = 1, + .audit_skip = 1, }, [IORING_OP_SPLICE] = { .needs_file = 1, .hash_reg_file = 1, .unbound_nonreg_file = 1, + .audit_skip = 1, + }, + [IORING_OP_PROVIDE_BUFFERS] = { + .audit_skip = 1, + }, + [IORING_OP_REMOVE_BUFFERS] = { + .audit_skip = 1, }, - [IORING_OP_PROVIDE_BUFFERS] = {}, - [IORING_OP_REMOVE_BUFFERS] = {}, [IORING_OP_TEE] = { .needs_file = 1, .hash_reg_file = 1, .unbound_nonreg_file = 1, + .audit_skip = 1, }, [IORING_OP_SHUTDOWN] = { .needs_file = 1, @@ -6591,6 +6624,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if ((req->flags & REQ_F_CREDS) && req->creds != current_cred()) creds = override_creds(req->creds); + if (!io_op_defs[req->opcode].audit_skip) + audit_uring_entry(req->opcode); + switch (req->opcode) { case IORING_OP_NOP: ret = io_nop(req, issue_flags); @@ -6706,6 +6742,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) break; } + if (!io_op_defs[req->opcode].audit_skip) + audit_uring_exit(!ret, ret); + if (creds) revert_creds(creds); if (ret) @@ -7360,6 +7399,8 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; + audit_alloc_kernel(current); + mutex_lock(&sqd->lock); while (1) { bool cap_entries, sqt_spin = false; @@ -7425,6 +7466,8 @@ static int io_sq_thread(void *data) io_run_task_work(); mutex_unlock(&sqd->lock); + audit_free(current); + complete(&sqd->exited); do_exit(0); } diff --git a/include/linux/audit.h b/include/linux/audit.h index 82b7c1116a85..d656a06dd909 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -286,7 +286,10 @@ static inline int audit_signal_info(int sig, struct task_struct *t) /* These are defined in auditsc.c */ /* Public API */ extern int audit_alloc(struct task_struct *task); +extern int audit_alloc_kernel(struct task_struct *task); extern void __audit_free(struct task_struct *task); +extern void __audit_uring_entry(u8 op); +extern void __audit_uring_exit(int success, long code); extern void __audit_syscall_entry(int major, unsigned long a0, unsigned long a1, unsigned long a2, unsigned long a3); extern void __audit_syscall_exit(int ret_success, long ret_value); @@ -323,6 +326,21 @@ static inline void audit_free(struct task_struct *task) if (unlikely(task->audit_context)) __audit_free(task); } +static inline void audit_uring_entry(u8 op) +{ + /* + * We intentionally check audit_context() before audit_enabled as most + * Linux systems (as of ~2021) rely on systemd which forces audit to + * be enabled regardless of the user's audit configuration. + */ + if (unlikely(audit_context() && audit_enabled)) + __audit_uring_entry(op); +} +static inline void audit_uring_exit(int success, long code) +{ + if (unlikely(!audit_dummy_context())) + __audit_uring_exit(success, code); +} static inline void audit_syscall_entry(int major, unsigned long a0, unsigned long a1, unsigned long a2, unsigned long a3) @@ -554,8 +572,16 @@ static inline int audit_alloc(struct task_struct *task) { return 0; } +static inline int audit_alloc_kernel(struct task_struct *task) +{ + return 0; +} static inline void audit_free(struct task_struct *task) { } +static inline void audit_uring_entry(u8 op) +{ } +static inline void audit_uring_exit(int success, long code) +{ } static inline void audit_syscall_entry(int major, unsigned long a0, unsigned long a1, unsigned long a2, unsigned long a3) diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index daa481729e9b..a1997697c8b1 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -118,6 +118,7 @@ #define AUDIT_TIME_ADJNTPVAL 1333 /* NTP value adjustment */ #define AUDIT_BPF 1334 /* BPF subsystem */ #define AUDIT_EVENT_LISTENER 1335 /* Task joined multicast read socket */ +#define AUDIT_URINGOP 1336 /* io_uring operation */ #define AUDIT_AVC 1400 /* SE Linux avc denial or grant */ #define AUDIT_SELINUX_ERR 1401 /* Internal SE Linux Errors */ diff --git a/kernel/audit.h b/kernel/audit.h index 13abc48de0bd..d1161e3b83e2 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -103,10 +103,12 @@ struct audit_context { enum { AUDIT_CTX_UNUSED, /* audit_context is currently unused */ AUDIT_CTX_SYSCALL, /* in use by syscall */ + AUDIT_CTX_URING, /* in use by io_uring */ } context; enum audit_state state, current_state; unsigned int serial; /* serial number for record */ int major; /* syscall number */ + int uring_op; /* uring operation */ struct timespec64 ctime; /* time of syscall entry */ unsigned long argv[4]; /* syscall arguments */ long return_code;/* syscall return code */ diff --git a/kernel/auditsc.c b/kernel/auditsc.c index f3d309b05c2d..6dda448fb826 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -959,6 +959,7 @@ static void audit_reset_context(struct audit_context *ctx) ctx->current_state = ctx->state; ctx->serial = 0; ctx->major = 0; + ctx->uring_op = 0; ctx->ctime = (struct timespec64){ .tv_sec = 0, .tv_nsec = 0 }; memset(ctx->argv, 0, sizeof(ctx->argv)); ctx->return_code = 0; @@ -1044,6 +1045,31 @@ int audit_alloc(struct task_struct *tsk) return 0; } +/** + * audit_alloc_kernel - allocate an audit_context for a kernel task + * @tsk: the kernel task + * + * Similar to the audit_alloc() function, but intended for kernel private + * threads. Returns zero on success, negative values on failure. + */ +int audit_alloc_kernel(struct task_struct *tsk) +{ + /* + * At the moment we are just going to call into audit_alloc() to + * simplify the code, but there two things to keep in mind with this + * approach: + * + * 1. Filtering internal kernel tasks is a bit laughable in almost all + * cases, but there is at least one case where there is a benefit: + * the '-a task,never' case allows the admin to effectively disable + * task auditing at runtime. + * + * 2. The {set,clear}_task_syscall_work() ops likely have zero effect + * on these internal kernel tasks, but they probably don't hurt either. + */ + return audit_alloc(tsk); +} + static inline void audit_free_context(struct audit_context *context) { /* resetting is extra work, but it is likely just noise */ @@ -1546,6 +1572,44 @@ out: audit_log_end(ab); } +/** + * audit_log_uring - generate a AUDIT_URINGOP record + * @ctx: the audit context + */ +static void audit_log_uring(struct audit_context *ctx) +{ + struct audit_buffer *ab; + const struct cred *cred; + + ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_URINGOP); + if (!ab) + return; + cred = current_cred(); + audit_log_format(ab, "uring_op=%d", ctx->uring_op); + if (ctx->return_valid != AUDITSC_INVALID) + audit_log_format(ab, " success=%s exit=%ld", + (ctx->return_valid == AUDITSC_SUCCESS ? + "yes" : "no"), + ctx->return_code); + audit_log_format(ab, + " items=%d" + " ppid=%d pid=%d uid=%u gid=%u euid=%u suid=%u" + " fsuid=%u egid=%u sgid=%u fsgid=%u", + ctx->name_count, + task_ppid_nr(current), task_tgid_nr(current), + from_kuid(&init_user_ns, cred->uid), + from_kgid(&init_user_ns, cred->gid), + from_kuid(&init_user_ns, cred->euid), + from_kuid(&init_user_ns, cred->suid), + from_kuid(&init_user_ns, cred->fsuid), + from_kgid(&init_user_ns, cred->egid), + from_kgid(&init_user_ns, cred->sgid), + from_kgid(&init_user_ns, cred->fsgid)); + audit_log_task_context(ab); + audit_log_key(ab, ctx->filterkey); + audit_log_end(ab); +} + static void audit_log_exit(void) { int i, call_panic = 0; @@ -1581,6 +1645,9 @@ static void audit_log_exit(void) audit_log_key(ab, context->filterkey); audit_log_end(ab); break; + case AUDIT_CTX_URING: + audit_log_uring(context); + break; default: BUG(); break; @@ -1751,6 +1818,105 @@ static void audit_return_fixup(struct audit_context *ctx, ctx->return_valid = (success ? AUDITSC_SUCCESS : AUDITSC_FAILURE); } +/** + * __audit_uring_entry - prepare the kernel task's audit context for io_uring + * @op: the io_uring opcode + * + * This is similar to audit_syscall_entry() but is intended for use by io_uring + * operations. This function should only ever be called from + * audit_uring_entry() as we rely on the audit context checking present in that + * function. + */ +void __audit_uring_entry(u8 op) +{ + struct audit_context *ctx = audit_context(); + + if (ctx->state == AUDIT_STATE_DISABLED) + return; + + /* + * NOTE: It's possible that we can be called from the process' context + * before it returns to userspace, and before audit_syscall_exit() + * is called. In this case there is not much to do, just record + * the io_uring details and return. + */ + ctx->uring_op = op; + if (ctx->context == AUDIT_CTX_SYSCALL) + return; + + ctx->dummy = !audit_n_rules; + if (!ctx->dummy && ctx->state == AUDIT_STATE_BUILD) + ctx->prio = 0; + + ctx->context = AUDIT_CTX_URING; + ctx->current_state = ctx->state; + ktime_get_coarse_real_ts64(&ctx->ctime); +} + +/** + * __audit_uring_exit - wrap up the kernel task's audit context after io_uring + * @success: true/false value to indicate if the operation succeeded or not + * @code: operation return code + * + * This is similar to audit_syscall_exit() but is intended for use by io_uring + * operations. This function should only ever be called from + * audit_uring_exit() as we rely on the audit context checking present in that + * function. + */ +void __audit_uring_exit(int success, long code) +{ + struct audit_context *ctx = audit_context(); + + /* + * TODO: At some point we will likely want to filter on io_uring ops + * and other things similar to what we do for syscalls, but that + * is something for another day; just record what we can here. + */ + + if (ctx->context == AUDIT_CTX_SYSCALL) { + /* + * NOTE: See the note in __audit_uring_entry() about the case + * where we may be called from process context before we + * return to userspace via audit_syscall_exit(). In this + * case we simply emit a URINGOP record and bail, the + * normal syscall exit handling will take care of + * everything else. + * It is also worth mentioning that when we are called, + * the current process creds may differ from the creds + * used during the normal syscall processing; keep that + * in mind if/when we move the record generation code. + */ + + /* + * We need to filter on the syscall info here to decide if we + * should emit a URINGOP record. I know it seems odd but this + * solves the problem where users have a filter to block *all* + * syscall records in the "exit" filter; we want to preserve + * the behavior here. + */ + audit_filter_syscall(current, ctx); + audit_filter_inodes(current, ctx); + if (ctx->current_state != AUDIT_STATE_RECORD) + return; + + audit_log_uring(ctx); + return; + } + + /* this may generate CONFIG_CHANGE records */ + if (!list_empty(&ctx->killed_trees)) + audit_kill_trees(ctx); + + audit_filter_inodes(current, ctx); + if (ctx->current_state != AUDIT_STATE_RECORD) + goto out; + audit_return_fixup(ctx, success, code); + audit_log_exit(); + +out: + audit_reset_context(ctx); +} + /** * __audit_syscall_entry - fill in an audit record at syscall entry * @major: major syscall type (function) From 67daf270cebcf7aab4b3292b36f9adf357b23ddc Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Sun, 18 Apr 2021 21:54:47 -0400 Subject: [PATCH 071/134] audit: add filtering for io_uring records This patch adds basic audit io_uring filtering, using as much of the existing audit filtering infrastructure as possible. In order to do this we reuse the audit filter rule's syscall mask for the io_uring operation and we create a new filter for io_uring operations as AUDIT_FILTER_URING_EXIT/audit_filter_list[7]. Thanks to Richard Guy Briggs for his review, feedback, and work on the corresponding audit userspace changes. Acked-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/uapi/linux/audit.h | 3 +- kernel/audit_tree.c | 3 +- kernel/audit_watch.c | 3 +- kernel/auditfilter.c | 15 ++++++++-- kernel/auditsc.c | 60 +++++++++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 20 deletions(-) diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index a1997697c8b1..ecf1edd2affa 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -167,8 +167,9 @@ #define AUDIT_FILTER_EXCLUDE 0x05 /* Apply rule before record creation */ #define AUDIT_FILTER_TYPE AUDIT_FILTER_EXCLUDE /* obsolete misleading naming */ #define AUDIT_FILTER_FS 0x06 /* Apply rule at __audit_inode_child */ +#define AUDIT_FILTER_URING_EXIT 0x07 /* Apply rule at io_uring op exit */ -#define AUDIT_NR_FILTERS 7 +#define AUDIT_NR_FILTERS 8 #define AUDIT_FILTER_PREPEND 0x10 /* Prepend to front of list */ diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 2cd7b5694422..338c53a961c5 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -726,7 +726,8 @@ int audit_make_tree(struct audit_krule *rule, char *pathname, u32 op) { if (pathname[0] != '/' || - rule->listnr != AUDIT_FILTER_EXIT || + (rule->listnr != AUDIT_FILTER_EXIT && + rule->listnr != AUDIT_FILTER_URING_EXIT) || op != Audit_equal || rule->inode_f || rule->watch || rule->tree) return -EINVAL; diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index 2acf7ca49154..698b62b4a2ec 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -183,7 +183,8 @@ int audit_to_watch(struct audit_krule *krule, char *path, int len, u32 op) return -EOPNOTSUPP; if (path[0] != '/' || path[len-1] == '/' || - krule->listnr != AUDIT_FILTER_EXIT || + (krule->listnr != AUDIT_FILTER_EXIT && + krule->listnr != AUDIT_FILTER_URING_EXIT) || op != Audit_equal || krule->inode_f || krule->watch || krule->tree) return -EINVAL; diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index db2c6b59dfc3..d75acb014ccd 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -44,7 +44,8 @@ struct list_head audit_filter_list[AUDIT_NR_FILTERS] = { LIST_HEAD_INIT(audit_filter_list[4]), LIST_HEAD_INIT(audit_filter_list[5]), LIST_HEAD_INIT(audit_filter_list[6]), -#if AUDIT_NR_FILTERS != 7 + LIST_HEAD_INIT(audit_filter_list[7]), +#if AUDIT_NR_FILTERS != 8 #error Fix audit_filter_list initialiser #endif }; @@ -56,6 +57,7 @@ static struct list_head audit_rules_list[AUDIT_NR_FILTERS] = { LIST_HEAD_INIT(audit_rules_list[4]), LIST_HEAD_INIT(audit_rules_list[5]), LIST_HEAD_INIT(audit_rules_list[6]), + LIST_HEAD_INIT(audit_rules_list[7]), }; DEFINE_MUTEX(audit_filter_mutex); @@ -151,7 +153,8 @@ char *audit_unpack_string(void **bufp, size_t *remain, size_t len) static inline int audit_to_inode(struct audit_krule *krule, struct audit_field *f) { - if (krule->listnr != AUDIT_FILTER_EXIT || + if ((krule->listnr != AUDIT_FILTER_EXIT && + krule->listnr != AUDIT_FILTER_URING_EXIT) || krule->inode_f || krule->watch || krule->tree || (f->op != Audit_equal && f->op != Audit_not_equal)) return -EINVAL; @@ -248,6 +251,7 @@ static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data * pr_err("AUDIT_FILTER_ENTRY is deprecated\n"); goto exit_err; case AUDIT_FILTER_EXIT: + case AUDIT_FILTER_URING_EXIT: case AUDIT_FILTER_TASK: #endif case AUDIT_FILTER_USER: @@ -332,6 +336,10 @@ static int audit_field_valid(struct audit_entry *entry, struct audit_field *f) if (entry->rule.listnr != AUDIT_FILTER_FS) return -EINVAL; break; + case AUDIT_PERM: + if (entry->rule.listnr == AUDIT_FILTER_URING_EXIT) + return -EINVAL; + break; } switch (entry->rule.listnr) { @@ -980,7 +988,8 @@ static inline int audit_add_rule(struct audit_entry *entry) } entry->rule.prio = ~0ULL; - if (entry->rule.listnr == AUDIT_FILTER_EXIT) { + if (entry->rule.listnr == AUDIT_FILTER_EXIT || + entry->rule.listnr == AUDIT_FILTER_URING_EXIT) { if (entry->rule.flags & AUDIT_FILTER_PREPEND) entry->rule.prio = ++prio_high; else diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 6dda448fb826..3c9fb842a8f0 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -805,6 +805,34 @@ static int audit_in_mask(const struct audit_krule *rule, unsigned long val) return rule->mask[word] & bit; } +/** + * audit_filter_uring - apply filters to an io_uring operation + * @tsk: associated task + * @ctx: audit context + */ +static void audit_filter_uring(struct task_struct *tsk, + struct audit_context *ctx) +{ + struct audit_entry *e; + enum audit_state state; + + if (auditd_test_task(tsk)) + return; + + rcu_read_lock(); + list_for_each_entry_rcu(e, &audit_filter_list[AUDIT_FILTER_URING_EXIT], + list) { + if (audit_in_mask(&e->rule, ctx->uring_op) && + audit_filter_rules(tsk, &e->rule, ctx, NULL, &state, + false)) { + rcu_read_unlock(); + ctx->current_state = state; + return; + } + } + rcu_read_unlock(); +} + /* At syscall exit time, this filter is called if the audit_state is * not low enough that auditing cannot take place, but is also not * high enough that we already know we have to write an audit record @@ -1757,7 +1785,7 @@ static void audit_log_exit(void) * __audit_free - free a per-task audit context * @tsk: task whose audit context block to free * - * Called from copy_process and do_exit + * Called from copy_process, do_exit, and the io_uring code */ void __audit_free(struct task_struct *tsk) { @@ -1775,15 +1803,21 @@ void __audit_free(struct task_struct *tsk) * random task_struct that doesn't doesn't have any meaningful data we * need to log via audit_log_exit(). */ - if (tsk == current && !context->dummy && - context->context == AUDIT_CTX_SYSCALL) { + if (tsk == current && !context->dummy) { context->return_valid = AUDITSC_INVALID; context->return_code = 0; - - audit_filter_syscall(tsk, context); - audit_filter_inodes(tsk, context); - if (context->current_state == AUDIT_STATE_RECORD) - audit_log_exit(); + if (context->context == AUDIT_CTX_SYSCALL) { + audit_filter_syscall(tsk, context); + audit_filter_inodes(tsk, context); + if (context->current_state == AUDIT_STATE_RECORD) + audit_log_exit(); + } else if (context->context == AUDIT_CTX_URING) { + /* TODO: verify this case is real and valid */ + audit_filter_uring(tsk, context); + audit_filter_inodes(tsk, context); + if (context->current_state == AUDIT_STATE_RECORD) + audit_log_uring(context); + } } audit_set_context(tsk, NULL); @@ -1867,12 +1901,6 @@ void __audit_uring_exit(int success, long code) { struct audit_context *ctx = audit_context(); - /* - * TODO: At some point we will likely want to filter on io_uring ops - * and other things similar to what we do for syscalls, but that - * is something for another day; just record what we can here. - */ - if (ctx->context == AUDIT_CTX_SYSCALL) { /* * NOTE: See the note in __audit_uring_entry() about the case @@ -1895,6 +1923,8 @@ void __audit_uring_exit(int success, long code) * the behavior here. */ audit_filter_syscall(current, ctx); + if (ctx->current_state != AUDIT_STATE_RECORD) + audit_filter_uring(current, ctx); audit_filter_inodes(current, ctx); if (ctx->current_state != AUDIT_STATE_RECORD) return; @@ -1907,6 +1937,8 @@ void __audit_uring_exit(int success, long code) if (!list_empty(&ctx->killed_trees)) audit_kill_trees(ctx); + /* run through both filters to ensure we set the filterkey properly */ + audit_filter_uring(current, ctx); audit_filter_inodes(current, ctx); if (ctx->current_state != AUDIT_STATE_RECORD) goto out; From 3a862cacf8670729b1ad8fc28e4f7e078f9c570c Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 1 Feb 2021 19:22:44 -0500 Subject: [PATCH 072/134] fs: add anon_inode_getfile_secure() similar to anon_inode_getfd_secure() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extending the secure anonymous inode support to other subsystems requires that we have a secure anon_inode_getfile() variant in addition to the existing secure anon_inode_getfd() variant. Thankfully we can reuse the existing __anon_inode_getfile() function and just wrap it with the proper arguments. Acked-by: Mickaël Salaün Signed-off-by: Paul Moore --- fs/anon_inodes.c | 29 +++++++++++++++++++++++++++++ include/linux/anon_inodes.h | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index a280156138ed..e0c3e33c4177 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -148,6 +148,35 @@ struct file *anon_inode_getfile(const char *name, } EXPORT_SYMBOL_GPL(anon_inode_getfile); +/** + * anon_inode_getfile_secure - Like anon_inode_getfile(), but creates a new + * !S_PRIVATE anon inode rather than reuse the + * singleton anon inode and calls the + * inode_init_security_anon() LSM hook. This + * allows for both the inode to have its own + * security context and for the LSM to enforce + * policy on the inode's creation. + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * @context_inode: + * [in] the logical relationship with the new inode (optional) + * + * The LSM may use @context_inode in inode_init_security_anon(), but a + * reference to it is not held. Returns the newly created file* or an error + * pointer. See the anon_inode_getfile() documentation for more information. + */ +struct file *anon_inode_getfile_secure(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode) +{ + return __anon_inode_getfile(name, fops, priv, flags, + context_inode, true); +} + static int __anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h index 71881a2b6f78..5deaddbd7927 100644 --- a/include/linux/anon_inodes.h +++ b/include/linux/anon_inodes.h @@ -15,6 +15,10 @@ struct inode; struct file *anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags); +struct file *anon_inode_getfile_secure(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode); int anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags); int anon_inode_getfd_secure(const char *name, From 91a9ab7c942aaa40ac5957eebe71ddae30b2a49c Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 1 Feb 2021 19:33:52 -0500 Subject: [PATCH 073/134] io_uring: convert io_uring to the secure anon inode interface Converting io_uring's anonymous inode to the secure anon inode API enables LSMs to enforce policy on the io_uring anonymous inodes if they chose to do so. This is an important first step towards providing the necessary mechanisms so that LSMs can apply security policy to io_uring operations. Signed-off-by: Paul Moore --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 388754b24785..56cc9aba0d01 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -10155,8 +10155,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx) return ERR_PTR(ret); #endif - file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx, - O_RDWR | O_CLOEXEC); + file = anon_inode_getfile_secure("[io_uring]", &io_uring_fops, ctx, + O_RDWR | O_CLOEXEC, NULL); #if defined(CONFIG_UNIX) if (IS_ERR(file)) { sock_release(ctx->ring_sock); From cdc1404a40461faba23c5a5ad40adcc7eecc1580 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 1 Feb 2021 19:56:49 -0500 Subject: [PATCH 074/134] lsm,io_uring: add LSM hooks to io_uring A full expalantion of io_uring is beyond the scope of this commit description, but in summary it is an asynchronous I/O mechanism which allows for I/O requests and the resulting data to be queued in memory mapped "rings" which are shared between the kernel and userspace. Optionally, io_uring offers the ability for applications to spawn kernel threads to dequeue I/O requests from the ring and submit the requests in the kernel, helping to minimize the syscall overhead. Rings are accessed in userspace by memory mapping a file descriptor provided by the io_uring_setup(2), and can be shared between applications as one might do with any open file descriptor. Finally, process credentials can be registered with a given ring and any process with access to that ring can submit I/O requests using any of the registered credentials. While the io_uring functionality is widely recognized as offering a vastly improved, and high performing asynchronous I/O mechanism, its ability to allow processes to submit I/O requests with credentials other than its own presents a challenge to LSMs. When a process creates a new io_uring ring the ring's credentials are inhertied from the calling process; if this ring is shared with another process operating with different credentials there is the potential to bypass the LSMs security policy. Similarly, registering credentials with a given ring allows any process with access to that ring to submit I/O requests with those credentials. In an effort to allow LSMs to apply security policy to io_uring I/O operations, this patch adds two new LSM hooks. These hooks, in conjunction with the LSM anonymous inode support previously submitted, allow an LSM to apply access control policy to the sharing of io_uring rings as well as any io_uring credential changes requested by a process. The new LSM hooks are described below: * int security_uring_override_creds(cred) Controls if the current task, executing an io_uring operation, is allowed to override it's credentials with @cred. In cases where the current task is a user application, the current credentials will be those of the user application. In cases where the current task is a kernel thread servicing io_uring requests the current credentials will be those of the io_uring ring (inherited from the process that created the ring). * int security_uring_sqpoll(void) Controls if the current task is allowed to create an io_uring polling thread (IORING_SETUP_SQPOLL). Without a SQPOLL thread in the kernel processes must submit I/O requests via io_uring_enter(2) which allows us to compare any requested credential changes against the application making the request. With a SQPOLL thread, we can no longer compare requested credential changes against the application making the request, the comparison is made against the ring's credentials. Signed-off-by: Paul Moore --- fs/io_uring.c | 10 ++++++++++ include/linux/lsm_hook_defs.h | 5 +++++ include/linux/lsm_hooks.h | 13 +++++++++++++ include/linux/security.h | 16 ++++++++++++++++ security/security.c | 12 ++++++++++++ 5 files changed, 56 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 56cc9aba0d01..f89d00af3a67 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -80,6 +80,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -7070,6 +7071,11 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (!req->creds) return -EINVAL; get_cred(req->creds); + ret = security_uring_override_creds(req->creds); + if (ret) { + put_cred(req->creds); + return ret; + } req->flags |= REQ_F_CREDS; } state = &ctx->submit_state; @@ -8566,6 +8572,10 @@ static int io_sq_offload_create(struct io_ring_ctx *ctx, struct io_sq_data *sqd; bool attached; + ret = security_uring_sqpoll(); + if (ret) + return ret; + sqd = io_get_sq_data(p, &attached); if (IS_ERR(sqd)) { ret = PTR_ERR(sqd); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 2adeea44c0d5..b3c525353769 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -402,3 +402,8 @@ LSM_HOOK(void, LSM_RET_VOID, perf_event_free, struct perf_event *event) LSM_HOOK(int, 0, perf_event_read, struct perf_event *event) LSM_HOOK(int, 0, perf_event_write, struct perf_event *event) #endif /* CONFIG_PERF_EVENTS */ + +#ifdef CONFIG_IO_URING +LSM_HOOK(int, 0, uring_override_creds, const struct cred *new) +LSM_HOOK(int, 0, uring_sqpoll, void) +#endif /* CONFIG_IO_URING */ diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 5c4c5c0602cb..0eb0ae95c4c4 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1557,6 +1557,19 @@ * Read perf_event security info if allowed. * @perf_event_write: * Write perf_event security info if allowed. + * + * Security hooks for io_uring + * + * @uring_override_creds: + * Check if the current task, executing an io_uring operation, is allowed + * to override it's credentials with @new. + * + * @new: the new creds to use + * + * @uring_sqpoll: + * Check whether the current task is allowed to spawn a io_uring polling + * thread (IORING_SETUP_SQPOLL). + * */ union security_list_options { #define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__); diff --git a/include/linux/security.h b/include/linux/security.h index 5b7288521300..7979b9629a42 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -2038,4 +2038,20 @@ static inline int security_perf_event_write(struct perf_event *event) #endif /* CONFIG_SECURITY */ #endif /* CONFIG_PERF_EVENTS */ +#ifdef CONFIG_IO_URING +#ifdef CONFIG_SECURITY +extern int security_uring_override_creds(const struct cred *new); +extern int security_uring_sqpoll(void); +#else +static inline int security_uring_override_creds(const struct cred *new) +{ + return 0; +} +static inline int security_uring_sqpoll(void) +{ + return 0; +} +#endif /* CONFIG_SECURITY */ +#endif /* CONFIG_IO_URING */ + #endif /* ! __LINUX_SECURITY_H */ diff --git a/security/security.c b/security/security.c index 9ffa9e9c5c55..c49a2c0cc1c1 100644 --- a/security/security.c +++ b/security/security.c @@ -2625,3 +2625,15 @@ int security_perf_event_write(struct perf_event *event) return call_int_hook(perf_event_write, 0, event); } #endif /* CONFIG_PERF_EVENTS */ + +#ifdef CONFIG_IO_URING +int security_uring_override_creds(const struct cred *new) +{ + return call_int_hook(uring_override_creds, 0, new); +} + +int security_uring_sqpoll(void) +{ + return call_int_hook(uring_sqpoll, 0); +} +#endif /* CONFIG_IO_URING */ From 740b03414b20e7f1879cd99aae27d8c401bbcbf9 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 23 Feb 2021 18:16:45 -0500 Subject: [PATCH 075/134] selinux: add support for the io_uring access controls This patch implements two new io_uring access controls, specifically support for controlling the io_uring "personalities" and IORING_SETUP_SQPOLL. Controlling the sharing of io_urings themselves is handled via the normal file/inode labeling and sharing mechanisms. The io_uring { override_creds } permission restricts which domains the subject domain can use to override it's own credentials. Granting a domain the io_uring { override_creds } permission allows it to impersonate another domain in io_uring operations. The io_uring { sqpoll } permission restricts which domains can create asynchronous io_uring polling threads. This is important from a security perspective as operations queued by this asynchronous thread inherit the credentials of the thread creator by default; if an io_uring is shared across process/domain boundaries this could result in one domain impersonating another. Controlling the creation of sqpoll threads, and the sharing of io_urings across processes, allow policy authors to restrict the ability of one domain to impersonate another via io_uring. As a quick summary, this patch adds a new object class with two permissions: io_uring { override_creds sqpoll } These permissions can be seen in the two simple policy statements below: allow domA_t domB_t : io_uring { override_creds }; allow domA_t self : io_uring { sqpoll }; Signed-off-by: Paul Moore --- security/selinux/hooks.c | 34 +++++++++++++++++++++++++++++ security/selinux/include/classmap.h | 2 ++ 2 files changed, 36 insertions(+) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 6517f221d52c..012e8504ed9e 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -7111,6 +7111,35 @@ static int selinux_perf_event_write(struct perf_event *event) } #endif +#ifdef CONFIG_IO_URING +/** + * selinux_uring_override_creds - check the requested cred override + * @new: the target creds + * + * Check to see if the current task is allowed to override it's credentials + * to service an io_uring operation. + */ +static int selinux_uring_override_creds(const struct cred *new) +{ + return avc_has_perm(&selinux_state, current_sid(), cred_sid(new), + SECCLASS_IO_URING, IO_URING__OVERRIDE_CREDS, NULL); +} + +/** + * selinux_uring_sqpoll - check if a io_uring polling thread can be created + * + * Check to see if the current task is allowed to create a new io_uring + * kernel polling thread. + */ +static int selinux_uring_sqpoll(void) +{ + int sid = current_sid(); + + return avc_has_perm(&selinux_state, sid, sid, + SECCLASS_IO_URING, IO_URING__SQPOLL, NULL); +} +#endif /* CONFIG_IO_URING */ + /* * IMPORTANT NOTE: When adding new hooks, please be careful to keep this order: * 1. any hooks that don't belong to (2.) or (3.) below, @@ -7349,6 +7378,11 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(perf_event_write, selinux_perf_event_write), #endif +#ifdef CONFIG_IO_URING + LSM_HOOK_INIT(uring_override_creds, selinux_uring_override_creds), + LSM_HOOK_INIT(uring_sqpoll, selinux_uring_sqpoll), +#endif + LSM_HOOK_INIT(locked_down, selinux_lockdown), /* diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index 084757ff4390..698ccfdaf82d 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -254,6 +254,8 @@ struct security_class_mapping secclass_map[] = { { "integrity", "confidentiality", NULL } }, { "anon_inode", { COMMON_FILE_PERMS, NULL } }, + { "io_uring", + { "override_creds", "sqpoll", NULL } }, { NULL } }; From d9d8c93938c40e12de91650d04fceb99d92dad8a Mon Sep 17 00:00:00 2001 From: Casey Schaufler Date: Fri, 21 May 2021 11:21:16 -0700 Subject: [PATCH 076/134] Smack: Brutalist io_uring support Add Smack privilege checks for io_uring. Use CAP_MAC_OVERRIDE for the override_creds case and CAP_MAC_ADMIN for creating a polling thread. These choices are based on conjecture regarding the intent of the surrounding code. Signed-off-by: Casey Schaufler [PM: make the smack_uring_* funcs static, remove debug code] Signed-off-by: Paul Moore --- security/smack/smack_lsm.c | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index cacbe7518519..f90ab1efeb6d 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -4691,6 +4691,48 @@ static int smack_dentry_create_files_as(struct dentry *dentry, int mode, return 0; } +#ifdef CONFIG_IO_URING +/** + * smack_uring_override_creds - Is io_uring cred override allowed? + * @new: the target creds + * + * Check to see if the current task is allowed to override it's credentials + * to service an io_uring operation. + */ +static int smack_uring_override_creds(const struct cred *new) +{ + struct task_smack *tsp = smack_cred(current_cred()); + struct task_smack *nsp = smack_cred(new); + + /* + * Allow the degenerate case where the new Smack value is + * the same as the current Smack value. + */ + if (tsp->smk_task == nsp->smk_task) + return 0; + + if (smack_privileged_cred(CAP_MAC_OVERRIDE, current_cred())) + return 0; + + return -EPERM; +} + +/** + * smack_uring_sqpoll - check if a io_uring polling thread can be created + * + * Check to see if the current task is allowed to create a new io_uring + * kernel polling thread. + */ +static int smack_uring_sqpoll(void) +{ + if (smack_privileged_cred(CAP_MAC_ADMIN, current_cred())) + return 0; + + return -EPERM; +} + +#endif /* CONFIG_IO_URING */ + struct lsm_blob_sizes smack_blob_sizes __lsm_ro_after_init = { .lbs_cred = sizeof(struct task_smack), .lbs_file = sizeof(struct smack_known *), @@ -4843,6 +4885,10 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(inode_copy_up, smack_inode_copy_up), LSM_HOOK_INIT(inode_copy_up_xattr, smack_inode_copy_up_xattr), LSM_HOOK_INIT(dentry_create_files_as, smack_dentry_create_files_as), +#ifdef CONFIG_IO_URING + LSM_HOOK_INIT(uring_override_creds, smack_uring_override_creds), + LSM_HOOK_INIT(uring_sqpoll, smack_uring_sqpoll), +#endif }; From 8e71168e2cc760e2d016ad99a59773169211472f Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 14 Sep 2021 15:15:16 +0200 Subject: [PATCH 077/134] lsm_audit: avoid overloading the "key" audit field The "key" field is used to associate records with the rule that triggered them, os it's not a good idea to overload it with an additional IPC key semantic. Moreover, as the classic "key" field is a text field, while the IPC key is numeric, AVC records containing the IPC key info actually confuse audit userspace, which tries to interpret the number as a hex-encoded string, thus showing garbage for example in the ausearch "interpret" output mode. Hence, change it to "ipc_key" to fix both issues and also make the meaning of this field more clear. Signed-off-by: Ondrej Mosnacek Reviewed-by: Richard Guy Briggs Signed-off-by: Paul Moore --- security/lsm_audit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/lsm_audit.c b/security/lsm_audit.c index 5a5016ef43b0..1897cbf6fc69 100644 --- a/security/lsm_audit.c +++ b/security/lsm_audit.c @@ -224,7 +224,7 @@ static void dump_common_audit_data(struct audit_buffer *ab, case LSM_AUDIT_DATA_NONE: return; case LSM_AUDIT_DATA_IPC: - audit_log_format(ab, " key=%d ", a->u.ipc_id); + audit_log_format(ab, " ipc_key=%d ", a->u.ipc_id); break; case LSM_AUDIT_DATA_CAP: audit_log_format(ab, " capability=%d ", a->u.cap); From f20311cc9c58052e0b215013046cbf390937910c Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Thu, 16 Sep 2021 00:03:07 +0200 Subject: [PATCH 078/134] crypto: caam - disable pkc for non-E SoCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On newer CAAM versions, not all accelerators are disabled if the SoC is a non-E variant. While the driver checks most of the modules for availability, there is one - PKHA - which sticks out. On non-E variants it is still reported as available, that is the number of instances is non-zero, but it has limited functionality. In particular it doesn't support encryption and decryption, but just signing and verifying. This is indicated by a bit in the PKHA_MISC field. Take this bit into account if we are checking for availability. This will the following error: [ 8.167817] caam_jr 8020000.jr: 20000b0f: CCB: desc idx 11: : Invalid CHA selected. Tested on an NXP LS1028A (non-E) SoC. Fixes: d239b10d4ceb ("crypto: caam - add register map changes cf. Era 10") Signed-off-by: Michael Walle Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caampkc.c | 19 +++++++++++++++---- drivers/crypto/caam/regs.h | 3 +++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index e313233ec6de..bf6275ffc4aa 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -1153,16 +1153,27 @@ static struct caam_akcipher_alg caam_rsa = { int caam_pkc_init(struct device *ctrldev) { struct caam_drv_private *priv = dev_get_drvdata(ctrldev); - u32 pk_inst; + u32 pk_inst, pkha; int err; init_done = false; /* Determine public key hardware accelerator presence. */ - if (priv->era < 10) + if (priv->era < 10) { pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT; - else - pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK; + } else { + pkha = rd_reg32(&priv->ctrl->vreg.pkha); + pk_inst = pkha & CHA_VER_NUM_MASK; + + /* + * Newer CAAMs support partially disabled functionality. If this is the + * case, the number is non-zero, but this bit is set to indicate that + * no encryption or decryption is supported. Only signing and verifying + * is supported. + */ + if (pkha & CHA_VER_MISC_PKHA_NO_CRYPT) + pk_inst = 0; + } /* Do not register algorithms if PKHA is not present. */ if (!pk_inst) diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index af61f3a2c0d4..3738625c0250 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -322,6 +322,9 @@ struct version_regs { /* CHA Miscellaneous Information - AESA_MISC specific */ #define CHA_VER_MISC_AES_GCM BIT(1 + CHA_VER_MISC_SHIFT) +/* CHA Miscellaneous Information - PKHA_MISC specific */ +#define CHA_VER_MISC_PKHA_NO_CRYPT BIT(7 + CHA_VER_MISC_SHIFT) + /* * caam_perfmon - Performance Monitor/Secure Memory Status/ * CAAM Global Status/Component Version IDs From ca605f97dae4bf070b7c584aec23c1c922e4d823 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Thu, 16 Sep 2021 15:45:41 +0100 Subject: [PATCH 079/134] crypto: qat - power up 4xxx device After reset or boot, QAT 4xxx devices are inactive and require to be explicitly activated. This is done by writing the DRV_ACTIVE bit in the PM_INTERRUPT register and polling the PM_INIT_STATE to make sure that the transaction has completed properly. If this is not done, the driver will fail the initialization sequence reporting the following message: [ 22.081193] 4xxx 0000:f7:00.0: enabling device (0140 -> 0142) [ 22.720285] QAT: AE0 is inactive!! [ 22.720287] QAT: failed to get device out of reset [ 22.720288] 4xxx 0000:f7:00.0: qat_hal_clr_reset error [ 22.720290] 4xxx 0000:f7:00.0: Failed to init the AEs [ 22.720290] 4xxx 0000:f7:00.0: Failed to initialise Acceleration Engine [ 22.720789] 4xxx 0000:f7:00.0: Resetting device qat_dev0 [ 22.825099] 4xxx: probe of 0000:f7:00.0 failed with error -14 The patch also temporarily disables the power management source of interrupt, to avoid possible spurious interrupts as the power management feature is not fully supported. The device init function has been added to adf_dev_init(), and not in the probe of 4xxx to make sure that the device is re-enabled in case of reset. Note that the error code reported by hw_data->init_device() in adf_dev_init() has been shadowed for consistency with the other calls in the same function. Fixes: 8c8268166e83 ("crypto: qat - add qat_4xxx driver") Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 31 +++++++++++++++++++ .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 10 ++++++ .../crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_init.c | 5 +++ 4 files changed, 47 insertions(+) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 33d8e50dcbda..88c0ded411f1 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ +#include #include #include #include @@ -161,6 +162,35 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); } +static int adf_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + u32 status; + u32 csr; + int ret; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2); + csr |= ADF_4XXX_PM_SOU; + ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_4XXX_PM_INIT_STATE, + ADF_4XXX_PM_POLL_DELAY_US, + ADF_4XXX_PM_POLL_TIMEOUT_US, true, addr, + ADF_4XXX_PM_STATUS); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + + return ret; +} + static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) { return 0; @@ -215,6 +245,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->exit_arb = adf_exit_arb; hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; + hw_data->init_device = adf_init_device; hw_data->reset_device = adf_reset_flr; hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; hw_data->uof_get_num_objs = uof_get_num_objs; diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index 4fe2a776293c..924bac6feb37 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -62,6 +62,16 @@ #define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) #define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) +/* Power management */ +#define ADF_4XXX_PM_POLL_DELAY_US 20 +#define ADF_4XXX_PM_POLL_TIMEOUT_US USEC_PER_SEC +#define ADF_4XXX_PM_STATUS (0x50A00C) +#define ADF_4XXX_PM_INTERRUPT (0x50A028) +#define ADF_4XXX_PM_DRV_ACTIVE BIT(20) +#define ADF_4XXX_PM_INIT_STATE BIT(21) +/* Power management source in ERRSOU2 and ERRMSK2 */ +#define ADF_4XXX_PM_SOU BIT(18) + /* Firmware Binaries */ #define ADF_4XXX_FW "qat_4xxx.bin" #define ADF_4XXX_MMP "qat_4xxx_mmp.bin" diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index b62b34fccc90..ca8e23f0bcc4 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -170,6 +170,7 @@ struct adf_hw_device_data { int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); const u32 *(*get_arb_mapping)(void); + int (*init_device)(struct adf_accel_dev *accel_dev); void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, bool enable); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index 60bc7b991d35..e3749e5817d9 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -79,6 +79,11 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) return -EFAULT; } + if (hw_data->init_device && hw_data->init_device(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n"); + return -EFAULT; + } + if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); return -EFAULT; From 7c5329697ed4e0e1bf9a4e4fc9f0053f2f58935d Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 16 Sep 2021 19:05:22 +0200 Subject: [PATCH 080/134] crypto: marvell/cesa - drop unneeded MODULE_ALIAS The MODULE_DEVICE_TABLE already creates proper alias for platform driver. Having another MODULE_ALIAS causes the alias to be duplicated. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/cesa.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index f14aac532f53..5cd332880653 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -615,7 +615,6 @@ static struct platform_driver marvell_cesa = { }; module_platform_driver(marvell_cesa); -MODULE_ALIAS("platform:mv_crypto"); MODULE_AUTHOR("Boris Brezillon "); MODULE_AUTHOR("Arnaud Ebalard "); MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); From adad556efcdd42a1d9e060cbe5f6161cccf1fa28 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 17 Sep 2021 08:26:19 +0800 Subject: [PATCH 081/134] crypto: api - Fix built-in testing dependency failures When complex algorithms that depend on other algorithms are built into the kernel, the order of registration must be done such that the underlying algorithms are ready before the ones on top are registered. As otherwise they would fail during the self-test which is required during registration. In the past we have used subsystem initialisation ordering to guarantee this. The number of such precedence levels are limited and they may cause ripple effects in other subsystems. This patch solves this problem by delaying all self-tests during boot-up for built-in algorithms. They will be tested either when something else in the kernel requests for them, or when we have finished registering all built-in algorithms, whichever comes earlier. Reported-by: Vladis Dronov Signed-off-by: Herbert Xu --- crypto/algapi.c | 73 +++++++++++++++++++++++++++++++++-------------- crypto/api.c | 52 +++++++++++++++++++++++++++++---- crypto/internal.h | 10 +++++++ 3 files changed, 108 insertions(+), 27 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 43f999dba4dc..422bdca214e1 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -389,29 +389,10 @@ void crypto_remove_final(struct list_head *list) } EXPORT_SYMBOL_GPL(crypto_remove_final); -static void crypto_wait_for_test(struct crypto_larval *larval) -{ - int err; - - err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); - if (err != NOTIFY_STOP) { - if (WARN_ON(err != NOTIFY_DONE)) - goto out; - crypto_alg_tested(larval->alg.cra_driver_name, 0); - } - - err = wait_for_completion_killable(&larval->completion); - WARN_ON(err); - if (!err) - crypto_notify(CRYPTO_MSG_ALG_LOADED, larval); - -out: - crypto_larval_kill(&larval->alg); -} - int crypto_register_alg(struct crypto_alg *alg) { struct crypto_larval *larval; + bool test_started; int err; alg->cra_flags &= ~CRYPTO_ALG_DEAD; @@ -421,12 +402,15 @@ int crypto_register_alg(struct crypto_alg *alg) down_write(&crypto_alg_sem); larval = __crypto_register_alg(alg); + test_started = static_key_enabled(&crypto_boot_test_finished); + larval->test_started = test_started; up_write(&crypto_alg_sem); if (IS_ERR(larval)) return PTR_ERR(larval); - crypto_wait_for_test(larval); + if (test_started) + crypto_wait_for_test(larval); return 0; } EXPORT_SYMBOL_GPL(crypto_register_alg); @@ -633,6 +617,8 @@ int crypto_register_instance(struct crypto_template *tmpl, if (IS_ERR(larval)) goto unlock; + larval->test_started = true; + hlist_add_head(&inst->list, &tmpl->instances); inst->tmpl = tmpl; @@ -1261,9 +1247,48 @@ void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt); #endif +static void __init crypto_start_tests(void) +{ + for (;;) { + struct crypto_larval *larval = NULL; + struct crypto_alg *q; + + down_write(&crypto_alg_sem); + + list_for_each_entry(q, &crypto_alg_list, cra_list) { + struct crypto_larval *l; + + if (!crypto_is_larval(q)) + continue; + + l = (void *)q; + + if (!crypto_is_test_larval(l)) + continue; + + if (l->test_started) + continue; + + l->test_started = true; + larval = l; + break; + } + + up_write(&crypto_alg_sem); + + if (!larval) + break; + + crypto_wait_for_test(larval); + } + + static_branch_enable(&crypto_boot_test_finished); +} + static int __init crypto_algapi_init(void) { crypto_init_proc(); + crypto_start_tests(); return 0; } @@ -1272,7 +1297,11 @@ static void __exit crypto_algapi_exit(void) crypto_exit_proc(); } -module_init(crypto_algapi_init); +/* + * We run this at late_initcall so that all the built-in algorithms + * have had a chance to register themselves first. + */ +late_initcall(crypto_algapi_init); module_exit(crypto_algapi_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/api.c b/crypto/api.c index c4eda56cff89..1cf1f03347cc 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,8 @@ EXPORT_SYMBOL_GPL(crypto_alg_sem); BLOCKING_NOTIFIER_HEAD(crypto_chain); EXPORT_SYMBOL_GPL(crypto_chain); +DEFINE_STATIC_KEY_FALSE(crypto_boot_test_finished); + static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg); struct crypto_alg *crypto_mod_get(struct crypto_alg *alg) @@ -47,11 +50,6 @@ void crypto_mod_put(struct crypto_alg *alg) } EXPORT_SYMBOL_GPL(crypto_mod_put); -static inline int crypto_is_test_larval(struct crypto_larval *larval) -{ - return larval->alg.cra_driver_name[0]; -} - static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type, u32 mask) { @@ -163,11 +161,55 @@ void crypto_larval_kill(struct crypto_alg *alg) } EXPORT_SYMBOL_GPL(crypto_larval_kill); +void crypto_wait_for_test(struct crypto_larval *larval) +{ + int err; + + err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); + if (err != NOTIFY_STOP) { + if (WARN_ON(err != NOTIFY_DONE)) + goto out; + crypto_alg_tested(larval->alg.cra_driver_name, 0); + } + + err = wait_for_completion_killable(&larval->completion); + WARN_ON(err); + if (!err) + crypto_notify(CRYPTO_MSG_ALG_LOADED, larval); + +out: + crypto_larval_kill(&larval->alg); +} +EXPORT_SYMBOL_GPL(crypto_wait_for_test); + +static void crypto_start_test(struct crypto_larval *larval) +{ + if (!crypto_is_test_larval(larval)) + return; + + if (larval->test_started) + return; + + down_write(&crypto_alg_sem); + if (larval->test_started) { + up_write(&crypto_alg_sem); + return; + } + + larval->test_started = true; + up_write(&crypto_alg_sem); + + crypto_wait_for_test(larval); +} + static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg) { struct crypto_larval *larval = (void *)alg; long timeout; + if (!static_branch_likely(&crypto_boot_test_finished)) + crypto_start_test(larval); + timeout = wait_for_completion_killable_timeout( &larval->completion, 60 * HZ); diff --git a/crypto/internal.h b/crypto/internal.h index f00869af689f..c08385571853 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ struct crypto_larval { struct crypto_alg *adult; struct completion completion; u32 mask; + bool test_started; }; enum { @@ -45,6 +47,8 @@ extern struct list_head crypto_alg_list; extern struct rw_semaphore crypto_alg_sem; extern struct blocking_notifier_head crypto_chain; +DECLARE_STATIC_KEY_FALSE(crypto_boot_test_finished); + #ifdef CONFIG_PROC_FS void __init crypto_init_proc(void); void __exit crypto_exit_proc(void); @@ -70,6 +74,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask); struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask); void crypto_larval_kill(struct crypto_alg *alg); +void crypto_wait_for_test(struct crypto_larval *larval); void crypto_alg_tested(const char *name, int err); void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, @@ -156,5 +161,10 @@ static inline void crypto_yield(u32 flags) cond_resched(); } +static inline int crypto_is_test_larval(struct crypto_larval *larval) +{ + return larval->alg.cra_driver_name[0]; +} + #endif /* _CRYPTO_INTERNAL_H */ From 8a764ef1bd43fb2bb4ff3290746e5c820a3a9716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20G=C3=B6ttsche?= Date: Tue, 28 Sep 2021 17:39:31 +0200 Subject: [PATCH 082/134] selinux: enable genfscon labeling for securityfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for genfscon per-file labeling of securityfs files. This allows for separate labels and thereby access control for different files. For example a genfscon statement genfscon securityfs /integrity/ima/policy \ system_u:object_r:ima_policy_t:s0 will set a private label to the IMA policy file and thus allow to control the ability to set the IMA policy. Setting labels directly with setxattr(2), e.g. by chcon(1) or setfiles(8), is still not supported. Signed-off-by: Christian Göttsche [PM: line width fixes in the commit description] Signed-off-by: Paul Moore --- security/selinux/hooks.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 012e8504ed9e..549f631e9832 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -760,7 +760,8 @@ static int selinux_set_mnt_opts(struct super_block *sb, !strcmp(sb->s_type->name, "tracefs") || !strcmp(sb->s_type->name, "binder") || !strcmp(sb->s_type->name, "bpf") || - !strcmp(sb->s_type->name, "pstore")) + !strcmp(sb->s_type->name, "pstore") || + !strcmp(sb->s_type->name, "securityfs")) sbsec->flags |= SE_SBGENFS; if (!strcmp(sb->s_type->name, "sysfs") || From f5d0e5e9d72d3a06018efbfa3adccc0e09a129f9 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 28 Sep 2021 18:22:26 -0400 Subject: [PATCH 083/134] selinux: remove the SELinux lockdown implementation NOTE: This patch intentionally omits any "Fixes:" metadata or stable tagging since it removes a SELinux access control check; while removing the control point is the right thing to do moving forward, removing it in stable kernels could be seen as a regression. The original SELinux lockdown implementation in 59438b46471a ("security,lockdown,selinux: implement SELinux lockdown") used the current task's credentials as both the subject and object in the SELinux lockdown hook, selinux_lockdown(). Unfortunately that proved to be incorrect in a number of cases as the core kernel was calling the LSM lockdown hook in places where the credentials from the "current" task_struct were not the correct credentials to use in the SELinux access check. Attempts were made to resolve this by adding a credential pointer to the LSM lockdown hook as well as suggesting that the single hook be split into two: one for user tasks, one for kernel tasks; however neither approach was deemed acceptable by Linus. Faced with the prospect of either changing the subj/obj in the access check to a constant context (likely the kernel's label) or removing the SELinux lockdown check entirely, the SELinux community decided that removing the lockdown check was preferable. The supporting changes to the general LSM layer are left intact, this patch only removes the SELinux implementation. Acked-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/hooks.c | 30 ----------------------------- security/selinux/include/classmap.h | 2 -- 2 files changed, 32 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 549f631e9832..7297d1f0a9df 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -7014,34 +7014,6 @@ static void selinux_bpf_prog_free(struct bpf_prog_aux *aux) } #endif -static int selinux_lockdown(enum lockdown_reason what) -{ - struct common_audit_data ad; - u32 sid = current_sid(); - int invalid_reason = (what <= LOCKDOWN_NONE) || - (what == LOCKDOWN_INTEGRITY_MAX) || - (what >= LOCKDOWN_CONFIDENTIALITY_MAX); - - if (WARN(invalid_reason, "Invalid lockdown reason")) { - audit_log(audit_context(), - GFP_ATOMIC, AUDIT_SELINUX_ERR, - "lockdown_reason=invalid"); - return -EINVAL; - } - - ad.type = LSM_AUDIT_DATA_LOCKDOWN; - ad.u.reason = what; - - if (what <= LOCKDOWN_INTEGRITY_MAX) - return avc_has_perm(&selinux_state, - sid, sid, SECCLASS_LOCKDOWN, - LOCKDOWN__INTEGRITY, &ad); - else - return avc_has_perm(&selinux_state, - sid, sid, SECCLASS_LOCKDOWN, - LOCKDOWN__CONFIDENTIALITY, &ad); -} - struct lsm_blob_sizes selinux_blob_sizes __lsm_ro_after_init = { .lbs_cred = sizeof(struct task_security_struct), .lbs_file = sizeof(struct file_security_struct), @@ -7384,8 +7356,6 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(uring_sqpoll, selinux_uring_sqpoll), #endif - LSM_HOOK_INIT(locked_down, selinux_lockdown), - /* * PUT "CLONING" (ACCESSING + ALLOCATING) HOOKS HERE */ diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index 698ccfdaf82d..35aac62a662e 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -250,8 +250,6 @@ struct security_class_mapping secclass_map[] = { { COMMON_SOCK_PERMS, NULL } }, { "perf_event", { "open", "cpu", "kernel", "tracepoint", "read", "write", NULL } }, - { "lockdown", - { "integrity", "confidentiality", NULL } }, { "anon_inode", { COMMON_FILE_PERMS, NULL } }, { "io_uring", From f7324d4ba9e846e96ac85fbe74afe3fbdacf3b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 20 Sep 2021 09:44:05 +0200 Subject: [PATCH 084/134] hwrng: meson - Improve error handling for core clock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -ENOENT (ie. "there is no clock") is fine to ignore for an optional clock, other values are not supposed to be ignored and should be escalated to the caller (e.g. -EPROBE_DEFER). Ignore -ENOENT by using devm_clk_get_optional(). While touching this code also add an error message for the fatal errors. Signed-off-by: Uwe Kleine-König Reviewed-by: Neil Armstrong Reviewed-by: Martin Blumenstingl Tested-by: Martin Blumenstingl Signed-off-by: Herbert Xu --- drivers/char/hw_random/meson-rng.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c index e446236e81f2..8bb30282ca46 100644 --- a/drivers/char/hw_random/meson-rng.c +++ b/drivers/char/hw_random/meson-rng.c @@ -54,9 +54,10 @@ static int meson_rng_probe(struct platform_device *pdev) if (IS_ERR(data->base)) return PTR_ERR(data->base); - data->core_clk = devm_clk_get(dev, "core"); + data->core_clk = devm_clk_get_optional(dev, "core"); if (IS_ERR(data->core_clk)) - data->core_clk = NULL; + return dev_err_probe(dev, PTR_ERR(data->core_clk), + "Failed to get core clock\n"); if (data->core_clk) { ret = clk_prepare_enable(data->core_clk); From 38aa192a05f22f9778f9420e630f0322525ef12e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 20 Sep 2021 12:05:35 +0200 Subject: [PATCH 085/134] crypto: ecc - fix CRYPTO_DEFAULT_RNG dependency The ecc.c file started out as part of the ECDH algorithm but got moved out into a standalone module later. It does not build without CRYPTO_DEFAULT_RNG, so now that other modules are using it as well we can run into this link error: aarch64-linux-ld: ecc.c:(.text+0xfc8): undefined reference to `crypto_default_rng' aarch64-linux-ld: ecc.c:(.text+0xff4): undefined reference to `crypto_put_default_rng' Move the 'select CRYPTO_DEFAULT_RNG' statement into the correct symbol. Fixes: 0d7a78643f69 ("crypto: ecrdsa - add EC-RDSA (GOST 34.10) algorithm") Fixes: 4e6602916bc6 ("crypto: ecdsa - Add support for ECDSA signature verification") Signed-off-by: Arnd Bergmann Reviewed-by: Stefan Berger Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 536df4b6b825..285f82647d2b 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -233,12 +233,12 @@ config CRYPTO_DH config CRYPTO_ECC tristate + select CRYPTO_RNG_DEFAULT config CRYPTO_ECDH tristate "ECDH algorithm" select CRYPTO_ECC select CRYPTO_KPP - select CRYPTO_RNG_DEFAULT help Generic implementation of the ECDH algorithm From 6e96dbe7c40a66a1dac3cdc8d29e9172d937a7b1 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 20 Sep 2021 19:28:38 +0100 Subject: [PATCH 086/134] crypto: hisilicon/zip - Fix spelling mistake "COMSUMED" -> "CONSUMED" There is a spelling mistake in a literal string. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 7148201ce76e..873971ef9aee 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -218,7 +218,7 @@ static const struct debugfs_reg32 hzip_dfx_regs[] = { {"HZIP_AVG_DELAY ", 0x28ull}, {"HZIP_MEM_VISIBLE_DATA ", 0x30ull}, {"HZIP_MEM_VISIBLE_ADDR ", 0x34ull}, - {"HZIP_COMSUMED_BYTE ", 0x38ull}, + {"HZIP_CONSUMED_BYTE ", 0x38ull}, {"HZIP_PRODUCED_BYTE ", 0x40ull}, {"HZIP_COMP_INF ", 0x70ull}, {"HZIP_PRE_OUT ", 0x78ull}, From e42dff467ee688fe6b5a083f1837d06e3b27d8c0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 27 Sep 2021 19:23:42 +0800 Subject: [PATCH 087/134] crypto: api - Export crypto_boot_test_finished We need to export crypto_boot_test_finished in case api.c is built-in while algapi.c is built as a module. Fixes: adad556efcdd ("crypto: api - Fix built-in testing dependency failures") Reported-by: Stephen Rothwell Signed-off-by: Herbert Xu Tested-by: Stephen Rothwell # ppc32 build Signed-off-by: Herbert Xu --- crypto/api.c | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/api.c b/crypto/api.c index 1cf1f03347cc..ee5991fe11f8 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -32,6 +32,7 @@ BLOCKING_NOTIFIER_HEAD(crypto_chain); EXPORT_SYMBOL_GPL(crypto_chain); DEFINE_STATIC_KEY_FALSE(crypto_boot_test_finished); +EXPORT_SYMBOL_GPL(crypto_boot_test_finished); static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg); From 42f355ef59a2f98fa4affb4265d3ba3e2d86baf1 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 19 May 2021 16:00:20 -0400 Subject: [PATCH 088/134] audit: replace magic audit syscall class numbers with macros Replace audit syscall class magic numbers with macros. This required putting the macros into new header file include/linux/audit_arch.h since the syscall macros were included for both 64 bit and 32 bit in any compat code, causing redefinition warnings. Link: https://lore.kernel.org/r/2300b1083a32aade7ae7efb95826e8f3f260b1df.1621363275.git.rgb@redhat.com Signed-off-by: Richard Guy Briggs Acked-by: Christian Brauner [PM: renamed header to audit_arch.h after consulting with Richard] Signed-off-by: Paul Moore --- MAINTAINERS | 1 + arch/alpha/kernel/audit.c | 8 ++++---- arch/ia64/kernel/audit.c | 8 ++++---- arch/parisc/kernel/audit.c | 8 ++++---- arch/parisc/kernel/compat_audit.c | 9 +++++---- arch/powerpc/kernel/audit.c | 10 +++++----- arch/powerpc/kernel/compat_audit.c | 11 ++++++----- arch/s390/kernel/audit.c | 10 +++++----- arch/s390/kernel/compat_audit.c | 11 ++++++----- arch/sparc/kernel/audit.c | 10 +++++----- arch/sparc/kernel/compat_audit.c | 11 ++++++----- arch/x86/ia32/audit.c | 11 ++++++----- arch/x86/kernel/audit_64.c | 8 ++++---- include/linux/audit.h | 1 + include/linux/audit_arch.h | 23 +++++++++++++++++++++++ kernel/auditsc.c | 12 ++++++------ lib/audit.c | 10 +++++----- lib/compat_audit.c | 11 ++++++----- 18 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 include/linux/audit_arch.h diff --git a/MAINTAINERS b/MAINTAINERS index eeb4c70b3d5b..94c88bc0502f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3113,6 +3113,7 @@ W: https://github.com/linux-audit T: git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git F: include/asm-generic/audit_*.h F: include/linux/audit.h +F: include/linux/audit_arch.h F: include/uapi/linux/audit.h F: kernel/audit* F: lib/*audit.c diff --git a/arch/alpha/kernel/audit.c b/arch/alpha/kernel/audit.c index 96a9d18ff4c4..81cbd804e375 100644 --- a/arch/alpha/kernel/audit.c +++ b/arch/alpha/kernel/audit.c @@ -37,13 +37,13 @@ int audit_classify_syscall(int abi, unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/ia64/kernel/audit.c b/arch/ia64/kernel/audit.c index 5192ca899fe6..dba6a74c9ab3 100644 --- a/arch/ia64/kernel/audit.c +++ b/arch/ia64/kernel/audit.c @@ -38,13 +38,13 @@ int audit_classify_syscall(int abi, unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/parisc/kernel/audit.c b/arch/parisc/kernel/audit.c index 9eb47b2225d2..14244e83db75 100644 --- a/arch/parisc/kernel/audit.c +++ b/arch/parisc/kernel/audit.c @@ -47,13 +47,13 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/parisc/kernel/compat_audit.c b/arch/parisc/kernel/compat_audit.c index 20c39c9d86a9..1991b99f92ba 100644 --- a/arch/parisc/kernel/compat_audit.c +++ b/arch/parisc/kernel/compat_audit.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include unsigned int parisc32_dir_class[] = { @@ -30,12 +31,12 @@ int parisc32_classify_syscall(unsigned syscall) { switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/powerpc/kernel/audit.c b/arch/powerpc/kernel/audit.c index a2dddd7f3d09..6eb18ef77dff 100644 --- a/arch/powerpc/kernel/audit.c +++ b/arch/powerpc/kernel/audit.c @@ -47,15 +47,15 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/powerpc/kernel/compat_audit.c b/arch/powerpc/kernel/compat_audit.c index 55c6ccda0a85..216a54f85a12 100644 --- a/arch/powerpc/kernel/compat_audit.c +++ b/arch/powerpc/kernel/compat_audit.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #undef __powerpc64__ +#include #include unsigned ppc32_dir_class[] = { @@ -31,14 +32,14 @@ int ppc32_classify_syscall(unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/s390/kernel/audit.c b/arch/s390/kernel/audit.c index d395c6c9944c..7e331e1831d4 100644 --- a/arch/s390/kernel/audit.c +++ b/arch/s390/kernel/audit.c @@ -47,15 +47,15 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/s390/kernel/compat_audit.c b/arch/s390/kernel/compat_audit.c index 444fb1f66944..acacc96c57cb 100644 --- a/arch/s390/kernel/compat_audit.c +++ b/arch/s390/kernel/compat_audit.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #undef __s390x__ +#include #include #include "audit.h" @@ -32,14 +33,14 @@ int s390_classify_syscall(unsigned syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c index a6e91bf34d48..50fab35bdaba 100644 --- a/arch/sparc/kernel/audit.c +++ b/arch/sparc/kernel/audit.c @@ -48,15 +48,15 @@ int audit_classify_syscall(int abi, unsigned int syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c index 10eeb4f15b20..12515bd655c3 100644 --- a/arch/sparc/kernel/compat_audit.c +++ b/arch/sparc/kernel/compat_audit.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #define __32bit_syscall_numbers__ +#include #include #include "kernel.h" @@ -32,14 +33,14 @@ int sparc32_classify_syscall(unsigned int syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/x86/ia32/audit.c b/arch/x86/ia32/audit.c index 6efe6cb3768a..a5fc3b1385e0 100644 --- a/arch/x86/ia32/audit.c +++ b/arch/x86/ia32/audit.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -31,15 +32,15 @@ int ia32_classify_syscall(unsigned syscall) { switch (syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: case __NR_execveat: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/x86/kernel/audit_64.c b/arch/x86/kernel/audit_64.c index 83d9cad4e68b..2a6cc9c9c881 100644 --- a/arch/x86/kernel/audit_64.c +++ b/arch/x86/kernel/audit_64.c @@ -47,14 +47,14 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_execve: case __NR_execveat: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/include/linux/audit.h b/include/linux/audit.h index 82b7c1116a85..5fbeeeb6b726 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -11,6 +11,7 @@ #include #include +#include #include #include diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h new file mode 100644 index 000000000000..d4a506faabb0 --- /dev/null +++ b/include/linux/audit_arch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* audit_arch.h -- Arch layer specific support for audit + * + * Copyright 2021 Red Hat Inc., Durham, North Carolina. + * All Rights Reserved. + * + * Author: Richard Guy Briggs + */ +#ifndef _LINUX_AUDIT_ARCH_H_ +#define _LINUX_AUDIT_ARCH_H_ + +enum auditsc_class_t { + AUDITSC_NATIVE = 0, + AUDITSC_COMPAT, + AUDITSC_OPEN, + AUDITSC_OPENAT, + AUDITSC_SOCKETCALL, + AUDITSC_EXECVE, + + AUDITSC_NVALS /* count */ +}; + +#endif diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 1af4e7d0c16e..3f9108101598 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -153,7 +153,7 @@ static int audit_match_perm(struct audit_context *ctx, int mask) n = ctx->major; switch (audit_classify_syscall(ctx->arch, n)) { - case 0: /* native */ + case AUDITSC_NATIVE: if ((mask & AUDIT_PERM_WRITE) && audit_match_class(AUDIT_CLASS_WRITE, n)) return 1; @@ -164,7 +164,7 @@ static int audit_match_perm(struct audit_context *ctx, int mask) audit_match_class(AUDIT_CLASS_CHATTR, n)) return 1; return 0; - case 1: /* 32bit on biarch */ + case AUDITSC_COMPAT: /* 32bit on biarch */ if ((mask & AUDIT_PERM_WRITE) && audit_match_class(AUDIT_CLASS_WRITE_32, n)) return 1; @@ -175,13 +175,13 @@ static int audit_match_perm(struct audit_context *ctx, int mask) audit_match_class(AUDIT_CLASS_CHATTR_32, n)) return 1; return 0; - case 2: /* open */ + case AUDITSC_OPEN: return mask & ACC_MODE(ctx->argv[1]); - case 3: /* openat */ + case AUDITSC_OPENAT: return mask & ACC_MODE(ctx->argv[2]); - case 4: /* socketcall */ + case AUDITSC_SOCKETCALL: return ((mask & AUDIT_PERM_WRITE) && ctx->argv[0] == SYS_BIND); - case 5: /* execve */ + case AUDITSC_EXECVE: return mask & AUDIT_PERM_EXEC; default: return 0; diff --git a/lib/audit.c b/lib/audit.c index 5004bff928a7..3ec1a94d8d64 100644 --- a/lib/audit.c +++ b/lib/audit.c @@ -45,23 +45,23 @@ int audit_classify_syscall(int abi, unsigned syscall) switch(syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif #ifdef __NR_execveat case __NR_execveat: #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 77eabad69b4a..d6567d9e8b99 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include unsigned compat_dir_class[] = { @@ -33,19 +34,19 @@ int audit_classify_compat_syscall(int abi, unsigned syscall) switch (syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; default: - return 1; + return AUDITSC_COMPAT; } } From 1c30e3af8a79260cdba833a719209b01e6b92300 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 19 May 2021 16:00:21 -0400 Subject: [PATCH 089/134] audit: add support for the openat2 syscall The openat2(2) syscall was added in kernel v5.6 with commit fddb5d430ad9 ("open: introduce openat2(2) syscall"). Add the openat2(2) syscall to the audit syscall classifier. Link: https://github.com/linux-audit/audit-kernel/issues/67 Link: https://lore.kernel.org/r/f5f1a4d8699613f8c02ce762807228c841c2e26f.1621363275.git.rgb@redhat.com Signed-off-by: Richard Guy Briggs Acked-by: Christian Brauner [PM: merge fuzz due to previous header rename, commit line wraps] Signed-off-by: Paul Moore --- arch/alpha/kernel/audit.c | 2 ++ arch/ia64/kernel/audit.c | 2 ++ arch/parisc/kernel/audit.c | 2 ++ arch/parisc/kernel/compat_audit.c | 2 ++ arch/powerpc/kernel/audit.c | 2 ++ arch/powerpc/kernel/compat_audit.c | 2 ++ arch/s390/kernel/audit.c | 2 ++ arch/s390/kernel/compat_audit.c | 2 ++ arch/sparc/kernel/audit.c | 2 ++ arch/sparc/kernel/compat_audit.c | 2 ++ arch/x86/ia32/audit.c | 2 ++ arch/x86/kernel/audit_64.c | 2 ++ include/linux/audit_arch.h | 1 + kernel/auditsc.c | 3 +++ lib/audit.c | 4 ++++ lib/compat_audit.c | 4 ++++ 16 files changed, 36 insertions(+) diff --git a/arch/alpha/kernel/audit.c b/arch/alpha/kernel/audit.c index 81cbd804e375..3ab04709784a 100644 --- a/arch/alpha/kernel/audit.c +++ b/arch/alpha/kernel/audit.c @@ -42,6 +42,8 @@ int audit_classify_syscall(int abi, unsigned syscall) return AUDITSC_OPENAT; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/ia64/kernel/audit.c b/arch/ia64/kernel/audit.c index dba6a74c9ab3..ec61f20ca61f 100644 --- a/arch/ia64/kernel/audit.c +++ b/arch/ia64/kernel/audit.c @@ -43,6 +43,8 @@ int audit_classify_syscall(int abi, unsigned syscall) return AUDITSC_OPENAT; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/parisc/kernel/audit.c b/arch/parisc/kernel/audit.c index 14244e83db75..f420b5552140 100644 --- a/arch/parisc/kernel/audit.c +++ b/arch/parisc/kernel/audit.c @@ -52,6 +52,8 @@ int audit_classify_syscall(int abi, unsigned syscall) return AUDITSC_OPENAT; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/parisc/kernel/compat_audit.c b/arch/parisc/kernel/compat_audit.c index 1991b99f92ba..539b16891bdf 100644 --- a/arch/parisc/kernel/compat_audit.c +++ b/arch/parisc/kernel/compat_audit.c @@ -36,6 +36,8 @@ int parisc32_classify_syscall(unsigned syscall) return AUDITSC_OPENAT; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_COMPAT; } diff --git a/arch/powerpc/kernel/audit.c b/arch/powerpc/kernel/audit.c index 6eb18ef77dff..1bcfca5fdf67 100644 --- a/arch/powerpc/kernel/audit.c +++ b/arch/powerpc/kernel/audit.c @@ -54,6 +54,8 @@ int audit_classify_syscall(int abi, unsigned syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/powerpc/kernel/compat_audit.c b/arch/powerpc/kernel/compat_audit.c index 216a54f85a12..d92ffe4e5dc1 100644 --- a/arch/powerpc/kernel/compat_audit.c +++ b/arch/powerpc/kernel/compat_audit.c @@ -39,6 +39,8 @@ int ppc32_classify_syscall(unsigned syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_COMPAT; } diff --git a/arch/s390/kernel/audit.c b/arch/s390/kernel/audit.c index 7e331e1831d4..02051a596b87 100644 --- a/arch/s390/kernel/audit.c +++ b/arch/s390/kernel/audit.c @@ -54,6 +54,8 @@ int audit_classify_syscall(int abi, unsigned syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/s390/kernel/compat_audit.c b/arch/s390/kernel/compat_audit.c index acacc96c57cb..a7c46e8310f0 100644 --- a/arch/s390/kernel/compat_audit.c +++ b/arch/s390/kernel/compat_audit.c @@ -40,6 +40,8 @@ int s390_classify_syscall(unsigned syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_COMPAT; } diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c index 50fab35bdaba..b092274eca79 100644 --- a/arch/sparc/kernel/audit.c +++ b/arch/sparc/kernel/audit.c @@ -55,6 +55,8 @@ int audit_classify_syscall(int abi, unsigned int syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c index 12515bd655c3..f1ea0005a729 100644 --- a/arch/sparc/kernel/compat_audit.c +++ b/arch/sparc/kernel/compat_audit.c @@ -40,6 +40,8 @@ int sparc32_classify_syscall(unsigned int syscall) return AUDITSC_SOCKETCALL; case __NR_execve: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_COMPAT; } diff --git a/arch/x86/ia32/audit.c b/arch/x86/ia32/audit.c index a5fc3b1385e0..59e19549e759 100644 --- a/arch/x86/ia32/audit.c +++ b/arch/x86/ia32/audit.c @@ -40,6 +40,8 @@ int ia32_classify_syscall(unsigned syscall) case __NR_execve: case __NR_execveat: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_COMPAT; } diff --git a/arch/x86/kernel/audit_64.c b/arch/x86/kernel/audit_64.c index 2a6cc9c9c881..44c3601cfdc4 100644 --- a/arch/x86/kernel/audit_64.c +++ b/arch/x86/kernel/audit_64.c @@ -53,6 +53,8 @@ int audit_classify_syscall(int abi, unsigned syscall) case __NR_execve: case __NR_execveat: return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: return AUDITSC_NATIVE; } diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index d4a506faabb0..8fdb1afe251a 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -16,6 +16,7 @@ enum auditsc_class_t { AUDITSC_OPENAT, AUDITSC_SOCKETCALL, AUDITSC_EXECVE, + AUDITSC_OPENAT2, AUDITSC_NVALS /* count */ }; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 3f9108101598..8c4335a35274 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -63,6 +63,7 @@ #include #include #include +#include #include "audit.h" @@ -183,6 +184,8 @@ static int audit_match_perm(struct audit_context *ctx, int mask) return ((mask & AUDIT_PERM_WRITE) && ctx->argv[0] == SYS_BIND); case AUDITSC_EXECVE: return mask & AUDIT_PERM_EXEC; + case AUDITSC_OPENAT2: + return mask & ACC_MODE((u32)((struct open_how *)ctx->argv[2])->flags); default: return 0; } diff --git a/lib/audit.c b/lib/audit.c index 3ec1a94d8d64..738bda22dd39 100644 --- a/lib/audit.c +++ b/lib/audit.c @@ -60,6 +60,10 @@ int audit_classify_syscall(int abi, unsigned syscall) #endif case __NR_execve: return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: return AUDITSC_NATIVE; } diff --git a/lib/compat_audit.c b/lib/compat_audit.c index d6567d9e8b99..3d6b8996f027 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -46,6 +46,10 @@ int audit_classify_compat_syscall(int abi, unsigned syscall) #endif case __NR_execve: return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: return AUDITSC_COMPAT; } From 571e5c0efcb29c5dac8cf2949d3eed84ec43056c Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 19 May 2021 16:00:22 -0400 Subject: [PATCH 090/134] audit: add OPENAT2 record to list "how" info Since the openat2(2) syscall uses a struct open_how pointer to communicate its parameters they are not usefully recorded by the audit SYSCALL record's four existing arguments. Add a new audit record type OPENAT2 that reports the parameters in its third argument, struct open_how with fields oflag, mode and resolve. The new record in the context of an event would look like: time->Wed Mar 17 16:28:53 2021 type=PROCTITLE msg=audit(1616012933.531:184): proctitle= 73797363616C6C735F66696C652F6F70656E617432002F746D702F61756469742D 7465737473756974652D737641440066696C652D6F70656E617432 type=PATH msg=audit(1616012933.531:184): item=1 name="file-openat2" inode=29 dev=00:1f mode=0100600 ouid=0 ogid=0 rdev=00:00 obj=unconfined_u:object_r:user_tmp_t:s0 nametype=CREATE cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0 type=PATH msg=audit(1616012933.531:184): item=0 name="/root/rgb/git/audit-testsuite/tests" inode=25 dev=00:1f mode=040700 ouid=0 ogid=0 rdev=00:00 obj=unconfined_u:object_r:user_tmp_t:s0 nametype=PARENT cap_fp=0 cap_fi=0 cap_fe=0 cap_fver=0 cap_frootid=0 type=CWD msg=audit(1616012933.531:184): cwd="/root/rgb/git/audit-testsuite/tests" type=OPENAT2 msg=audit(1616012933.531:184): oflag=0100302 mode=0600 resolve=0xa type=SYSCALL msg=audit(1616012933.531:184): arch=c000003e syscall=437 success=yes exit=4 a0=3 a1=7ffe315f1c53 a2=7ffe315f1550 a3=18 items=2 ppid=528 pid=540 auid=0 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=ttyS0 ses=1 comm="openat2" exe="/root/rgb/git/audit-testsuite/tests/syscalls_file/openat2" subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 key="testsuite-1616012933-bjAUcEPO" Link: https://lore.kernel.org/r/d23fbb89186754487850367224b060e26f9b7181.1621363275.git.rgb@redhat.com Signed-off-by: Richard Guy Briggs Acked-by: Christian Brauner [PM: tweak subject, wrap example, move AUDIT_OPENAT2 to 1337] Signed-off-by: Paul Moore --- fs/open.c | 2 ++ include/linux/audit.h | 10 ++++++++++ include/uapi/linux/audit.h | 1 + kernel/audit.h | 2 ++ kernel/auditsc.c | 18 +++++++++++++++++- 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/fs/open.c b/fs/open.c index daa324606a41..a7f6cab81267 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1248,6 +1248,8 @@ SYSCALL_DEFINE4(openat2, int, dfd, const char __user *, filename, if (err) return err; + audit_openat2_how(&tmp); + /* O_LARGEFILE is only allowed for non-O_PATH. */ if (!(tmp.flags & O_PATH) && force_o_largefile()) tmp.flags |= O_LARGEFILE; diff --git a/include/linux/audit.h b/include/linux/audit.h index 5fbeeeb6b726..0e0eb4c0fa10 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -399,6 +399,7 @@ extern int __audit_log_bprm_fcaps(struct linux_binprm *bprm, const struct cred *old); extern void __audit_log_capset(const struct cred *new, const struct cred *old); extern void __audit_mmap_fd(int fd, int flags); +extern void __audit_openat2_how(struct open_how *how); extern void __audit_log_kern_module(char *name); extern void __audit_fanotify(unsigned int response); extern void __audit_tk_injoffset(struct timespec64 offset); @@ -495,6 +496,12 @@ static inline void audit_mmap_fd(int fd, int flags) __audit_mmap_fd(fd, flags); } +static inline void audit_openat2_how(struct open_how *how) +{ + if (unlikely(!audit_dummy_context())) + __audit_openat2_how(how); +} + static inline void audit_log_kern_module(char *name) { if (!audit_dummy_context()) @@ -646,6 +653,9 @@ static inline void audit_log_capset(const struct cred *new, static inline void audit_mmap_fd(int fd, int flags) { } +static inline void audit_openat2_how(struct open_how *how) +{ } + static inline void audit_log_kern_module(char *name) { } diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index daa481729e9b..afa2472ad5d6 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -118,6 +118,7 @@ #define AUDIT_TIME_ADJNTPVAL 1333 /* NTP value adjustment */ #define AUDIT_BPF 1334 /* BPF subsystem */ #define AUDIT_EVENT_LISTENER 1335 /* Task joined multicast read socket */ +#define AUDIT_OPENAT2 1337 /* Record showing openat2 how args */ #define AUDIT_AVC 1400 /* SE Linux avc denial or grant */ #define AUDIT_SELINUX_ERR 1401 /* Internal SE Linux Errors */ diff --git a/kernel/audit.h b/kernel/audit.h index d6a2c899a8db..3b64a97f6091 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -14,6 +14,7 @@ #include #include #include +#include // struct open_how /* AUDIT_NAMES is the number of slots we reserve in the audit_context * for saving names from getname(). If we get more names we will allocate @@ -188,6 +189,7 @@ struct audit_context { int fd; int flags; } mmap; + struct open_how openat2; struct { int argc; } execve; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 8c4335a35274..a4ba53f5354e 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -63,7 +63,7 @@ #include #include #include -#include +#include // struct open_how #include "audit.h" @@ -1306,6 +1306,12 @@ static void show_special(struct audit_context *context, int *call_panic) audit_log_format(ab, "fd=%d flags=0x%x", context->mmap.fd, context->mmap.flags); break; + case AUDIT_OPENAT2: + audit_log_format(ab, "oflag=0%llo mode=0%llo resolve=0x%llx", + context->openat2.flags, + context->openat2.mode, + context->openat2.resolve); + break; case AUDIT_EXECVE: audit_log_execve_info(context, &ab); break; @@ -2536,6 +2542,16 @@ void __audit_mmap_fd(int fd, int flags) context->type = AUDIT_MMAP; } +void __audit_openat2_how(struct open_how *how) +{ + struct audit_context *context = audit_context(); + + context->openat2.flags = how->flags; + context->openat2.mode = how->mode; + context->openat2.resolve = how->resolve; + context->type = AUDIT_OPENAT2; +} + void __audit_log_kern_module(char *name) { struct audit_context *context = audit_context(); From 7663ad9a5dbcc27f3090e6bfd192c7e59222709f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 28 Sep 2021 10:40:21 +0200 Subject: [PATCH 091/134] rcu: Always inline rcu_dynticks_task*_{enter,exit}() RCU managed to grow a few noinstr violations: vmlinux.o: warning: objtool: rcu_dynticks_eqs_enter()+0x0: call to rcu_dynticks_task_trace_enter() leaves .noinstr.text section vmlinux.o: warning: objtool: rcu_dynticks_eqs_exit()+0xe: call to rcu_dynticks_task_trace_exit() leaves .noinstr.text section Fix them by adding __always_inline to the relevant trivial functions. Also replace the noinstr with __always_inline for the existing rcu_dynticks_task_*() functions since noinstr would force noinline them, even when empty, which seems silly. Fixes: 7d0c9c50c5a1 ("rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built") Reported-by: Stephen Rothwell Reviewed-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 1a6fdb03d0a5..5199559fbbf0 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1479,7 +1479,7 @@ static void rcu_bind_gp_kthread(void) } /* Record the current task on dyntick-idle entry. */ -static void noinstr rcu_dynticks_task_enter(void) +static __always_inline void rcu_dynticks_task_enter(void) { #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id()); @@ -1487,7 +1487,7 @@ static void noinstr rcu_dynticks_task_enter(void) } /* Record no current task on dyntick-idle exit. */ -static void noinstr rcu_dynticks_task_exit(void) +static __always_inline void rcu_dynticks_task_exit(void) { #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) WRITE_ONCE(current->rcu_tasks_idle_cpu, -1); @@ -1495,7 +1495,7 @@ static void noinstr rcu_dynticks_task_exit(void) } /* Turn on heavyweight RCU tasks trace readers on idle/user entry. */ -static void rcu_dynticks_task_trace_enter(void) +static __always_inline void rcu_dynticks_task_trace_enter(void) { #ifdef CONFIG_TASKS_TRACE_RCU if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) @@ -1504,7 +1504,7 @@ static void rcu_dynticks_task_trace_enter(void) } /* Turn off heavyweight RCU tasks trace readers on idle/user exit. */ -static void rcu_dynticks_task_trace_exit(void) +static __always_inline void rcu_dynticks_task_trace_exit(void) { #ifdef CONFIG_TASKS_TRACE_RCU if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) From 74aece72f95f399dd29363669dc32a1344c8fab4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 28 Sep 2021 10:40:22 +0200 Subject: [PATCH 092/134] rcu: Fix rcu_dynticks_curr_cpu_in_eqs() vs noinstr vmlinux.o: warning: objtool: rcu_nmi_enter()+0x36: call to __kasan_check_read() leaves .noinstr.text section noinstr cannot have atomic_*() functions in because they're explicitly annotated, use arch_atomic_*(). Fixes: 2be57f732889 ("rcu: Weaken ->dynticks accesses and updates") Reported-by: Stephen Rothwell Reviewed-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 6a1e9d3374db..ef8d36f580fc 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -327,7 +327,7 @@ static void rcu_dynticks_eqs_online(void) */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); + return !(arch_atomic_read(this_cpu_ptr(&rcu_data.dynticks)) & 0x1); } /* From 183b60e005975d3c84c22199ca64a9221e620fb6 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Mon, 27 Sep 2021 16:16:55 +0800 Subject: [PATCH 093/134] crypto: hisilicon/qm - modify the uacce mode check As qm should register to uacce in UACCE_DEV_SVA mode, this patch modifies to checks uacce mode before doing uacce registration. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6c57149e28f0..fed52ae516ba 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3127,7 +3127,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (IS_ERR(uacce)) return PTR_ERR(uacce); - if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) { + if (uacce->flags & UACCE_DEV_SVA) { qm->use_sva = true; } else { /* only consider sva case */ @@ -3402,8 +3402,10 @@ void hisi_qm_uninit(struct hisi_qm *qm) qm_irq_unregister(qm); hisi_qm_pci_uninit(qm); - uacce_remove(qm->uacce); - qm->uacce = NULL; + if (qm->use_sva) { + uacce_remove(qm->uacce); + qm->uacce = NULL; + } up_write(&qm->qps_lock); } @@ -5823,9 +5825,11 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_irq_register; } - ret = qm_alloc_uacce(qm); - if (ret < 0) - dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + if (qm->mode == UACCE_MODE_SVA) { + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + } ret = hisi_qm_memory_init(qm); if (ret) @@ -5838,8 +5842,10 @@ int hisi_qm_init(struct hisi_qm *qm) return 0; err_alloc_uacce: - uacce_remove(qm->uacce); - qm->uacce = NULL; + if (qm->use_sva) { + uacce_remove(qm->uacce); + qm->uacce = NULL; + } err_irq_register: qm_irq_unregister(qm); err_pci_init: From cfd6fb45cfaf46fa9547421d8da387dc9c7997d4 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 27 Sep 2021 14:18:03 +0200 Subject: [PATCH 094/134] crypto: ccree - avoid out-of-range warnings from clang clang points out inconsistencies in the FIELD_PREP() invocation in this driver that result from the 'mask' being a 32-bit value: drivers/crypto/ccree/cc_driver.c:117:18: error: result of comparison of constant 18446744073709551615 with expression of type 'u32' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] cache_params |= FIELD_PREP(mask, val); ^~~~~~~~~~~~~~~~~~~~~ include/linux/bitfield.h:94:3: note: expanded from macro 'FIELD_PREP' __BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: "); \ ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ include/linux/bitfield.h:52:28: note: expanded from macro '__BF_FIELD_CHECK' BUILD_BUG_ON_MSG((_mask) > (typeof(_reg))~0ull, \ ~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This does not happen in other places that just pass a constant here. Work around the warnings by widening the type of the temporary variable. Fixes: 05c2a705917b ("crypto: ccree - rework cache parameters handling") Signed-off-by: Arnd Bergmann Acked-by: Gilad ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index e599ac6dc162..790fa9058a36 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -103,7 +103,8 @@ MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); static void init_cc_cache_params(struct cc_drvdata *drvdata) { struct device *dev = drvdata_to_dev(drvdata); - u32 cache_params, ace_const, val, mask; + u32 cache_params, ace_const, val; + u64 mask; /* compute CC_AXIM_CACHE_PARAMS */ cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); From 9b768e8a3909ac1ab39ed44a3933716da7761a6f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Tue, 28 Sep 2021 12:44:29 +0100 Subject: [PATCH 095/134] crypto: qat - detect PFVF collision after ACK Detect a PFVF collision between the local and the remote function by checking if the message on the PFVF CSR has been overwritten. This is done after the remote function confirms that the message has been received, by clearing the interrupt bit, or the maximum number of attempts (ADF_IOV_MSG_ACK_MAX_RETRY) to check the CSR has been exceeded. Fixes: ed8ccaef52fa ("crypto: qat - Add support for SRIOV") Signed-off-by: Giovanni Cabiddu Co-developed-by: Marco Chiappero Signed-off-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 976b9ab7617c..789a4135e28c 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -156,6 +156,13 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); } while ((val & int_bit) && (count++ < ADF_IOV_MSG_ACK_MAX_RETRY)); + if (val != msg) { + dev_dbg(&GET_DEV(accel_dev), + "Collision - PFVF CSR overwritten by remote function\n"); + ret = -EIO; + goto out; + } + if (val & int_bit) { dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); val &= ~int_bit; From 18fcba469ba5359c1de7e3fb16f7b9e8cd1b8e02 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Tue, 28 Sep 2021 12:44:30 +0100 Subject: [PATCH 096/134] crypto: qat - disregard spurious PFVF interrupts Upon receiving a PFVF message, check if the interrupt bit is set in the message. If it is not, that means that the interrupt was probably triggered by a collision. In this case, disregard the message and re-enable the interrupts. Fixes: ed8ccaef52fa ("crypto: qat - Add support for SRIOV") Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 6 ++++++ drivers/crypto/qat/qat_common/adf_vf_isr.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 789a4135e28c..5a41beb8f20f 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -211,6 +211,11 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) /* Read message from the VF */ msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); + if (!(msg & ADF_VF2PF_INT)) { + dev_info(&GET_DEV(accel_dev), + "Spurious VF2PF interrupt, msg %X. Ignored\n", msg); + goto out; + } /* To ACK, clear the VF2PFINT bit */ msg &= ~ADF_VF2PF_INT; @@ -294,6 +299,7 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) if (resp && adf_iov_putmsg(accel_dev, resp, vf_nr)) dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); +out: /* re-enable interrupt on PF from this VF */ adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr)); diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index 07f81682c19b..db5e7abbe5f3 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -96,6 +96,11 @@ static void adf_pf2vf_bh_handler(void *data) /* Read the message from PF */ msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0)); + if (!(msg & ADF_PF2VF_INT)) { + dev_info(&GET_DEV(accel_dev), + "Spurious PF2VF interrupt, msg %X. Ignored\n", msg); + goto out; + } if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) /* Ignore legacy non-system (non-kernel) PF2VF messages */ @@ -144,6 +149,7 @@ static void adf_pf2vf_bh_handler(void *data) msg &= ~ADF_PF2VF_INT; ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); +out: /* Re-enable PF2VF interrupts */ adf_enable_pf2vf_interrupts(accel_dev); return; From e17f49bb244a281fe39bfdad0306a38b3a02e7bf Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:31 +0100 Subject: [PATCH 097/134] crypto: qat - remove unnecessary collision prevention step in PFVF The initial version of the PFVF protocol included an initial "carrier sensing" to get ownership of the channel. Collisions can happen anyway, the extra wait and test does not prevent collisions, it instead slows the communication down, so remove it. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 5a41beb8f20f..d3f6ff68d45d 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -126,28 +126,10 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) goto out; } - /* Attempt to get ownership of PF2VF CSR */ msg &= ~local_in_use_mask; msg |= local_in_use_pattern; - ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg); - /* Wait in case remote func also attempting to get ownership */ - msleep(ADF_IOV_MSG_COLLISION_DETECT_DELAY); - - val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); - if ((val & local_in_use_mask) != local_in_use_pattern) { - dev_dbg(&GET_DEV(accel_dev), - "PF2VF CSR in use by remote - collision detected\n"); - ret = -EBUSY; - goto out; - } - - /* - * This function now owns the PV2VF CSR. The IN_USE_BY pattern must - * remain in the PF2VF CSR for all writes including ACK from remote - * until this local function relinquishes the CSR. Send the message - * by interrupting the remote. - */ + /* Attempt to get ownership of the PF2VF CSR */ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); /* Wait for confirmation from remote func it received the message */ From 993161d36ab5f0f8064751f157482d332a8fcf2c Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:32 +0100 Subject: [PATCH 098/134] crypto: qat - fix handling of VF to PF interrupts Currently, VF to PF interrupt handling is based on the DH895XCC device behavior, which is not entirely common to all devices. In order to make interrupt detection and handling correct for all of the supported devices, make the interrupt handling device specific by: - introducing get_vf2pf_sources() for getting a 32 bits long value where each bit represents a vf2pf interrupt; - adding the device [enable|disable]_vf2pf_interrupts to hw_data; - defining [enable|disable]_vf2pf_interrupts for all the devices that are currently supported, using only their required and specific ERRSOU|ERRMASK registers (DH895XCC has 32 interrupts spread across ERRSOU3 and ERRSOU5, C62X/C3XXX has 16 in ERRSOU3 only, etc). Code has been shared by different devices wherever possible. This patch is based on earlier work done by Salvatore Benedetto. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 3 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 3 + .../crypto/qat/qat_common/adf_accel_devices.h | 5 ++ .../crypto/qat/qat_common/adf_gen2_hw_data.c | 40 ++++++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 12 +++ drivers/crypto/qat/qat_common/adf_isr.c | 20 +---- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 76 +++++-------------- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 49 ++++++++++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 5 +- 9 files changed, 135 insertions(+), 78 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 3027c01bc89e..b9bd52eaa184 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -206,6 +206,9 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; + hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; + hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index b023c80873bb..f28dae0982bc 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -208,6 +208,9 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; + hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; + hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index ca8e23f0bcc4..57d9ca08e611 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -177,6 +177,11 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); int (*enable_pfvf_comms)(struct adf_accel_dev *accel_dev); + u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr); + void (*enable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr, + u32 vf_mask); + void (*disable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr, + u32 vf_mask); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); char *(*uof_get_name)(u32 obj_num); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 9e560c7d4163..566918012778 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -4,6 +4,46 @@ #include "icp_qat_hw.h" #include +u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) +{ + u32 errsou3, errmsk3, vf_int_mask; + + /* Get the interrupt sources triggered by VFs */ + errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); + vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3); + + /* To avoid adding duplicate entries to work queue, clear + * vf_int_mask_sets bits that are already masked in ERRMSK register. + */ + errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); + vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3); + + return vf_int_mask; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources); + +void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts); + +void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) + | ADF_GEN2_ERR_MSK_VF2PF(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 756b0ddfac5e..3486e51aad75 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -125,6 +125,18 @@ do { \ #define ADF_SSMWDT(i) (ADF_SSMWDT_OFFSET + ((i) * 0x4000)) #define ADF_SSMWDTPKE(i) (ADF_SSMWDTPKE_OFFSET + ((i) * 0x4000)) + /* VF2PF interrupts */ +#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) +#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) +#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C) +#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC) +#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) +#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9) + +u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); +void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); +void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index c55a9f14b0d2..40593c9449a2 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -16,12 +16,6 @@ #include "adf_transport_internal.h" #define ADF_MAX_NUM_VFS 32 -#define ADF_ERRSOU3 (0x3A000 + 0x0C) -#define ADF_ERRSOU5 (0x3A000 + 0xD8) -#define ADF_ERRMSK3 (0x3A000 + 0x1C) -#define ADF_ERRMSK5 (0x3A000 + 0xDC) -#define ADF_ERR_REG_VF2PF_L(vf_src) (((vf_src) & 0x01FFFE00) >> 9) -#define ADF_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) static int adf_enable_msix(struct adf_accel_dev *accel_dev) { @@ -71,22 +65,10 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_addr = pmisc->virt_addr; - u32 errsou3, errsou5, errmsk3, errmsk5; unsigned long vf_mask; /* Get the interrupt sources triggered by VFs */ - errsou3 = ADF_CSR_RD(pmisc_addr, ADF_ERRSOU3); - errsou5 = ADF_CSR_RD(pmisc_addr, ADF_ERRSOU5); - vf_mask = ADF_ERR_REG_VF2PF_L(errsou3); - vf_mask |= ADF_ERR_REG_VF2PF_U(errsou5); - - /* To avoid adding duplicate entries to work queue, clear - * vf_int_mask_sets bits that are already masked in ERRMSK register. - */ - errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_ERRMSK3); - errmsk5 = ADF_CSR_RD(pmisc_addr, ADF_ERRMSK5); - vf_mask &= ~ADF_ERR_REG_VF2PF_L(errmsk3); - vf_mask &= ~ADF_ERR_REG_VF2PF_U(errmsk5); + vf_mask = hw_data->get_vf2pf_sources(pmisc_addr); if (vf_mask) { struct adf_accel_vf_info *vf_info; diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index d3f6ff68d45d..cdef6c34524e 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -5,82 +5,42 @@ #include "adf_common_drv.h" #include "adf_pf2vf_msg.h" -#define ADF_DH895XCC_EP_OFFSET 0x3A000 -#define ADF_DH895XCC_ERRMSK3 (ADF_DH895XCC_EP_OFFSET + 0x1C) -#define ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask) ((vf_mask & 0xFFFF) << 9) -#define ADF_DH895XCC_ERRMSK5 (ADF_DH895XCC_EP_OFFSET + 0xDC) -#define ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask) (vf_mask >> 16) - -static void __adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, - u32 vf_mask) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; - u32 reg; - - /* Enable VF2PF Messaging Ints - VFs 1 through 16 per vf_mask[15:0] */ - if (vf_mask & 0xFFFF) { - reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3); - reg &= ~ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); - } - - /* Enable VF2PF Messaging Ints - VFs 17 through 32 per vf_mask[31:16] */ - if (vf_mask >> 16) { - reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5); - reg &= ~ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); - } -} - void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - __adf_enable_vf2pf_interrupts(accel_dev, vf_mask); + hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } -static void __adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, - u32 vf_mask) -{ - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; - u32 reg; - - /* Disable VF2PF interrupts for VFs 1 through 16 per vf_mask[15:0] */ - if (vf_mask & 0xFFFF) { - reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3) | - ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); - } - - /* Disable VF2PF interrupts for VFs 17 through 32 per vf_mask[31:16] */ - if (vf_mask >> 16) { - reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5) | - ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); - ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); - } -} - void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; unsigned long flags; spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags); - __adf_disable_vf2pf_interrupts(accel_dev, vf_mask); + hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags); } -void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, u32 vf_mask) +void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, + u32 vf_mask) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + spin_lock(&accel_dev->pf.vf2pf_ints_lock); - __adf_disable_vf2pf_interrupts(accel_dev, vf_mask); + hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask); spin_unlock(&accel_dev->pf.vf2pf_ints_lock); } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 0a9ce365a544..b496032c992b 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -175,6 +175,52 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) ADF_DH895XCC_SMIA1_MASK); } +static u32 get_vf2pf_sources(void __iomem *pmisc_bar) +{ + u32 errsou5, errmsk5, vf_int_mask; + + vf_int_mask = adf_gen2_get_vf2pf_sources(pmisc_bar); + + /* Get the interrupt sources triggered by VFs, but to avoid duplicates + * in the work queue, clear vf_int_mask_sets bits that are already + * masked in ERRMSK register. + */ + errsou5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRSOU5); + errmsk5 = ADF_CSR_RD(pmisc_bar, ADF_GEN2_ERRMSK5); + vf_int_mask |= ADF_DH895XCC_ERR_REG_VF2PF_U(errsou5); + vf_int_mask &= ~ADF_DH895XCC_ERR_REG_VF2PF_U(errmsk5); + + return vf_int_mask; +} + +static void enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ + adf_gen2_enable_vf2pf_interrupts(pmisc_addr, vf_mask); + + /* Enable VF2PF Messaging Ints - VFs 16 through 31 per vf_mask[31:16] */ + if (vf_mask >> 16) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) + & ~ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask); + + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); + } +} + +static void disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) +{ + /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ + adf_gen2_disable_vf2pf_interrupts(pmisc_addr, vf_mask); + + /* Disable VF2PF interrupts for VFs 16 through 31 per vf_mask[31:16] */ + if (vf_mask >> 16) { + u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK5) + | ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask); + + ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK5, val); + } +} + static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) { spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); @@ -226,6 +272,9 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_sbr; hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vf2pf_sources = get_vf2pf_sources; + hw_data->enable_vf2pf_interrupts = enable_vf2pf_interrupts; + hw_data->disable_vf2pf_interrupts = disable_vf2pf_interrupts; hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index f99319cd4543..0f9f24b44663 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -34,7 +34,10 @@ #define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) #define ADF_DH895XCC_ERRSSMSH_EN BIT(3) -#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +/* Masks for VF2PF interrupts */ +#define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) +#define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16) +#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) /* AE to function mapping */ #define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 From b79c7532dc337c87365fda62914eaeb0e038160d Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:33 +0100 Subject: [PATCH 099/134] crypto: qat - remove duplicated logic across GEN2 drivers QAT GEN2 devices share most of the behavior which means a number of device specific functions can be shared too and some differences abstracted away by simple parameters. The functions adf_enable_error_correction(), get_num_accels(), get_num_aes() and get_pf2vf_offset() for c3xxx, c62x and dh895xx have been reworked and moved to the GEN2 file, adf_gen2_hw_data.c. The definitions of tx_rx_gap and tx_rings_mask have been moved to adf_gen2_hw_data.h. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 79 ++----------------- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 13 +-- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 77 ++---------------- .../crypto/qat/qat_c62x/adf_c62x_hw_data.h | 12 --- .../crypto/qat/qat_common/adf_gen2_hw_data.c | 70 ++++++++++++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 15 ++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 75 ++---------------- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 11 --- 8 files changed, 107 insertions(+), 245 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index b9bd52eaa184..6a39d2e7f4c0 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -48,34 +48,6 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; } -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->accel_mask) - return 0; - - for (i = 0; i < ADF_C3XXX_MAX_ACCELERATORS; i++) { - if (self->accel_mask & (1 << i)) - ctr++; - } - return ctr; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->ae_mask) - return 0; - - for (i = 0; i < ADF_C3XXX_MAX_ACCELENGINES; i++) { - if (self->ae_mask & (1 << i)) - ctr++; - } - return ctr; -} - static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_C3XXX_PMISC_BAR; @@ -88,12 +60,12 @@ static u32 get_etr_bar_id(struct adf_hw_device_data *self) static u32 get_sram_bar_id(struct adf_hw_device_data *self) { - return 0; + return ADF_C3XXX_SRAM_BAR; } static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int aes = get_num_aes(self); + int aes = self->get_num_aes(self); if (aes == 6) return DEV_SKU_4; @@ -106,41 +78,6 @@ static const u32 *adf_get_arbiter_mapping(void) return thrd_to_arb_map; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_C3XXX_PF2VF_OFFSET(i); -} - -static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR]; - unsigned long accel_mask = hw_device->accel_mask; - unsigned long ae_mask = hw_device->ae_mask; - void __iomem *csr = misc_bar->virt_addr; - unsigned int val, i; - - /* Enable Accel Engine error detection & correction */ - for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { - val = ADF_CSR_RD(csr, ADF_C3XXX_AE_CTX_ENABLES(i)); - val |= ADF_C3XXX_ENABLE_AE_ECC_ERR; - ADF_CSR_WR(csr, ADF_C3XXX_AE_CTX_ENABLES(i), val); - val = ADF_CSR_RD(csr, ADF_C3XXX_AE_MISC_CONTROL(i)); - val |= ADF_C3XXX_ENABLE_AE_ECC_PARITY_CORR; - ADF_CSR_WR(csr, ADF_C3XXX_AE_MISC_CONTROL(i), val); - } - - /* Enable shared memory error detection & correction */ - for_each_set_bit(i, &accel_mask, ADF_C3XXX_MAX_ACCELERATORS) { - val = ADF_CSR_RD(csr, ADF_C3XXX_UERRSSMSH(i)); - val |= ADF_C3XXX_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_C3XXX_UERRSSMSH(i), val); - val = ADF_CSR_RD(csr, ADF_C3XXX_CERRSSMSH(i)); - val |= ADF_C3XXX_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_C3XXX_CERRSSMSH(i), val); - } -} - static void adf_enable_ints(struct adf_accel_dev *accel_dev) { void __iomem *addr; @@ -177,16 +114,16 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_C3XXX_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_C3XXX_TX_RINGS_MASK; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_accel_cap = adf_gen2_get_accel_cap; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; @@ -205,7 +142,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 86ee02a86789..1b86f828725c 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -6,8 +6,7 @@ /* PCIe configuration space */ #define ADF_C3XXX_PMISC_BAR 0 #define ADF_C3XXX_ETR_BAR 1 -#define ADF_C3XXX_RX_RINGS_OFFSET 8 -#define ADF_C3XXX_TX_RINGS_MASK 0xFF +#define ADF_C3XXX_SRAM_BAR 0 #define ADF_C3XXX_MAX_ACCELERATORS 3 #define ADF_C3XXX_MAX_ACCELENGINES 6 #define ADF_C3XXX_ACCELERATORS_REG_OFFSET 16 @@ -19,16 +18,6 @@ #define ADF_C3XXX_SMIA0_MASK 0xFFFF #define ADF_C3XXX_SMIA1_MASK 0x1 #define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC -/* Error detection and correction */ -#define ADF_C3XXX_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) -#define ADF_C3XXX_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) -#define ADF_C3XXX_ENABLE_AE_ECC_ERR BIT(28) -#define ADF_C3XXX_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) -#define ADF_C3XXX_UERRSSMSH(i) (i * 0x4000 + 0x18) -#define ADF_C3XXX_CERRSSMSH(i) (i * 0x4000 + 0x10) -#define ADF_C3XXX_ERRSSMSH_EN BIT(3) - -#define ADF_C3XXX_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) /* AE to function mapping */ #define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index f28dae0982bc..e259ca38a653 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -48,34 +48,6 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; } -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->accel_mask) - return 0; - - for (i = 0; i < ADF_C62X_MAX_ACCELERATORS; i++) { - if (self->accel_mask & (1 << i)) - ctr++; - } - return ctr; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->ae_mask) - return 0; - - for (i = 0; i < ADF_C62X_MAX_ACCELENGINES; i++) { - if (self->ae_mask & (1 << i)) - ctr++; - } - return ctr; -} - static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_C62X_PMISC_BAR; @@ -93,7 +65,7 @@ static u32 get_sram_bar_id(struct adf_hw_device_data *self) static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int aes = get_num_aes(self); + int aes = self->get_num_aes(self); if (aes == 8) return DEV_SKU_2; @@ -108,41 +80,6 @@ static const u32 *adf_get_arbiter_mapping(void) return thrd_to_arb_map; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_C62X_PF2VF_OFFSET(i); -} - -static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR]; - unsigned long accel_mask = hw_device->accel_mask; - unsigned long ae_mask = hw_device->ae_mask; - void __iomem *csr = misc_bar->virt_addr; - unsigned int val, i; - - /* Enable Accel Engine error detection & correction */ - for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { - val = ADF_CSR_RD(csr, ADF_C62X_AE_CTX_ENABLES(i)); - val |= ADF_C62X_ENABLE_AE_ECC_ERR; - ADF_CSR_WR(csr, ADF_C62X_AE_CTX_ENABLES(i), val); - val = ADF_CSR_RD(csr, ADF_C62X_AE_MISC_CONTROL(i)); - val |= ADF_C62X_ENABLE_AE_ECC_PARITY_CORR; - ADF_CSR_WR(csr, ADF_C62X_AE_MISC_CONTROL(i), val); - } - - /* Enable shared memory error detection & correction */ - for_each_set_bit(i, &accel_mask, ADF_C62X_MAX_ACCELERATORS) { - val = ADF_CSR_RD(csr, ADF_C62X_UERRSSMSH(i)); - val |= ADF_C62X_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_C62X_UERRSSMSH(i), val); - val = ADF_CSR_RD(csr, ADF_C62X_CERRSSMSH(i)); - val |= ADF_C62X_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_C62X_CERRSSMSH(i), val); - } -} - static void adf_enable_ints(struct adf_accel_dev *accel_dev) { void __iomem *addr; @@ -179,16 +116,16 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_C62X_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_C62X_TX_RINGS_MASK; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_accel_cap = adf_gen2_get_accel_cap; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; @@ -207,7 +144,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_flr; hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index e6664bd20c91..68c3436bd3aa 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -7,8 +7,6 @@ #define ADF_C62X_SRAM_BAR 0 #define ADF_C62X_PMISC_BAR 1 #define ADF_C62X_ETR_BAR 2 -#define ADF_C62X_RX_RINGS_OFFSET 8 -#define ADF_C62X_TX_RINGS_MASK 0xFF #define ADF_C62X_MAX_ACCELERATORS 5 #define ADF_C62X_MAX_ACCELENGINES 10 #define ADF_C62X_ACCELERATORS_REG_OFFSET 16 @@ -20,16 +18,6 @@ #define ADF_C62X_SMIA0_MASK 0xFFFF #define ADF_C62X_SMIA1_MASK 0x1 #define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC -/* Error detection and correction */ -#define ADF_C62X_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) -#define ADF_C62X_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) -#define ADF_C62X_ENABLE_AE_ECC_ERR BIT(28) -#define ADF_C62X_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) -#define ADF_C62X_UERRSSMSH(i) (i * 0x4000 + 0x18) -#define ADF_C62X_CERRSSMSH(i) (i * 0x4000 + 0x10) -#define ADF_C62X_ERRSSMSH_EN BIT(3) - -#define ADF_C62X_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) /* AE to function mapping */ #define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 566918012778..1deeeaed9a8c 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -4,6 +4,14 @@ #include "icp_qat_hw.h" #include +#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) + +u32 adf_gen2_get_pf2vf_offset(u32 i) +{ + return ADF_GEN2_PF2VF_OFFSET(i); +} +EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset); + u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr) { u32 errsou3, errmsk3, vf_int_mask; @@ -44,6 +52,68 @@ void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) } EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); +u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->accel_mask) + return 0; + + for (i = 0; i < self->num_accel; i++) + if (self->accel_mask & (1 << i)) + ctr++; + + return ctr; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_num_accels); + +u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->ae_mask) + return 0; + + for (i = 0; i < self->num_engines; i++) + if (self->ae_mask & (1 << i)) + ctr++; + + return ctr; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes); + +void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *misc_bar = &GET_BARS(accel_dev) + [hw_data->get_misc_bar_id(hw_data)]; + unsigned long accel_mask = hw_data->accel_mask; + unsigned long ae_mask = hw_data->ae_mask; + void __iomem *csr = misc_bar->virt_addr; + unsigned int val, i; + + /* Enable Accel Engine error detection & correction */ + for_each_set_bit(i, &ae_mask, hw_data->num_engines) { + val = ADF_CSR_RD(csr, ADF_GEN2_AE_CTX_ENABLES(i)); + val |= ADF_GEN2_ENABLE_AE_ECC_ERR; + ADF_CSR_WR(csr, ADF_GEN2_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(csr, ADF_GEN2_AE_MISC_CONTROL(i)); + val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR; + ADF_CSR_WR(csr, ADF_GEN2_AE_MISC_CONTROL(i), val); + } + + /* Enable shared memory error detection & correction */ + for_each_set_bit(i, &accel_mask, hw_data->num_accel) { + val = ADF_CSR_RD(csr, ADF_GEN2_UERRSSMSH(i)); + val |= ADF_GEN2_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_GEN2_UERRSSMSH(i), val); + val = ADF_CSR_RD(csr, ADF_GEN2_CERRSSMSH(i)); + val |= ADF_GEN2_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_GEN2_CERRSSMSH(i), val); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction); + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) { diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 3486e51aad75..c169d704097d 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -22,6 +22,8 @@ #define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 #define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 #define ADF_RING_BUNDLE_SIZE 0x1000 +#define ADF_GEN2_RX_RINGS_OFFSET 8 +#define ADF_GEN2_TX_RINGS_MASK 0xFF #define BUILD_RING_BASE_ADDR(addr, size) \ (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) @@ -125,6 +127,15 @@ do { \ #define ADF_SSMWDT(i) (ADF_SSMWDT_OFFSET + ((i) * 0x4000)) #define ADF_SSMWDTPKE(i) (ADF_SSMWDTPKE_OFFSET + ((i) * 0x4000)) +/* Error detection and correction */ +#define ADF_GEN2_AE_CTX_ENABLES(i) ((i) * 0x1000 + 0x20818) +#define ADF_GEN2_AE_MISC_CONTROL(i) ((i) * 0x1000 + 0x20960) +#define ADF_GEN2_ENABLE_AE_ECC_ERR BIT(28) +#define ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) +#define ADF_GEN2_UERRSSMSH(i) ((i) * 0x4000 + 0x18) +#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10) +#define ADF_GEN2_ERRSSMSH_EN BIT(3) + /* VF2PF interrupts */ #define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C) #define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8) @@ -133,10 +144,14 @@ do { \ #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) #define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9) +u32 adf_gen2_get_pf2vf_offset(u32 i); u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar); void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask); +u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self); +u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self); +void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev); void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b496032c992b..e5e64f880fbf 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -35,34 +35,6 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->accel_mask) - return 0; - - for (i = 0; i < ADF_DH895XCC_MAX_ACCELERATORS; i++) { - if (self->accel_mask & (1 << i)) - ctr++; - } - return ctr; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - u32 i, ctr = 0; - - if (!self || !self->ae_mask) - return 0; - - for (i = 0; i < ADF_DH895XCC_MAX_ACCELENGINES; i++) { - if (self->ae_mask & (1 << i)) - ctr++; - } - return ctr; -} - static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_DH895XCC_PMISC_BAR; @@ -126,41 +98,6 @@ static const u32 *adf_get_arbiter_mapping(void) return thrd_to_arb_map; } -static u32 get_pf2vf_offset(u32 i) -{ - return ADF_DH895XCC_PF2VF_OFFSET(i); -} - -static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; - unsigned long accel_mask = hw_device->accel_mask; - unsigned long ae_mask = hw_device->ae_mask; - void __iomem *csr = misc_bar->virt_addr; - unsigned int val, i; - - /* Enable Accel Engine error detection & correction */ - for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { - val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_CTX_ENABLES(i)); - val |= ADF_DH895XCC_ENABLE_AE_ECC_ERR; - ADF_CSR_WR(csr, ADF_DH895XCC_AE_CTX_ENABLES(i), val); - val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_MISC_CONTROL(i)); - val |= ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR; - ADF_CSR_WR(csr, ADF_DH895XCC_AE_MISC_CONTROL(i), val); - } - - /* Enable shared memory error detection & correction */ - for_each_set_bit(i, &accel_mask, ADF_DH895XCC_MAX_ACCELERATORS) { - val = ADF_CSR_RD(csr, ADF_DH895XCC_UERRSSMSH(i)); - val |= ADF_DH895XCC_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_DH895XCC_UERRSSMSH(i), val); - val = ADF_CSR_RD(csr, ADF_DH895XCC_CERRSSMSH(i)); - val |= ADF_DH895XCC_ERRSSMSH_EN; - ADF_CSR_WR(csr, ADF_DH895XCC_CERRSSMSH(i), val); - } -} - static void adf_enable_ints(struct adf_accel_dev *accel_dev) { void __iomem *addr; @@ -244,16 +181,16 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; - hw_data->tx_rx_gap = ADF_DH895XCC_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_DH895XCC_TX_RINGS_MASK; + hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->enable_error_correction = adf_gen2_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_accel_cap = get_accel_cap; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; + hw_data->get_num_accels = adf_gen2_get_num_accels; + hw_data->get_num_aes = adf_gen2_get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_admin_info = adf_gen2_get_admin_info; @@ -271,7 +208,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->reset_device = adf_reset_sbr; - hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset; hw_data->get_vf2pf_sources = get_vf2pf_sources; hw_data->enable_vf2pf_interrupts = enable_vf2pf_interrupts; hw_data->disable_vf2pf_interrupts = disable_vf2pf_interrupts; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 0f9f24b44663..0af34dd8708a 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -7,8 +7,6 @@ #define ADF_DH895XCC_SRAM_BAR 0 #define ADF_DH895XCC_PMISC_BAR 1 #define ADF_DH895XCC_ETR_BAR 2 -#define ADF_DH895XCC_RX_RINGS_OFFSET 8 -#define ADF_DH895XCC_TX_RINGS_MASK 0xFF #define ADF_DH895XCC_FUSECTL_SKU_MASK 0x300000 #define ADF_DH895XCC_FUSECTL_SKU_SHIFT 20 #define ADF_DH895XCC_FUSECTL_SKU_1 0x0 @@ -25,19 +23,10 @@ #define ADF_DH895XCC_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_DH895XCC_SMIA0_MASK 0xFFFFFFFF #define ADF_DH895XCC_SMIA1_MASK 0x1 -/* Error detection and correction */ -#define ADF_DH895XCC_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) -#define ADF_DH895XCC_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) -#define ADF_DH895XCC_ENABLE_AE_ECC_ERR BIT(28) -#define ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) -#define ADF_DH895XCC_UERRSSMSH(i) (i * 0x4000 + 0x18) -#define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) -#define ADF_DH895XCC_ERRSSMSH_EN BIT(3) /* Masks for VF2PF interrupts */ #define ADF_DH895XCC_ERR_REG_VF2PF_U(vf_src) (((vf_src) & 0x0000FFFF) << 16) #define ADF_DH895XCC_ERR_MSK_VF2PF_U(vf_mask) ((vf_mask) >> 16) -#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) /* AE to function mapping */ #define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 From c3878a786be09d3f7df17936c922be430cdd4e8e Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Tue, 28 Sep 2021 12:44:34 +0100 Subject: [PATCH 100/134] crypto: qat - use hweight for bit counting Replace homegrown bit counting logic in adf_gen2_get_num_accels() and adf_gen2_get_num_aes() with the functions hweight16() and hweight32(), respectively. Signed-off-by: Giovanni Cabiddu Reviewed-by: Marco Chiappero Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 1deeeaed9a8c..262bdc05dab4 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -54,31 +54,19 @@ EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts); u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self) { - u32 i, ctr = 0; - if (!self || !self->accel_mask) return 0; - for (i = 0; i < self->num_accel; i++) - if (self->accel_mask & (1 << i)) - ctr++; - - return ctr; + return hweight16(self->accel_mask); } EXPORT_SYMBOL_GPL(adf_gen2_get_num_accels); u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self) { - u32 i, ctr = 0; - if (!self || !self->ae_mask) return 0; - for (i = 0; i < self->num_engines; i++) - if (self->ae_mask & (1 << i)) - ctr++; - - return ctr; + return hweight32(self->ae_mask); } EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes); From 6e680f94bc31d0fd0ff01123c964d895ea8040fa Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:35 +0100 Subject: [PATCH 101/134] crypto: qat - make pfvf send message direction agnostic The functions adf_iov_putmsg() and __adf_iov_putmsg() are shared by both PF and VF. Any logging or documentation should not refer to any specific direction. Make comments and log messages direction agnostic by replacing PF2VF with PFVF. Also fix the wording for some related comments. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index cdef6c34524e..41f4b5643dbb 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -77,11 +77,11 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) mutex_lock(lock); - /* Check if PF2VF CSR is in use by remote function */ + /* Check if the PFVF CSR is in use by remote function */ val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); if ((val & remote_in_use_mask) == remote_in_use_pattern) { dev_dbg(&GET_DEV(accel_dev), - "PF2VF CSR in use by remote function\n"); + "PFVF CSR in use by remote function\n"); ret = -EBUSY; goto out; } @@ -89,7 +89,7 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) msg &= ~local_in_use_mask; msg |= local_in_use_pattern; - /* Attempt to get ownership of the PF2VF CSR */ + /* Attempt to get ownership of the PFVF CSR */ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); /* Wait for confirmation from remote func it received the message */ @@ -111,7 +111,7 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) ret = -EIO; } - /* Finished with PF2VF CSR; relinquish it and leave msg in CSR */ + /* Finished with the PFVF CSR; relinquish it and leave msg in CSR */ ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask); out: mutex_unlock(lock); @@ -119,12 +119,13 @@ out: } /** - * adf_iov_putmsg() - send PF2VF message + * adf_iov_putmsg() - send PFVF message * @accel_dev: Pointer to acceleration device. * @msg: Message to send - * @vf_nr: VF number to which the message will be sent + * @vf_nr: VF number to which the message will be sent if on PF, ignored + * otherwise * - * Function sends a message from the PF to a VF + * Function sends a message through the PFVF channel * * Return: 0 on success, error code otherwise. */ From 21db65edb6a5a160a402311d1fad0c8ae050eec1 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:36 +0100 Subject: [PATCH 102/134] crypto: qat - move pfvf collision detection values Keep adf_pf2vf_msg.h as much as possible focused on the protocol definition. Instead, collision parameters are an implementation detail which should stay close to the code consuming them, therefore move them to adf_pf2vf_msg.c. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 9 +++++++++ drivers/crypto/qat/qat_common/adf_pf2vf_msg.h | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 41f4b5643dbb..22977001271c 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -5,6 +5,15 @@ #include "adf_common_drv.h" #include "adf_pf2vf_msg.h" +#define ADF_IOV_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_IOV_MSG_ACK_DELAY 2 +#define ADF_IOV_MSG_ACK_MAX_RETRY 100 +#define ADF_IOV_MSG_RETRY_DELAY 5 +#define ADF_IOV_MSG_MAX_RETRIES 3 +#define ADF_IOV_MSG_RESP_TIMEOUT (ADF_IOV_MSG_ACK_DELAY * \ + ADF_IOV_MSG_ACK_MAX_RETRY + \ + ADF_IOV_MSG_COLLISION_DETECT_DELAY) + void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h index ffd43aa50b57..a7d8f8367345 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h @@ -90,13 +90,4 @@ /* VF->PF Compatible Version Request */ #define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 -/* Collision detection */ -#define ADF_IOV_MSG_COLLISION_DETECT_DELAY 10 -#define ADF_IOV_MSG_ACK_DELAY 2 -#define ADF_IOV_MSG_ACK_MAX_RETRY 100 -#define ADF_IOV_MSG_RETRY_DELAY 5 -#define ADF_IOV_MSG_MAX_RETRIES 3 -#define ADF_IOV_MSG_RESP_TIMEOUT (ADF_IOV_MSG_ACK_DELAY * \ - ADF_IOV_MSG_ACK_MAX_RETRY + \ - ADF_IOV_MSG_COLLISION_DETECT_DELAY) #endif /* ADF_IOV_MSG_H */ From 71b5f2ab5e52df6f1a927e91243251bfc35c50e4 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:37 +0100 Subject: [PATCH 103/134] crypto: qat - rename pfvf collision constants Replace any reference of "IOV" with PFVF in the collision constants. Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 22977001271c..5459f295fcd9 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -5,14 +5,14 @@ #include "adf_common_drv.h" #include "adf_pf2vf_msg.h" -#define ADF_IOV_MSG_COLLISION_DETECT_DELAY 10 -#define ADF_IOV_MSG_ACK_DELAY 2 -#define ADF_IOV_MSG_ACK_MAX_RETRY 100 -#define ADF_IOV_MSG_RETRY_DELAY 5 -#define ADF_IOV_MSG_MAX_RETRIES 3 -#define ADF_IOV_MSG_RESP_TIMEOUT (ADF_IOV_MSG_ACK_DELAY * \ - ADF_IOV_MSG_ACK_MAX_RETRY + \ - ADF_IOV_MSG_COLLISION_DETECT_DELAY) +#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_PFVF_MSG_ACK_DELAY 2 +#define ADF_PFVF_MSG_ACK_MAX_RETRY 100 +#define ADF_PFVF_MSG_RETRY_DELAY 5 +#define ADF_PFVF_MSG_MAX_RETRIES 3 +#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \ + ADF_PFVF_MSG_ACK_MAX_RETRY + \ + ADF_PFVF_MSG_COLLISION_DETECT_DELAY) void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { @@ -103,9 +103,9 @@ static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) /* Wait for confirmation from remote func it received the message */ do { - msleep(ADF_IOV_MSG_ACK_DELAY); + msleep(ADF_PFVF_MSG_ACK_DELAY); val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); - } while ((val & int_bit) && (count++ < ADF_IOV_MSG_ACK_MAX_RETRY)); + } while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY)); if (val != msg) { dev_dbg(&GET_DEV(accel_dev), @@ -146,8 +146,8 @@ int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) do { ret = __adf_iov_putmsg(accel_dev, msg, vf_nr); if (ret) - msleep(ADF_IOV_MSG_RETRY_DELAY); - } while (ret && (count++ < ADF_IOV_MSG_MAX_RETRIES)); + msleep(ADF_PFVF_MSG_RETRY_DELAY); + } while (ret && (count++ < ADF_PFVF_MSG_MAX_RETRIES)); return ret; } @@ -277,7 +277,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) { - unsigned long timeout = msecs_to_jiffies(ADF_IOV_MSG_RESP_TIMEOUT); + unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msg = 0; int ret; From 7a73c4622aaa8a7a3820800c5c6b53e1097527ed Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:38 +0100 Subject: [PATCH 104/134] crypto: qat - add VF and PF wrappers to common send function The send function, adf_iov_putmsg(), is shared by both PF and VF. This commit provides two direction specific APIs, adf_send_pf2vf_msg() and adf_send_vf2pf_msg() which decouple the implementation, which can change and evolve over time, from the user. With this change, the adf_iov_putmsg() is now isolated inside the file adf_pf2vf_msg.c and has been marked as static. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 3 +- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 37 +++++++++++++++++-- drivers/crypto/qat/qat_common/adf_vf2pf_msg.c | 4 +- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 4261749fae8d..dd82272019ec 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -62,7 +62,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev); void adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); -int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info); @@ -200,7 +199,7 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info); - +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg); int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev); void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev); int adf_init_pf_wq(void); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 5459f295fcd9..23bcbb2e22e2 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -138,7 +138,7 @@ out: * * Return: 0 on success, error code otherwise. */ -int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { u32 count = 0; int ret; @@ -152,6 +152,35 @@ int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) return ret; } +/** + * adf_send_pf2vf_msg() - send PF to VF message + * @accel_dev: Pointer to acceleration device + * @vf_nr: VF number to which the message will be sent + * @msg: Message to send + * + * This function allows the PF to send a message to a specific VF. + * + * Return: 0 on success, error code otherwise. + */ +static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg) +{ + return adf_iov_putmsg(accel_dev, msg, vf_nr); +} + +/** + * adf_send_vf2pf_msg() - send VF to PF message + * @accel_dev: Pointer to acceleration device + * @msg: Message to send + * + * This function allows the VF to send a message to the PF. + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) +{ + return adf_iov_putmsg(accel_dev, msg, 0); +} + void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) { struct adf_accel_dev *accel_dev = vf_info->accel_dev; @@ -248,7 +277,7 @@ void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) goto err; } - if (resp && adf_iov_putmsg(accel_dev, resp, vf_nr)) + if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp)) dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); out: @@ -269,7 +298,7 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { - if (vf->init && adf_iov_putmsg(accel_dev, msg, i)) + if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg)) dev_err(&GET_DEV(accel_dev), "Failed to send restarting msg to VF%d\n", i); } @@ -290,7 +319,7 @@ static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) reinit_completion(&accel_dev->vf.iov_msg_completion); /* Send request from VF to PF */ - ret = adf_iov_putmsg(accel_dev, msg, 0); + ret = adf_send_vf2pf_msg(accel_dev, msg); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to send Compatibility Version Request.\n"); diff --git a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c index 3e25fac051b2..8d11bb24cea0 100644 --- a/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_vf2pf_msg.c @@ -17,7 +17,7 @@ int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev) u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); - if (adf_iov_putmsg(accel_dev, msg, 0)) { + if (adf_send_vf2pf_msg(accel_dev, msg)) { dev_err(&GET_DEV(accel_dev), "Failed to send Init event to PF\n"); return -EFAULT; @@ -41,7 +41,7 @@ void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev) (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); if (test_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status)) - if (adf_iov_putmsg(accel_dev, msg, 0)) + if (adf_send_vf2pf_msg(accel_dev, msg)) dev_err(&GET_DEV(accel_dev), "Failed to send Shutdown event to PF\n"); } From aa3c68634df86280087e8a4f2d3ba751eee3c6b4 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:39 +0100 Subject: [PATCH 105/134] crypto: qat - extract send and wait from adf_vf2pf_request_version() In the function adf_vf2pf_request_version(), the VF sends a request to the PF and waits for a response before parsing and handling it. Since this pattern will be used by other requests, define a new function, adf_send_vf2pf_req(), that only deals with sending a VF2PF request and waiting for a response. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 23bcbb2e22e2..711f6e3f6673 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -181,6 +181,42 @@ int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg) return adf_iov_putmsg(accel_dev, msg, 0); } +/** + * adf_send_vf2pf_req() - send VF2PF request message + * @accel_dev: Pointer to acceleration device. + * @msg: Request message to send + * + * This function sends a message that requires a response from the VF to the PF + * and waits for a reply. + * + * Return: 0 on success, error code otherwise. + */ +static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg) +{ + unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); + int ret; + + reinit_completion(&accel_dev->vf.iov_msg_completion); + + /* Send request from VF to PF */ + ret = adf_send_vf2pf_msg(accel_dev, msg); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send request msg to PF\n"); + return ret; + } + + /* Wait for response */ + if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, + timeout)) { + dev_err(&GET_DEV(accel_dev), + "PFVF request/response message timeout expired\n"); + return -EIO; + } + + return 0; +} + void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) { struct adf_accel_dev *accel_dev = vf_info->accel_dev; @@ -306,7 +342,6 @@ void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) { - unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT); struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msg = 0; int ret; @@ -316,24 +351,13 @@ static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255); - reinit_completion(&accel_dev->vf.iov_msg_completion); - - /* Send request from VF to PF */ - ret = adf_send_vf2pf_msg(accel_dev, msg); + ret = adf_send_vf2pf_req(accel_dev, msg); if (ret) { dev_err(&GET_DEV(accel_dev), "Failed to send Compatibility Version Request.\n"); return ret; } - /* Wait for response */ - if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, - timeout)) { - dev_err(&GET_DEV(accel_dev), - "IOV request/response message timeout expired\n"); - return -EIO; - } - /* Response from PF received, check compatibility */ switch (accel_dev->vf.compatible) { case ADF_PF2VF_VF_COMPATIBLE: From 32dfef6f92dd0aa287a4798a4a358c351f78aa32 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 28 Sep 2021 12:44:40 +0100 Subject: [PATCH 106/134] crypto: qat - share adf_enable_pf2vf_comms() from adf_pf2vf_msg.c The PFVF protocol "enable" functions are direction specific but not device specific. Move the protocol enable function for the PF into the PF specific protocol file for better file organization and duplicated code reduction. NOTE: the patch keeps gen4 disabled as it doesn't have full PFVF support yet. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 4 ++-- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 7 ------- drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 7 ------- drivers/crypto/qat/qat_common/adf_common_drv.h | 6 ++++++ drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 18 ++++++++++++++++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 7 ------- 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 88c0ded411f1..fa768f10635f 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -191,7 +191,7 @@ static int adf_init_device(struct adf_accel_dev *accel_dev) return ret; } -static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) +static int pfvf_comms_disabled(struct adf_accel_dev *accel_dev) { return 0; } @@ -253,7 +253,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->uof_get_ae_mask = uof_get_ae_mask; hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; - hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms; + hw_data->enable_pfvf_comms = pfvf_comms_disabled; hw_data->disable_iov = adf_disable_sriov; hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 6a39d2e7f4c0..1fa690219d92 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -91,13 +91,6 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) ADF_C3XXX_SMIA1_MASK); } -static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) -{ - spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); - - return 0; -} - static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) { adf_gen2_cfg_iov_thds(accel_dev, enable, diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index e259ca38a653..0613db077689 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -93,13 +93,6 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) ADF_C62X_SMIA1_MASK); } -static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) -{ - spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); - - return 0; -} - static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) { adf_gen2_cfg_iov_thds(accel_dev, enable, diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index dd82272019ec..2cc6622833c4 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -196,6 +196,7 @@ void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev, u32 vf_mask); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask); +int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev); void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info); @@ -210,6 +211,11 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev); #else #define adf_sriov_configure NULL +static inline int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) { } diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 711f6e3f6673..59860bdaedb6 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -396,3 +396,21 @@ int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return adf_vf2pf_request_version(accel_dev); } EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); + +/** + * adf_enable_pf2vf_comms() - Function enables communication from pf to vf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * This function carries out the necessary steps to setup and start the PFVF + * communication channel, if any. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) +{ + spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index e5e64f880fbf..8e2e1554dcf6 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -158,13 +158,6 @@ static void disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) } } -static int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev) -{ - spin_lock_init(&accel_dev->pf.vf2pf_ints_lock); - - return 0; -} - static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) { adf_gen2_cfg_iov_thds(accel_dev, enable, From 82e269ad8afe1c73aa6c28dca3c23fecc6c8b616 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 28 Sep 2021 13:54:01 +0200 Subject: [PATCH 107/134] crypto: testmgr - Only disable migration in crypto_disable_simd_for_test() crypto_disable_simd_for_test() disables preemption in order to receive a stable per-CPU variable which it needs to modify in order to alter crypto_simd_usable() results. This can also be achived by migrate_disable() which forbidds CPU migrations but allows the task to be preempted. The latter is important for PREEMPT_RT since operation like skcipher_walk_first() may allocate memory which must not happen with disabled preemption on PREEMPT_RT. Use migrate_disable() in crypto_disable_simd_for_test() to achieve a stable per-CPU pointer. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Herbert Xu --- crypto/testmgr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 70f69f0910c9..58eee8eab4bf 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1061,14 +1061,14 @@ static void generate_random_testvec_config(struct testvec_config *cfg, static void crypto_disable_simd_for_test(void) { - preempt_disable(); + migrate_disable(); __this_cpu_write(crypto_simd_disabled_for_test, true); } static void crypto_reenable_simd_for_test(void) { __this_cpu_write(crypto_simd_disabled_for_test, false); - preempt_enable(); + migrate_enable(); } /* From b6f5f0c8f72d348b2d07b20d7b680ef13a7ffe98 Mon Sep 17 00:00:00 2001 From: Markus Schneider-Pargmann Date: Thu, 30 Sep 2021 21:12:42 +0200 Subject: [PATCH 108/134] hwrng: mtk - Force runtime pm ops for sleep ops Currently mtk_rng_runtime_suspend/resume is called for both runtime pm and system sleep operations. This is wrong as these should only be runtime ops as the name already suggests. Currently freezing the system will lead to a call to mtk_rng_runtime_suspend even if the device currently isn't active. This leads to a clock warning because it is disabled/unprepared although it isn't enabled/prepared currently. This patch fixes this by only setting the runtime pm ops and forces to call the runtime pm ops from the system sleep ops as well if active but not otherwise. Fixes: 81d2b34508c6 ("hwrng: mtk - add runtime PM support") Signed-off-by: Markus Schneider-Pargmann Signed-off-by: Herbert Xu --- drivers/char/hw_random/mtk-rng.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c index 8ad7b515a51b..6c00ea008555 100644 --- a/drivers/char/hw_random/mtk-rng.c +++ b/drivers/char/hw_random/mtk-rng.c @@ -166,8 +166,13 @@ static int mtk_rng_runtime_resume(struct device *dev) return mtk_rng_init(&priv->rng); } -static UNIVERSAL_DEV_PM_OPS(mtk_rng_pm_ops, mtk_rng_runtime_suspend, - mtk_rng_runtime_resume, NULL); +static const struct dev_pm_ops mtk_rng_pm_ops = { + SET_RUNTIME_PM_OPS(mtk_rng_runtime_suspend, + mtk_rng_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) +}; + #define MTK_RNG_PM_OPS (&mtk_rng_pm_ops) #else /* CONFIG_PM */ #define MTK_RNG_PM_OPS NULL From 4342f70538b929b188c6e370fe24a155e6532eb2 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Mon, 11 Oct 2021 22:22:29 +0200 Subject: [PATCH 109/134] selinux: remove unneeded ipv6 hook wrappers Netfilter places the protocol number the hook function is getting called from in state->pf, so we can use that instead of an extra wrapper. While at it, remove one-line wrappers too and make selinux_ip_{out,forward,postroute} useable as hook function. Signed-off-by: Florian Westphal Message-Id: <20211011202229.28289-1-fw@strlen.de> Signed-off-by: Paul Moore --- security/selinux/hooks.c | 80 +++++++++------------------------------- 1 file changed, 18 insertions(+), 62 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 7297d1f0a9df..4210831d5ade 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -5689,10 +5689,11 @@ static int selinux_tun_dev_open(void *security) #ifdef CONFIG_NETFILTER -static unsigned int selinux_ip_forward(struct sk_buff *skb, - const struct net_device *indev, - u16 family) +static unsigned int selinux_ip_forward(void *priv, struct sk_buff *skb, + const struct nf_hook_state *state) { + const struct net_device *indev = state->in; + u16 family = state->pf; int err; char *addrp; u32 peer_sid; @@ -5747,25 +5748,10 @@ static unsigned int selinux_ip_forward(struct sk_buff *skb, return NF_ACCEPT; } -static unsigned int selinux_ipv4_forward(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_forward(skb, state->in, PF_INET); -} - -#if IS_ENABLED(CONFIG_IPV6) -static unsigned int selinux_ipv6_forward(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_forward(skb, state->in, PF_INET6); -} -#endif /* IPV6 */ - -static unsigned int selinux_ip_output(struct sk_buff *skb, - u16 family) +static unsigned int selinux_ip_output(void *priv, struct sk_buff *skb, + const struct nf_hook_state *state) { + u16 family = state->pf; struct sock *sk; u32 sid; @@ -5805,21 +5791,6 @@ static unsigned int selinux_ip_output(struct sk_buff *skb, return NF_ACCEPT; } -static unsigned int selinux_ipv4_output(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_output(skb, PF_INET); -} - -#if IS_ENABLED(CONFIG_IPV6) -static unsigned int selinux_ipv6_output(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_output(skb, PF_INET6); -} -#endif /* IPV6 */ static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb, int ifindex, @@ -5855,10 +5826,12 @@ static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb, return NF_ACCEPT; } -static unsigned int selinux_ip_postroute(struct sk_buff *skb, - const struct net_device *outdev, - u16 family) +static unsigned int selinux_ip_postroute(void *priv, + struct sk_buff *skb, + const struct nf_hook_state *state) { + const struct net_device *outdev = state->out; + u16 family = state->pf; u32 secmark_perm; u32 peer_sid; int ifindex = outdev->ifindex; @@ -5994,23 +5967,6 @@ static unsigned int selinux_ip_postroute(struct sk_buff *skb, return NF_ACCEPT; } - -static unsigned int selinux_ipv4_postroute(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_postroute(skb, state->out, PF_INET); -} - -#if IS_ENABLED(CONFIG_IPV6) -static unsigned int selinux_ipv6_postroute(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - return selinux_ip_postroute(skb, state->out, PF_INET6); -} -#endif /* IPV6 */ - #endif /* CONFIG_NETFILTER */ static int selinux_netlink_send(struct sock *sk, struct sk_buff *skb) @@ -7475,38 +7431,38 @@ DEFINE_LSM(selinux) = { static const struct nf_hook_ops selinux_nf_ops[] = { { - .hook = selinux_ipv4_postroute, + .hook = selinux_ip_postroute, .pf = NFPROTO_IPV4, .hooknum = NF_INET_POST_ROUTING, .priority = NF_IP_PRI_SELINUX_LAST, }, { - .hook = selinux_ipv4_forward, + .hook = selinux_ip_forward, .pf = NFPROTO_IPV4, .hooknum = NF_INET_FORWARD, .priority = NF_IP_PRI_SELINUX_FIRST, }, { - .hook = selinux_ipv4_output, + .hook = selinux_ip_output, .pf = NFPROTO_IPV4, .hooknum = NF_INET_LOCAL_OUT, .priority = NF_IP_PRI_SELINUX_FIRST, }, #if IS_ENABLED(CONFIG_IPV6) { - .hook = selinux_ipv6_postroute, + .hook = selinux_ip_postroute, .pf = NFPROTO_IPV6, .hooknum = NF_INET_POST_ROUTING, .priority = NF_IP6_PRI_SELINUX_LAST, }, { - .hook = selinux_ipv6_forward, + .hook = selinux_ip_forward, .pf = NFPROTO_IPV6, .hooknum = NF_INET_FORWARD, .priority = NF_IP6_PRI_SELINUX_FIRST, }, { - .hook = selinux_ipv6_output, + .hook = selinux_ip_output, .pf = NFPROTO_IPV6, .hooknum = NF_INET_LOCAL_OUT, .priority = NF_IP6_PRI_SELINUX_FIRST, From cbfcd13be5cb2a07868afe67520ed181956579a7 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Wed, 28 Jul 2021 16:03:13 +0200 Subject: [PATCH 110/134] selinux: fix race condition when computing ocontext SIDs Current code contains a lot of racy patterns when converting an ocontext's context structure to an SID. This is being done in a "lazy" fashion, such that the SID is looked up in the SID table only when it's first needed and then cached in the "sid" field of the ocontext structure. However, this is done without any locking or memory barriers and is thus unsafe. Between commits 24ed7fdae669 ("selinux: use separate table for initial SID lookup") and 66f8e2f03c02 ("selinux: sidtab reverse lookup hash table"), this race condition lead to an actual observable bug, because a pointer to the shared sid field was passed directly to sidtab_context_to_sid(), which was using this location to also store an intermediate value, which could have been read by other threads and interpreted as an SID. In practice this caused e.g. new mounts to get a wrong (seemingly random) filesystem context, leading to strange denials. This bug has been spotted in the wild at least twice, see [1] and [2]. Fix the race condition by making all the racy functions use a common helper that ensures the ocontext::sid accesses are made safely using the appropriate SMP constructs. Note that security_netif_sid() was populating the sid field of both contexts stored in the ocontext, but only the first one was actually used. The SELinux wiki's documentation on the "netifcon" policy statement [3] suggests that using only the first context is intentional. I kept only the handling of the first context here, as there is really no point in doing the SID lookup for the unused one. I wasn't able to reproduce the bug mentioned above on any kernel that includes commit 66f8e2f03c02, even though it has been reported that the issue occurs with that commit, too, just less frequently. Thus, I wasn't able to verify that this patch fixes the issue, but it makes sense to avoid the race condition regardless. [1] https://github.com/containers/container-selinux/issues/89 [2] https://lists.fedoraproject.org/archives/list/selinux@lists.fedoraproject.org/thread/6DMTAMHIOAOEMUAVTULJD45JZU7IBAFM/ [3] https://selinuxproject.org/page/NetworkStatements#netifcon Cc: stable@vger.kernel.org Cc: Xinjie Zheng Reported-by: Sujithra Periasamy Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/services.c | 162 ++++++++++++++++----------------- 1 file changed, 77 insertions(+), 85 deletions(-) diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index e5f1b2757a83..c4931bf6f92a 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -2376,6 +2376,43 @@ err_policy: return rc; } +/** + * ocontext_to_sid - Helper to safely get sid for an ocontext + * @sidtab: SID table + * @c: ocontext structure + * @index: index of the context entry (0 or 1) + * @out_sid: pointer to the resulting SID value + * + * For all ocontexts except OCON_ISID the SID fields are populated + * on-demand when needed. Since updating the SID value is an SMP-sensitive + * operation, this helper must be used to do that safely. + * + * WARNING: This function may return -ESTALE, indicating that the caller + * must retry the operation after re-acquiring the policy pointer! + */ +static int ocontext_to_sid(struct sidtab *sidtab, struct ocontext *c, + size_t index, u32 *out_sid) +{ + int rc; + u32 sid; + + /* Ensure the associated sidtab entry is visible to this thread. */ + sid = smp_load_acquire(&c->sid[index]); + if (!sid) { + rc = sidtab_context_to_sid(sidtab, &c->context[index], &sid); + if (rc) + return rc; + + /* + * Ensure the new sidtab entry is visible to other threads + * when they see the SID. + */ + smp_store_release(&c->sid[index], sid); + } + *out_sid = sid; + return 0; +} + /** * security_port_sid - Obtain the SID for a port. * @state: SELinux state @@ -2414,17 +2451,13 @@ retry: } if (c) { - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, out_sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - *out_sid = c->sid[0]; + if (rc) + goto out; } else { *out_sid = SECINITSID_PORT; } @@ -2473,18 +2506,13 @@ retry: } if (c) { - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, - &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, out_sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - *out_sid = c->sid[0]; + if (rc) + goto out; } else *out_sid = SECINITSID_UNLABELED; @@ -2533,17 +2561,13 @@ retry: } if (c) { - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, out_sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - *out_sid = c->sid[0]; + if (rc) + goto out; } else *out_sid = SECINITSID_UNLABELED; @@ -2587,25 +2611,13 @@ retry: } if (c) { - if (!c->sid[0] || !c->sid[1]) { - rc = sidtab_context_to_sid(sidtab, &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; - rc = sidtab_context_to_sid(sidtab, &c->context[1], - &c->sid[1]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, if_sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - *if_sid = c->sid[0]; + if (rc) + goto out; } else *if_sid = SECINITSID_NETIF; @@ -2697,18 +2709,13 @@ retry: } if (c) { - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, - &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, out_sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - *out_sid = c->sid[0]; + if (rc) + goto out; } else { *out_sid = SECINITSID_NODE; } @@ -2873,7 +2880,7 @@ static inline int __security_genfs_sid(struct selinux_policy *policy, u16 sclass; struct genfs *genfs; struct ocontext *c; - int rc, cmp = 0; + int cmp = 0; while (path[0] == '/' && path[1] == '/') path++; @@ -2887,9 +2894,8 @@ static inline int __security_genfs_sid(struct selinux_policy *policy, break; } - rc = -ENOENT; if (!genfs || cmp) - goto out; + return -ENOENT; for (c = genfs->head; c; c = c->next) { len = strlen(c->u.name); @@ -2898,20 +2904,10 @@ static inline int __security_genfs_sid(struct selinux_policy *policy, break; } - rc = -ENOENT; if (!c) - goto out; + return -ENOENT; - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); - if (rc) - goto out; - } - - *sid = c->sid[0]; - rc = 0; -out: - return rc; + return ocontext_to_sid(sidtab, c, 0, sid); } /** @@ -2996,17 +2992,13 @@ retry: if (c) { sbsec->behavior = c->v.behavior; - if (!c->sid[0]) { - rc = sidtab_context_to_sid(sidtab, &c->context[0], - &c->sid[0]); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out; + rc = ocontext_to_sid(sidtab, c, 0, &sbsec->sid); + if (rc == -ESTALE) { + rcu_read_unlock(); + goto retry; } - sbsec->sid = c->sid[0]; + if (rc) + goto out; } else { rc = __security_genfs_sid(policy, fstype, "/", SECCLASS_DIR, &sbsec->sid); From 1d1e1ded13568be81a0e19d228e310a48997bec8 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 11 Oct 2021 17:50:48 -0400 Subject: [PATCH 111/134] selinux: make better use of the nf_hook_state passed to the NF hooks This patch builds on a previous SELinux/netfilter patch by Florian Westphal and makes better use of the nf_hook_state variable passed into the SELinux/netfilter hooks as well as a number of other small cleanups in the related code. Signed-off-by: Paul Moore --- security/selinux/hooks.c | 52 +++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 4210831d5ade..49d52d7a7714 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -5692,38 +5692,38 @@ static int selinux_tun_dev_open(void *security) static unsigned int selinux_ip_forward(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { - const struct net_device *indev = state->in; - u16 family = state->pf; - int err; + int ifindex; + u16 family; char *addrp; u32 peer_sid; struct common_audit_data ad; struct lsm_network_audit net = {0,}; - u8 secmark_active; - u8 netlbl_active; - u8 peerlbl_active; + int secmark_active, peerlbl_active; if (!selinux_policycap_netpeer()) return NF_ACCEPT; secmark_active = selinux_secmark_enabled(); - netlbl_active = netlbl_enabled(); peerlbl_active = selinux_peerlbl_enabled(); if (!secmark_active && !peerlbl_active) return NF_ACCEPT; + family = state->pf; if (selinux_skb_peerlbl_sid(skb, family, &peer_sid) != 0) return NF_DROP; + ifindex = state->in->ifindex; ad.type = LSM_AUDIT_DATA_NET; ad.u.net = &net; - ad.u.net->netif = indev->ifindex; + ad.u.net->netif = ifindex; ad.u.net->family = family; if (selinux_parse_skb(skb, &ad, &addrp, 1, NULL) != 0) return NF_DROP; if (peerlbl_active) { - err = selinux_inet_sys_rcv_skb(dev_net(indev), indev->ifindex, + int err; + + err = selinux_inet_sys_rcv_skb(state->net, ifindex, addrp, family, peer_sid, &ad); if (err) { selinux_netlbl_err(skb, family, err, 1); @@ -5737,7 +5737,7 @@ static unsigned int selinux_ip_forward(void *priv, struct sk_buff *skb, SECCLASS_PACKET, PACKET__FORWARD_IN, &ad)) return NF_DROP; - if (netlbl_active) + if (netlbl_enabled()) /* we do this in the FORWARD path and not the POST_ROUTING * path because we want to make sure we apply the necessary * labeling before IPsec is applied so we can leverage AH @@ -5751,7 +5751,6 @@ static unsigned int selinux_ip_forward(void *priv, struct sk_buff *skb, static unsigned int selinux_ip_output(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { - u16 family = state->pf; struct sock *sk; u32 sid; @@ -5785,7 +5784,7 @@ static unsigned int selinux_ip_output(void *priv, struct sk_buff *skb, sid = sksec->sid; } else sid = SECINITSID_KERNEL; - if (selinux_netlbl_skbuff_setsid(skb, family, sid) != 0) + if (selinux_netlbl_skbuff_setsid(skb, state->pf, sid) != 0) return NF_DROP; return NF_ACCEPT; @@ -5793,25 +5792,24 @@ static unsigned int selinux_ip_output(void *priv, struct sk_buff *skb, static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb, - int ifindex, - u16 family) + const struct nf_hook_state *state) { - struct sock *sk = skb_to_full_sk(skb); + struct sock *sk; struct sk_security_struct *sksec; struct common_audit_data ad; struct lsm_network_audit net = {0,}; - char *addrp; u8 proto; - if (sk == NULL) + if (state->sk == NULL) return NF_ACCEPT; + sk = skb_to_full_sk(skb); sksec = sk->sk_security; ad.type = LSM_AUDIT_DATA_NET; ad.u.net = &net; - ad.u.net->netif = ifindex; - ad.u.net->family = family; - if (selinux_parse_skb(skb, &ad, &addrp, 0, &proto)) + ad.u.net->netif = state->out->ifindex; + ad.u.net->family = state->pf; + if (selinux_parse_skb(skb, &ad, NULL, 0, &proto)) return NF_DROP; if (selinux_secmark_enabled()) @@ -5830,24 +5828,22 @@ static unsigned int selinux_ip_postroute(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { - const struct net_device *outdev = state->out; - u16 family = state->pf; + u16 family; u32 secmark_perm; u32 peer_sid; - int ifindex = outdev->ifindex; + int ifindex; struct sock *sk; struct common_audit_data ad; struct lsm_network_audit net = {0,}; char *addrp; - u8 secmark_active; - u8 peerlbl_active; + int secmark_active, peerlbl_active; /* If any sort of compatibility mode is enabled then handoff processing * to the selinux_ip_postroute_compat() function to deal with the * special handling. We do this in an attempt to keep this function * as fast and as clean as possible. */ if (!selinux_policycap_netpeer()) - return selinux_ip_postroute_compat(skb, ifindex, family); + return selinux_ip_postroute_compat(skb, state); secmark_active = selinux_secmark_enabled(); peerlbl_active = selinux_peerlbl_enabled(); @@ -5873,6 +5869,7 @@ static unsigned int selinux_ip_postroute(void *priv, return NF_ACCEPT; #endif + family = state->pf; if (sk == NULL) { /* Without an associated socket the packet is either coming * from the kernel or it is being forwarded; check the packet @@ -5933,6 +5930,7 @@ static unsigned int selinux_ip_postroute(void *priv, secmark_perm = PACKET__SEND; } + ifindex = state->out->ifindex; ad.type = LSM_AUDIT_DATA_NET; ad.u.net = &net; ad.u.net->netif = ifindex; @@ -5950,7 +5948,7 @@ static unsigned int selinux_ip_postroute(void *priv, u32 if_sid; u32 node_sid; - if (sel_netif_sid(dev_net(outdev), ifindex, &if_sid)) + if (sel_netif_sid(state->net, ifindex, &if_sid)) return NF_DROP; if (avc_has_perm(&selinux_state, peer_sid, if_sid, From e9fd7292935906c09824a10bc27b48fd3992c366 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Wed, 18 Nov 2020 21:15:08 -0500 Subject: [PATCH 112/134] selinux: fix all of the W=1 build warnings There were a number of places in the code where the function definition did not match the associated comment block as well at least one file where the appropriate header files were not included (missing function declaration/prototype); this patch fixes all of these issue such that building the SELinux code with "W=1" is now warning free. % make W=1 security/selinux/ Signed-off-by: Paul Moore --- security/selinux/avc.c | 13 ++++++++++++- security/selinux/netlabel.c | 7 +++++-- security/selinux/netport.c | 2 +- security/selinux/ss/hashtab.c | 1 + security/selinux/ss/mls.c | 4 ++++ security/selinux/ss/services.c | 14 ++++++++++---- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 97f4c944a20f..abcd9740d10f 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -547,6 +547,7 @@ static inline struct avc_node *avc_search_node(struct selinux_avc *avc, /** * avc_lookup - Look up an AVC entry. + * @avc: the access vector cache * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class @@ -597,6 +598,7 @@ static int avc_latest_notif_update(struct selinux_avc *avc, /** * avc_insert - Insert an AVC entry. + * @avc: the access vector cache * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class @@ -825,9 +827,14 @@ out: /** * avc_update_node - Update an AVC entry + * @avc: the access vector cache * @event : Updating event * @perms : Permission mask bits - * @ssid,@tsid,@tclass : identifier of an AVC entry + * @driver: xperm driver information + * @xperm: xperm permissions + * @ssid: AVC entry source sid + * @tsid: AVC entry target sid + * @tclass : AVC entry target object class * @seqno : sequence number when decision was made * @xpd: extended_perms_decision to be added to the node * @flags: the AVC_* flags, e.g. AVC_EXTENDED_PERMS, or 0. @@ -928,6 +935,7 @@ out: /** * avc_flush - Flush the cache + * @avc: the access vector cache */ static void avc_flush(struct selinux_avc *avc) { @@ -956,6 +964,7 @@ static void avc_flush(struct selinux_avc *avc) /** * avc_ss_reset - Flush the cache and revalidate migrated permissions. + * @avc: the access vector cache * @seqno: policy sequence number */ int avc_ss_reset(struct selinux_avc *avc, u32 seqno) @@ -1105,6 +1114,7 @@ decision: /** * avc_has_perm_noaudit - Check permissions but perform no auditing. + * @state: SELinux state * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class @@ -1156,6 +1166,7 @@ inline int avc_has_perm_noaudit(struct selinux_state *state, /** * avc_has_perm - Check permissions and perform any appropriate auditing. + * @state: SELinux state * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class diff --git a/security/selinux/netlabel.c b/security/selinux/netlabel.c index abaab7683840..29b88e81869b 100644 --- a/security/selinux/netlabel.c +++ b/security/selinux/netlabel.c @@ -29,6 +29,7 @@ /** * selinux_netlbl_sidlookup_cached - Cache a SID lookup * @skb: the packet + * @family: the packet's address family * @secattr: the NetLabel security attributes * @sid: the SID * @@ -128,6 +129,7 @@ void selinux_netlbl_cache_invalidate(void) /** * selinux_netlbl_err - Handle a NetLabel packet error * @skb: the packet + * @family: the packet's address family * @error: the error code * @gateway: true if host is acting as a gateway, false otherwise * @@ -160,7 +162,6 @@ void selinux_netlbl_sk_security_free(struct sk_security_struct *sksec) /** * selinux_netlbl_sk_security_reset - Reset the NetLabel fields * @sksec: the sk_security_struct - * @family: the socket family * * Description: * Called when the NetLabel state of a sk_security_struct needs to be reset. @@ -313,6 +314,7 @@ assoc_request_return: /** * selinux_netlbl_inet_conn_request - Label an incoming stream connection * @req: incoming connection request socket + * @family: the request socket's address family * * Description: * A new incoming connection request is represented by @req, we need to label @@ -343,6 +345,7 @@ inet_conn_request_return: /** * selinux_netlbl_inet_csk_clone - Initialize the newly created sock * @sk: the new sock + * @family: the sock's address family * * Description: * A new connection has been established using @sk, we've already labeled the @@ -378,7 +381,7 @@ void selinux_netlbl_sctp_sk_clone(struct sock *sk, struct sock *newsk) /** * selinux_netlbl_socket_post_create - Label a socket using NetLabel - * @sock: the socket to label + * @sk: the sock to label * @family: protocol family * * Description: diff --git a/security/selinux/netport.c b/security/selinux/netport.c index b8bc3897891d..9ba09d11c0f5 100644 --- a/security/selinux/netport.c +++ b/security/selinux/netport.c @@ -73,7 +73,7 @@ static unsigned int sel_netport_hashfn(u16 pnum) /** * sel_netport_find - Search for a port record * @protocol: protocol - * @port: pnum + * @pnum: port * * Description: * Search the network port table and return the matching record. If an entry diff --git a/security/selinux/ss/hashtab.c b/security/selinux/ss/hashtab.c index b8f6b3e0a921..727c3b484bd3 100644 --- a/security/selinux/ss/hashtab.c +++ b/security/selinux/ss/hashtab.c @@ -8,6 +8,7 @@ #include #include #include "hashtab.h" +#include "security.h" static struct kmem_cache *hashtab_node_cachep __ro_after_init; diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c index d338962fb0c4..3f5fd124342c 100644 --- a/security/selinux/ss/mls.c +++ b/security/selinux/ss/mls.c @@ -553,6 +553,7 @@ int mls_compute_sid(struct policydb *p, #ifdef CONFIG_NETLABEL /** * mls_export_netlbl_lvl - Export the MLS sensitivity levels to NetLabel + * @p: the policy * @context: the security context * @secattr: the NetLabel security attributes * @@ -574,6 +575,7 @@ void mls_export_netlbl_lvl(struct policydb *p, /** * mls_import_netlbl_lvl - Import the NetLabel MLS sensitivity levels + * @p: the policy * @context: the security context * @secattr: the NetLabel security attributes * @@ -595,6 +597,7 @@ void mls_import_netlbl_lvl(struct policydb *p, /** * mls_export_netlbl_cat - Export the MLS categories to NetLabel + * @p: the policy * @context: the security context * @secattr: the NetLabel security attributes * @@ -622,6 +625,7 @@ int mls_export_netlbl_cat(struct policydb *p, /** * mls_import_netlbl_cat - Import the MLS categories from NetLabel + * @p: the policy * @context: the security context * @secattr: the NetLabel security attributes * diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index c4931bf6f92a..8e92af7dd284 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1102,7 +1102,7 @@ allow: * @state: SELinux state * @ssid: source security identifier * @tsid: target security identifier - * @tclass: target security class + * @orig_tclass: target security class * @avd: access vector decisions * @xperms: extended permissions * @@ -1626,6 +1626,7 @@ int security_context_str_to_sid(struct selinux_state *state, * @scontext_len: length in bytes * @sid: security identifier, SID * @def_sid: default SID to assign on error + * @gfp_flags: the allocator get-free-page (GFP) flags * * Obtains a SID associated with the security context that * has the string representation specified by @scontext. @@ -1919,6 +1920,7 @@ out: * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class + * @qstr: object name * @out_sid: security identifier for new subject/object * * Compute a SID to use for labeling a new subject or object in the @@ -1947,6 +1949,7 @@ int security_transition_sid_user(struct selinux_state *state, /** * security_member_sid - Compute the SID for member selection. + * @state: SELinux state * @ssid: source security identifier * @tsid: target security identifier * @tclass: target security class @@ -2273,6 +2276,7 @@ void selinux_policy_commit(struct selinux_state *state, * @state: SELinux state * @data: binary policy data * @len: length of data in bytes + * @load_state: policy load state * * Load a new set of security policy configuration data, * validate it and convert the SID table as necessary. @@ -2525,7 +2529,7 @@ out: * security_ib_endport_sid - Obtain the SID for a subnet management interface. * @state: SELinux state * @dev_name: device name - * @port: port number + * @port_num: port number * @out_sid: security identifier */ int security_ib_endport_sid(struct selinux_state *state, @@ -2856,9 +2860,10 @@ out_unlock: /** * __security_genfs_sid - Helper to obtain a SID for a file in a filesystem + * @policy: policy * @fstype: filesystem type * @path: path from root of mount - * @sclass: file security class + * @orig_sclass: file security class * @sid: SID for path * * Obtain a SID to use for a file in a filesystem that @@ -2915,7 +2920,7 @@ static inline int __security_genfs_sid(struct selinux_policy *policy, * @state: SELinux state * @fstype: filesystem type * @path: path from root of mount - * @sclass: file security class + * @orig_sclass: file security class * @sid: SID for path * * Acquire policy_rwlock before calling __security_genfs_sid() and release @@ -3297,6 +3302,7 @@ out_unlock: * @nlbl_sid: NetLabel SID * @nlbl_type: NetLabel labeling protocol type * @xfrm_sid: XFRM SID + * @peer_sid: network peer sid * * Description: * Compare the @nlbl_sid and @xfrm_sid values and if the two SIDs can be From 86dd9fd52e1449084c58c1ebb1f099a2ccc85091 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 13 Oct 2021 10:28:48 -0700 Subject: [PATCH 113/134] LSM: Avoid warnings about potentially unused hook variables Building with W=1 shows many unused const variable warnings. These can be silenced, as we're well aware of their being potentially unused: ./include/linux/lsm_hook_defs.h:36:18: error: 'ptrace_access_check_default' defined but not used [-Werror=unused-const-variable=] 36 | LSM_HOOK(int, 0, ptrace_access_check, struct task_struct *child, | ^~~~~~~~~~~~~~~~~~~ security/security.c:706:32: note: in definition of macro 'LSM_RET_DEFAULT' 706 | #define LSM_RET_DEFAULT(NAME) (NAME##_default) | ^~~~ security/security.c:711:9: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int' 711 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME) | ^~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/lsm_hook_defs.h:36:1: note: in expansion of macro 'LSM_HOOK' 36 | LSM_HOOK(int, 0, ptrace_access_check, struct task_struct *child, | ^~~~~~~~ Cc: James Morris Cc: "Serge E. Hallyn" Cc: Paul Moore Cc: Casey Schaufler Cc: KP Singh Cc: linux-security-module@vger.kernel.org Reported-by: kernel test robot Link: https://lore.kernel.org/linux-mm/202110131608.zms53FPR-lkp@intel.com/ Fixes: 98e828a0650f ("security: Refactor declaration of LSM hooks") Signed-off-by: Kees Cook Acked-by: James Morris Signed-off-by: Paul Moore --- security/security.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/security.c b/security/security.c index c49a2c0cc1c1..40518c340571 100644 --- a/security/security.c +++ b/security/security.c @@ -706,7 +706,7 @@ static int lsm_superblock_alloc(struct super_block *sb) #define LSM_RET_DEFAULT(NAME) (NAME##_default) #define DECLARE_LSM_RET_DEFAULT_void(DEFAULT, NAME) #define DECLARE_LSM_RET_DEFAULT_int(DEFAULT, NAME) \ - static const int LSM_RET_DEFAULT(NAME) = (DEFAULT); + static const int __maybe_unused LSM_RET_DEFAULT(NAME) = (DEFAULT); #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME) From 29bc22ac5e5bc63275e850f0c8fc549e3d0e306b Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 12 Oct 2021 09:56:12 -0700 Subject: [PATCH 114/134] binder: use euid from cred instead of using task Save the 'struct cred' associated with a binder process at initial open to avoid potential race conditions when converting to an euid. Set a transaction's sender_euid from the 'struct cred' saved at binder_open() instead of looking up the euid from the binder proc's 'struct task'. This ensures the euid is associated with the security context that of the task that opened binder. Cc: stable@vger.kernel.org # 4.4+ Fixes: 457b9a6f09f0 ("Staging: android: add binder driver") Signed-off-by: Todd Kjos Suggested-by: Stephen Smalley Suggested-by: Jann Horn Acked-by: Casey Schaufler Signed-off-by: Paul Moore --- drivers/android/binder.c | 4 +++- drivers/android/binder_internal.h | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index d9030cb6b1e4..231cff9b3b75 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2702,7 +2702,7 @@ static void binder_transaction(struct binder_proc *proc, t->from = thread; else t->from = NULL; - t->sender_euid = task_euid(proc->tsk); + t->sender_euid = proc->cred->euid; t->to_proc = target_proc; t->to_thread = target_thread; t->code = tr->code; @@ -4343,6 +4343,7 @@ static void binder_free_proc(struct binder_proc *proc) } binder_alloc_deferred_release(&proc->alloc); put_task_struct(proc->tsk); + put_cred(proc->cred); binder_stats_deleted(BINDER_STAT_PROC); kfree(proc); } @@ -5021,6 +5022,7 @@ static int binder_open(struct inode *nodp, struct file *filp) spin_lock_init(&proc->outer_lock); get_task_struct(current->group_leader); proc->tsk = current->group_leader; + proc->cred = get_cred(filp->f_cred); INIT_LIST_HEAD(&proc->todo); init_waitqueue_head(&proc->freeze_wait); proc->default_priority = task_nice(current); diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h index 810c0b84d3f8..e7d4920b3368 100644 --- a/drivers/android/binder_internal.h +++ b/drivers/android/binder_internal.h @@ -364,6 +364,9 @@ struct binder_ref { * (invariant after initialized) * @tsk task_struct for group_leader of process * (invariant after initialized) + * @cred struct cred associated with the `struct file` + * in binder_open() + * (invariant after initialized) * @deferred_work_node: element for binder_deferred_list * (protected by binder_deferred_lock) * @deferred_work: bitmap of deferred work to perform @@ -424,6 +427,7 @@ struct binder_proc { struct list_head waiting_threads; int pid; struct task_struct *tsk; + const struct cred *cred; struct hlist_node deferred_work_node; int deferred_work; int outstanding_txns; From 52f88693378a58094c538662ba652aff0253c4fe Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 12 Oct 2021 09:56:13 -0700 Subject: [PATCH 115/134] binder: use cred instead of task for selinux checks Since binder was integrated with selinux, it has passed 'struct task_struct' associated with the binder_proc to represent the source and target of transactions. The conversion of task to SID was then done in the hook implementations. It turns out that there are race conditions which can result in an incorrect security context being used. Fix by using the 'struct cred' saved during binder_open and pass it to the selinux subsystem. Cc: stable@vger.kernel.org # 5.14 (need backport for earlier stables) Fixes: 79af73079d75 ("Add security hooks to binder and implement the hooks for SELinux.") Suggested-by: Jann Horn Signed-off-by: Todd Kjos Acked-by: Casey Schaufler Signed-off-by: Paul Moore --- drivers/android/binder.c | 12 ++++----- include/linux/lsm_hook_defs.h | 14 +++++----- include/linux/lsm_hooks.h | 14 +++++----- include/linux/security.h | 28 ++++++++++---------- security/security.c | 14 +++++----- security/selinux/hooks.c | 48 ++++++++++------------------------- 6 files changed, 54 insertions(+), 76 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 231cff9b3b75..1571e01cfa52 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2047,7 +2047,7 @@ static int binder_translate_binder(struct flat_binder_object *fp, ret = -EINVAL; goto done; } - if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + if (security_binder_transfer_binder(proc->cred, target_proc->cred)) { ret = -EPERM; goto done; } @@ -2093,7 +2093,7 @@ static int binder_translate_handle(struct flat_binder_object *fp, proc->pid, thread->pid, fp->handle); return -EINVAL; } - if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) { + if (security_binder_transfer_binder(proc->cred, target_proc->cred)) { ret = -EPERM; goto done; } @@ -2181,7 +2181,7 @@ static int binder_translate_fd(u32 fd, binder_size_t fd_offset, ret = -EBADF; goto err_fget; } - ret = security_binder_transfer_file(proc->tsk, target_proc->tsk, file); + ret = security_binder_transfer_file(proc->cred, target_proc->cred, file); if (ret < 0) { ret = -EPERM; goto err_security; @@ -2586,8 +2586,8 @@ static void binder_transaction(struct binder_proc *proc, return_error_line = __LINE__; goto err_invalid_target_handle; } - if (security_binder_transaction(proc->tsk, - target_proc->tsk) < 0) { + if (security_binder_transaction(proc->cred, + target_proc->cred) < 0) { return_error = BR_FAILED_REPLY; return_error_param = -EPERM; return_error_line = __LINE__; @@ -4555,7 +4555,7 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp, ret = -EBUSY; goto out; } - ret = security_binder_set_context_mgr(proc->tsk); + ret = security_binder_set_context_mgr(proc->cred); if (ret < 0) goto out; if (uid_valid(context->binder_context_mgr_uid)) { diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index b3c525353769..4c7ed0268ce3 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -26,13 +26,13 @@ * #undef LSM_HOOK * }; */ -LSM_HOOK(int, 0, binder_set_context_mgr, struct task_struct *mgr) -LSM_HOOK(int, 0, binder_transaction, struct task_struct *from, - struct task_struct *to) -LSM_HOOK(int, 0, binder_transfer_binder, struct task_struct *from, - struct task_struct *to) -LSM_HOOK(int, 0, binder_transfer_file, struct task_struct *from, - struct task_struct *to, struct file *file) +LSM_HOOK(int, 0, binder_set_context_mgr, const struct cred *mgr) +LSM_HOOK(int, 0, binder_transaction, const struct cred *from, + const struct cred *to) +LSM_HOOK(int, 0, binder_transfer_binder, const struct cred *from, + const struct cred *to) +LSM_HOOK(int, 0, binder_transfer_file, const struct cred *from, + const struct cred *to, struct file *file) LSM_HOOK(int, 0, ptrace_access_check, struct task_struct *child, unsigned int mode) LSM_HOOK(int, 0, ptrace_traceme, struct task_struct *parent) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 0eb0ae95c4c4..528554e9b90c 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1313,22 +1313,22 @@ * * @binder_set_context_mgr: * Check whether @mgr is allowed to be the binder context manager. - * @mgr contains the task_struct for the task being registered. + * @mgr contains the struct cred for the current binder process. * Return 0 if permission is granted. * @binder_transaction: * Check whether @from is allowed to invoke a binder transaction call * to @to. - * @from contains the task_struct for the sending task. - * @to contains the task_struct for the receiving task. + * @from contains the struct cred for the sending process. + * @to contains the struct cred for the receiving process. * @binder_transfer_binder: * Check whether @from is allowed to transfer a binder reference to @to. - * @from contains the task_struct for the sending task. - * @to contains the task_struct for the receiving task. + * @from contains the struct cred for the sending process. + * @to contains the struct cred for the receiving process. * @binder_transfer_file: * Check whether @from is allowed to transfer @file to @to. - * @from contains the task_struct for the sending task. + * @from contains the struct cred for the sending process. * @file contains the struct file being transferred. - * @to contains the task_struct for the receiving task. + * @to contains the struct cred for the receiving process. * * @ptrace_access_check: * Check permission before allowing the current process to trace the diff --git a/include/linux/security.h b/include/linux/security.h index 7979b9629a42..9be72166e859 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -258,13 +258,13 @@ extern int security_init(void); extern int early_security_init(void); /* Security operations */ -int security_binder_set_context_mgr(struct task_struct *mgr); -int security_binder_transaction(struct task_struct *from, - struct task_struct *to); -int security_binder_transfer_binder(struct task_struct *from, - struct task_struct *to); -int security_binder_transfer_file(struct task_struct *from, - struct task_struct *to, struct file *file); +int security_binder_set_context_mgr(const struct cred *mgr); +int security_binder_transaction(const struct cred *from, + const struct cred *to); +int security_binder_transfer_binder(const struct cred *from, + const struct cred *to); +int security_binder_transfer_file(const struct cred *from, + const struct cred *to, struct file *file); int security_ptrace_access_check(struct task_struct *child, unsigned int mode); int security_ptrace_traceme(struct task_struct *parent); int security_capget(struct task_struct *target, @@ -508,25 +508,25 @@ static inline int early_security_init(void) return 0; } -static inline int security_binder_set_context_mgr(struct task_struct *mgr) +static inline int security_binder_set_context_mgr(const struct cred *mgr) { return 0; } -static inline int security_binder_transaction(struct task_struct *from, - struct task_struct *to) +static inline int security_binder_transaction(const struct cred *from, + const struct cred *to) { return 0; } -static inline int security_binder_transfer_binder(struct task_struct *from, - struct task_struct *to) +static inline int security_binder_transfer_binder(const struct cred *from, + const struct cred *to) { return 0; } -static inline int security_binder_transfer_file(struct task_struct *from, - struct task_struct *to, +static inline int security_binder_transfer_file(const struct cred *from, + const struct cred *to, struct file *file) { return 0; diff --git a/security/security.c b/security/security.c index 40518c340571..d9d53c1e466a 100644 --- a/security/security.c +++ b/security/security.c @@ -747,25 +747,25 @@ static int lsm_superblock_alloc(struct super_block *sb) /* Security operations */ -int security_binder_set_context_mgr(struct task_struct *mgr) +int security_binder_set_context_mgr(const struct cred *mgr) { return call_int_hook(binder_set_context_mgr, 0, mgr); } -int security_binder_transaction(struct task_struct *from, - struct task_struct *to) +int security_binder_transaction(const struct cred *from, + const struct cred *to) { return call_int_hook(binder_transaction, 0, from, to); } -int security_binder_transfer_binder(struct task_struct *from, - struct task_struct *to) +int security_binder_transfer_binder(const struct cred *from, + const struct cred *to) { return call_int_hook(binder_transfer_binder, 0, from, to); } -int security_binder_transfer_file(struct task_struct *from, - struct task_struct *to, struct file *file) +int security_binder_transfer_file(const struct cred *from, + const struct cred *to, struct file *file) { return call_int_hook(binder_transfer_file, 0, from, to, file); } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 49d52d7a7714..b4a1bde20261 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -255,29 +255,6 @@ static inline u32 task_sid_obj(const struct task_struct *task) return sid; } -/* - * get the security ID of a task for use with binder - */ -static inline u32 task_sid_binder(const struct task_struct *task) -{ - /* - * In many case where this function is used we should be using the - * task's subjective SID, but we can't reliably access the subjective - * creds of a task other than our own so we must use the objective - * creds/SID, which are safe to access. The downside is that if a task - * is temporarily overriding it's creds it will not be reflected here; - * however, it isn't clear that binder would handle that case well - * anyway. - * - * If this ever changes and we can safely reference the subjective - * creds/SID of another task, this function will make it easier to - * identify the various places where we make use of the task SIDs in - * the binder code. It is also likely that we will need to adjust - * the main drivers/android binder code as well. - */ - return task_sid_obj(task); -} - static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dentry); /* @@ -2067,18 +2044,19 @@ static inline u32 open_file_to_av(struct file *file) /* Hook functions begin here. */ -static int selinux_binder_set_context_mgr(struct task_struct *mgr) +static int selinux_binder_set_context_mgr(const struct cred *mgr) { return avc_has_perm(&selinux_state, - current_sid(), task_sid_binder(mgr), SECCLASS_BINDER, + current_sid(), cred_sid(mgr), SECCLASS_BINDER, BINDER__SET_CONTEXT_MGR, NULL); } -static int selinux_binder_transaction(struct task_struct *from, - struct task_struct *to) +static int selinux_binder_transaction(const struct cred *from, + const struct cred *to) { u32 mysid = current_sid(); - u32 fromsid = task_sid_binder(from); + u32 fromsid = cred_sid(from); + u32 tosid = cred_sid(to); int rc; if (mysid != fromsid) { @@ -2089,24 +2067,24 @@ static int selinux_binder_transaction(struct task_struct *from, return rc; } - return avc_has_perm(&selinux_state, fromsid, task_sid_binder(to), + return avc_has_perm(&selinux_state, fromsid, tosid, SECCLASS_BINDER, BINDER__CALL, NULL); } -static int selinux_binder_transfer_binder(struct task_struct *from, - struct task_struct *to) +static int selinux_binder_transfer_binder(const struct cred *from, + const struct cred *to) { return avc_has_perm(&selinux_state, - task_sid_binder(from), task_sid_binder(to), + cred_sid(from), cred_sid(to), SECCLASS_BINDER, BINDER__TRANSFER, NULL); } -static int selinux_binder_transfer_file(struct task_struct *from, - struct task_struct *to, +static int selinux_binder_transfer_file(const struct cred *from, + const struct cred *to, struct file *file) { - u32 sid = task_sid_binder(to); + u32 sid = cred_sid(to); struct file_security_struct *fsec = selinux_file(file); struct dentry *dentry = file->f_path.dentry; struct inode_security_struct *isec; From 4d5b5539742d2554591751b4248b0204d20dcc9d Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Tue, 12 Oct 2021 09:56:14 -0700 Subject: [PATCH 116/134] binder: use cred instead of task for getsecid Use the 'struct cred' saved at binder_open() to lookup the security ID via security_cred_getsecid(). This ensures that the security context that opened binder is the one used to generate the secctx. Cc: stable@vger.kernel.org # 5.4+ Fixes: ec74136ded79 ("binder: create node flag to request sender's security context") Signed-off-by: Todd Kjos Suggested-by: Stephen Smalley Reported-by: kernel test robot Acked-by: Casey Schaufler Signed-off-by: Paul Moore --- drivers/android/binder.c | 11 +---------- include/linux/security.h | 5 +++++ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 1571e01cfa52..49b08c04fa09 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2713,16 +2713,7 @@ static void binder_transaction(struct binder_proc *proc, u32 secid; size_t added_size; - /* - * Arguably this should be the task's subjective LSM secid but - * we can't reliably access the subjective creds of a task - * other than our own so we must use the objective creds, which - * are safe to access. The downside is that if a task is - * temporarily overriding it's creds it will not be reflected - * here; however, it isn't clear that binder would handle that - * case well anyway. - */ - security_task_getsecid_obj(proc->tsk, &secid); + security_cred_getsecid(proc->cred, &secid); ret = security_secid_to_secctx(secid, &secctx, &secctx_sz); if (ret) { return_error = BR_FAILED_REPLY; diff --git a/include/linux/security.h b/include/linux/security.h index 9be72166e859..cc6d39358336 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1041,6 +1041,11 @@ static inline void security_transfer_creds(struct cred *new, { } +static inline void security_cred_getsecid(const struct cred *c, u32 *secid) +{ + *secid = 0; +} + static inline int security_kernel_act_as(struct cred *cred, u32 secid) { return 0; From b97c2b219b56d30e7f6c482a246c191df016e502 Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Tue, 5 Oct 2021 12:51:31 -0700 Subject: [PATCH 117/134] crypto: ccp - Fix whitespace in sev_cmd_buffer_len() Extra tab in sev_cmd_buffer_len(). Signed-off-by: Peter Gonda Cc: Tom Lendacky Cc: Brijesh Singh Cc: Marc Orr Cc: Joerg Roedel Cc: Herbert Xu Cc: David Rientjes Cc: John Allen Cc: "David S. Miller" Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Marc Orr Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sev-dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 2ecb0e1f65d8..e09925d86bf3 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -134,7 +134,7 @@ static int sev_cmd_buffer_len(int cmd) case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware); case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id); case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report); - case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel); + case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel); default: return 0; } From 06f6e365e2ecf799c249bb464aa9d5f055e88b56 Mon Sep 17 00:00:00 2001 From: Ovidiu Panait Date: Sun, 10 Oct 2021 19:36:42 +0300 Subject: [PATCH 118/134] crypto: octeontx2 - set assoclen in aead_do_fallback() Currently, in case of aead fallback, no associated data info is set in the fallback request. To fix this, call aead_request_set_ad() to pass the assoclen. Fixes: 6f03f0e8b6c8 ("crypto: octeontx2 - register with linux crypto framework") Signed-off-by: Ovidiu Panait Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c index a72723455df7..877a948469bd 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -1274,6 +1274,7 @@ static int aead_do_fallback(struct aead_request *req, bool is_enc) req->base.complete, req->base.data); aead_request_set_crypt(&rctx->fbk_req, req->src, req->dst, req->cryptlen, req->iv); + aead_request_set_ad(&rctx->fbk_req, req->assoclen); ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : crypto_aead_decrypt(&rctx->fbk_req); } else { From d9516f346e8b8e9c7dd37976a06a5bde1a871d6f Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sat, 16 Oct 2021 15:23:51 +0800 Subject: [PATCH 119/134] audit: return early if the filter rule has a lower priority It is not necessary for audit_filter_rules() functions to check audit fileds of the rule with a lower priority, and if we did, there might be some unintended effects, such as the ctx->ppid may be changed unexpectedly, so return early if the rule has a lower priority. Signed-off-by: Gaosheng Cui [PM: slight tweak to the subject line] Signed-off-by: Paul Moore --- kernel/auditsc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index a4ba53f5354e..6efb0bb909d0 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -470,6 +470,9 @@ static int audit_filter_rules(struct task_struct *tsk, u32 sid; unsigned int sessionid; + if (ctx && rule->prio <= ctx->prio) + return 0; + cred = rcu_dereference_check(tsk->cred, tsk == current || task_creation); for (i = 0; i < rule->field_count; i++) { @@ -737,8 +740,6 @@ static int audit_filter_rules(struct task_struct *tsk, } if (ctx) { - if (rule->prio <= ctx->prio) - return 0; if (rule->filterkey) { kfree(ctx->filterkey); ctx->filterkey = kstrdup(rule->filterkey, GFP_ATOMIC); From 1c73213ba991d26a91282e775d1f5a60e41e5184 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 19 Oct 2021 12:19:44 -0400 Subject: [PATCH 120/134] selinux: fix a sock regression in selinux_ip_postroute_compat() Unfortunately we can't rely on nf_hook_state->sk being the proper originating socket so revert to using skb_to_full_sk(skb). Fixes: 1d1e1ded1356 ("selinux: make better use of the nf_hook_state passed to the NF hooks") Reported-by: Linux Kernel Functional Testing Suggested-by: Florian Westphal Signed-off-by: Paul Moore --- security/selinux/hooks.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index b4a1bde20261..6f08cd2fc6a8 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -5778,9 +5778,9 @@ static unsigned int selinux_ip_postroute_compat(struct sk_buff *skb, struct lsm_network_audit net = {0,}; u8 proto; - if (state->sk == NULL) - return NF_ACCEPT; sk = skb_to_full_sk(skb); + if (sk == NULL) + return NF_ACCEPT; sksec = sk->sk_security; ad.type = LSM_AUDIT_DATA_NET; From 15bf32398ad488c0df1cbaf16431422c87e4feea Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Tue, 12 Oct 2021 09:23:07 -0400 Subject: [PATCH 121/134] security: Return xattr name from security_dentry_init_security() Right now security_dentry_init_security() only supports single security label and is used by SELinux only. There are two users of this hook, namely ceph and nfs. NFS does not care about xattr name. Ceph hardcodes the xattr name to security.selinux (XATTR_NAME_SELINUX). I am making changes to fuse/virtiofs to send security label to virtiofsd and I need to send xattr name as well. I also hardcoded the name of xattr to security.selinux. Stephen Smalley suggested that it probably is a good idea to modify security_dentry_init_security() to also return name of xattr so that we can avoid this hardcoding in the callers. This patch adds a new parameter "const char **xattr_name" to security_dentry_init_security() and LSM puts the name of xattr too if caller asked for it (xattr_name != NULL). Signed-off-by: Vivek Goyal Reviewed-by: Jeff Layton Reviewed-by: Christian Brauner Acked-by: James Morris [PM: fixed typos in the commit description] Signed-off-by: Paul Moore --- fs/ceph/xattr.c | 3 +-- fs/nfs/nfs4proc.c | 3 ++- include/linux/lsm_hook_defs.h | 3 ++- include/linux/lsm_hooks.h | 3 +++ include/linux/security.h | 6 ++++-- security/security.c | 7 ++++--- security/selinux/hooks.c | 6 +++++- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 159a1ffa4f4b..fcf7dfdecf96 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -1311,7 +1311,7 @@ int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, int err; err = security_dentry_init_security(dentry, mode, &dentry->d_name, - &as_ctx->sec_ctx, + &name, &as_ctx->sec_ctx, &as_ctx->sec_ctxlen); if (err < 0) { WARN_ON_ONCE(err != -EOPNOTSUPP); @@ -1335,7 +1335,6 @@ int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, * It only supports single security module and only selinux has * dentry_init_security hook. */ - name = XATTR_NAME_SELINUX; name_len = strlen(name); err = ceph_pagelist_reserve(pagelist, 4 * 2 + name_len + as_ctx->sec_ctxlen); diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index e1214bb6b7ee..459860aa8fd7 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -127,7 +127,8 @@ nfs4_label_init_security(struct inode *dir, struct dentry *dentry, return NULL; err = security_dentry_init_security(dentry, sattr->ia_mode, - &dentry->d_name, (void **)&label->label, &label->len); + &dentry->d_name, NULL, + (void **)&label->label, &label->len); if (err == 0) return label; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 4c7ed0268ce3..a9ac70ae01ab 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -83,7 +83,8 @@ LSM_HOOK(int, 0, sb_add_mnt_opt, const char *option, const char *val, LSM_HOOK(int, 0, move_mount, const struct path *from_path, const struct path *to_path) LSM_HOOK(int, 0, dentry_init_security, struct dentry *dentry, - int mode, const struct qstr *name, void **ctx, u32 *ctxlen) + int mode, const struct qstr *name, const char **xattr_name, + void **ctx, u32 *ctxlen) LSM_HOOK(int, 0, dentry_create_files_as, struct dentry *dentry, int mode, struct qstr *name, const struct cred *old, struct cred *new) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 528554e9b90c..0bada4df23fc 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -196,6 +196,9 @@ * @dentry dentry to use in calculating the context. * @mode mode used to determine resource type. * @name name of the last path component used to create file + * @xattr_name pointer to place the pointer to security xattr name. + * Caller does not have to free the resulting pointer. Its + * a pointer to static string. * @ctx pointer to place the pointer to the resulting context in. * @ctxlen point to place the length of the resulting context. * @dentry_create_files_as: diff --git a/include/linux/security.h b/include/linux/security.h index cc6d39358336..7e0ba63b5dde 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -317,8 +317,9 @@ int security_add_mnt_opt(const char *option, const char *val, int len, void **mnt_opts); int security_move_mount(const struct path *from_path, const struct path *to_path); int security_dentry_init_security(struct dentry *dentry, int mode, - const struct qstr *name, void **ctx, - u32 *ctxlen); + const struct qstr *name, + const char **xattr_name, void **ctx, + u32 *ctxlen); int security_dentry_create_files_as(struct dentry *dentry, int mode, struct qstr *name, const struct cred *old, @@ -739,6 +740,7 @@ static inline void security_inode_free(struct inode *inode) static inline int security_dentry_init_security(struct dentry *dentry, int mode, const struct qstr *name, + const char **xattr_name, void **ctx, u32 *ctxlen) { diff --git a/security/security.c b/security/security.c index d9d53c1e466a..95e30fadba78 100644 --- a/security/security.c +++ b/security/security.c @@ -1052,11 +1052,12 @@ void security_inode_free(struct inode *inode) } int security_dentry_init_security(struct dentry *dentry, int mode, - const struct qstr *name, void **ctx, - u32 *ctxlen) + const struct qstr *name, + const char **xattr_name, void **ctx, + u32 *ctxlen) { return call_int_hook(dentry_init_security, -EOPNOTSUPP, dentry, mode, - name, ctx, ctxlen); + name, xattr_name, ctx, ctxlen); } EXPORT_SYMBOL(security_dentry_init_security); diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 6f08cd2fc6a8..1af2fbc08588 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -2927,7 +2927,8 @@ static void selinux_inode_free_security(struct inode *inode) } static int selinux_dentry_init_security(struct dentry *dentry, int mode, - const struct qstr *name, void **ctx, + const struct qstr *name, + const char **xattr_name, void **ctx, u32 *ctxlen) { u32 newsid; @@ -2940,6 +2941,9 @@ static int selinux_dentry_init_security(struct dentry *dentry, int mode, if (rc) return rc; + if (xattr_name) + *xattr_name = XATTR_NAME_SELINUX; + return security_sid_to_context(&selinux_state, newsid, (char **)ctx, ctxlen); } From 7e75c33756c980d0ec1cca83082fef960f1dcc1f Mon Sep 17 00:00:00 2001 From: Qing Wang Date: Tue, 12 Oct 2021 20:26:01 -0700 Subject: [PATCH 122/134] hwrng: s390 - replace snprintf in show functions with sysfs_emit coccicheck complains about the use of snprintf() in sysfs show functions. Fix the following coccicheck warning: drivers/char/hw_random/s390-trng.c:114:8-16: WARNING: use scnprintf or sprintf. drivers/char/hw_random/s390-trng.c:122:8-16: WARNING: use scnprintf or sprintf. Use sysfs_emit instead of scnprintf or sprintf makes more sense. Signed-off-by: Qing Wang Signed-off-by: Herbert Xu --- drivers/char/hw_random/s390-trng.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 7c673afd7241..2beaa35c0d74 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -111,7 +111,7 @@ static ssize_t trng_counter_show(struct device *dev, #if IS_ENABLED(CONFIG_ARCH_RANDOM) u64 arch_counter = atomic64_read(&s390_arch_random_counter); - return snprintf(buf, PAGE_SIZE, + return sysfs_emit(buf, "trng: %llu\n" "hwrng: %llu\n" "arch: %llu\n" @@ -119,7 +119,7 @@ static ssize_t trng_counter_show(struct device *dev, dev_counter, hwrng_counter, arch_counter, dev_counter + hwrng_counter + arch_counter); #else - return snprintf(buf, PAGE_SIZE, + return sysfs_emit(buf, "trng: %llu\n" "hwrng: %llu\n" "total: %llu\n", From 3ae88f676aa63366ffa9eebb8ae787c7e19f0c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Fri, 15 Oct 2021 10:39:18 +0300 Subject: [PATCH 123/134] crypto: tcrypt - fix skcipher multi-buffer tests for 1420B blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit ad6d66bcac77e ("crypto: tcrypt - include 1420 byte blocks in aead and skcipher benchmarks") mentions: > power-of-2 block size. So let's add 1420 bytes explicitly, and round > it up to the next blocksize multiple of the algo in question if it > does not support 1420 byte blocks. but misses updating skcipher multi-buffer tests. Fix this by using the proper (rounded) input size. Fixes: ad6d66bcac77e ("crypto: tcrypt - include 1420 byte blocks in aead and skcipher benchmarks") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 82b0400985a5..00149657a4bc 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1333,7 +1333,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, if (bs > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", - *b_size, XBUFSIZE * PAGE_SIZE); + bs, XBUFSIZE * PAGE_SIZE); goto out; } @@ -1386,8 +1386,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, memset(cur->xbuf[p], 0xff, k); skcipher_request_set_crypt(cur->req, cur->sg, - cur->sg, *b_size, - iv); + cur->sg, bs, iv); } if (secs) { From cad439fc040efe5f4381e3a7d583c5c200dbc186 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 19 Oct 2021 21:28:02 +0800 Subject: [PATCH 124/134] crypto: api - Do not create test larvals if manager is disabled The delayed boot-time testing patch created a dependency loop between api.c and algapi.c because it added a crypto_alg_tested call to the former when the crypto manager is disabled. We could instead avoid creating the test larvals if the crypto manager is disabled. This avoids the dependency loop as well as saving some unnecessary work, albeit in a very unlikely case. Reported-by: Nathan Chancellor Reported-by: Naresh Kamboju Reported-by: kernel test robot Fixes: adad556efcdd ("crypto: api - Fix built-in testing dependency failures") Signed-off-by: Herbert Xu --- crypto/algapi.c | 56 +++++++++++++++++++++++++++++++------------------ crypto/api.c | 7 ++----- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/crypto/algapi.c b/crypto/algapi.c index 422bdca214e1..d379fd91fb7b 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -216,6 +216,32 @@ void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, } EXPORT_SYMBOL_GPL(crypto_remove_spawns); +static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) +{ + struct crypto_larval *larval; + + if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER)) + return NULL; + + larval = crypto_larval_alloc(alg->cra_name, + alg->cra_flags | CRYPTO_ALG_TESTED, 0); + if (IS_ERR(larval)) + return larval; + + larval->adult = crypto_mod_get(alg); + if (!larval->adult) { + kfree(larval); + return ERR_PTR(-ENOENT); + } + + refcount_set(&larval->alg.cra_refcnt, 1); + memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, + CRYPTO_MAX_ALG_NAME); + larval->alg.cra_priority = alg->cra_priority; + + return larval; +} + static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) { struct crypto_alg *q; @@ -250,31 +276,20 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) goto err; } - larval = crypto_larval_alloc(alg->cra_name, - alg->cra_flags | CRYPTO_ALG_TESTED, 0); + larval = crypto_alloc_test_larval(alg); if (IS_ERR(larval)) goto out; - ret = -ENOENT; - larval->adult = crypto_mod_get(alg); - if (!larval->adult) - goto free_larval; - - refcount_set(&larval->alg.cra_refcnt, 1); - memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, - CRYPTO_MAX_ALG_NAME); - larval->alg.cra_priority = alg->cra_priority; - list_add(&alg->cra_list, &crypto_alg_list); - list_add(&larval->alg.cra_list, &crypto_alg_list); + + if (larval) + list_add(&larval->alg.cra_list, &crypto_alg_list); crypto_stats_init(alg); out: return larval; -free_larval: - kfree(larval); err: larval = ERR_PTR(ret); goto out; @@ -403,10 +418,11 @@ int crypto_register_alg(struct crypto_alg *alg) down_write(&crypto_alg_sem); larval = __crypto_register_alg(alg); test_started = static_key_enabled(&crypto_boot_test_finished); - larval->test_started = test_started; + if (!IS_ERR_OR_NULL(larval)) + larval->test_started = test_started; up_write(&crypto_alg_sem); - if (IS_ERR(larval)) + if (IS_ERR_OR_NULL(larval)) return PTR_ERR(larval); if (test_started) @@ -616,8 +632,8 @@ int crypto_register_instance(struct crypto_template *tmpl, larval = __crypto_register_alg(&inst->alg); if (IS_ERR(larval)) goto unlock; - - larval->test_started = true; + else if (larval) + larval->test_started = true; hlist_add_head(&inst->list, &tmpl->instances); inst->tmpl = tmpl; @@ -626,7 +642,7 @@ unlock: up_write(&crypto_alg_sem); err = PTR_ERR(larval); - if (IS_ERR(larval)) + if (IS_ERR_OR_NULL(larval)) goto err; crypto_wait_for_test(larval); diff --git a/crypto/api.c b/crypto/api.c index ee5991fe11f8..cf0869dd130b 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -167,11 +167,8 @@ void crypto_wait_for_test(struct crypto_larval *larval) int err; err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); - if (err != NOTIFY_STOP) { - if (WARN_ON(err != NOTIFY_DONE)) - goto out; - crypto_alg_tested(larval->alg.cra_driver_name, 0); - } + if (WARN_ON_ONCE(err != NOTIFY_STOP)) + goto out; err = wait_for_completion_killable(&larval->completion); WARN_ON(err); From 1730c5aa3b158b15af567eb3aae84f5cf6ca66f2 Mon Sep 17 00:00:00 2001 From: Prabhjot Khurana Date: Wed, 20 Oct 2021 11:35:34 +0100 Subject: [PATCH 125/134] crypto: engine - Add KPP Support to Crypto Engine Add KPP support to the crypto engine queue manager, so that it can be used to simplify the logic of KPP device drivers as done for other crypto drivers. Signed-off-by: Prabhjot Khurana Signed-off-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- Documentation/crypto/crypto_engine.rst | 4 ++++ crypto/crypto_engine.c | 26 ++++++++++++++++++++++++++ include/crypto/engine.h | 5 +++++ 3 files changed, 35 insertions(+) diff --git a/Documentation/crypto/crypto_engine.rst b/Documentation/crypto/crypto_engine.rst index 25cf9836c336..d562ea17d994 100644 --- a/Documentation/crypto/crypto_engine.rst +++ b/Documentation/crypto/crypto_engine.rst @@ -69,6 +69,8 @@ the crypto engine via one of: * crypto_transfer_hash_request_to_engine() +* crypto_transfer_kpp_request_to_engine() + * crypto_transfer_skcipher_request_to_engine() At the end of the request process, a call to one of the following functions is needed: @@ -79,4 +81,6 @@ At the end of the request process, a call to one of the following functions is n * crypto_finalize_hash_request() +* crypto_finalize_kpp_request() + * crypto_finalize_skcipher_request() diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index cff21f4e03e3..fb07da9920ee 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -327,6 +327,19 @@ int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine, } EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine); +/** + * crypto_transfer_kpp_request_to_engine - transfer one kpp_request to list + * into the engine queue + * @engine: the hardware engine + * @req: the request need to be listed into the engine queue + */ +int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine, + struct kpp_request *req) +{ + return crypto_transfer_request_to_engine(engine, &req->base); +} +EXPORT_SYMBOL_GPL(crypto_transfer_kpp_request_to_engine); + /** * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request * to list into the engine queue @@ -382,6 +395,19 @@ void crypto_finalize_hash_request(struct crypto_engine *engine, } EXPORT_SYMBOL_GPL(crypto_finalize_hash_request); +/** + * crypto_finalize_kpp_request - finalize one kpp_request if the request is done + * @engine: the hardware engine + * @req: the request need to be finalized + * @err: error number + */ +void crypto_finalize_kpp_request(struct crypto_engine *engine, + struct kpp_request *req, int err) +{ + return crypto_finalize_request(engine, &req->base, err); +} +EXPORT_SYMBOL_GPL(crypto_finalize_kpp_request); + /** * crypto_finalize_skcipher_request - finalize one skcipher_request if * the request is done diff --git a/include/crypto/engine.h b/include/crypto/engine.h index 26cac19b0f46..fd4f2fa23f51 100644 --- a/include/crypto/engine.h +++ b/include/crypto/engine.h @@ -16,6 +16,7 @@ #include #include #include +#include #define ENGINE_NAME_LEN 30 /* @@ -96,6 +97,8 @@ int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine, struct akcipher_request *req); int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine, struct ahash_request *req); +int crypto_transfer_kpp_request_to_engine(struct crypto_engine *engine, + struct kpp_request *req); int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine, struct skcipher_request *req); void crypto_finalize_aead_request(struct crypto_engine *engine, @@ -104,6 +107,8 @@ void crypto_finalize_akcipher_request(struct crypto_engine *engine, struct akcipher_request *req, int err); void crypto_finalize_hash_request(struct crypto_engine *engine, struct ahash_request *req, int err); +void crypto_finalize_kpp_request(struct crypto_engine *engine, + struct kpp_request *req, int err); void crypto_finalize_skcipher_request(struct crypto_engine *engine, struct skcipher_request *req, int err); int crypto_engine_start(struct crypto_engine *engine); From a745d3ace3fd65ada44d61dafa64a2a69679ac35 Mon Sep 17 00:00:00 2001 From: Daniele Alessandrelli Date: Wed, 20 Oct 2021 11:35:35 +0100 Subject: [PATCH 126/134] crypto: ecc - Move ecc.h to include/crypto/internal Move ecc.h header file to 'include/crypto/internal' so that it can be easily imported from everywhere in the kernel tree. This change is done to allow crypto device drivers to re-use the symbols exported by 'crypto/ecc.c', thus avoiding code duplication. Signed-off-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- crypto/ecc.c | 2 +- crypto/ecdh.c | 2 +- crypto/ecdsa.c | 2 +- crypto/ecrdsa.c | 2 +- crypto/ecrdsa_defs.h | 2 +- {crypto => include/crypto/internal}/ecc.h | 0 6 files changed, 5 insertions(+), 5 deletions(-) rename {crypto => include/crypto/internal}/ecc.h (100%) diff --git a/crypto/ecc.c b/crypto/ecc.c index afc6cefdc1d9..80efc9b4eb69 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -32,10 +32,10 @@ #include #include #include +#include #include #include -#include "ecc.h" #include "ecc_curve_defs.h" typedef struct { diff --git a/crypto/ecdh.c b/crypto/ecdh.c index c6f61c2211dc..e4857d534344 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -6,11 +6,11 @@ */ #include +#include #include #include #include #include -#include "ecc.h" struct ecdh_ctx { unsigned int curve_id; diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index 1e7b15009bf6..b3a8a6b572ba 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -5,12 +5,12 @@ #include #include +#include #include #include #include #include -#include "ecc.h" #include "ecdsasignature.asn1.h" struct ecc_ctx { diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index 6a3fd09057d0..b32ffcaad9ad 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -20,12 +20,12 @@ #include #include #include +#include #include #include #include #include "ecrdsa_params.asn1.h" #include "ecrdsa_pub_key.asn1.h" -#include "ecc.h" #include "ecrdsa_defs.h" #define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) diff --git a/crypto/ecrdsa_defs.h b/crypto/ecrdsa_defs.h index 170baf039007..0056335b9d03 100644 --- a/crypto/ecrdsa_defs.h +++ b/crypto/ecrdsa_defs.h @@ -13,7 +13,7 @@ #ifndef _CRYTO_ECRDSA_DEFS_H #define _CRYTO_ECRDSA_DEFS_H -#include "ecc.h" +#include #define ECRDSA_MAX_SIG_SIZE (2 * 512 / 8) #define ECRDSA_MAX_DIGITS (512 / 64) diff --git a/crypto/ecc.h b/include/crypto/internal/ecc.h similarity index 100% rename from crypto/ecc.h rename to include/crypto/internal/ecc.h From eaffe377e168d25c52091cf31f5a7a6511897857 Mon Sep 17 00:00:00 2001 From: Daniele Alessandrelli Date: Wed, 20 Oct 2021 11:35:36 +0100 Subject: [PATCH 127/134] crypto: ecc - Export additional helper functions Export the following additional ECC helper functions: - ecc_alloc_point() - ecc_free_point() - vli_num_bits() - ecc_point_is_zero() This is done to allow future ECC device drivers to re-use existing code, thus simplifying their implementation. Functions are exported using EXPORT_SYMBOL() (instead of EXPORT_SYMBOL_GPL()) to be consistent with the functions already exported by crypto/ecc.c. Exported functions are documented in include/crypto/internal/ecc.h. Signed-off-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- crypto/ecc.c | 12 ++++++++---- include/crypto/internal/ecc.h | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/crypto/ecc.c b/crypto/ecc.c index 80efc9b4eb69..7315217c8f73 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -81,7 +81,7 @@ static void ecc_free_digits_space(u64 *space) kfree_sensitive(space); } -static struct ecc_point *ecc_alloc_point(unsigned int ndigits) +struct ecc_point *ecc_alloc_point(unsigned int ndigits) { struct ecc_point *p = kmalloc(sizeof(*p), GFP_KERNEL); @@ -106,8 +106,9 @@ err_alloc_x: kfree(p); return NULL; } +EXPORT_SYMBOL(ecc_alloc_point); -static void ecc_free_point(struct ecc_point *p) +void ecc_free_point(struct ecc_point *p) { if (!p) return; @@ -116,6 +117,7 @@ static void ecc_free_point(struct ecc_point *p) kfree_sensitive(p->y); kfree_sensitive(p); } +EXPORT_SYMBOL(ecc_free_point); static void vli_clear(u64 *vli, unsigned int ndigits) { @@ -165,7 +167,7 @@ static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits) } /* Counts the number of bits required for vli. */ -static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) +unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) { unsigned int i, num_digits; u64 digit; @@ -180,6 +182,7 @@ static unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits) return ((num_digits - 1) * 64 + i); } +EXPORT_SYMBOL(vli_num_bits); /* Set dest from unaligned bit string src. */ void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits) @@ -1062,11 +1065,12 @@ EXPORT_SYMBOL(vli_mod_inv); /* ------ Point operations ------ */ /* Returns true if p_point is the point at infinity, false otherwise. */ -static bool ecc_point_is_zero(const struct ecc_point *point) +bool ecc_point_is_zero(const struct ecc_point *point) { return (vli_is_zero(point->x, point->ndigits) && vli_is_zero(point->y, point->ndigits)); } +EXPORT_SYMBOL(ecc_point_is_zero); /* Point multiplication algorithm using Montgomery's ladder with co-Z * coordinates. From https://eprint.iacr.org/2011/338.pdf diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h index 1350e8eb6ac2..4f6c1a68882f 100644 --- a/include/crypto/internal/ecc.h +++ b/include/crypto/internal/ecc.h @@ -225,6 +225,41 @@ void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod, void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, const u64 *mod, unsigned int ndigits); +/** + * vli_num_bits() - Counts the number of bits required for vli. + * + * @vli: vli to check. + * @ndigits: Length of the @vli + * + * Return: The number of bits required to represent @vli. + */ +unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits); + +/** + * ecc_aloc_point() - Allocate ECC point. + * + * @ndigits: Length of vlis in u64 qwords. + * + * Return: Pointer to the allocated point or NULL if allocation failed. + */ +struct ecc_point *ecc_alloc_point(unsigned int ndigits); + +/** + * ecc_free_point() - Free ECC point. + * + * @p: The point to free. + */ +void ecc_free_point(struct ecc_point *p); + +/** + * ecc_point_is_zero() - Check if point is zero. + * + * @p: Point to check for zero. + * + * Return: true if point is the point at infinity, false otherwise. + */ +bool ecc_point_is_zero(const struct ecc_point *point); + /** * ecc_point_mult_shamir() - Add two points multiplied by scalars * @@ -242,4 +277,5 @@ void ecc_point_mult_shamir(const struct ecc_point *result, const u64 *x, const struct ecc_point *p, const u64 *y, const struct ecc_point *q, const struct ecc_curve *curve); + #endif From cadddc89a0445bbd5133f4f4523e07a9ce4c6e52 Mon Sep 17 00:00:00 2001 From: Prabhjot Khurana Date: Wed, 20 Oct 2021 11:35:37 +0100 Subject: [PATCH 128/134] dt-bindings: crypto: Add Keem Bay ECC bindings Add Keem Bay Offload and Crypto Subsystem (OCS) Elliptic Curve Cryptography (ECC) device tree bindings. Signed-off-by: Prabhjot Khurana Signed-off-by: Daniele Alessandrelli Reviewed-by: Rob Herring Signed-off-by: Herbert Xu --- .../crypto/intel,keembay-ocs-ecc.yaml | 47 +++++++++++++++++++ MAINTAINERS | 7 +++ 2 files changed, 54 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml new file mode 100644 index 000000000000..a3c16451b1ad --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-ecc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay OCS ECC Device Tree Bindings + +maintainers: + - Daniele Alessandrelli + - Prabhjot Khurana + +description: + The Intel Keem Bay Offload and Crypto Subsystem (OCS) Elliptic Curve + Cryptography (ECC) device provides hardware acceleration for elliptic curve + cryptography using the NIST P-256 and NIST P-384 elliptic curves. + +properties: + compatible: + const: intel,keembay-ocs-ecc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + crypto@30001000 { + compatible = "intel,keembay-ocs-ecc"; + reg = <0x30001000 0x1000>; + interrupts = ; + clocks = <&scmi_clk 95>; + }; diff --git a/MAINTAINERS b/MAINTAINERS index eeb4c70b3d5b..c588801a7b12 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9512,6 +9512,13 @@ F: drivers/crypto/keembay/keembay-ocs-aes-core.c F: drivers/crypto/keembay/ocs-aes.c F: drivers/crypto/keembay/ocs-aes.h +INTEL KEEM BAY OCS ECC CRYPTO DRIVER +M: Daniele Alessandrelli +M: Prabhjot Khurana +M: Mark Gross +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml + INTEL KEEM BAY OCS HCU CRYPTO DRIVER M: Daniele Alessandrelli M: Declan Murphy From c9f608c38009062d7a7c8c48c7d43a328a4d9eee Mon Sep 17 00:00:00 2001 From: Prabhjot Khurana Date: Wed, 20 Oct 2021 11:35:38 +0100 Subject: [PATCH 129/134] crypto: keembay-ocs-ecc - Add Keem Bay OCS ECC Driver The Intel Keem Bay SoC can provide hardware acceleration of Elliptic Curve Cryptography (ECC) by means of its Offload and Crypto Subsystem (OCS). Add the Keem Bay OCS ECC driver which leverages such hardware capabilities to provide hardware-acceleration of ECDH-256 and ECDH-384. Signed-off-by: Prabhjot Khurana Co-developed-by: Daniele Alessandrelli Signed-off-by: Daniele Alessandrelli Signed-off-by: Herbert Xu --- MAINTAINERS | 4 + drivers/crypto/keembay/Kconfig | 19 + drivers/crypto/keembay/Makefile | 2 + drivers/crypto/keembay/keembay-ocs-ecc.c | 1017 ++++++++++++++++++++++ 4 files changed, 1042 insertions(+) create mode 100644 drivers/crypto/keembay/keembay-ocs-ecc.c diff --git a/MAINTAINERS b/MAINTAINERS index c588801a7b12..030c2142de23 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9518,6 +9518,10 @@ M: Prabhjot Khurana M: Mark Gross S: Maintained F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-ecc.c +F: drivers/crypto/keembay/ocs-ecc-curve-defs.h INTEL KEEM BAY OCS HCU CRYPTO DRIVER M: Daniele Alessandrelli diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index 00cf8f028cb9..7942b48dd55a 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -39,6 +39,25 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS Intel does not recommend use of CTS mode with AES/SM4. +config CRYPTO_DEV_KEEMBAY_OCS_ECC + tristate "Support for Intel Keem Bay OCS ECC HW acceleration" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF || COMPILE_TEST + depends on HAS_IOMEM + select CRYPTO_ECDH + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + Elliptic Curve Cryptography (ECC) hardware acceleration for use with + Crypto API. + + Provides OCS acceleration for ECDH-256 and ECDH-384. + + Say Y or M if you are compiling for the Intel Keem Bay SoC. The + module will be called keembay-ocs-ecc. + + If unsure, say N. + config CRYPTO_DEV_KEEMBAY_OCS_HCU tristate "Support for Intel Keem Bay OCS HCU HW acceleration" select CRYPTO_HASH diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile index aea03d4432c4..7c12c3c138bd 100644 --- a/drivers/crypto/keembay/Makefile +++ b/drivers/crypto/keembay/Makefile @@ -4,5 +4,7 @@ obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o + obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/keembay/keembay-ocs-ecc.c b/drivers/crypto/keembay/keembay-ocs-ecc.c new file mode 100644 index 000000000000..679e6ae295e0 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-ecc.c @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS ECC Crypto Driver. + * + * Copyright (C) 2019-2021 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#define DRV_NAME "keembay-ocs-ecc" + +#define KMB_OCS_ECC_PRIORITY 350 + +#define HW_OFFS_OCS_ECC_COMMAND 0x00000000 +#define HW_OFFS_OCS_ECC_STATUS 0x00000004 +#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080 +#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100 +#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180 +#define HW_OFFS_OCS_ECC_ISR 0x00000400 +#define HW_OFFS_OCS_ECC_IER 0x00000404 + +#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0) +#define HW_OCS_ECC_COMMAND_INS_BP BIT(0) + +#define HW_OCS_ECC_COMMAND_START_VAL BIT(0) + +#define OCS_ECC_OP_SIZE_384 BIT(8) +#define OCS_ECC_OP_SIZE_256 0 + +/* ECC Instruction : for ECC_COMMAND */ +#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP) +#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP) + +#define ECC_ENABLE_INTR 1 + +#define POLL_USEC 100 +#define TIMEOUT_USEC 10000 + +#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS +#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \ + << ECC_DIGITS_TO_BYTES_SHIFT) + +#define POW_CUBE 3 + +/** + * struct ocs_ecc_dev - ECC device context + * @list: List of device contexts + * @dev: OCS ECC device + * @base_reg: IO base address of OCS ECC + * @engine: Crypto engine for the device + * @irq_done: IRQ done completion. + * @irq: IRQ number + */ +struct ocs_ecc_dev { + struct list_head list; + struct device *dev; + void __iomem *base_reg; + struct crypto_engine *engine; + struct completion irq_done; + int irq; +}; + +/** + * struct ocs_ecc_ctx - Transformation context. + * @engine_ctx: Crypto engine ctx. + * @ecc_dev: The ECC driver associated with this context. + * @curve: The elliptic curve used by this transformation. + * @private_key: The private key. + */ +struct ocs_ecc_ctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_ecc_dev *ecc_dev; + const struct ecc_curve *curve; + u64 private_key[KMB_ECC_VLI_MAX_DIGITS]; +}; + +/* Driver data. */ +struct ocs_ecc_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +/* Global variable holding the list of OCS ECC devices (only one expected). */ +static struct ocs_ecc_drv ocs_ecc = { + .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock), +}; + +/* Get OCS ECC tfm context from kpp_request. */ +static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req) +{ + return kpp_tfm_ctx(crypto_kpp_reqtfm(req)); +} + +/* Converts number of digits to number of bytes. */ +static inline unsigned int digits_to_bytes(unsigned int n) +{ + return n << ECC_DIGITS_TO_BYTES_SHIFT; +} + +/* + * Wait for ECC idle i.e when an operation (other than write operations) + * is done. + */ +static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev) +{ + u32 value; + + return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS), + value, + !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE), + POLL_USEC, TIMEOUT_USEC); +} + +static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size) +{ + iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL, + ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); +} + +/* Direct write of u32 buffer to ECC engine with associated instruction. */ +static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev, + u32 op_size, + u32 inst, + const void *data_in, + size_t data_size) +{ + iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + /* MMIO Write src uint32 to dst. */ + memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in, + data_size); +} + +/* Start OCS ECC operation and wait for its completion. */ +static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size, + u32 inst) +{ + reinit_completion(&ecc_dev->irq_done); + + iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER); + iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND); + + return wait_for_completion_interruptible(&ecc_dev->irq_done); +} + +/** + * ocs_ecc_read_cx_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cx_out: The buffer where to store the CX value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out, + size_t byte_count) +{ + memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT, + byte_count); +} + +/** + * ocs_ecc_read_cy_out() - Read the CX data output buffer. + * @dev: The OCS ECC device to read from. + * @cy_out: The buffer where to store the CY value. Must be at least + * @byte_count byte long. + * @byte_count: The amount of data to read. + */ +static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out, + size_t byte_count) +{ + memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT, + byte_count); +} + +static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx) +{ + if (tctx->ecc_dev) + return tctx->ecc_dev; + + spin_lock(&ocs_ecc.lock); + + /* Only a single OCS device available. */ + tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev, + list); + + spin_unlock(&ocs_ecc.lock); + + return tctx->ecc_dev; +} + +/* Do point multiplication using OCS ECC HW. */ +static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev, + struct ecc_point *result, + const struct ecc_point *point, + u64 *scalar, + const struct ecc_curve *curve) +{ + u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */ + u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(curve->g.ndigits); + int rc = 0; + + /* Generate random nbytes for Simple and Differential SCA protection. */ + rc = crypto_get_default_rng(); + if (rc) + return rc; + + rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes); + crypto_put_default_rng(); + if (rc) + return rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param; Base point (Gx). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + point->x, nbytes); + + /* Write ay param; Base point (Gy). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + point->y, nbytes); + + /* + * Write the private key into DATA_IN reg. + * + * Since DATA_IN register is used to write different values during the + * computation private Key value is overwritten with + * side-channel-resistance value. + */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D, + scalar, nbytes); + + /* Write operand by/l. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L, + sca, nbytes); + memzero_explicit(sca, sizeof(sca)); + + /* Write p = curve prime(GF modulus). */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Write a = curve coefficient. */ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A, + curve->a, nbytes); + + /* Make hardware perform the multiplication. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A); + if (rc) + return rc; + + /* Read result. */ + ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes); + ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes); + + return 0; +} + +/** + * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW. + * @ecc_dev: The OCS ECC device to use. + * @scalar_out: Where to store the output scalar. + * @scalar_a: Input scalar operand 'a'. + * @scalar_b: Input scalar operand 'b' + * @curve: The curve on which the operation is performed. + * @ndigits: The size of the operands (in digits). + * @inst: The operation to perform (as an OCS ECC instruction). + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out, + const u64 *scalar_a, const u64 *scalar_b, + const struct ecc_curve *curve, + unsigned int ndigits, const u32 inst) +{ + u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ? + OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256; + size_t nbytes = digits_to_bytes(ndigits); + int rc; + + /* Wait engine to be idle before starting new operation. */ + rc = ocs_ecc_wait_idle(ecc_dev); + if (rc) + return rc; + + /* Send ecc_start pulse as well as indicating operation size. */ + ocs_ecc_cmd_start(ecc_dev, op_size); + + /* Write ax param (Base point (Gx).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX, + scalar_a, nbytes); + + /* Write ay param Base point (Gy).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY, + scalar_b, nbytes); + + /* Write p = curve prime(GF modulus).*/ + ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P, + curve->p, nbytes); + + /* Give instruction A.B or A+B to ECC engine. */ + rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst); + if (rc) + return rc; + + ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes); + + if (vli_is_zero(scalar_out, ndigits)) + return -EINVAL; + + return 0; +} + +/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */ +static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 }; + int rc; + + if (WARN_ON(pk->ndigits != curve->g.ndigits)) + return -EINVAL; + + /* Check 1: Verify key is not the zero point. */ + if (ecc_point_is_zero(pk)) + return -EINVAL; + + /* Check 2: Verify key is in the range [0, p-1]. */ + if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1) + return -EINVAL; + + if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1) + return -EINVAL; + + /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */ + + /* y^2 */ + /* Compute y^2 -> store in yy */ + rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* x^3 */ + /* Assigning w = 3, used for calculating x^3. */ + w[0] = POW_CUBE; + /* Load the next stage.*/ + rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_POW_B_MODP); + if (rc) + goto exit; + + /* Do a*x -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_MUL_B_MODP); + if (rc) + goto exit; + + /* Do ax + b == w + b; store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve, + pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* x^3 + ax + b == x^3 + w -> store in w. */ + rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits, + OCS_ECC_INST_CALC_A_ADD_B_MODP); + if (rc) + goto exit; + + /* Compare y^2 == x^3 + a·x + b. */ + rc = vli_cmp(yy, w, pk->ndigits); + if (rc) + rc = -EINVAL; + +exit: + memzero_explicit(xxx, sizeof(xxx)); + memzero_explicit(yy, sizeof(yy)); + memzero_explicit(w, sizeof(w)); + + return rc; +} + +/* SP800-56A section 5.6.2.3.3 full verification */ +static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev, + const struct ecc_curve *curve, + struct ecc_point *pk) +{ + struct ecc_point *nQ; + int rc; + + /* Checks 1 through 3 */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + return rc; + + /* Check 4: Verify that nQ is the zero point. */ + nQ = ecc_alloc_point(pk->ndigits); + if (!nQ) + return -ENOMEM; + + rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve); + if (rc) + goto exit; + + if (!ecc_point_is_zero(nQ)) + rc = -EINVAL; + +exit: + ecc_free_point(nQ); + + return rc; +} + +static int kmb_ecc_is_key_valid(const struct ecc_curve *curve, + const u64 *private_key, size_t private_key_len) +{ + size_t ndigits = curve->g.ndigits; + u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1}; + u64 res[KMB_ECC_VLI_MAX_DIGITS]; + + if (private_key_len != digits_to_bytes(ndigits)) + return -EINVAL; + + if (!private_key) + return -EINVAL; + + /* Make sure the private key is in the range [2, n-3]. */ + if (vli_cmp(one, private_key, ndigits) != -1) + return -EINVAL; + + vli_sub(res, curve->n, one, ndigits); + vli_sub(res, res, one, ndigits); + if (vli_cmp(res, private_key, ndigits) != 1) + return -EINVAL; + + return 0; +} + +/* + * ECC private keys are generated using the method of extra random bits, + * equivalent to that described in FIPS 186-4, Appendix B.4.1. + * + * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer + * than requested + * 0 <= c mod(n-1) <= n-2 and implies that + * 1 <= d <= n-1 + * + * This method generates a private key uniformly distributed in the range + * [1, n-1]. + */ +static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey) +{ + size_t nbytes = digits_to_bytes(curve->g.ndigits); + u64 priv[KMB_ECC_VLI_MAX_DIGITS]; + size_t nbits; + int rc; + + nbits = vli_num_bits(curve->n, curve->g.ndigits); + + /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */ + if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv)) + return -EINVAL; + + /* + * FIPS 186-4 recommends that the private key should be obtained from a + * RBG with a security strength equal to or greater than the security + * strength associated with N. + * + * The maximum security strength identified by NIST SP800-57pt1r4 for + * ECC is 256 (N >= 512). + * + * This condition is met by the default RNG because it selects a favored + * DRBG with a security strength of 256. + */ + if (crypto_get_default_rng()) + return -EFAULT; + + rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes); + crypto_put_default_rng(); + if (rc) + goto cleanup; + + rc = kmb_ecc_is_key_valid(curve, priv, nbytes); + if (rc) + goto cleanup; + + ecc_swap_digits(priv, privkey, curve->g.ndigits); + +cleanup: + memzero_explicit(&priv, sizeof(priv)); + + return rc; +} + +static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + struct ecdh params; + int rc = 0; + + rc = crypto_ecdh_decode_key(buf, len, ¶ms); + if (rc) + goto cleanup; + + /* Ensure key size is not bigger then expected. */ + if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) { + rc = -EINVAL; + goto cleanup; + } + + /* Auto-generate private key is not provided. */ + if (!params.key || !params.key_size) { + rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key); + goto cleanup; + } + + rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key, + params.key_size); + if (rc) + goto cleanup; + + ecc_swap_digits((const u64 *)params.key, tctx->private_key, + tctx->curve->g.ndigits); +cleanup: + memzero_explicit(¶ms, sizeof(params)); + + if (rc) + tctx->curve = NULL; + + return rc; +} + +/* Compute shared secret. */ +static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + const struct ecc_curve *curve = tctx->curve; + u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS]; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + size_t copied, nbytes, pubk_len; + struct ecc_point *pk, *result; + int rc; + + nbytes = digits_to_bytes(curve->g.ndigits); + + /* Public key is a point, thus it has two coordinates */ + pubk_len = 2 * nbytes; + + /* Copy public key from SG list to pubk_buf. */ + copied = sg_copy_to_buffer(req->src, + sg_nents_for_len(req->src, pubk_len), + pubk_buf, pubk_len); + if (copied != pubk_len) + return -EINVAL; + + /* Allocate and initialize public key point. */ + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits); + ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits); + + /* + * Check the public key for following + * Check 1: Verify key is not the zero point. + * Check 2: Verify key is in the range [1, p-1]. + * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p + */ + rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk); + if (rc) + goto exit_free_pk; + + /* Allocate point for storing computed shared secret. */ + result = ecc_alloc_point(pk->ndigits); + if (!result) { + rc = -ENOMEM; + goto exit_free_pk; + } + + /* Calculate the shared secret.*/ + rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve); + if (rc) + goto exit_free_result; + + if (ecc_point_is_zero(result)) { + rc = -EFAULT; + goto exit_free_result; + } + + /* Copy shared secret from point to buffer. */ + ecc_swap_digits(result->x, shared_secret, result->ndigits); + + /* Request might ask for less bytes than what we have. */ + nbytes = min_t(size_t, nbytes, req->dst_len); + + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, nbytes), + shared_secret, nbytes); + + if (copied != nbytes) + rc = -EINVAL; + + memzero_explicit(shared_secret, sizeof(shared_secret)); + +exit_free_result: + ecc_free_point(result); + +exit_free_pk: + ecc_free_point(pk); + + return rc; +} + +/* Compute public key. */ +static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx, + struct kpp_request *req) +{ + const struct ecc_curve *curve = tctx->curve; + u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2]; + struct ecc_point *pk; + size_t pubk_len; + size_t copied; + int rc; + + /* Public key is a point, so it has double the digits. */ + pubk_len = 2 * digits_to_bytes(curve->g.ndigits); + + pk = ecc_alloc_point(curve->g.ndigits); + if (!pk) + return -ENOMEM; + + /* Public Key(pk) = priv * G. */ + rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key, + curve); + if (rc) + goto exit; + + /* SP800-56A rev 3 5.6.2.1.3 key check */ + if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) { + rc = -EAGAIN; + goto exit; + } + + /* Copy public key from point to buffer. */ + ecc_swap_digits(pk->x, pubk_buf, pk->ndigits); + ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits); + + /* Copy public key to req->dst. */ + copied = sg_copy_from_buffer(req->dst, + sg_nents_for_len(req->dst, pubk_len), + pubk_buf, pubk_len); + + if (copied != pubk_len) + rc = -EINVAL; + +exit: + ecc_free_point(pk); + + return rc; +} + +static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct kpp_request *req = container_of(areq, struct kpp_request, base); + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev; + int rc; + + if (req->src) + rc = kmb_ecc_do_shared_secret(tctx, req); + else + rc = kmb_ecc_do_public_key(tctx, req); + + crypto_finalize_kpp_request(ecc_dev->engine, req, rc); + + return 0; +} + +static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Check the request dst is big enough to hold the public key. */ + if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits))) + return -EINVAL; + + /* 'src' is not supposed to be present when generate pubk is called. */ + if (req->src) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req); + const struct ecc_curve *curve = tctx->curve; + + /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */ + if (!tctx->curve) + return -EINVAL; + + /* Ensure dst is present. */ + if (!req->dst) + return -EINVAL; + + /* Ensure src is present. */ + if (!req->src) + return -EINVAL; + + /* + * req->src is expected to the (other-side) public key, so its length + * must be 2 * coordinate size (in bytes). + */ + if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits)) + return -EINVAL; + + return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine, + req); +} + +static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id) +{ + memset(tctx, 0, sizeof(*tctx)); + + tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx); + + if (IS_ERR(tctx->ecc_dev)) { + pr_err("Failed to find the device : %ld\n", + PTR_ERR(tctx->ecc_dev)); + return PTR_ERR(tctx->ecc_dev); + } + + tctx->curve = ecc_get_curve(curve_id); + if (!tctx->curve) + return -EOPNOTSUPP; + + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256); +} + +static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384); +} + +static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + memzero_explicit(tctx->private_key, sizeof(*tctx->private_key)); +} + +static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm); + + /* Public key is made of two coordinates, so double the digits. */ + return digits_to_bytes(tctx->curve->g.ndigits) * 2; +} + +static struct kpp_alg ocs_ecdh_p256 = { + .set_secret = kmb_ocs_ecdh_set_secret, + .generate_public_key = kmb_ocs_ecdh_generate_public_key, + .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .init = kmb_ocs_ecdh_nist_p256_init_tfm, + .exit = kmb_ocs_ecdh_exit_tfm, + .max_size = kmb_ocs_ecdh_max_size, + .base = { + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "ecdh-nist-p256-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, +}; + +static struct kpp_alg ocs_ecdh_p384 = { + .set_secret = kmb_ocs_ecdh_set_secret, + .generate_public_key = kmb_ocs_ecdh_generate_public_key, + .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret, + .init = kmb_ocs_ecdh_nist_p384_init_tfm, + .exit = kmb_ocs_ecdh_exit_tfm, + .max_size = kmb_ocs_ecdh_max_size, + .base = { + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "ecdh-nist-p384-keembay-ocs", + .cra_priority = KMB_OCS_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ocs_ecc_ctx), + }, +}; + +static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id) +{ + struct ocs_ecc_dev *ecc_dev = dev_id; + u32 status; + + /* + * Read the status register and write it back to clear the + * DONE_INT_STATUS bit. + */ + status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR); + + if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE)) + return IRQ_NONE; + + complete(&ecc_dev->irq_done); + + return IRQ_HANDLED; +} + +static int kmb_ocs_ecc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_ecc_dev *ecc_dev; + int rc; + + ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL); + if (!ecc_dev) + return -ENOMEM; + + ecc_dev->dev = dev; + + platform_set_drvdata(pdev, ecc_dev); + + INIT_LIST_HEAD(&ecc_dev->list); + init_completion(&ecc_dev->irq_done); + + /* Get base register address. */ + ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ecc_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + rc = PTR_ERR(ecc_dev->base_reg); + goto list_del; + } + + /* Get and request IRQ */ + ecc_dev->irq = platform_get_irq(pdev, 0); + if (ecc_dev->irq < 0) { + rc = ecc_dev->irq; + goto list_del; + } + + rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler, + NULL, 0, "keembay-ocs-ecc", ecc_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + goto list_del; + } + + /* Add device to the list of OCS ECC devices. */ + spin_lock(&ocs_ecc.lock); + list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list); + spin_unlock(&ocs_ecc.lock); + + /* Initialize crypto engine. */ + ecc_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!ecc_dev->engine) { + dev_err(dev, "Could not allocate crypto engine\n"); + goto list_del; + } + + rc = crypto_engine_start(ecc_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + /* Register the KPP algo. */ + rc = crypto_register_kpp(&ocs_ecdh_p256); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + rc = crypto_register_kpp(&ocs_ecdh_p384); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto ocs_ecdh_p384_error; + } + + return 0; + +ocs_ecdh_p384_error: + crypto_unregister_kpp(&ocs_ecdh_p256); + +cleanup: + crypto_engine_exit(ecc_dev->engine); + +list_del: + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + return rc; +} + +static int kmb_ocs_ecc_remove(struct platform_device *pdev) +{ + struct ocs_ecc_dev *ecc_dev; + + ecc_dev = platform_get_drvdata(pdev); + if (!ecc_dev) + return -ENODEV; + + crypto_unregister_kpp(&ocs_ecdh_p384); + crypto_unregister_kpp(&ocs_ecdh_p256); + + spin_lock(&ocs_ecc.lock); + list_del(&ecc_dev->list); + spin_unlock(&ocs_ecc.lock); + + crypto_engine_exit(ecc_dev->engine); + + return 0; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_ecc_of_match[] = { + { + .compatible = "intel,keembay-ocs-ecc", + }, + {} +}; + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_ecc_driver = { + .probe = kmb_ocs_ecc_probe, + .remove = kmb_ocs_ecc_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_ecc_of_match, + }, +}; +module_platform_driver(kmb_ocs_ecc_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs"); From a472cc0dde3eb057db71c80f102556eeced03805 Mon Sep 17 00:00:00 2001 From: Tang Bin Date: Thu, 21 Oct 2021 09:34:22 +0800 Subject: [PATCH 130/134] crypto: s5p-sss - Add error handling in s5p_aes_probe() The function s5p_aes_probe() does not perform sufficient error checking after executing platform_get_resource(), thus fix it. Fixes: c2afad6c6105 ("crypto: s5p-sss - Add HASH support for Exynos") Cc: Signed-off-by: Tang Bin Reviewed-by: Krzysztof Kozlowski Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 55aa3a71169b..7717e9e5977b 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -2171,6 +2171,8 @@ static int s5p_aes_probe(struct platform_device *pdev) variant = find_s5p_sss_version(pdev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; /* * Note: HASH and PRNG uses the same registers in secss, avoid From 284340a368a034243d304bd64e5f6923780e3708 Mon Sep 17 00:00:00 2001 From: Tang Bin Date: Thu, 21 Oct 2021 10:16:23 +0800 Subject: [PATCH 131/134] crypto: sa2ul - Use the defined variable to clean code Use the defined variable "dev" to make the code cleaner. Signed-off-by: Tang Bin Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 544d7040cfc5..bcbc38dc6ae8 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -2412,8 +2412,7 @@ static int sa_ul_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) { - dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, - ret); + dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret); pm_runtime_disable(dev); return ret; } @@ -2435,16 +2434,16 @@ static int sa_ul_probe(struct platform_device *pdev) sa_register_algos(dev_data); - ret = of_platform_populate(node, NULL, NULL, &pdev->dev); + ret = of_platform_populate(node, NULL, NULL, dev); if (ret) goto release_dma; - device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child); + device_for_each_child(dev, dev, sa_link_child); return 0; release_dma: - sa_unregister_algos(&pdev->dev); + sa_unregister_algos(dev); dma_release_channel(dev_data->dma_rx2); dma_release_channel(dev_data->dma_rx1); @@ -2453,8 +2452,8 @@ release_dma: destroy_dma_pool: dma_pool_destroy(dev_data->sc_pool); - pm_runtime_put_sync(&pdev->dev); - pm_runtime_disable(&pdev->dev); + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); return ret; } From 83bff109616433d4cfd999e14f1ffc4759c3c1e0 Mon Sep 17 00:00:00 2001 From: Cai Huoqing Date: Thu, 21 Oct 2021 16:38:54 +0800 Subject: [PATCH 132/134] crypto: ccp - Make use of the helper macro kthread_run() Repalce kthread_create/wake_up_process() with kthread_run() to simplify the code. Signed-off-by: Cai Huoqing Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v3.c | 5 ++--- drivers/crypto/ccp/ccp-dev-v5.c | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index 0d5576f6ad21..fe69053b2394 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -467,8 +467,8 @@ static int ccp_init(struct ccp_device *ccp) cmd_q = &ccp->cmd_q[i]; - kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, - "%s-q%u", ccp->name, cmd_q->id); + kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, + "%s-q%u", ccp->name, cmd_q->id); if (IS_ERR(kthread)) { dev_err(dev, "error creating queue thread (%ld)\n", PTR_ERR(kthread)); @@ -477,7 +477,6 @@ static int ccp_init(struct ccp_device *ccp) } cmd_q->kthread = kthread; - wake_up_process(kthread); } dev_dbg(dev, "Enabling interrupts...\n"); diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index 7838f63bab32..7b73332d6aa1 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -950,8 +950,8 @@ static int ccp5_init(struct ccp_device *ccp) cmd_q = &ccp->cmd_q[i]; - kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, - "%s-q%u", ccp->name, cmd_q->id); + kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, + "%s-q%u", ccp->name, cmd_q->id); if (IS_ERR(kthread)) { dev_err(dev, "error creating queue thread (%ld)\n", PTR_ERR(kthread)); @@ -960,7 +960,6 @@ static int ccp5_init(struct ccp_device *ccp) } cmd_q->kthread = kthread; - wake_up_process(kthread); } dev_dbg(dev, "Enabling interrupts...\n"); From 68b6dea802cea0dbdd8bd7ccc60716b5a32a5d8a Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 21 Oct 2021 14:30:28 -0400 Subject: [PATCH 133/134] crypto: pcrypt - Delay write to padata->info These three events can race when pcrypt is used multiple times in a template ("pcrypt(pcrypt(...))"): 1. [taskA] The caller makes the crypto request via crypto_aead_encrypt() 2. [kworkerB] padata serializes the inner pcrypt request 3. [kworkerC] padata serializes the outer pcrypt request 3 might finish before the call to crypto_aead_encrypt() returns in 1, resulting in two possible issues. First, a use-after-free of the crypto request's memory when, for example, taskA writes to the outer pcrypt request's padata->info in pcrypt_aead_enc() after kworkerC completes the request. Second, the outer pcrypt request overwrites the inner pcrypt request's return code with -EINPROGRESS, making a successful request appear to fail. For instance, kworkerB writes the outer pcrypt request's padata->info in pcrypt_aead_done() and then taskA overwrites it in pcrypt_aead_enc(). Avoid both situations by delaying the write of padata->info until after the inner crypto request's return code is checked. This prevents the use-after-free by not touching the crypto request's memory after the next-inner crypto request is made, and stops padata->info from being overwritten. Fixes: 5068c7a883d16 ("crypto: pcrypt - Add pcrypt crypto parallelization wrapper") Reported-by: syzbot+b187b77c8474f9648fae@syzkaller.appspotmail.com Signed-off-by: Daniel Jordan Signed-off-by: Herbert Xu --- crypto/pcrypt.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index d569c7ed6c80..9d10b846ccf7 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -78,12 +78,14 @@ static void pcrypt_aead_enc(struct padata_priv *padata) { struct pcrypt_request *preq = pcrypt_padata_request(padata); struct aead_request *req = pcrypt_request_ctx(preq); + int ret; - padata->info = crypto_aead_encrypt(req); + ret = crypto_aead_encrypt(req); - if (padata->info == -EINPROGRESS) + if (ret == -EINPROGRESS) return; + padata->info = ret; padata_do_serial(padata); } @@ -123,12 +125,14 @@ static void pcrypt_aead_dec(struct padata_priv *padata) { struct pcrypt_request *preq = pcrypt_padata_request(padata); struct aead_request *req = pcrypt_request_ctx(preq); + int ret; - padata->info = crypto_aead_decrypt(req); + ret = crypto_aead_decrypt(req); - if (padata->info == -EINPROGRESS) + if (ret == -EINPROGRESS) return; + padata->info = ret; padata_do_serial(padata); } From 39ef08517082a424b5b65c3dbaa6c0fa9d3303b9 Mon Sep 17 00:00:00 2001 From: Lei He Date: Fri, 22 Oct 2021 20:44:43 +0800 Subject: [PATCH 134/134] crypto: testmgr - fix wrong key length for pkcs1pad Fix wrong test data at testmgr.h, it seems to be caused by ignoring the last '\0' when calling sizeof. Signed-off-by: Lei He Signed-off-by: Herbert Xu --- crypto/testmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/testmgr.h b/crypto/testmgr.h index e6fca34b5b25..779720bf9364 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1201,7 +1201,7 @@ static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { "\xd1\x86\x48\x55\xce\x83\xee\x8e\x51\xc7\xde\x32\x12\x47\x7d\x46" "\xb8\x35\xdf\x41\x02\x01\x00\x02\x01\x00\x02\x01\x00\x02\x01\x00" "\x02\x01\x00", - .key_len = 804, + .key_len = 803, /* * m is SHA256 hash of following message: * "\x49\x41\xbe\x0a\x0c\xc9\xf6\x35\x51\xe4\x27\x56\x13\x71\x4b\xd0"