From a70dd213296b2f2524c2b4440cb60d46f85ab2e4 Mon Sep 17 00:00:00 2001 From: Connor O'Brien Date: Wed, 8 Nov 2023 07:00:23 +0000 Subject: [PATCH] ANDROID: sched: Add find_exec_ctx helper Add a helper to find the runnable owner down a chain of blocked waiters This patch was broken out from a larger chain migration patch originally by Connor O'Brien. Cc: Joel Fernandes Cc: Qais Yousef Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Juri Lelli Cc: Vincent Guittot Cc: Dietmar Eggemann Cc: Valentin Schneider Cc: Steven Rostedt Cc: Ben Segall Cc: Zimuzo Ezeozue Cc: Mel Gorman Cc: Will Deacon Cc: Waiman Long Cc: Boqun Feng Cc: "Paul E. McKenney" Cc: Metin Kaya Cc: Xuewen Yan Cc: K Prateek Nayak Cc: Thomas Gleixner Cc: Daniel Lezcano Cc: kernel-team@android.com Change-Id: I0d490e9360b1e3e12498c7d05cc7f335e9c2c296 Signed-off-by: Connor O'Brien [jstultz: split out from larger chain migration patch] Signed-off-by: John Stultz Bug: 306081722 --- v8 * Minor formatting changes suggested by Metin Kaya * Fix for null owner case, uncovered by enabling optimistic spinning * Add null exec_ctx check to this patch from an earlier patch, as suggested by Metin v10: * Cleanup suggested by Metin to go directly to mutex owner * Simplify logic to task_queued_on_rq suggested by Metin --- kernel/sched/core.c | 43 +++++++++++++++++++++++++++++++++++++++++ kernel/sched/deadline.c | 16 ++++++++++++--- kernel/sched/rt.c | 11 +++++++++-- kernel/sched/sched.h | 12 +++++++++++- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8a98ae4421fe..405a2f032efb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3952,6 +3952,49 @@ static void activate_blocked_waiters(struct rq *target_rq, raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); } } + +/* XXX: Add detailed comment! */ +static inline bool task_queued_on_rq(struct rq *rq, struct task_struct *task) +{ + if (!task_on_rq_queued(task)) + return false; + + smp_rmb(); + + if (task_rq(task) != rq) + return false; + + smp_rmb(); + + return task_on_rq_queued(task); +} + +/* + * Returns the unblocked task at the end of the blocked chain starting with p + * if that chain is composed entirely of tasks enqueued on rq, or NULL otherwise. + */ +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p) +{ + struct task_struct *exec_ctx, *owner; + + if (!sched_proxy_exec()) + return p; + + lockdep_assert_rq_held(rq); + + for (exec_ctx = p; task_is_blocked(exec_ctx) && !task_on_cpu(rq, exec_ctx); + exec_ctx = owner) { + owner = __mutex_owner(exec_ctx->blocked_on); + if (!owner || owner == exec_ctx) + break; + + if (!task_queued_on_rq(rq, owner) || task_current_donor(rq, owner)) { + exec_ctx = NULL; + break; + } + } + return exec_ctx; +} #else /* !CONFIG_SCHED_PROXY_EXEC */ static inline void proxy_remove_from_sleeping_owner(struct task_struct *p) { diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 6799122eb765..0ad99fd124e7 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2298,6 +2298,8 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) { + struct task_struct *exec_ctx; + /* * Current can't be migrated, useless to reschedule, * let's hope p can move out. @@ -2306,12 +2308,16 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) !cpudl_find(&rq->rd->cpudl, rq->donor, rq->curr, NULL)) return; + exec_ctx = find_exec_ctx(rq, p); + if (task_current(rq, exec_ctx)) + return; + /* * p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. */ if (p->nr_cpus_allowed != 1 && - cpudl_find(&rq->rd->cpudl, p, p, NULL)) + cpudl_find(&rq->rd->cpudl, p, exec_ctx, NULL)) return; resched_curr(rq); @@ -2532,7 +2538,7 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec if (unlikely(!later_mask)) return -1; - if (exec_ctx->nr_cpus_allowed == 1) + if (exec_ctx && exec_ctx->nr_cpus_allowed == 1) return -1; /* @@ -2611,12 +2617,16 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec /* Locks the rq it finds */ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) { + struct task_struct *exec_ctx; struct rq *later_rq = NULL; int tries; int cpu; for (tries = 0; tries < DL_MAX_TRIES; tries++) { - cpu = find_later_rq(task, task); + exec_ctx = find_exec_ctx(rq, task); + if (!exec_ctx) + break; + cpu = find_later_rq(task, exec_ctx); if ((cpu == -1) || (cpu == rq->cpu)) break; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index c6635e150161..3d590e518e41 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1709,6 +1709,11 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) !cpupri_find(&rq->rd->cpupri, rq->donor, rq->curr, NULL)) return; + /* No reason to preempt since rq->curr wouldn't change anyway */ + exec_ctx = find_exec_ctx(rq, p); + if (task_current(rq, exec_ctx)) + return; + /* * p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. @@ -1911,7 +1916,7 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe if (unlikely(!lowest_mask)) return -1; - if (exec_ctx->nr_cpus_allowed == 1) + if (exec_ctx && exec_ctx->nr_cpus_allowed == 1) return -1; /* No other targets possible */ /* @@ -2002,12 +2007,14 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe /* Will lock the rq it finds */ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) { + struct task_struct *exec_ctx; struct rq *lowest_rq = NULL; int tries; int cpu; for (tries = 0; tries < RT_MAX_TRIES; tries++) { - cpu = find_lowest_rq(task, task); + exec_ctx = find_exec_ctx(rq, task); + cpu = find_lowest_rq(task, exec_ctx); if ((cpu == -1) || (cpu == rq->cpu)) break; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 63546055f54a..5521f3c46440 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3888,7 +3888,17 @@ bool task_is_pushable(struct rq *rq, struct task_struct *p, int cpu) return false; } -#endif + +#ifdef CONFIG_SCHED_PROXY_EXEC +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p); +#else /* !CONFIG_SCHED_PROXY_EXEC */ +static inline +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p) +{ + return p; +} +#endif /* CONFIG_SCHED_PROXY_EXEC */ +#endif /* CONFIG_SMP */ #ifdef CONFIG_RT_MUTEXES