diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8a98ae4421fe..405a2f032efb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3952,6 +3952,49 @@ static void activate_blocked_waiters(struct rq *target_rq, raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); } } + +/* XXX: Add detailed comment! */ +static inline bool task_queued_on_rq(struct rq *rq, struct task_struct *task) +{ + if (!task_on_rq_queued(task)) + return false; + + smp_rmb(); + + if (task_rq(task) != rq) + return false; + + smp_rmb(); + + return task_on_rq_queued(task); +} + +/* + * Returns the unblocked task at the end of the blocked chain starting with p + * if that chain is composed entirely of tasks enqueued on rq, or NULL otherwise. + */ +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p) +{ + struct task_struct *exec_ctx, *owner; + + if (!sched_proxy_exec()) + return p; + + lockdep_assert_rq_held(rq); + + for (exec_ctx = p; task_is_blocked(exec_ctx) && !task_on_cpu(rq, exec_ctx); + exec_ctx = owner) { + owner = __mutex_owner(exec_ctx->blocked_on); + if (!owner || owner == exec_ctx) + break; + + if (!task_queued_on_rq(rq, owner) || task_current_donor(rq, owner)) { + exec_ctx = NULL; + break; + } + } + return exec_ctx; +} #else /* !CONFIG_SCHED_PROXY_EXEC */ static inline void proxy_remove_from_sleeping_owner(struct task_struct *p) { diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 6799122eb765..0ad99fd124e7 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2298,6 +2298,8 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) { + struct task_struct *exec_ctx; + /* * Current can't be migrated, useless to reschedule, * let's hope p can move out. @@ -2306,12 +2308,16 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) !cpudl_find(&rq->rd->cpudl, rq->donor, rq->curr, NULL)) return; + exec_ctx = find_exec_ctx(rq, p); + if (task_current(rq, exec_ctx)) + return; + /* * p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. */ if (p->nr_cpus_allowed != 1 && - cpudl_find(&rq->rd->cpudl, p, p, NULL)) + cpudl_find(&rq->rd->cpudl, p, exec_ctx, NULL)) return; resched_curr(rq); @@ -2532,7 +2538,7 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec if (unlikely(!later_mask)) return -1; - if (exec_ctx->nr_cpus_allowed == 1) + if (exec_ctx && exec_ctx->nr_cpus_allowed == 1) return -1; /* @@ -2611,12 +2617,16 @@ static int find_later_rq(struct task_struct *sched_ctx, struct task_struct *exec /* Locks the rq it finds */ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) { + struct task_struct *exec_ctx; struct rq *later_rq = NULL; int tries; int cpu; for (tries = 0; tries < DL_MAX_TRIES; tries++) { - cpu = find_later_rq(task, task); + exec_ctx = find_exec_ctx(rq, task); + if (!exec_ctx) + break; + cpu = find_later_rq(task, exec_ctx); if ((cpu == -1) || (cpu == rq->cpu)) break; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index c6635e150161..3d590e518e41 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1709,6 +1709,11 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) !cpupri_find(&rq->rd->cpupri, rq->donor, rq->curr, NULL)) return; + /* No reason to preempt since rq->curr wouldn't change anyway */ + exec_ctx = find_exec_ctx(rq, p); + if (task_current(rq, exec_ctx)) + return; + /* * p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. @@ -1911,7 +1916,7 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe if (unlikely(!lowest_mask)) return -1; - if (exec_ctx->nr_cpus_allowed == 1) + if (exec_ctx && exec_ctx->nr_cpus_allowed == 1) return -1; /* No other targets possible */ /* @@ -2002,12 +2007,14 @@ static int find_lowest_rq(struct task_struct *sched_ctx, struct task_struct *exe /* Will lock the rq it finds */ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) { + struct task_struct *exec_ctx; struct rq *lowest_rq = NULL; int tries; int cpu; for (tries = 0; tries < RT_MAX_TRIES; tries++) { - cpu = find_lowest_rq(task, task); + exec_ctx = find_exec_ctx(rq, task); + cpu = find_lowest_rq(task, exec_ctx); if ((cpu == -1) || (cpu == rq->cpu)) break; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 63546055f54a..5521f3c46440 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3888,7 +3888,17 @@ bool task_is_pushable(struct rq *rq, struct task_struct *p, int cpu) return false; } -#endif + +#ifdef CONFIG_SCHED_PROXY_EXEC +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p); +#else /* !CONFIG_SCHED_PROXY_EXEC */ +static inline +struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p) +{ + return p; +} +#endif /* CONFIG_SCHED_PROXY_EXEC */ +#endif /* CONFIG_SMP */ #ifdef CONFIG_RT_MUTEXES