diff --git a/include/linux/sched.h b/include/linux/sched.h index 0d385fffe271..5d4a425e7659 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1213,6 +1213,9 @@ struct task_struct { enum blocked_on_state blocked_on_state; struct mutex *blocked_on; /* lock we're blocked on */ struct task_struct *blocked_donor; /* task that is boosting this task */ +#ifdef CONFIG_SCHED_PROXY_EXEC + struct list_head migration_node; +#endif raw_spinlock_t blocked_lock; #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/kernel/fork.c b/kernel/fork.c index f830d2163bdc..0caff64e929e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2366,6 +2366,9 @@ __latent_entropy struct task_struct *copy_process( p->blocked_on_state = BO_RUNNABLE; p->blocked_on = NULL; /* not blocked yet */ p->blocked_donor = NULL; /* nobody is boosting p yet */ +#ifdef CONFIG_SCHED_PROXY_EXEC + INIT_LIST_HEAD(&p->migration_node); +#endif #ifdef CONFIG_BCACHE p->sequential_io = 0; p->sequential_io_avg = 0; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b739ed894278..37873a47510c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6851,6 +6851,7 @@ static bool proxy_deactivate(struct rq *rq, struct task_struct *donor) static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int target_cpu) { + LIST_HEAD(migrate_list); struct rq *target_rq; lockdep_assert_rq_held(rq); @@ -6880,19 +6881,27 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, rq_set_donor(rq, rq->curr); set_next_task(rq, rq->curr); - WARN_ON(p == rq->curr); - - deactivate_task(rq, p, 0); - proxy_set_task_cpu(p, target_cpu); - + for (; p; p = p->blocked_donor) { + WARN_ON(p == rq->curr); + deactivate_task(rq, p, 0); + proxy_set_task_cpu(p, target_cpu); + /* + * We can abuse blocked_node to migrate the thing, + * because @p was still on the rq. + */ + list_add(&p->migration_node, &migrate_list); + } zap_balance_callbacks(rq); rq_unpin_lock(rq, rf); raw_spin_rq_unlock(rq); raw_spin_rq_lock(target_rq); + while (!list_empty(&migrate_list)) { + p = list_first_entry(&migrate_list, struct task_struct, migration_node); + list_del_init(&p->migration_node); - activate_task(target_rq, p, 0); - wakeup_preempt(target_rq, p, 0); - + activate_task(target_rq, p, 0); + wakeup_preempt(target_rq, p, 0); + } raw_spin_rq_unlock(target_rq); raw_spin_rq_lock(rq); rq_repin_lock(rq, rf);