diff --git a/include/linux/sched.h b/include/linux/sched.h index 5d4a425e7659..4a95575eb508 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1215,6 +1215,11 @@ struct task_struct { struct task_struct *blocked_donor; /* task that is boosting this task */ #ifdef CONFIG_SCHED_PROXY_EXEC struct list_head migration_node; + struct list_head blocked_head; /* tasks blocked on this task */ + struct list_head blocked_node; /* our entry on someone elses blocked_head */ + /* Node for list of tasks to process blocked_head list for blocked entitiy activations */ + struct list_head blocked_activation_node; + struct task_struct *sleeping_owner; /* task our blocked_node is enqueued on */ #endif raw_spinlock_t blocked_lock; diff --git a/kernel/fork.c b/kernel/fork.c index 0caff64e929e..e6f25678f367 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2368,6 +2368,10 @@ __latent_entropy struct task_struct *copy_process( p->blocked_donor = NULL; /* nobody is boosting p yet */ #ifdef CONFIG_SCHED_PROXY_EXEC INIT_LIST_HEAD(&p->migration_node); + INIT_LIST_HEAD(&p->blocked_head); + INIT_LIST_HEAD(&p->blocked_node); + INIT_LIST_HEAD(&p->blocked_activation_node); + p->sleeping_owner = NULL; #endif #ifdef CONFIG_BCACHE p->sequential_io = 0; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 37873a47510c..6d182c3cc86f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3768,6 +3768,50 @@ static inline void ttwu_do_wakeup(struct task_struct *p) } #ifdef CONFIG_SCHED_PROXY_EXEC +static inline +void __proxy_remove_from_sleeping_owner(struct task_struct *owner, struct task_struct *p) +{ + lockdep_assert_held(&owner->blocked_lock); + + if (p->sleeping_owner == owner) { + list_del_init(&p->blocked_node); + WRITE_ONCE(p->sleeping_owner, NULL); + put_task_struct(owner); // matches get in proxy_enqueue_on_owner + } +} + +static inline void proxy_remove_from_sleeping_owner(struct task_struct *p) +{ + struct task_struct *owner = READ_ONCE(p->sleeping_owner); + + if (owner) { + raw_spin_lock(&owner->blocked_lock); + __proxy_remove_from_sleeping_owner(owner, p); + raw_spin_unlock(&owner->blocked_lock); + } +} + +static void do_activate_task(struct rq *rq, struct task_struct *p, int en_flags) +{ + if (!sched_proxy_exec()) { + activate_task(rq, p, en_flags); + return; + } + + lockdep_assert_rq_held(rq); + proxy_remove_from_sleeping_owner(p); + /* + * By calling activate_task with blocked_lock held, we + * order against the find_proxy_task() blocked_task case + * such that no more blocked tasks will be enqueued on p + * once we release p->blocked_lock. + */ + raw_spin_lock(&p->blocked_lock); + WARN_ON(task_cpu(p) != cpu_of(rq)); + activate_task(rq, p, en_flags); + raw_spin_unlock(&p->blocked_lock); +} + #ifdef CONFIG_SMP static inline void proxy_set_task_cpu(struct task_struct *p, int cpu) { @@ -3791,6 +3835,138 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu) __set_task_cpu(p, cpu); } #endif /* CONFIG_SMP */ + +static void do_activate_blocked_waiter(struct rq *target_rq, struct task_struct *p, int en_flags) +{ + unsigned long flags; + unsigned int state; + struct rq_flags rf; + int target_cpu = cpu_of(target_rq); + + raw_spin_lock_irqsave(&p->pi_lock, flags); + state = READ_ONCE(p->__state); + /* Avoid racing with ttwu */ + if (state == TASK_WAKING) + goto out; + + if (READ_ONCE(p->on_rq)) { + /* + * We raced with a non mutex handoff activation of p. + * That activation will also take care of activating + * all of the tasks after p in the blocked_head list, + * so we're done here. + */ + goto out; + } + + proxy_set_task_cpu(p, target_cpu); + rq_lock_irqsave(target_rq, &rf); + update_rq_clock(target_rq); + do_activate_task(target_rq, p, en_flags); + resched_curr(target_rq); + rq_unlock_irqrestore(target_rq, &rf); +out: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); +} + +static void activate_blocked_waiters(struct rq *target_rq, + struct task_struct *owner, + int wake_flags) +{ + struct list_head bal_head; + unsigned long flags; + int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; + + if (!sched_proxy_exec()) + return; + + INIT_LIST_HEAD(&bal_head); + + if (wake_flags & WF_MIGRATED) + en_flags |= ENQUEUE_MIGRATED; + + /* + * A whole bunch of 'proxy' tasks back this blocked task, wake + * them all up to give this task its 'fair' share. + */ + /* + * This is a little unique here and the locking is messy. + * At this point we only hold the blocked_lock, so the + * owner task may be able to run and do all sorts of + * things while we are processing the blocked_head list, + * including going back to sleep, which can cause tasks + * to be added to the owners->blocked_head while we are + * processing it! + * Thus, we pull the entire list off the owner->blocked_head + * here so that we will only process a finite amount of + * tasks. Tasks added after this will be processed by the + * future wake events. + * Even though we have pulled the list off the blocked_head + * the removed list is *still* "owned" and serialized by + * the owner->blocked_lock! As we have to serialize against + * mid-chain wakeups, who may try to remove themselves from + * the list. + */ + raw_spin_lock_irqsave(&owner->blocked_lock, flags); + if (!list_empty(&owner->blocked_activation_node)) { + raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); + return; + } + get_task_struct(owner); + list_add_tail(&owner->blocked_activation_node, &bal_head); + raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); + + while (!list_empty(&bal_head)) { + struct list_head tmp_head; + + INIT_LIST_HEAD(&tmp_head); + owner = list_first_entry(&bal_head, struct task_struct, blocked_activation_node); + + raw_spin_lock_irqsave(&owner->blocked_lock, flags); + list_replace_init(&owner->blocked_head, &tmp_head); + list_del_init(&owner->blocked_activation_node); + while (!list_empty(&tmp_head)) { + struct task_struct *p; + + p = list_first_entry(&tmp_head, + struct task_struct, + blocked_node); + WARN_ON(p == owner); + WARN_ON(p->sleeping_owner != owner); + __proxy_remove_from_sleeping_owner(owner, p); + raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); + + do_activate_blocked_waiter(target_rq, p, en_flags); + + raw_spin_lock_irqsave(&p->blocked_lock, flags); + if (list_empty(&p->blocked_activation_node)) { + get_task_struct(p); + list_add_tail(&p->blocked_activation_node, &bal_head); + } + raw_spin_unlock_irqrestore(&p->blocked_lock, flags); + + raw_spin_lock_irqsave(&owner->blocked_lock, flags); + } + put_task_struct(owner); // put matches get prior to adding to local bal_head + raw_spin_unlock_irqrestore(&owner->blocked_lock, flags); + } +} +#else /* !CONFIG_SCHED_PROXY_EXEC */ +static inline void proxy_remove_from_sleeping_owner(struct task_struct *p) +{ +} + +static inline void do_activate_task(struct rq *rq, struct task_struct *p, + int en_flags) +{ + activate_task(rq, p, en_flags); +} + +static inline void activate_blocked_waiters(struct rq *target_rq, + struct task_struct *owner, + int wake_flags) +{ +} #endif /* CONFIG_SCHED_PROXY_EXEC */ #ifdef CONFIG_SMP @@ -3850,7 +4026,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, atomic_dec(&task_rq(p)->nr_iowait); } - activate_task(rq, p, en_flags); + do_activate_task(rq, p, en_flags); wakeup_preempt(rq, p, wake_flags); ttwu_do_wakeup(p); @@ -3914,8 +4090,10 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags) rq = __task_rq_lock(p, &rf); if (task_on_rq_queued(p)) { update_rq_clock(rq); - if (p->se.sched_delayed) + if (p->se.sched_delayed) { + proxy_remove_from_sleeping_owner(p); enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); + } if (!task_on_cpu(rq, p)) { /* * When on_rq && !on_cpu the task is preempted, see if @@ -3949,13 +4127,19 @@ void sched_ttwu_pending(void *arg) update_rq_clock(rq); llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { + int wake_flags; if (WARN_ON_ONCE(p->on_cpu)) smp_cond_load_acquire(&p->on_cpu, !VAL); if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) set_task_cpu(p, cpu_of(rq)); - ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); + wake_flags = p->sched_remote_wakeup ? WF_MIGRATED : 0; + ttwu_do_activate(rq, p, wake_flags, &rf); + rq_unlock(rq, &rf); + activate_blocked_waiters(rq, p, wake_flags); + rq_lock(rq, &rf); + update_rq_clock(rq); } /* @@ -4475,10 +4659,10 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) #else cpu = task_cpu(p); #endif /* CONFIG_SMP */ - ttwu_queue(p, cpu, wake_flags); } out: + activate_blocked_waiters(cpu_rq(task_cpu(p)), p, wake_flags); if (success) { trace_android_rvh_try_to_wake_up_success(p); ttwu_stat(p, task_cpu(p), wake_flags); @@ -6815,26 +6999,6 @@ proxy_resched_idle(struct rq *rq) return rq->idle; } -static bool proxy_deactivate(struct rq *rq, struct task_struct *donor) -{ - unsigned long state = READ_ONCE(donor->__state); - - /* Don't deactivate if the state has been changed to TASK_RUNNING */ - if (state == TASK_RUNNING) - return false; - /* - * Because we got donor from pick_next_task, it is *crucial* - * that we call proxy_resched_idle before we deactivate it. - * As once we deactivate donor, donor->on_rq is set to zero, - * which allows ttwu to immediately try to wake the task on - * another rq. So we cannot use *any* references to donor - * after that point. So things like cfs_rq->curr or rq->donor - * need to be changed from next *before* we deactivate. - */ - proxy_resched_idle(rq); - return try_to_block_task(rq, donor, state, true); -} - #ifdef CONFIG_SMP /* * If the blocked-on relationship crosses CPUs, migrate @p to the @@ -6920,6 +7084,28 @@ void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, } #endif /* CONFIG_SMP */ +static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner, + struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + lockdep_assert_held(&owner->blocked_lock); + /* + * ttwu_activate() will pick them up and place them on whatever rq + * @owner will run next. + */ + WARN_ON(p == owner); + WARN_ON(!p->on_rq); + WARN_ON(p->sleeping_owner); + get_task_struct(owner); + WRITE_ONCE(p->sleeping_owner, owner); + /* + * ttwu_do_activate must not have a chance to activate p + * elsewhere before it's fully extricated from its old rq. + */ + list_add(&p->blocked_node, &owner->blocked_head); + block_task(rq, p, 0); +} + /* * Find runnable lock owner to proxy for mutex blocked donor * @@ -7026,18 +7212,38 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) return proxy_resched_idle(rq); } - if (!owner->on_rq) { - /* XXX Don't handle blocked owners yet */ - if (!proxy_deactivate(rq, donor)) - goto needs_return; - goto out; - } + if (!owner->on_rq || owner->se.sched_delayed) { + /* + * rq->curr must not be added to the blocked_head list or else + * ttwu_do_activate could enqueue it elsewhere before it switches + * out here. The approach to avoid this is the same as in the + * migrate_task case. + */ + if (curr_in_chain) { + raw_spin_unlock(&p->blocked_lock); + raw_spin_unlock(&mutex->wait_lock); + return proxy_resched_idle(rq); + } - if (owner->se.sched_delayed) { - /* XXX Don't handle delayed dequeue yet */ - if (!proxy_deactivate(rq, donor)) - goto needs_return; - goto out; + /* + * If !@owner->on_rq, holding @rq->lock will not pin the task, + * so we cannot drop @mutex->wait_lock until we're sure its a blocked + * task on this rq. + * + * We use @owner->blocked_lock to serialize against ttwu_activate(). + * Either we see its new owner->on_rq or it will see our list_add(). + */ + if (owner != p) { + raw_spin_unlock(&p->blocked_lock); + raw_spin_lock(&owner->blocked_lock); + } + + proxy_resched_idle(rq); + proxy_enqueue_on_owner(rq, owner, p); + + raw_spin_unlock(&owner->blocked_lock); + raw_spin_unlock(&mutex->wait_lock); + return NULL; /* retry task selection */ } /*