tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge android16-6.12 into android16-6.12-lts

Browse Source 
This merges the android16-6.12 branch into the -lts branch, catching
it up with the latest changes in there.

It contains the following commits:

* 21ed84930c UPSTREAM: Revert "usb: xhci: Implement xhci_handshake_check_state() helper"
* 5b3ae3bcbe BACKPORT: usb: xhci: Skip xhci_reset in xhci_resume if xhci is being removed
* 5c72e9faba ANDROID: rust_binder: adjust errors from death notifications
* 9e02edea7f ANDROID: rust_binder: use u64 for death cookie
* 4317f0aeff ANDROID: f2fs: fixup ABI break due to reserved_pin_section
* 25bdb4a624 Revert "ANDROID: ABI: update symbol list for honor"
* a76eb2b67b ANDROID: GKI: Update oplus symbol list
* 6222007a04 ANDROID: mm/readahead: add for bypass high order allocation
* 659d7bb454 ANDROID: ABI: Update symbol list for exynos
* 26937a37f5 ANDROID: MODVERSIONS: hide type definition in drivers/usb/core/driver.c
* 8760b6e4f5 ANDROID: usb: Add vendor hook for usb suspend and resume
* da662aecc8 FROMLIST: KVM: Avoid synchronize_srcu() in kvm_io_bus_register_dev()
* 4be05c6524 FROMLIST: KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering
* d6045efc66 FROMLIST: KVM: arm64: vgic-init: Remove vgic_ready() macro
* f06dd0cd35 ANDROID: rust_binder: release threads before refs
* 5bbd30a60b ANDROID: ABI: Update pixel symbol list
* bafbebf2ab ANDROID: GKI: Update symbol list for xiaomi
* b7b130b7cc ANDROID: export folio_deactivate() for GKI purpose.
* 41f730f9c4 ANDROID: GKI: update exynos symbol list
* 766ecae19f UPSTREAM: xhci: dbctty: disable ECHO flag by default
* 8ea40f5243 ANDROID: GKI: Update xiaomi symbol list.
* 5594b4731d ANDROID: vendor_hooks: export tracepoint symbols
* 0d4cc1daff ANDROID: KVM: arm64: Don't update IOMMU under memory pressure
* 672185e575 ANDROID: iommu/iommu: Handle multi-page deferred sg mappings
* 740d42d181 ANDROID: vendor_hooks: Add vendor_hook in futex to fix the OEM scheduling priority bug
* 6eb6f346ac ANDROID: ABI: Update symbol list for mtk
* c302079179 ANDROID: vendor_hooks: Add vendor hook for GenieZone demand paging
* 5c1cddc983 ANDROID: vendor_hooks: Add vendor hook for GenieZone para-virtualization
* d893caf112 ANDROID: ashmem_rust: Add support for retrieving an ashmem area's vmfile
* 0be74214c0 ANDROID: ashmem_rust: Add support for querying the size of an ashmem region
* eb50f663c4 ANDROID: ashmem_rust: Add support for providing an ashmem region's name
* 6bdbae6ea9 ANDROID: ashmem_rust: Add is_ashmem_file()
* 0d890f867e ANDROID: ABI: update symbol list for honor
* 12727f8a4b FROMGIT: f2fs: introduce reserved_pin_section sysfs entry
* 286cd9d628 ANDROID: GKI: Update RTK STB KMI symbol list
* 7b4f7682b5 ANDROID: GKI: Update symbol list for Amlogic
* 862ce4b2c4 ANDROID: KVM: arm64: iommu: Fix power tracking
* 61184996a8 ANDROID: drivers/iommu: Fix return value in iommu_map_sg
* acad0cd51d ANDROID: ABI: update symbol list for galaxy
* 393dbad32c ANDROID: vendor_hook: add condition to call for freezing fail
* b62fe47ba2 ANDROID: fix ashmem_rust return EINVAL bug in ashmem_rust.rs
* a7e1300b95 ANDROID: Revert "cpufreq: Avoid using inconsistent policy->min and policy->max"
* 15d2fe0544 ANDROID: qcom: Update the ABI symbol list
* f6ca783ba2 UPSTREAM: scsi: ufs: qcom: Check gear against max gear in vop freq_to_gear()
* 237708e9d3 ANDROID: GKI: Update symbols list file for honor White list the vm_normal_folio_pmd
* f18e354aa9 ANDROID: mm: export vm_normal_folio_pmd to allow vendors to implement simplified smaps
* c181c478b0 ANDROID: vendor_hooks: add hook to record slab free
* d2e452e197 ANDROID: Build fixups with PROXY_EXEC v18 + !CONFIG_SMP
* 4f9e4406e4 ANDROID: Update proxy-exec logic from v14 to v18
* 3fa8dabe1a ANDROID: GKI: update asr symbols list
* 94310b3f77 ANDROID: Add the dma header to aarch64 allowlist
* 880d6538c5 UPSTREAM: usb: gadget: u_serial: Fix race condition in TTY wakeup
* b115bf2302 ANDROID: ABI: Update symbol list for mtk
* e87018c5f9 FROMGIT: sched/deadline: Fix dl_server runtime calculation formula
* e2bf362ee2 FROMGIT: sched/core: Fix migrate_swap() vs. hotplug
* 06ca12d7d2 ANDROID: GKI: update the ABI symbol list
* 55972ed83a ANDROID: Fixup init_user_ns CRC change
* 4e873ad607 ANDROID: user: Add vendor hook to user for GKI purpose
* a097cd9c30 ANDROID: export find_user() for GKI purpose.
* 85b8233f7e ANDROID: rust_binder: use euid from the task
* 969c904869 ANDROID: ashmem: rename VmAreaNew->VmaNew
* 2ab3e5f283 ANDROID: rust_binder: rename VmAreaNew->VmaNew
* 2ef75ab83a ANDROID: rust_binder: use tgid_nr_ns for getting pid
* 6a2be11026 UPSTREAM: task: rust: rework how current is accessed
* 602e2300de UPSTREAM: rust: add PidNamespace
* 12dfc1d9cb UPSTREAM: rust: miscdevice: add mmap support
* 8e67cb756f UPSTREAM: mm: rust: add VmaNew for f_ops->mmap()
* bd140ddf75 UPSTREAM: mm: rust: add mmput_async support
* 0c50773076 UPSTREAM: mm: rust: add lock_vma_under_rcu
* 0b5465bb31 UPSTREAM: mm: rust: add vm_insert_page
* d7f52612c5 UPSTREAM: mm: rust: add vm_area_struct methods that require read access
* f03d4f7490 UPSTREAM: mm: rust: add abstraction for struct mm_struct
* 2ef6dbc73e BACKPORT: rust: miscdevice: change how f_ops vtable is constructed
* 1acd3b312f Revert "FROMLIST: mm: rust: add abstraction for struct mm_struct"
* a012c15566 Revert "FROMLIST: mm: rust: add vm_area_struct methods that require read access"
* 3be00a9bf8 Revert "FROMLIST: mm: rust: add vm_insert_page"
* 3aed88205e Revert "FROMLIST: mm: rust: add lock_vma_under_rcu"
* a121b6e72f Revert "FROMLIST: mm: rust: add mmput_async support"
* 9248564a81 Revert "FROMLIST: mm: rust: add VmAreaNew for f_ops->mmap()"
* 6de3ace5b5 Revert "FROMLIST: rust: miscdevice: add mmap support"
* b7f54dd23b Revert "BACKPORT: FROMLIST: task: rust: rework how current is accessed"
* 5913c80b22 ANDROID: iommu/arm-smmu-v3-kvm: Fix idmap free_leaf
* c40c54e669 UPSTREAM: erofs: impersonate the opener's credentials when accessing backing file
* 4d0200d0a9 BACKPORT: erofs: add 'fsoffset' mount option to specify filesystem offset
* 399deda7b5 ANDROID: scsi: ufs: add UFSHCD_ANDROID_QUIRK_NO_IS_READ_ON_H8
* f6b1ab83f6 ANDROID: rust_binder: remove binder_logs/procs/pid immediately
* dd35623c83 ANDROID: ABI: update symbol list for mtktv
* 58beebb30f FROMLIST: fuse: give wakeup hints to the scheduler
* 0f917e4066 ANDROID: virt: gunyah: Replace arm_smccc_1_1_smc with arm_smccc_1_1_invoke
* 33429dd323 UPSTREAM: posix-cpu-timers: fix race between handle_posix_cpu_timers() and posix_cpu_timer_del()
* 6483832947 ANDROID: GKI: Update symbol list file for xiaomi
* 668635cd34 UPSTREAM: usb: gadget: uvc: dont call usb_composite_setup_continue when not streaming

Change-Id: I64074144d1a6da9fdd3b4dd5f8314ccea4f9d9e8
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
This commit is contained in:
Greg Kroah-Hartman
2025-07-13 12:16:44 +00:00
parent b0291e9ade 21ed84930c
commit 1741b1e583
99 changed files with 58457 additions and 999 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/sched/core.c
+269 -245
View File
@@ -2141,7 +2141,7 @@ inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
return dequeue_task_result;
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
static inline void __activate_task(struct rq *rq, struct task_struct *p, int flags)
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
@@ -2155,6 +2155,61 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
}
EXPORT_SYMBOL_GPL(activate_task);
#ifdef CONFIG_SCHED_PROXY_EXEC
static inline
void __proxy_remove_from_sleeping_owner(struct task_struct *owner, struct task_struct *p)
{
lockdep_assert_held(&owner->blocked_lock);
if (p->sleeping_owner == owner) {
list_del_init(&p->blocked_node);
WRITE_ONCE(p->sleeping_owner, NULL);
put_task_struct(owner); // matches get in proxy_enqueue_on_owner
}
}
static inline void proxy_remove_from_sleeping_owner(struct task_struct *p)
{
struct task_struct *owner = READ_ONCE(p->sleeping_owner);
if (owner) {
raw_spin_lock(&owner->blocked_lock);
__proxy_remove_from_sleeping_owner(owner, p);
raw_spin_unlock(&owner->blocked_lock);
}
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
if (!sched_proxy_exec()) {
__activate_task(rq, p, flags);
return;
}
lockdep_assert_rq_held(rq);
proxy_remove_from_sleeping_owner(p);
/*
* By calling __activate_task() with blocked_lock held, we
* order against the find_proxy_task() blocked_task case
* such that no more blocked tasks will be enqueued on p
* once we release p->blocked_lock.
*/
raw_spin_lock(&p->blocked_lock);
WARN_ON(task_cpu(p) != cpu_of(rq));
__activate_task(rq, p, flags);
raw_spin_unlock(&p->blocked_lock);
}
#else
static inline void proxy_remove_from_sleeping_owner(struct task_struct *p)
{
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
__activate_task(rq, p, flags);
}
#endif
void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
SCHED_WARN_ON(flags & DEQUEUE_SLEEP);
@@ -3794,68 +3849,14 @@ static inline void ttwu_do_wakeup(struct task_struct *p)
}
#ifdef CONFIG_SCHED_PROXY_EXEC
static inline
void __proxy_remove_from_sleeping_owner(struct task_struct *owner, struct task_struct *p)
{
lockdep_assert_held(&owner->blocked_lock);
if (p->sleeping_owner == owner) {
list_del_init(&p->blocked_node);
WRITE_ONCE(p->sleeping_owner, NULL);
put_task_struct(owner); // matches get in proxy_enqueue_on_owner
}
}
static inline void proxy_remove_from_sleeping_owner(struct task_struct *p)
{
struct task_struct *owner = READ_ONCE(p->sleeping_owner);
if (owner) {
raw_spin_lock(&owner->blocked_lock);
__proxy_remove_from_sleeping_owner(owner, p);
raw_spin_unlock(&owner->blocked_lock);
}
}
static void do_activate_task(struct rq *rq, struct task_struct *p, int en_flags)
{
if (!sched_proxy_exec()) {
activate_task(rq, p, en_flags);
return;
}
lockdep_assert_rq_held(rq);
proxy_remove_from_sleeping_owner(p);
/*
* By calling activate_task with blocked_lock held, we
* order against the find_proxy_task() blocked_task case
* such that no more blocked tasks will be enqueued on p
* once we release p->blocked_lock.
*/
raw_spin_lock(&p->blocked_lock);
WARN_ON(task_cpu(p) != cpu_of(rq));
activate_task(rq, p, en_flags);
raw_spin_unlock(&p->blocked_lock);
}
static bool proxy_task_runnable_but_waking(struct task_struct *p)
{
if (!sched_proxy_exec())
return false;
return (READ_ONCE(p->__state) == TASK_RUNNING &&
READ_ONCE(p->blocked_on_state) == BO_WAKING);
}
#ifdef CONFIG_SMP
static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
{
unsigned int wake_cpu;
/* Sanity check to make sure we can return safely */
WARN_ON(!is_cpu_allowed(p, p->wake_cpu));
/*
* Since we enqueuing blocked tasks on a cpu it may not
* be able to run on, preserve wake_cpu when we
* Since we are enqueuing a blocked task on a cpu it may
* not be able to run on, preserve wake_cpu when we
* __set_task_cpu so we can return the task to where it
* was previously runnable.
*/
@@ -3869,38 +3870,53 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
__set_task_cpu(p, cpu);
}
#endif /* CONFIG_SMP */
static bool proxy_task_runnable_but_waking(struct task_struct *p)
{
if (!sched_proxy_exec())
return false;
return (READ_ONCE(p->__state) == TASK_RUNNING &&
READ_ONCE(p->blocked_on_state) == BO_WAKING);
}
static void do_activate_blocked_waiter(struct rq *target_rq, struct task_struct *p, int en_flags)
{
unsigned long flags;
unsigned int state;
struct rq_flags rf;
int target_cpu = cpu_of(target_rq);
raw_spin_lock_irqsave(&p->pi_lock, flags);
state = READ_ONCE(p->__state);
/* Avoid racing with ttwu */
if (state == TASK_WAKING)
goto out;
scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
state = READ_ONCE(p->__state);
/* Avoid racing with ttwu */
if (state == TASK_WAKING)
return;
if (READ_ONCE(p->on_rq)) {
/*
* We raced with a non mutex handoff activation of p.
* That activation will also take care of activating
* all of the tasks after p in the blocked_head list,
* so we're done here.
*/
goto out;
if (READ_ONCE(p->on_rq)) {
/*
* We raced with a non mutex handoff activation of p.
* That activation will also take care of activating
* all of the tasks after p in the blocked_head list,
* so we're done here.
*/
return;
}
if (task_on_cpu(task_rq(p), p)) {
/*
* Its possible this activation is very late, and
* we already were woken up and are running on a
* different cpu. If that task blocked, it could be
* dequeued (so on_rq == 0), but still on_cpu.
* Bail in this case, as we definitely don't want to
* activate a task when its on_cpu elsewhere.
*/
return;
}
proxy_set_task_cpu(p, target_cpu);
rq_lock_irqsave(target_rq, &rf);
update_rq_clock(target_rq);
activate_task(target_rq, p, en_flags);
resched_curr(target_rq);
rq_unlock_irqrestore(target_rq, &rf);
}
proxy_set_task_cpu(p, target_cpu);
rq_lock_irqsave(target_rq, &rf);
update_rq_clock(target_rq);
do_activate_task(target_rq, p, en_flags);
resched_curr(target_rq);
rq_unlock_irqrestore(target_rq, &rf);
out:
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
}
static void activate_blocked_waiters(struct rq *target_rq,
@@ -3920,10 +3936,10 @@ static void activate_blocked_waiters(struct rq *target_rq,
en_flags |= ENQUEUE_MIGRATED;
/*
* A whole bunch of 'proxy' tasks back this blocked task, wake
* them all up to give this task its 'fair' share.
*/
/*
* A whole bunch of waiting donor tasks back this blocked
* lock owner task, wake them all up to give this task its
* 'fair' share.
*
* This is a little unique here and the locking is messy.
* At this point we only hold the blocked_lock, so the
* owner task may be able to run and do all sorts of
@@ -4105,16 +4121,6 @@ void move_queued_task_locked(struct rq *src_rq, struct rq *dst_rq, struct task_s
}
#endif /* CONFIG_SMP */
#else /* !CONFIG_SCHED_PROXY_EXEC */
static inline void proxy_remove_from_sleeping_owner(struct task_struct *p)
{
}
static inline void do_activate_task(struct rq *rq, struct task_struct *p,
int en_flags)
{
activate_task(rq, p, en_flags);
}
static bool proxy_task_runnable_but_waking(struct task_struct *p)
{
return false;
@@ -4128,6 +4134,13 @@ static inline void activate_blocked_waiters(struct rq *target_rq,
#endif /* CONFIG_SCHED_PROXY_EXEC */
#ifdef CONFIG_SMP
/*
* Checks to see if task p has been proxy-migrated to another rq
* and needs to be returned. If so, we deactivate the task here
* so that it can be properly woken up on the p->wake_cpu
* (or whichever cpu select_task_rq() picks at the bottom of
* try_to_wake_up()
*/
static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
bool ret = false;
@@ -4136,7 +4149,7 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
return false;
raw_spin_lock(&p->blocked_lock);
if (get_task_blocked_on(p) && p->blocked_on_state == BO_WAKING) {
if (__get_task_blocked_on(p) && p->blocked_on_state == BO_WAKING) {
if (!task_current(rq, p) && (p->wake_cpu != cpu_of(rq))) {
if (task_current_donor(rq, p)) {
put_prev_task(rq, p);
@@ -4161,6 +4174,7 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
{
return false;
}
static inline void _trace_sched_pe_return_migration(struct task_struct *p)
{
}
@@ -4192,7 +4206,7 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
atomic_dec(&task_rq(p)->nr_iowait);
}
do_activate_task(rq, p, en_flags);
activate_task(rq, p, en_flags);
wakeup_preempt(rq, p, wake_flags);
ttwu_do_wakeup(p);
@@ -4260,6 +4274,10 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags)
proxy_remove_from_sleeping_owner(p);
enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED);
}
if (proxy_needs_return(rq, p)) {
_trace_sched_pe_return_migration(p);
goto out;
}
if (!task_on_cpu(rq, p)) {
/*
* When on_rq && !on_cpu the task is preempted, see if
@@ -4267,10 +4285,6 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags)
*/
wakeup_preempt(rq, p, wake_flags);
}
if (proxy_needs_return(rq, p)) {
_trace_sched_pe_return_migration(p);
goto out;
}
ttwu_do_wakeup(p);
ret = 1;
}
@@ -4411,6 +4425,11 @@ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
if (task_on_scx(p))
return false;
#ifdef CONFIG_SMP
if (p->sched_class == &stop_sched_class)
return false;
#endif
/*
* Do not complicate things with the async wake_list while the CPU is
* in hotplug state.
@@ -5541,6 +5560,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
}
}
#ifdef CONFIG_SCHED_PROXY_EXEC
/*
* Only called from __schedule context
*
@@ -5568,6 +5588,7 @@ static void zap_balance_callbacks(struct rq *rq)
}
rq->balance_callback = found ? &balance_push_callback : NULL;
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
static void balance_push(struct rq *rq);
@@ -5637,9 +5658,11 @@ void balance_callbacks(struct rq *rq, struct balance_callback *head)
#else
#ifdef CONFIG_SCHED_PROXY_EXEC
static inline void zap_balance_callbacks(struct rq *rq)
{
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
static inline void __balance_callbacks(struct rq *rq)
{
@@ -7162,13 +7185,10 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
}
#ifdef CONFIG_SCHED_PROXY_EXEC
static inline struct task_struct *
proxy_resched_idle(struct rq *rq)
static inline struct task_struct *proxy_resched_idle(struct rq *rq)
{
put_prev_task(rq, rq->donor);
put_prev_set_next_task(rq, rq->donor, rq->idle);
rq_set_donor(rq, rq->idle);
set_next_task(rq, rq->idle);
set_tsk_need_resched(rq->idle);
return rq->idle;
}
@@ -7189,11 +7209,10 @@ proxy_resched_idle(struct rq *rq)
static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
struct task_struct *p, int target_cpu)
{
struct rq *target_rq = cpu_rq(target_cpu);
LIST_HEAD(migrate_list);
struct rq *target_rq;
lockdep_assert_rq_held(rq);
target_rq = cpu_rq(target_cpu);
/*
* Since we're going to drop @rq, we have to put(@rq->donor) first,
@@ -7216,8 +7235,8 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
/* XXX - Added to address problems with changed dl_server semantics - double check */
__put_prev_set_next_dl_server(rq, rq->donor, rq->curr);
put_prev_task(rq, rq->donor);
rq_set_donor(rq, rq->curr);
set_next_task(rq, rq->curr);
rq_set_donor(rq, rq->idle);
set_next_task(rq, rq->idle);
for (; p; p = p->blocked_donor) {
WARN_ON(p == rq->curr);
@@ -7243,12 +7262,41 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
raw_spin_rq_unlock(target_rq);
raw_spin_rq_lock(rq);
rq_repin_lock(rq, rf);
}
/*
* Ok, now we have the lock again, put rq->curr and
* set_next_task() to idle
*/
proxy_resched_idle(rq);
static void proxy_force_return(struct rq *rq, struct rq_flags *rf,
struct task_struct *p)
{
lockdep_assert_rq_held(rq);
_trace_sched_pe_return_migration(p);
put_prev_task(rq, rq->donor);
rq_set_donor(rq, rq->idle);
set_next_task(rq, rq->idle);
WARN_ON(p == rq->curr);
p->blocked_on_state = BO_WAKING;
get_task_struct(p);
block_task(rq, p, 0);
zap_balance_callbacks(rq);
rq_unpin_lock(rq, rf);
raw_spin_rq_unlock(rq);
wake_up_process(p);
put_task_struct(p);
raw_spin_rq_lock(rq);
rq_repin_lock(rq, rf);
}
static inline bool proxy_can_run_here(struct rq *rq, struct task_struct *p)
{
if (p == rq->curr || p->wake_cpu == cpu_of(rq))
return true;
return false;
}
#else /* !CONFIG_SMP */
static inline
@@ -7256,6 +7304,17 @@ void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
struct task_struct *p, int target_cpu)
{
}
static inline
void proxy_force_return(struct rq *rq, struct rq_flags *rf,
struct task_struct *p)
{
}
static inline bool proxy_can_run_here(struct rq *rq, struct task_struct *p)
{
return true;
}
#endif /* CONFIG_SMP */
static void proxy_enqueue_on_owner(struct rq *rq, struct task_struct *owner,
@@ -7311,7 +7370,6 @@ static struct task_struct *
find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
{
struct task_struct *owner = NULL;
struct task_struct *ret = NULL;
bool curr_in_chain = false;
int this_cpu = cpu_of(rq);
struct task_struct *p;
@@ -7328,18 +7386,48 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* By taking mutex->wait_lock we hold off concurrent mutex_unlock()
* and ensure @owner sticks around.
*/
raw_spin_lock(&mutex->wait_lock);
raw_spin_lock(&p->blocked_lock);
guard(raw_spinlock)(&mutex->wait_lock);
guard(raw_spinlock)(&p->blocked_lock);
/* Check again that p is blocked with blocked_lock held */
if (mutex != get_task_blocked_on(p)) {
if (mutex != __get_task_blocked_on(p)) {
/*
* Something changed in the blocked_on chain and
* we don't know if only at this level. So, let's
* just bail out completely and let __schedule
* just bail out completely and let __schedule()
* figure things out (pick_again loop).
*/
goto out;
return NULL;
}
/* Double check blocked_on_state now we're holding the lock */
if (p->blocked_on_state == BO_RUNNABLE)
return p;
/*
* If a ww_mutex hits the die/wound case, it marks the task as
* BO_WAKING and calls try_to_wake_up(), so that the mutex
* cycle can be broken and we avoid a deadlock.
*
* However, if at that moment, we are here on the cpu which the
* die/wounded task is enqueued, we might loop on the cycle as
* BO_WAKING still causes task_is_blocked() to return true
* (since we want return migration to occur before we run the
* task).
*
* Unfortunately since we hold the rq lock, it will block
* try_to_wake_up from completing and doing the return
* migration.
*
* So when we hit a BO_WAKING task try to wake it up ourselves.
*/
if (p->blocked_on_state == BO_WAKING) {
if (task_current(rq, p)) {
/* If its current just set it runnable */
__force_blocked_on_runnable(p);
return p;
}
goto needs_return;
}
if (task_current(rq, p))
@@ -7348,61 +7436,22 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
owner = __mutex_owner(mutex);
if (!owner) {
/* If the owner is null, we may have some work to do */
if (!proxy_can_run_here(rq, p))
goto needs_return;
/* First if p is no longer blocked, just return it to run */
if (!task_is_blocked(p)) {
ret = p;
goto out;
}
goto needs_return;
__force_blocked_on_runnable(p);
return p;
}
owner_cpu = task_cpu(owner);
if (owner_cpu != this_cpu) {
trace_sched_pe_migration(donor, owner);
/*
* @owner can disappear, simply migrate to @owner_cpu and leave that CPU
* to sort things out.
*/
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
if (curr_in_chain)
return proxy_resched_idle(rq);
proxy_migrate_task(rq, rf, p, owner_cpu);
return NULL;
}
if (task_on_rq_migrating(owner)) {
trace_sched_pe_owner_is_migrating(owner, p);
/*
* One of the chain of mutex owners is currently migrating to this
* CPU, but has not yet been enqueued because we are holding the
* rq lock. As a simple solution, just schedule rq->idle to give
* the migration a chance to complete. Much like the migrate_task
* case we should end up back in find_proxy_task(), this time
* hopefully with all relevant tasks already enqueued.
*/
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
return proxy_resched_idle(rq);
}
if (!owner->on_rq || owner->se.sched_delayed) {
if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
/*
* rq->curr must not be added to the blocked_head list or else
* ttwu_do_activate could enqueue it elsewhere before it switches
* out here. The approach to avoid this is the same as in the
* migrate_task case.
*/
if (curr_in_chain) {
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
if (curr_in_chain)
return proxy_resched_idle(rq);
}
/*
* If !@owner->on_rq, holding @rq->lock will not pin the task,
@@ -7412,26 +7461,52 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
* We use @owner->blocked_lock to serialize against ttwu_activate().
* Either we see its new owner->on_rq or it will see our list_add().
*/
if (owner != p) {
raw_spin_unlock(&p->blocked_lock);
raw_spin_lock(&owner->blocked_lock);
}
WARN_ON(owner == p);
raw_spin_unlock(&p->blocked_lock);
raw_spin_lock(&owner->blocked_lock);
proxy_resched_idle(rq);
proxy_enqueue_on_owner(rq, owner, p);
raw_spin_unlock(&owner->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
raw_spin_lock(&p->blocked_lock);
return NULL; /* retry task selection */
}
owner_cpu = task_cpu(owner);
if (owner_cpu != this_cpu) {
trace_sched_pe_migration(donor, owner);
/*
* @owner can disappear, simply migrate to @owner_cpu
* and leave that CPU to sort things out.
*/
if (curr_in_chain)
return proxy_resched_idle(rq);
goto migrate;
}
if (task_on_rq_migrating(owner)) {
trace_sched_pe_owner_is_migrating(owner, p);
/*
* One of the chain of mutex owners is currently migrating to this
* CPU, but has not yet been enqueued because we are holding the
* rq lock. As a simple solution, just schedule rq->idle to give
* the migration a chance to complete. Much like the migrate_task
* case we should end up back in find_proxy_task(), this time
* hopefully with all relevant tasks already enqueued.
*/
return proxy_resched_idle(rq);
}
/*
* We could race with ttwu's return migration, so holding the
* rq lock, double check owner is both on_rq & on this cpu, as
* it might not even be on our RQ still
* Its possible to race where after we check owner->on_rq
* but before we check (owner_cpu != this_cpu) that the
* task on another cpu was migrated back to this cpu. In
* that case it could slip by our checks. So double check
* we are still on this cpu and not migrating. If we get
* inconsistent results, try again.
*/
if (!(task_on_rq_queued(owner) && task_cpu(owner) == this_cpu))
goto out;
if (!task_on_rq_queued(owner) || task_cpu(owner) != this_cpu)
return NULL;
if (owner == p) {
/*
@@ -7453,81 +7528,34 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
*
* Which leaves us to finish the ttwu_runnable() and make it go.
*
* So schedule rq->idle so that ttwu_runnable can get the rq lock
* and mark owner as running.
* So schedule rq->idle so that ttwu_runnable() can get the rq
* lock and mark owner as running.
*/
if (p->blocked_on_state == BO_WAKING)
goto needs_return;
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
return proxy_resched_idle(rq);
}
/*
* If a ww_mutex hits the die/wound case, it marks the task as
* BO_WAKING and calls try_to_wake_up(), so that the mutex
* cycle can be broken and we avoid a deadlock.
*
* However, if at that moment, we are here on the cpu which the
* die/wounded task is enqueued, we might loop on the cycle as
* BO_WAKING still causes task_is_blocked() to return true
* (since we want return migration to occur before we run the
* task).
*
* Unfortunately since we hold the rq lock, it will block
* try_to_wake_up from completing and doing the return
* migration.
*
* So when we hit a BO_WAKING task that has a valid mutex, and
* that mutex has an owner, we're hitting a mid-chain wakeup,
* so we can briefly schedule idle so we release the rq and
* let the wakeup complete.
*/
if (p->blocked_on_state == BO_WAKING)
goto needs_return;
/*
* OK, now we're absolutely sure @owner is on this
* rq, therefore holding @rq->lock is sufficient to
* guarantee its existence, as per ttwu_remote().
*/
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
owner->blocked_donor = p;
}
WARN_ON_ONCE(owner && !owner->on_rq);
return owner;
needs_return:
#ifdef CONFIG_SMP
WARN_ON(!is_cpu_allowed(p, p->wake_cpu));
if (p->wake_cpu == this_cpu) {
/* We can actually run here fine */
p->blocked_on_state = BO_RUNNABLE;
ret = p;
goto out;
}
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
if (curr_in_chain)
return proxy_resched_idle(rq);
p->blocked_on_state = BO_RUNNABLE;
_trace_sched_pe_return_migration(p);
proxy_migrate_task(rq, rf, p, p->wake_cpu);
/*
* NOTE: This logic is down here, because we need to call
* the functions with the mutex wait_lock and task
* blocked_lock released, so we have to get out of the
* guard() scope.
*/
migrate:
proxy_migrate_task(rq, rf, p, owner_cpu);
return NULL;
needs_return:
proxy_force_return(rq, rf, p);
return NULL;
#else
/* Nowhere else to migrate on UP */
p->blocked_on_state = BO_RUNNABLE;
ret = p;
#endif
out:
raw_spin_unlock(&p->blocked_lock);
raw_spin_unlock(&mutex->wait_lock);
return ret;
}
#else /* SCHED_PROXY_EXEC */
static struct task_struct *
@@ -7609,7 +7637,6 @@ static void __sched notrace __schedule(int sched_mode)
struct rq *rq;
bool prev_not_proxied;
int cpu;
bool preserve_need_resched = false;
cpu = smp_processor_id();
rq = cpu_rq(cpu);
@@ -7678,19 +7705,16 @@ pick_again:
next->blocked_donor = NULL;
if (unlikely(task_is_blocked(next))) {
next = find_proxy_task(rq, next, &rf);
if (!next) {
/* zap the balance_callbacks before picking again */
zap_balance_callbacks(rq);
if (!next)
goto pick_again;
}
if (next == rq->idle)
preserve_need_resched = true;
goto keep_resched;
}
trace_sched_finish_task_selection(rq->donor, next, cpu);
picked:
if (!preserve_need_resched)
clear_tsk_need_resched(prev);
clear_tsk_need_resched(prev);
clear_preempt_need_resched();
keep_resched:
#ifdef CONFIG_SCHED_DEBUG
rq->last_seen_need_resched_ns = 0;
#endif
kernel/sched/deadline.c
+12 -24
View File
@@ -1488,7 +1488,9 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64
if (dl_entity_is_special(dl_se))
return;
scaled_delta_exec = dl_scaled_delta_exec(rq, dl_se, delta_exec);
scaled_delta_exec = delta_exec;
if (!dl_server(dl_se))
scaled_delta_exec = dl_scaled_delta_exec(rq, dl_se, delta_exec);
dl_se->runtime -= scaled_delta_exec;
@@ -1595,7 +1597,7 @@ throttle:
*/
void dl_server_update_idle_time(struct rq *rq, struct task_struct *p)
{
s64 delta_exec, scaled_delta_exec;
s64 delta_exec;
if (!rq->fair_server.dl_defer)
return;
@@ -1608,9 +1610,7 @@ void dl_server_update_idle_time(struct rq *rq, struct task_struct *p)
if (delta_exec < 0)
return;
scaled_delta_exec = dl_scaled_delta_exec(rq, &rq->fair_server, delta_exec);
rq->fair_server.runtime -= scaled_delta_exec;
rq->fair_server.runtime -= delta_exec;
if (rq->fair_server.runtime < 0) {
rq->fair_server.dl_defer_running = 0;
@@ -2485,6 +2485,10 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p, struct task_s
update_curr_dl(rq);
update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 1);
if (task_is_blocked(p))
return;
if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -2679,34 +2683,18 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)
}
static inline bool __dl_revalidate_rq_state(struct task_struct *task, struct rq *rq,
struct rq *later, bool *retry)
struct rq *later)
{
if (task_rq(task) != rq)
return false;
if (!cpumask_test_cpu(later->cpu, &task->cpus_mask))
return false;
if (task_on_cpu(rq, task))
return false;
if (!dl_task(task))
return false;
if (is_migration_disabled(task))
return false;
if (!task_on_rq_queued(task))
return false;
return true;
return __revalidate_rq_state(task, rq, later);
}
static inline bool dl_revalidate_rq_state(struct task_struct *task, struct rq *rq,
struct rq *later, bool *retry)
{
if (!sched_proxy_exec())
return __dl_revalidate_rq_state(task, rq, later, retry);
return __dl_revalidate_rq_state(task, rq, later);
if (!dl_task(task) || is_migration_disabled(task))
return false;
kernel/sched/fair.c
+17 -20
View File
@@ -1192,7 +1192,7 @@ static void update_tg_load_avg(struct cfs_rq *cfs_rq)
}
#endif /* CONFIG_SMP */
static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
static s64 update_se(struct rq *rq, struct sched_entity *se)
{
u64 now = rq_clock_task(rq);
s64 delta_exec;
@@ -1203,6 +1203,7 @@ static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
se->exec_start = now;
if (entity_is_task(se)) {
struct task_struct *donor = task_of(se);
struct task_struct *running = rq->curr;
/*
* If se is a task, we account the time against the running
@@ -1210,8 +1211,14 @@ static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
*/
running->se.exec_start = now;
running->se.sum_exec_runtime += delta_exec;
trace_sched_stat_runtime(running, delta_exec);
account_group_exec_runtime(running, delta_exec);
/* cgroup time is always accounted against the donor */
cgroup_account_cputime(donor, delta_exec);
} else {
/* If not task, account the time against se */
/* If not task, account the time against donor se */
se->sum_exec_runtime += delta_exec;
}
@@ -1226,13 +1233,6 @@ static s64 update_curr_se(struct rq *rq, struct sched_entity *se)
return delta_exec;
}
static inline void update_curr_task(struct task_struct *p, s64 delta_exec)
{
trace_sched_stat_runtime(p, delta_exec);
account_group_exec_runtime(p, delta_exec);
cgroup_account_cputime(p, delta_exec);
}
static inline bool did_preempt_short(struct cfs_rq *cfs_rq, struct sched_entity *curr)
{
if (!sched_feat(PREEMPT_SHORT))
@@ -1271,13 +1271,8 @@ static inline bool do_preempt_short(struct cfs_rq *cfs_rq,
s64 update_curr_common(struct rq *rq)
{
struct task_struct *donor = rq->donor;
s64 delta_exec;
delta_exec = update_curr_se(rq, &donor->se);
if (likely(delta_exec > 0))
update_curr_task(donor, delta_exec);
return delta_exec;
return update_se(rq, &donor->se);
}
/*
@@ -1285,6 +1280,12 @@ s64 update_curr_common(struct rq *rq)
*/
static void update_curr(struct cfs_rq *cfs_rq)
{
/*
* Note: cfs_rq->curr corresponds to the task picked to
* run (ie: rq->donor.se) which due to proxy-exec may
* not necessarily be the actual task running
* (rq->curr.se). This is easy to confuse!
*/
struct sched_entity *curr = cfs_rq->curr;
struct rq *rq = rq_of(cfs_rq);
s64 delta_exec;
@@ -1293,7 +1294,7 @@ static void update_curr(struct cfs_rq *cfs_rq)
if (unlikely(!curr))
return;
delta_exec = update_curr_se(rq, curr);
delta_exec = update_se(rq, curr);
if (unlikely(delta_exec <= 0))
return;
@@ -1302,10 +1303,6 @@ static void update_curr(struct cfs_rq *cfs_rq)
update_min_vruntime(cfs_rq);
if (entity_is_task(curr)) {
struct task_struct *p = task_of(curr);
update_curr_task(p, delta_exec);
/*
* If the fair_server is active, we need to account for the
* fair_server time whether or not the task is running on
kernel/sched/rt.c
+5 -40
View File
@@ -1511,25 +1511,14 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
enqueue_rt_entity(rt_se, flags);
/*
* Current can't be pushed away. Selected is tied to current,
* so don't push it either.
*/
if (task_current(rq, p) || task_current_donor(rq, p))
return;
/*
* Pinned tasks can't be pushed.
*/
if (p->nr_cpus_allowed == 1)
return;
if (should_honor_rt_sync(rq, p, sync))
return;
if (task_is_blocked(p))
return;
enqueue_pushable_task(rq, p);
if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
enqueue_pushable_task(rq, p);
}
static bool dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
@@ -2035,42 +2024,18 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq)
}
static inline bool __rt_revalidate_rq_state(struct task_struct *task, struct rq *rq,
struct rq *lowest, bool *retry)
struct rq *lowest)
{
/*
* We had to unlock the run queue. In the mean time, task could have
* migrated already or had its affinity changed. Also make sure that it
* wasn't scheduled on its rq. It is possible the task was scheduled,
* set "migrate_disabled" and then got preempted, so we must check the
* task migration disable flag here too.
*/
if (task_rq(task) != rq)
return false;
if (!cpumask_test_cpu(lowest->cpu, &task->cpus_mask))
return false;
if (task_on_cpu(rq, task))
return false;
if (!rt_task(task))
return false;
if (is_migration_disabled(task))
return false;
if (!task_on_rq_queued(task))
return false;
return true;
return __revalidate_rq_state(task, rq, lowest);
}
/* XXX: TODO: Consolidate this w/ dl_revalidate_rq_state */
static inline bool rt_revalidate_rq_state(struct task_struct *task, struct rq *rq,
struct rq *lowest, bool *retry)
{
if (!sched_proxy_exec())
return __rt_revalidate_rq_state(task, rq, lowest, retry);
return __rt_revalidate_rq_state(task, rq, lowest);
/*
* Releasing the rq lock means we need to re-check pushability.
* Some scenarios:
kernel/sched/sched.h
+33 -4
View File
@@ -2336,7 +2336,7 @@ static inline bool task_is_blocked(struct task_struct *p)
static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
{
#ifdef CONFIG_SMP
return p->on_cpu;
return READ_ONCE(p->on_cpu);
#else
return task_current(rq, p);
#endif
@@ -2344,7 +2344,7 @@ static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
static inline int task_on_rq_queued(struct task_struct *p)
{
return p->on_rq == TASK_ON_RQ_QUEUED;
return READ_ONCE(p->on_rq) == TASK_ON_RQ_QUEUED;
}
static inline int task_on_rq_migrating(struct task_struct *p)
@@ -3169,6 +3169,34 @@ extern void set_rq_offline(struct rq *rq);
extern bool sched_smp_initialized;
static inline bool __revalidate_rq_state(struct task_struct *task, struct rq *rq,
struct rq *lowest)
{
/*
* We had to unlock the run queue. In the mean time, task could have
* migrated already or had its affinity changed. Also make sure that it
* wasn't scheduled on its rq. It is possible the task was scheduled,
* set "migrate_disabled" and then got preempted, so we must check the
* task migration disable flag here too.
*/
if (task_rq(task) != rq)
return false;
if (!cpumask_test_cpu(lowest->cpu, &task->cpus_mask))
return false;
if (task_on_cpu(rq, task))
return false;
if (is_migration_disabled(task))
return false;
if (!task_on_rq_queued(task))
return false;
return true;
}
#else /* !CONFIG_SMP: */
/*
@@ -3922,12 +3950,14 @@ int __task_is_pushable(struct rq *rq, struct task_struct *p, int cpu)
return 0;
}
#endif /* CONFIG_SMP */
#ifdef CONFIG_SCHED_PROXY_EXEC
void move_queued_task_locked(struct rq *rq, struct rq *dst_rq, struct task_struct *task);
int task_is_pushable(struct rq *rq, struct task_struct *p, int cpu);
struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p);
#else /* !CONFIG_SCHED_PROXY_EXEC */
#ifdef CONFIG_SMP
static inline
void move_queued_task_locked(struct rq *rq, struct rq *dst_rq, struct task_struct *task)
{
@@ -3939,14 +3969,13 @@ int task_is_pushable(struct rq *rq, struct task_struct *p, int cpu)
{
return __task_is_pushable(rq, p, cpu);
}
#endif
static inline
struct task_struct *find_exec_ctx(struct rq *rq, struct task_struct *p)
{
return p;
}
#endif /* CONFIG_SCHED_PROXY_EXEC */
#endif /* CONFIG_SMP */
#ifdef CONFIG_RT_MUTEXES