diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 338ae43c8a07..87e84bbac155 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -385,6 +385,28 @@ DECLARE_RESTRICTED_HOOK(android_rvh_set_task_comm, TP_PROTO(struct task_struct *tsk, bool exec), TP_ARGS(tsk, exec), 1); +DECLARE_HOOK(android_vh_task_should_scx, + TP_PROTO(int *should_scx, int policy, int prio), + TP_ARGS(should_scx, policy, prio)); +DECLARE_HOOK(android_vh_scx_ops_consider_migration, + TP_PROTO(bool *consider_migration), + TP_ARGS(consider_migration)); +DECLARE_HOOK(android_vh_scx_fix_prev_slice, + TP_PROTO(struct task_struct *p), + TP_ARGS(p)); +DECLARE_HOOK(android_vh_scx_ops_enable_state, + TP_PROTO(int state), + TP_ARGS(state)); +DECLARE_HOOK(android_vh_scx_enabled, + TP_PROTO(int enabled), + TP_ARGS(enabled)); +DECLARE_HOOK(android_vh_scx_set_cpus_allowed, + TP_PROTO(struct task_struct *p, struct affinity_context *ac, int *done), + TP_ARGS(p, ac, done)); +DECLARE_HOOK(android_vh_scx_task_switch_finish, + TP_PROTO(struct task_struct *p, int enable), + TP_ARGS(p, enable)); + struct sugov_policy; DECLARE_RESTRICTED_HOOK(android_rvh_set_sugov_update, TP_PROTO(struct sugov_policy *sg_policy, unsigned int next_freq, bool *should_update), diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6c349bb203b1..36fbea88b48a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5201,6 +5201,8 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + int should_scx = 0; + trace_android_rvh_sched_fork(p); __sched_fork(clone_flags, p); @@ -5247,10 +5249,11 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) scx_pre_fork(p); - if (rt_prio(p->prio)) { + trace_android_vh_task_should_scx(&should_scx, p->policy, p->prio); + if (rt_prio(p->prio) && !should_scx) { p->sched_class = &rt_sched_class; #ifdef CONFIG_SCHED_CLASS_EXT - } else if (task_should_scx(p->policy)) { + } else if (task_should_scx(p->policy) || should_scx) { p->sched_class = &ext_sched_class; #endif } else { @@ -8075,9 +8078,19 @@ EXPORT_SYMBOL(default_wake_function); const struct sched_class *__setscheduler_class(int policy, int prio) { +#ifdef CONFIG_SCHED_CLASS_EXT + int should_scx = 0; +#endif + if (dl_prio(prio)) return &dl_sched_class; +#ifdef CONFIG_SCHED_CLASS_EXT + trace_android_vh_task_should_scx(&should_scx, policy, prio); + if (should_scx) + return &ext_sched_class; +#endif + if (rt_prio(prio)) return &rt_sched_class; diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 2e31172e985a..5f4b70e34913 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2080,6 +2080,7 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) { + bool consider_migration = false; int sticky_cpu = p->scx.sticky_cpu; if (enq_flags & ENQUEUE_WAKEUP) @@ -2109,8 +2110,11 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags rq->scx.nr_running++; add_nr_running(rq, 1); - if (SCX_HAS_OP(runnable) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); + if (SCX_HAS_OP(runnable)) { + trace_android_vh_scx_ops_consider_migration(&consider_migration); + if (consider_migration || !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); + } if (enq_flags & SCX_ENQ_WAKEUP) touch_core_sched(rq, p); @@ -2169,6 +2173,8 @@ static void ops_dequeue(struct task_struct *p, u64 deq_flags) static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) { + bool consider_migration = false; + if (!(p->scx.flags & SCX_TASK_QUEUED)) { WARN_ON_ONCE(task_runnable(p)); return true; @@ -2193,8 +2199,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); } - if (SCX_HAS_OP(quiescent) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); + if (SCX_HAS_OP(quiescent)) { + trace_android_vh_scx_ops_consider_migration(&consider_migration); + if (consider_migration || !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); + } if (deq_flags & SCX_DEQ_SLEEP) p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; @@ -3090,8 +3099,10 @@ static struct task_struct *pick_task_scx(struct rq *rq) */ if (keep_prev) { p = prev; - if (!p->scx.slice) + if (!p->scx.slice) { p->scx.slice = SCX_SLICE_DFL; + trace_android_vh_scx_fix_prev_slice(p); + } } else { p = first_local_task(rq); if (!p) { @@ -3317,6 +3328,12 @@ static void task_woken_scx(struct rq *rq, struct task_struct *p) static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ac) { + int done = 0; + + trace_android_vh_scx_set_cpus_allowed(p, ac, &done); + if (done) + return; + set_cpus_allowed_common(p, ac); /* @@ -4619,6 +4636,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) default: break; } + trace_android_vh_scx_ops_enable_state(SCX_OPS_DISABLING); /* * Here, every runnable task is guaranteed to make forward progress and @@ -4660,6 +4678,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); + trace_android_vh_scx_task_switch_finish(p, 0); sched_enq_and_set_task(&ctx); @@ -4671,6 +4690,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_ops_enabled); + trace_android_vh_scx_enabled(0); for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) static_branch_disable(&scx_has_op[i]); static_branch_disable(&scx_ops_enq_last); @@ -4732,6 +4752,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != SCX_OPS_DISABLING); + trace_android_vh_scx_ops_enable_state(SCX_OPS_DISABLED); done: scx_ops_bypass(false); } @@ -5207,6 +5228,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) != SCX_OPS_DISABLED); + trace_android_vh_scx_ops_enable_state(SCX_OPS_ENABLING); atomic_set(&scx_exit_kind, SCX_EXIT_NONE); scx_warned_zero_slice = false; @@ -5354,6 +5376,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) */ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); static_branch_enable(&__scx_ops_enabled); + trace_android_vh_scx_enabled(1); /* * We're fully committed and can't fail. The task READY -> ENABLED @@ -5376,6 +5399,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) p->scx.slice = SCX_SLICE_DFL; p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); + trace_android_vh_scx_task_switch_finish(p, 1); sched_enq_and_set_task(&ctx); @@ -5390,6 +5414,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); goto err_disable; } + trace_android_vh_scx_ops_enable_state(SCX_OPS_ENABLED); if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all); diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 472d0bc271f8..77ee8a7072a4 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -103,6 +103,13 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_remove_entity_load_avg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_cpus_allowed_ptr); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_blocked_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_comm); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_task_should_scx); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_ops_consider_migration); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_fix_prev_slice); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_ops_enable_state); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_enabled); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_set_cpus_allowed); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_task_switch_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_rq_clock_pelt); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_load_avg_blocked_se);