From 58ac22a089569f8cbaa644a2f6c68c65ca6f3eff Mon Sep 17 00:00:00 2001 From: Dao Huang Date: Thu, 13 Mar 2025 16:08:01 +0800 Subject: [PATCH] ANDROID: Add vendorhooks to enable sched_ext in OGKI kernel. As discussed in 401436689, Add some vendor hooks to facilitate enabling sched_ext in subsequent OGKI kernels. For the newly added android_vh_task_should_scx in kernel/sched/core.c, we decided to switch all rt tasks to ext. In fact, this is not a risky attempt. We have already done this in our released products (hmbird scheduler), and it works well. The concept of this hook is not to raise the priority of sched_ext above RT, but to let sched_ext manage all tasks. Inside sched_ext, we will still give the original rt task a very high priority and regroup the priority of rt. The grouping basis may be heavy rt, light rt, user experience related rt, kernel rt, etc., and make some scheduling distinctions. At the same time, the main reason for incorporating rt into sched_ext management is that we can easily do core ctrl, task aggregation, task placement, interaction with the idle system, and targeted frequency adjustment (if there is a stable expectation for the execution of tasks on the CPU, frequency adjustment will be easier). Bug: 401436689 Change-Id: I78321c918e03ff28a2c74c668e86436c0e416d66 Signed-off-by: Dao Huang [jstultz: Slightly tweaked commit message] Signed-off-by: John Stultz --- include/trace/hooks/sched.h | 22 ++++++++++++++++++++++ kernel/sched/core.c | 17 +++++++++++++++-- kernel/sched/ext.c | 35 ++++++++++++++++++++++++++++++----- kernel/sched/vendor_hooks.c | 7 +++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 338ae43c8a07..87e84bbac155 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -385,6 +385,28 @@ DECLARE_RESTRICTED_HOOK(android_rvh_set_task_comm, TP_PROTO(struct task_struct *tsk, bool exec), TP_ARGS(tsk, exec), 1); +DECLARE_HOOK(android_vh_task_should_scx, + TP_PROTO(int *should_scx, int policy, int prio), + TP_ARGS(should_scx, policy, prio)); +DECLARE_HOOK(android_vh_scx_ops_consider_migration, + TP_PROTO(bool *consider_migration), + TP_ARGS(consider_migration)); +DECLARE_HOOK(android_vh_scx_fix_prev_slice, + TP_PROTO(struct task_struct *p), + TP_ARGS(p)); +DECLARE_HOOK(android_vh_scx_ops_enable_state, + TP_PROTO(int state), + TP_ARGS(state)); +DECLARE_HOOK(android_vh_scx_enabled, + TP_PROTO(int enabled), + TP_ARGS(enabled)); +DECLARE_HOOK(android_vh_scx_set_cpus_allowed, + TP_PROTO(struct task_struct *p, struct affinity_context *ac, int *done), + TP_ARGS(p, ac, done)); +DECLARE_HOOK(android_vh_scx_task_switch_finish, + TP_PROTO(struct task_struct *p, int enable), + TP_ARGS(p, enable)); + struct sugov_policy; DECLARE_RESTRICTED_HOOK(android_rvh_set_sugov_update, TP_PROTO(struct sugov_policy *sg_policy, unsigned int next_freq, bool *should_update), diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6c349bb203b1..36fbea88b48a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5201,6 +5201,8 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + int should_scx = 0; + trace_android_rvh_sched_fork(p); __sched_fork(clone_flags, p); @@ -5247,10 +5249,11 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) scx_pre_fork(p); - if (rt_prio(p->prio)) { + trace_android_vh_task_should_scx(&should_scx, p->policy, p->prio); + if (rt_prio(p->prio) && !should_scx) { p->sched_class = &rt_sched_class; #ifdef CONFIG_SCHED_CLASS_EXT - } else if (task_should_scx(p->policy)) { + } else if (task_should_scx(p->policy) || should_scx) { p->sched_class = &ext_sched_class; #endif } else { @@ -8075,9 +8078,19 @@ EXPORT_SYMBOL(default_wake_function); const struct sched_class *__setscheduler_class(int policy, int prio) { +#ifdef CONFIG_SCHED_CLASS_EXT + int should_scx = 0; +#endif + if (dl_prio(prio)) return &dl_sched_class; +#ifdef CONFIG_SCHED_CLASS_EXT + trace_android_vh_task_should_scx(&should_scx, policy, prio); + if (should_scx) + return &ext_sched_class; +#endif + if (rt_prio(prio)) return &rt_sched_class; diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 2e31172e985a..5f4b70e34913 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2080,6 +2080,7 @@ static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at) static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags) { + bool consider_migration = false; int sticky_cpu = p->scx.sticky_cpu; if (enq_flags & ENQUEUE_WAKEUP) @@ -2109,8 +2110,11 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags rq->scx.nr_running++; add_nr_running(rq, 1); - if (SCX_HAS_OP(runnable) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); + if (SCX_HAS_OP(runnable)) { + trace_android_vh_scx_ops_consider_migration(&consider_migration); + if (consider_migration || !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags); + } if (enq_flags & SCX_ENQ_WAKEUP) touch_core_sched(rq, p); @@ -2169,6 +2173,8 @@ static void ops_dequeue(struct task_struct *p, u64 deq_flags) static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags) { + bool consider_migration = false; + if (!(p->scx.flags & SCX_TASK_QUEUED)) { WARN_ON_ONCE(task_runnable(p)); return true; @@ -2193,8 +2199,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false); } - if (SCX_HAS_OP(quiescent) && !task_on_rq_migrating(p)) - SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); + if (SCX_HAS_OP(quiescent)) { + trace_android_vh_scx_ops_consider_migration(&consider_migration); + if (consider_migration || !task_on_rq_migrating(p)) + SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags); + } if (deq_flags & SCX_DEQ_SLEEP) p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP; @@ -3090,8 +3099,10 @@ static struct task_struct *pick_task_scx(struct rq *rq) */ if (keep_prev) { p = prev; - if (!p->scx.slice) + if (!p->scx.slice) { p->scx.slice = SCX_SLICE_DFL; + trace_android_vh_scx_fix_prev_slice(p); + } } else { p = first_local_task(rq); if (!p) { @@ -3317,6 +3328,12 @@ static void task_woken_scx(struct rq *rq, struct task_struct *p) static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ac) { + int done = 0; + + trace_android_vh_scx_set_cpus_allowed(p, ac, &done); + if (done) + return; + set_cpus_allowed_common(p, ac); /* @@ -4619,6 +4636,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) default: break; } + trace_android_vh_scx_ops_enable_state(SCX_OPS_DISABLING); /* * Here, every runnable task is guaranteed to make forward progress and @@ -4660,6 +4678,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); + trace_android_vh_scx_task_switch_finish(p, 0); sched_enq_and_set_task(&ctx); @@ -4671,6 +4690,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_ops_enabled); + trace_android_vh_scx_enabled(0); for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) static_branch_disable(&scx_has_op[i]); static_branch_disable(&scx_ops_enq_last); @@ -4732,6 +4752,7 @@ static void scx_ops_disable_workfn(struct kthread_work *work) WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) != SCX_OPS_DISABLING); + trace_android_vh_scx_ops_enable_state(SCX_OPS_DISABLED); done: scx_ops_bypass(false); } @@ -5207,6 +5228,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_ENABLING) != SCX_OPS_DISABLED); + trace_android_vh_scx_ops_enable_state(SCX_OPS_ENABLING); atomic_set(&scx_exit_kind, SCX_EXIT_NONE); scx_warned_zero_slice = false; @@ -5354,6 +5376,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) */ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); static_branch_enable(&__scx_ops_enabled); + trace_android_vh_scx_enabled(1); /* * We're fully committed and can't fail. The task READY -> ENABLED @@ -5376,6 +5399,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) p->scx.slice = SCX_SLICE_DFL; p->sched_class = new_class; check_class_changing(task_rq(p), p, old_class); + trace_android_vh_scx_task_switch_finish(p, 1); sched_enq_and_set_task(&ctx); @@ -5390,6 +5414,7 @@ static int scx_ops_enable(struct sched_ext_ops *ops, struct bpf_link *link) WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE); goto err_disable; } + trace_android_vh_scx_ops_enable_state(SCX_OPS_ENABLED); if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all); diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 472d0bc271f8..77ee8a7072a4 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -103,6 +103,13 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_remove_entity_load_avg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_cpus_allowed_ptr); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_blocked_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_comm); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_task_should_scx); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_ops_consider_migration); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_fix_prev_slice); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_ops_enable_state); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_enabled); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_set_cpus_allowed); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_scx_task_switch_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_rq_clock_pelt); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_load_avg_blocked_se);