From 6f8adeb75108670cf2c678e03827cd9fe4d83828 Mon Sep 17 00:00:00 2001 From: Ashay Jaiswal Date: Mon, 13 Jun 2022 16:40:17 +0530 Subject: [PATCH] ANDROID: Sched: Add restricted vendor hooks for scheduler Adding vendor hooks in several areas of the scheduler code. These are needed for implementing Qualcomm's load tracking scheme WALT via vendor modules. Due to WALT's proprietary nature, these changes cannot go upstream. This commit reverts commit dafc604d0dad ("Revert "Revert "Revert "ANDROID: Sched: Add restricted vendor hooks for scheduler""""). It was originally reverted multiple times to facilitate upstream merges. Effectively, bringing back the original change. Bug: 200103201 Bug: 243793188 Change-Id: Ib340c03ddccd4218cac1cca382a2c6e93064de02 Signed-off-by: Ashay Jaiswal Signed-off-by: Sai Harshini Nimmala --- include/trace/hooks/sched.h | 65 +++++++++++++++++++++++++++++++++++++ kernel/sched/core.c | 19 ++++++++++- kernel/sched/cputime.c | 11 +++++-- kernel/sched/fair.c | 7 +++- kernel/sched/topology.c | 1 + kernel/sched/vendor_hooks.c | 16 +++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/include/trace/hooks/sched.h b/include/trace/hooks/sched.h index 37b988996d97..bb96c4776633 100644 --- a/include/trace/hooks/sched.h +++ b/include/trace/hooks/sched.h @@ -117,6 +117,71 @@ DECLARE_RESTRICTED_HOOK(android_rvh_update_cpus_allowed, const struct cpumask *new_mask, int *ret), TP_ARGS(p, cpus_requested, new_mask, ret), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_set_task_cpu, + TP_PROTO(struct task_struct *p, unsigned int new_cpu), + TP_ARGS(p, new_cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_try_to_wake_up_success, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_fork, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_wake_up_new_task, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_new_task_stats, + TP_PROTO(struct task_struct *p), + TP_ARGS(p), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_flush_task, + TP_PROTO(struct task_struct *prev), + TP_ARGS(prev), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_tick_entry, + TP_PROTO(struct rq *rq), + TP_ARGS(rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_schedule, + TP_PROTO(struct task_struct *prev, struct task_struct *next, struct rq *rq), + TP_ARGS(prev, next, rq), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_starting, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_sched_cpu_dying, + TP_PROTO(int cpu), + TP_ARGS(cpu), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_account_irq, + TP_PROTO(struct task_struct *curr, int cpu, s64 delta, bool start), + TP_ARGS(curr, cpu, delta, start), 1); + +struct sched_entity; +DECLARE_RESTRICTED_HOOK(android_rvh_place_entity, + TP_PROTO(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial, u64 *vruntime), + TP_ARGS(cfs_rq, se, initial, vruntime), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_build_perf_domains, + TP_PROTO(bool *eas_check), + TP_ARGS(eas_check), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_cpu_capacity, + TP_PROTO(int cpu, unsigned long *capacity), + TP_ARGS(cpu, capacity), 1); + +DECLARE_RESTRICTED_HOOK(android_rvh_update_misfit_status, + TP_PROTO(struct task_struct *p, struct rq *rq, bool *need_update), + TP_ARGS(p, rq, need_update), 1); + DECLARE_HOOK(android_vh_build_sched_domains, TP_PROTO(bool has_asym), TP_ARGS(has_asym)); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fa092e6df44a..fd93768fa057 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3176,6 +3176,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->se.nr_migrations++; rseq_migrate(p); perf_event_task_migrate(p); + trace_android_rvh_set_task_cpu(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -4238,6 +4239,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + trace_android_rvh_try_to_wake_up(p); + cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); if (task_cpu(p) != cpu) { if (p->in_iowait) { @@ -4257,8 +4260,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) unlock: raw_spin_unlock_irqrestore(&p->pi_lock, flags); out: - if (success) + if (success) { + trace_android_rvh_try_to_wake_up_success(p); ttwu_stat(p, task_cpu(p), wake_flags); + } preempt_enable(); return success; @@ -4593,6 +4598,8 @@ late_initcall(sched_core_sysctl_init); */ int sched_fork(unsigned long clone_flags, struct task_struct *p) { + trace_android_rvh_sched_fork(p); + __sched_fork(clone_flags, p); /* * We mark the process as NEW here. This guarantees that @@ -4718,6 +4725,8 @@ void wake_up_new_task(struct task_struct *p) struct rq_flags rf; struct rq *rq; + trace_android_rvh_wake_up_new_task(p); + raw_spin_lock_irqsave(&p->pi_lock, rf.flags); WRITE_ONCE(p->__state, TASK_RUNNING); #ifdef CONFIG_SMP @@ -4736,6 +4745,7 @@ void wake_up_new_task(struct task_struct *p) rq = __task_rq_lock(p, &rf); update_rq_clock(rq); post_init_entity_util_avg(p); + trace_android_rvh_new_task_stats(p); activate_task(rq, p, ENQUEUE_NOCLOCK); trace_sched_wakeup_new(p); @@ -5151,6 +5161,8 @@ static struct rq *finish_task_switch(struct task_struct *prev) if (prev->sched_class->task_dead) prev->sched_class->task_dead(prev); + trace_android_rvh_flush_task(prev); + /* Task is done with its stack. */ put_task_stack(prev); @@ -5509,6 +5521,7 @@ void scheduler_tick(void) rq_lock(rq, &rf); update_rq_clock(rq); + trace_android_rvh_tick_entry(rq); thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq)); update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure); @@ -6526,6 +6539,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) rq->last_seen_need_resched_ns = 0; #endif + trace_android_rvh_schedule(prev, next, rq); if (likely(prev != next)) { rq->nr_switches++; /* @@ -9520,6 +9534,7 @@ int sched_cpu_starting(unsigned int cpu) sched_core_cpu_starting(cpu); sched_rq_cpu_starting(cpu); sched_tick_start(cpu); + trace_android_rvh_sched_cpu_starting(cpu); return 0; } @@ -9593,6 +9608,8 @@ int sched_cpu_dying(unsigned int cpu) } rq_unlock_irqrestore(rq, &rf); + trace_android_rvh_sched_cpu_dying(cpu); + calc_load_migrate(rq); update_max_interval(); hrtick_clear(rq); diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 3e0f79fee72e..a485f3b0201c 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -3,6 +3,7 @@ * Simple CPU accounting cgroup controller */ #include +#include #ifdef CONFIG_IRQ_TIME_ACCOUNTING @@ -54,6 +55,7 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) unsigned int pc; s64 delta; int cpu; + bool irq_start = true; if (!sched_clock_irqtime) return; @@ -69,10 +71,15 @@ void irqtime_account_irq(struct task_struct *curr, unsigned int offset) * in that case, so as not to confuse scheduler with a special task * that do not consume any time, but still wants to run. */ - if (pc & HARDIRQ_MASK) + if (pc & HARDIRQ_MASK) { irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); - else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) + irq_start = false; + } else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) { irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); + irq_start = false; + } + + trace_android_rvh_account_irq(curr, cpu, delta, irq_start); } static u64 irqtime_tick_accounted(u64 maxtime) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 88f5ec46c5d7..4ff33ed0bd49 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4274,7 +4274,10 @@ static inline int task_fits_capacity(struct task_struct *p, static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + bool need_update = true; + + trace_android_rvh_update_misfit_status(p, rq, &need_update); + if (!static_branch_unlikely(&sched_asym_cpucapacity) || !need_update) return; if (!p || p->nr_cpus_allowed == 1) { @@ -4383,6 +4386,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); + trace_android_rvh_place_entity(cfs_rq, se, initial, &vruntime); } static void check_enqueue_throttle(struct cfs_rq *cfs_rq); @@ -8590,6 +8594,7 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) if (!capacity) capacity = 1; + trace_android_rvh_update_cpu_capacity(cpu, &capacity); cpu_rq(cpu)->cpu_capacity = capacity; trace_sched_cpu_capacity_tp(cpu_rq(cpu)); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5275666b5eaa..c480058808bc 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -384,6 +384,7 @@ static bool build_perf_domains(const struct cpumask *cpu_map) * EAS is enabled for asymmetric CPU capacity topologies. * Allow vendor to override if desired. */ + trace_android_rvh_build_perf_domains(&eas_check); if (!per_cpu(sd_asym_cpucapacity, cpu) && !eas_check) { if (sched_debug()) { pr_info("rd %*pbl: CPUs do not have asymmetric capacities\n", diff --git a/kernel/sched/vendor_hooks.c b/kernel/sched/vendor_hooks.c index 0682fd4b1c22..5aacb10debf3 100644 --- a/kernel/sched/vendor_hooks.c +++ b/kernel/sched/vendor_hooks.c @@ -55,3 +55,19 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_dequeue_task_fair); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_post_init_entity_util_avg); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_setscheduler_uclamp); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_task_cpu); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_try_to_wake_up_success); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_fork); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_wake_up_new_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_new_task_stats); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_flush_task); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_tick_entry); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_schedule); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_starting); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_cpu_dying); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_account_irq); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_place_entity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_build_perf_domains); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpu_capacity); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_misfit_status);