diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 91c23908191c..87b78abce5c6 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5976,6 +5976,11 @@ sa1100ir [NET] See drivers/net/irda/sa1100_ir.c. + sched_proxy_exec= [KNL] + Enables or disables "proxy execution" style + solution to mutex-based priority inversion. + Format: + sched_verbose [KNL,EARLY] Enables verbose scheduler debug messages. schedstats= [KNL,X86] Enable or disable scheduled statistics. diff --git a/include/linux/sched.h b/include/linux/sched.h index b1105b2d435c..73458a21e787 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1619,6 +1619,19 @@ struct task_struct { */ }; +#ifdef CONFIG_SCHED_PROXY_EXEC +DECLARE_STATIC_KEY_TRUE(__sched_proxy_exec); +static inline bool sched_proxy_exec(void) +{ + return static_branch_likely(&__sched_proxy_exec); +} +#else +static inline bool sched_proxy_exec(void) +{ + return false; +} +#endif + #define TASK_REPORT_IDLE (TASK_REPORT + 1) #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1) diff --git a/init/Kconfig b/init/Kconfig index e2f27b0513ba..4c2f657841a2 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -863,6 +863,15 @@ config UCLAMP_BUCKETS_COUNT If in doubt, use the default value. +config SCHED_PROXY_EXEC + bool "Proxy Execution" + default n + # Avoid some build failures w/ PREEMPT_RT until it can be fixed + depends on !PREEMPT_RT + help + This option enables proxy execution, a mechanism for mutex-owning + tasks to inherit the scheduling context of higher priority waiters. + endmenu # diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d8497ca74461..a353f0924b3f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -125,6 +125,35 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(sched_switch); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); EXPORT_SYMBOL_GPL(runqueues); +#ifdef CONFIG_SCHED_PROXY_EXEC +DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec); +static int __init setup_proxy_exec(char *str) +{ + bool proxy_enable; + + if (kstrtobool(str, &proxy_enable)) { + pr_warn("Unable to parse sched_proxy_exec=\n"); + return 0; + } + + if (proxy_enable) { + pr_info("sched_proxy_exec enabled via boot arg\n"); + static_branch_enable(&__sched_proxy_exec); + } else { + pr_info("sched_proxy_exec disabled via boot arg\n"); + static_branch_disable(&__sched_proxy_exec); + } + return 1; +} +#else +static int __init setup_proxy_exec(char *str) +{ + pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n"); + return 0; +} +#endif +__setup("sched_proxy_exec=", setup_proxy_exec); + #ifdef CONFIG_SCHED_DEBUG /* * Debugging: various feature bits diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1779faa30baf..a1ddd055fbc9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1159,10 +1159,15 @@ struct rq { */ unsigned int nr_uninterruptible; +#ifdef CONFIG_SCHED_PROXY_EXEC + struct task_struct __rcu *donor; /* Scheduling context */ + struct task_struct __rcu *curr; /* Execution context */ +#else union { struct task_struct __rcu *donor; /* Scheduler context */ struct task_struct __rcu *curr; /* Execution context */ }; +#endif struct sched_dl_entity *dl_server; struct task_struct *idle; struct task_struct *stop; @@ -1360,10 +1365,17 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); #define cpu_curr(cpu) (cpu_rq(cpu)->curr) #define raw_rq() raw_cpu_ptr(&runqueues) +#ifdef CONFIG_SCHED_PROXY_EXEC +static inline void rq_set_donor(struct rq *rq, struct task_struct *t) +{ + rcu_assign_pointer(rq->donor, t); +} +#else static inline void rq_set_donor(struct rq *rq, struct task_struct *t) { /* Do nothing */ } +#endif #ifdef CONFIG_SCHED_CORE static inline struct cpumask *sched_group_span(struct sched_group *sg);