diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 7e163627149c..9367f1cb2e3c 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -580,12 +580,6 @@ enum }; #define SOFTIRQ_STOP_IDLE_MASK (~(1 << RCU_SOFTIRQ)) -/* Softirq's where the handling might be long: */ -#define LONG_SOFTIRQ_MASK ((1 << NET_TX_SOFTIRQ) | \ - (1 << NET_RX_SOFTIRQ) | \ - (1 << BLOCK_SOFTIRQ) | \ - (1 << IRQ_POLL_SOFTIRQ) | \ - (1 << TASKLET_SOFTIRQ)) /* map softirq index to softirq name. update 'softirq_to_name' in * kernel/softirq.c when adding a new softirq. @@ -612,7 +606,6 @@ extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); DECLARE_PER_CPU(struct task_struct *, ksoftirqd); -DECLARE_PER_CPU(__u32, active_softirqs); static inline struct task_struct *this_cpu_ksoftirqd(void) { diff --git a/include/linux/sched.h b/include/linux/sched.h index dff1b7740d45..03626ac2fe41 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1808,16 +1808,6 @@ current_restore_flags(unsigned long orig_flags, unsigned long flags) extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed); - -#ifdef CONFIG_RT_SOFTINT_OPTIMIZATION -extern bool cpupri_check_rt(void); -#else -static inline bool cpupri_check_rt(void) -{ - return false; -} -#endif - #ifdef CONFIG_SMP extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); diff --git a/include/trace/events/irq.h b/include/trace/events/irq.h index bb70f46656b2..eeceafaaea4c 100644 --- a/include/trace/events/irq.h +++ b/include/trace/events/irq.h @@ -160,51 +160,6 @@ DEFINE_EVENT(softirq, softirq_raise, TP_ARGS(vec_nr) ); -DECLARE_EVENT_CLASS(tasklet, - - TP_PROTO(void *func), - - TP_ARGS(func), - - TP_STRUCT__entry( - __field( void *, func) - ), - - TP_fast_assign( - __entry->func = func; - ), - - TP_printk("function=%ps", __entry->func) -); - -DEFINE_EVENT(tasklet, tasklet_entry, - - TP_PROTO(void *func), - - TP_ARGS(func) -); - -DEFINE_EVENT(tasklet, tasklet_exit, - - TP_PROTO(void *func), - - TP_ARGS(func) -); - -DEFINE_EVENT(tasklet, tasklet_hi_entry, - - TP_PROTO(void *func), - - TP_ARGS(func) -); - -DEFINE_EVENT(tasklet, tasklet_hi_exit, - - TP_PROTO(void *func), - - TP_ARGS(func) -); - #endif /* _TRACE_IRQ_H */ /* This part must be outside protection */ diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 6f5fd1d20119..1db2516d1bb4 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -356,16 +356,3 @@ void cpupri_cleanup(struct cpupri *cp) for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) free_cpumask_var(cp->pri_to_cpu[i].mask); } - -#ifdef CONFIG_RT_SOFTINT_OPTIMIZATION -/* - * cpupri_check_rt - check if CPU has a RT task - * should be called from rcu-sched read section. - */ -bool cpupri_check_rt(void) -{ - int cpu = raw_smp_processor_id(); - - return cpu_rq(cpu)->rd->cpupri.cpu_to_pri[cpu] > CPUPRI_NORMAL; -} -#endif diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 18bda93bc65b..80667fdf4b5e 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1571,20 +1571,15 @@ static int find_lowest_rq(struct task_struct *task); #ifdef CONFIG_RT_SOFTINT_OPTIMIZATION /* * Return whether the task on the given cpu is currently non-preemptible - * while handling a potentially long softint, or if the task is likely - * to block preemptions soon because it is a ksoftirq thread that is - * handling slow softints. + * while handling a softirq or is likely to block preemptions soon because + * it is a ksoftirq thread. */ bool task_may_not_preempt(struct task_struct *task, int cpu) { - __u32 softirqs = per_cpu(active_softirqs, cpu) | - local_softirq_pending(); - struct task_struct *cpu_ksoftirqd = per_cpu(ksoftirqd, cpu); - return ((softirqs & LONG_SOFTIRQ_MASK) && - (task == cpu_ksoftirqd || - task_thread_info(task)->preempt_count & SOFTIRQ_MASK)); + return (task_thread_info(task)->preempt_count & SOFTIRQ_MASK) || + task == cpu_ksoftirqd; } EXPORT_SYMBOL_GPL(task_may_not_preempt); #endif /* CONFIG_RT_SOFTINT_OPTIMIZATION */ diff --git a/kernel/softirq.c b/kernel/softirq.c index 5b49d16e4182..1cf952bdce35 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -63,13 +63,6 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp DEFINE_PER_CPU(struct task_struct *, ksoftirqd); EXPORT_PER_CPU_SYMBOL_GPL(ksoftirqd); -/* - * active_softirqs -- per cpu, a mask of softirqs that are being handled, - * with the expectation that approximate answers are acceptable and therefore - * no synchronization. - */ -DEFINE_PER_CPU(__u32, active_softirqs); - const char * const softirq_to_name[NR_SOFTIRQS] = { "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL", "TASKLET", "SCHED", "HRTIMER", "RCU" @@ -90,6 +83,21 @@ static void wakeup_softirqd(void) wake_up_process(tsk); } +/* + * If ksoftirqd is scheduled, we do not want to process pending softirqs + * right now. Let ksoftirqd handle this at its own rate, to get fairness, + * unless we're doing some of the synchronous softirqs. + */ +#define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ)) +static bool ksoftirqd_running(unsigned long pending) +{ + struct task_struct *tsk = __this_cpu_read(ksoftirqd); + + if (pending & SOFTIRQ_NOW_MASK) + return false; + return tsk && task_is_running(tsk) && !__kthread_should_park(tsk); +} + #ifdef CONFIG_TRACE_IRQFLAGS DEFINE_PER_CPU(int, hardirqs_enabled); DEFINE_PER_CPU(int, hardirq_context); @@ -231,7 +239,7 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) goto out; pending = local_softirq_pending(); - if (!pending) + if (!pending || ksoftirqd_running(pending)) goto out; /* @@ -414,6 +422,9 @@ static inline bool should_wake_ksoftirqd(void) static inline void invoke_softirq(void) { + if (ksoftirqd_running(local_softirq_pending())) + return; + if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) { #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* @@ -447,7 +458,7 @@ asmlinkage __visible void do_softirq(void) pending = local_softirq_pending(); - if (pending) + if (pending && !ksoftirqd_running(pending)) do_softirq_own_stack(); local_irq_restore(flags); @@ -504,17 +515,6 @@ static inline bool lockdep_softirq_start(void) { return false; } static inline void lockdep_softirq_end(bool in_hardirq) { } #endif -static inline __u32 softirq_deferred_for_rt(__u32 *pending) -{ - __u32 deferred = 0; - - if (cpupri_check_rt()) { - deferred = *pending & LONG_SOFTIRQ_MASK; - *pending &= ~LONG_SOFTIRQ_MASK; - } - return deferred; -} - asmlinkage __visible void __softirq_entry __do_softirq(void) { unsigned long end = jiffies + MAX_SOFTIRQ_TIME; @@ -522,7 +522,6 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) int max_restart = MAX_SOFTIRQ_RESTART; struct softirq_action *h; bool in_hardirq; - __u32 deferred; __u32 pending; int softirq_bit; @@ -535,15 +534,13 @@ asmlinkage __visible void __softirq_entry __do_softirq(void) pending = local_softirq_pending(); - deferred = softirq_deferred_for_rt(&pending); softirq_handle_begin(); in_hardirq = lockdep_softirq_start(); account_softirq_enter(current); restart: /* Reset the pending bitmask before enabling irqs */ - set_softirq_pending(deferred); - __this_cpu_write(active_softirqs, pending); + set_softirq_pending(0); local_irq_enable(); @@ -573,7 +570,6 @@ restart: pending >>= softirq_bit; } - __this_cpu_write(active_softirqs, 0); if (!IS_ENABLED(CONFIG_PREEMPT_RT) && __this_cpu_read(ksoftirqd) == current) rcu_softirq_qs(); @@ -581,16 +577,13 @@ restart: local_irq_disable(); pending = local_softirq_pending(); - deferred = softirq_deferred_for_rt(&pending); - if (pending) { if (time_before(jiffies, end) && !need_resched() && --max_restart) goto restart; - } - if (pending | deferred) wakeup_softirqd(); + } account_softirq_exit(current); lockdep_softirq_end(in_hardirq); @@ -790,15 +783,10 @@ static void tasklet_action_common(struct softirq_action *a, if (tasklet_trylock(t)) { if (!atomic_read(&t->count)) { if (tasklet_clear_sched(t)) { - if (t->use_callback) { - trace_tasklet_entry(t->callback); + if (t->use_callback) t->callback(t); - trace_tasklet_exit(t->callback); - } else { - trace_tasklet_entry(t->func); + else t->func(t->data); - trace_tasklet_exit(t->func); - } } tasklet_unlock(t); continue;