irq_work: Add generic hardirq context callbacks
Provide a mechanism that allows running code in IRQ context. It is most useful for NMI code that needs to interact with the rest of the system -- like wakeup a task to drain buffers. Perf currently has such a mechanism, so extract that and provide it as a generic feature, independent of perf so that others may also benefit. The IRQ context callback is generated through self-IPIs where possible, or on architectures like powerpc the decrementer (the built-in timer facility) is set to generate an interrupt immediately. Architectures that don't have anything like this get to do with a callback from the timer tick. These architectures can call irq_work_run() at the tail of any IRQ handlers that might enqueue such work (like the perf IRQ handler) to avoid undue latencies in processing the work. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Acked-by: Kyle McMartin <kyle@mcmartin.ca> Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [ various fixes ] Signed-off-by: Huang Ying <ying.huang@intel.com> LKML-Reference: <1287036094.7768.291.camel@yhuang-dev> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Peter Zijlstra
committed by
Ingo Molnar
Ingo Molnar
parent
8e5fc1a732
commit
e360adbe29
@@ -23,6 +23,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg
|
||||
CFLAGS_REMOVE_cgroup-debug.o = -pg
|
||||
CFLAGS_REMOVE_sched_clock.o = -pg
|
||||
CFLAGS_REMOVE_perf_event.o = -pg
|
||||
CFLAGS_REMOVE_irq_work.o = -pg
|
||||
endif
|
||||
|
||||
obj-$(CONFIG_FREEZER) += freezer.o
|
||||
@@ -100,6 +101,7 @@ obj-$(CONFIG_TRACING) += trace/
|
||||
obj-$(CONFIG_X86_DS) += trace/
|
||||
obj-$(CONFIG_RING_BUFFER) += trace/
|
||||
obj-$(CONFIG_SMP) += sched_cpupri.o
|
||||
obj-$(CONFIG_IRQ_WORK) += irq_work.o
|
||||
obj-$(CONFIG_PERF_EVENTS) += perf_event.o
|
||||
obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
|
||||
obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
|
||||
*
|
||||
* Provides a framework for enqueueing and running callbacks from hardirq
|
||||
* context. The enqueueing is NMI-safe.
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/irq_work.h>
|
||||
#include <linux/hardirq.h>
|
||||
|
||||
/*
|
||||
* An entry can be in one of four states:
|
||||
*
|
||||
* free NULL, 0 -> {claimed} : free to be used
|
||||
* claimed NULL, 3 -> {pending} : claimed to be enqueued
|
||||
* pending next, 3 -> {busy} : queued, pending callback
|
||||
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
|
||||
*
|
||||
* We use the lower two bits of the next pointer to keep PENDING and BUSY
|
||||
* flags.
|
||||
*/
|
||||
|
||||
#define IRQ_WORK_PENDING 1UL
|
||||
#define IRQ_WORK_BUSY 2UL
|
||||
#define IRQ_WORK_FLAGS 3UL
|
||||
|
||||
static inline bool irq_work_is_set(struct irq_work *entry, int flags)
|
||||
{
|
||||
return (unsigned long)entry->next & flags;
|
||||
}
|
||||
|
||||
static inline struct irq_work *irq_work_next(struct irq_work *entry)
|
||||
{
|
||||
unsigned long next = (unsigned long)entry->next;
|
||||
next &= ~IRQ_WORK_FLAGS;
|
||||
return (struct irq_work *)next;
|
||||
}
|
||||
|
||||
static inline struct irq_work *next_flags(struct irq_work *entry, int flags)
|
||||
{
|
||||
unsigned long next = (unsigned long)entry;
|
||||
next |= flags;
|
||||
return (struct irq_work *)next;
|
||||
}
|
||||
|
||||
static DEFINE_PER_CPU(struct irq_work *, irq_work_list);
|
||||
|
||||
/*
|
||||
* Claim the entry so that no one else will poke at it.
|
||||
*/
|
||||
static bool irq_work_claim(struct irq_work *entry)
|
||||
{
|
||||
struct irq_work *next, *nflags;
|
||||
|
||||
do {
|
||||
next = entry->next;
|
||||
if ((unsigned long)next & IRQ_WORK_PENDING)
|
||||
return false;
|
||||
nflags = next_flags(next, IRQ_WORK_FLAGS);
|
||||
} while (cmpxchg(&entry->next, next, nflags) != next);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void __weak arch_irq_work_raise(void)
|
||||
{
|
||||
/*
|
||||
* Lame architectures will get the timer tick callback
|
||||
*/
|
||||
}
|
||||
|
||||
/*
|
||||
* Queue the entry and raise the IPI if needed.
|
||||
*/
|
||||
static void __irq_work_queue(struct irq_work *entry)
|
||||
{
|
||||
struct irq_work **head, *next;
|
||||
|
||||
head = &get_cpu_var(irq_work_list);
|
||||
|
||||
do {
|
||||
next = *head;
|
||||
/* Can assign non-atomic because we keep the flags set. */
|
||||
entry->next = next_flags(next, IRQ_WORK_FLAGS);
|
||||
} while (cmpxchg(head, next, entry) != next);
|
||||
|
||||
/* The list was empty, raise self-interrupt to start processing. */
|
||||
if (!irq_work_next(entry))
|
||||
arch_irq_work_raise();
|
||||
|
||||
put_cpu_var(irq_work_list);
|
||||
}
|
||||
|
||||
/*
|
||||
* Enqueue the irq_work @entry, returns true on success, failure when the
|
||||
* @entry was already enqueued by someone else.
|
||||
*
|
||||
* Can be re-enqueued while the callback is still in progress.
|
||||
*/
|
||||
bool irq_work_queue(struct irq_work *entry)
|
||||
{
|
||||
if (!irq_work_claim(entry)) {
|
||||
/*
|
||||
* Already enqueued, can't do!
|
||||
*/
|
||||
return false;
|
||||
}
|
||||
|
||||
__irq_work_queue(entry);
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_queue);
|
||||
|
||||
/*
|
||||
* Run the irq_work entries on this cpu. Requires to be ran from hardirq
|
||||
* context with local IRQs disabled.
|
||||
*/
|
||||
void irq_work_run(void)
|
||||
{
|
||||
struct irq_work *list, **head;
|
||||
|
||||
head = &__get_cpu_var(irq_work_list);
|
||||
if (*head == NULL)
|
||||
return;
|
||||
|
||||
BUG_ON(!in_irq());
|
||||
BUG_ON(!irqs_disabled());
|
||||
|
||||
list = xchg(head, NULL);
|
||||
while (list != NULL) {
|
||||
struct irq_work *entry = list;
|
||||
|
||||
list = irq_work_next(list);
|
||||
|
||||
/*
|
||||
* Clear the PENDING bit, after this point the @entry
|
||||
* can be re-used.
|
||||
*/
|
||||
entry->next = next_flags(NULL, IRQ_WORK_BUSY);
|
||||
entry->func(entry);
|
||||
/*
|
||||
* Clear the BUSY bit and return to the free state if
|
||||
* no-one else claimed it meanwhile.
|
||||
*/
|
||||
cmpxchg(&entry->next, next_flags(NULL, IRQ_WORK_BUSY), NULL);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_run);
|
||||
|
||||
/*
|
||||
* Synchronize against the irq_work @entry, ensures the entry is not
|
||||
* currently in use.
|
||||
*/
|
||||
void irq_work_sync(struct irq_work *entry)
|
||||
{
|
||||
WARN_ON_ONCE(irqs_disabled());
|
||||
|
||||
while (irq_work_is_set(entry, IRQ_WORK_BUSY))
|
||||
cpu_relax();
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(irq_work_sync);
|
||||
+5
-99
@@ -2206,12 +2206,11 @@ static void free_event_rcu(struct rcu_head *head)
|
||||
kfree(event);
|
||||
}
|
||||
|
||||
static void perf_pending_sync(struct perf_event *event);
|
||||
static void perf_buffer_put(struct perf_buffer *buffer);
|
||||
|
||||
static void free_event(struct perf_event *event)
|
||||
{
|
||||
perf_pending_sync(event);
|
||||
irq_work_sync(&event->pending);
|
||||
|
||||
if (!event->parent) {
|
||||
atomic_dec(&nr_events);
|
||||
@@ -3162,16 +3161,7 @@ void perf_event_wakeup(struct perf_event *event)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Pending wakeups
|
||||
*
|
||||
* Handle the case where we need to wakeup up from NMI (or rq->lock) context.
|
||||
*
|
||||
* The NMI bit means we cannot possibly take locks. Therefore, maintain a
|
||||
* single linked list and use cmpxchg() to add entries lockless.
|
||||
*/
|
||||
|
||||
static void perf_pending_event(struct perf_pending_entry *entry)
|
||||
static void perf_pending_event(struct irq_work *entry)
|
||||
{
|
||||
struct perf_event *event = container_of(entry,
|
||||
struct perf_event, pending);
|
||||
@@ -3187,89 +3177,6 @@ static void perf_pending_event(struct perf_pending_entry *entry)
|
||||
}
|
||||
}
|
||||
|
||||
#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
|
||||
|
||||
static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
|
||||
PENDING_TAIL,
|
||||
};
|
||||
|
||||
static void perf_pending_queue(struct perf_pending_entry *entry,
|
||||
void (*func)(struct perf_pending_entry *))
|
||||
{
|
||||
struct perf_pending_entry **head;
|
||||
|
||||
if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
|
||||
return;
|
||||
|
||||
entry->func = func;
|
||||
|
||||
head = &get_cpu_var(perf_pending_head);
|
||||
|
||||
do {
|
||||
entry->next = *head;
|
||||
} while (cmpxchg(head, entry->next, entry) != entry->next);
|
||||
|
||||
set_perf_event_pending();
|
||||
|
||||
put_cpu_var(perf_pending_head);
|
||||
}
|
||||
|
||||
static int __perf_pending_run(void)
|
||||
{
|
||||
struct perf_pending_entry *list;
|
||||
int nr = 0;
|
||||
|
||||
list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
|
||||
while (list != PENDING_TAIL) {
|
||||
void (*func)(struct perf_pending_entry *);
|
||||
struct perf_pending_entry *entry = list;
|
||||
|
||||
list = list->next;
|
||||
|
||||
func = entry->func;
|
||||
entry->next = NULL;
|
||||
/*
|
||||
* Ensure we observe the unqueue before we issue the wakeup,
|
||||
* so that we won't be waiting forever.
|
||||
* -- see perf_not_pending().
|
||||
*/
|
||||
smp_wmb();
|
||||
|
||||
func(entry);
|
||||
nr++;
|
||||
}
|
||||
|
||||
return nr;
|
||||
}
|
||||
|
||||
static inline int perf_not_pending(struct perf_event *event)
|
||||
{
|
||||
/*
|
||||
* If we flush on whatever cpu we run, there is a chance we don't
|
||||
* need to wait.
|
||||
*/
|
||||
get_cpu();
|
||||
__perf_pending_run();
|
||||
put_cpu();
|
||||
|
||||
/*
|
||||
* Ensure we see the proper queue state before going to sleep
|
||||
* so that we do not miss the wakeup. -- see perf_pending_handle()
|
||||
*/
|
||||
smp_rmb();
|
||||
return event->pending.next == NULL;
|
||||
}
|
||||
|
||||
static void perf_pending_sync(struct perf_event *event)
|
||||
{
|
||||
wait_event(event->waitq, perf_not_pending(event));
|
||||
}
|
||||
|
||||
void perf_event_do_pending(void)
|
||||
{
|
||||
__perf_pending_run();
|
||||
}
|
||||
|
||||
/*
|
||||
* We assume there is only KVM supporting the callbacks.
|
||||
* Later on, we might change it to a list if there is
|
||||
@@ -3319,8 +3226,7 @@ static void perf_output_wakeup(struct perf_output_handle *handle)
|
||||
|
||||
if (handle->nmi) {
|
||||
handle->event->pending_wakeup = 1;
|
||||
perf_pending_queue(&handle->event->pending,
|
||||
perf_pending_event);
|
||||
irq_work_queue(&handle->event->pending);
|
||||
} else
|
||||
perf_event_wakeup(handle->event);
|
||||
}
|
||||
@@ -4356,8 +4262,7 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
|
||||
event->pending_kill = POLL_HUP;
|
||||
if (nmi) {
|
||||
event->pending_disable = 1;
|
||||
perf_pending_queue(&event->pending,
|
||||
perf_pending_event);
|
||||
irq_work_queue(&event->pending);
|
||||
} else
|
||||
perf_event_disable(event);
|
||||
}
|
||||
@@ -5374,6 +5279,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
|
||||
INIT_LIST_HEAD(&event->event_entry);
|
||||
INIT_LIST_HEAD(&event->sibling_list);
|
||||
init_waitqueue_head(&event->waitq);
|
||||
init_irq_work(&event->pending, perf_pending_event);
|
||||
|
||||
mutex_init(&event->mmap_mutex);
|
||||
|
||||
|
||||
+5
-2
@@ -37,7 +37,7 @@
|
||||
#include <linux/delay.h>
|
||||
#include <linux/tick.h>
|
||||
#include <linux/kallsyms.h>
|
||||
#include <linux/perf_event.h>
|
||||
#include <linux/irq_work.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)
|
||||
run_local_timers();
|
||||
rcu_check_callbacks(cpu, user_tick);
|
||||
printk_tick();
|
||||
perf_event_do_pending();
|
||||
#ifdef CONFIG_IRQ_WORK
|
||||
if (in_irq())
|
||||
irq_work_run();
|
||||
#endif
|
||||
scheduler_tick();
|
||||
run_posix_cpu_timers(p);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user