Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (45 commits) rcu: Move propagation of ->completed from rcu_start_gp() to rcu_report_qs_rsp() rcu: Remove rcu_needs_cpu_flush() to avoid false quiescent states rcu: Wire up RCU_BOOST_PRIO for rcutree rcu: Make rcu_torture_boost() exit loops at end of test rcu: Make rcu_torture_fqs() exit loops at end of test rcu: Permit rt_mutex_unlock() with irqs disabled rcu: Avoid having just-onlined CPU resched itself when RCU is idle rcu: Suppress NMI backtraces when stall ends before dump rcu: Prohibit grace periods during early boot rcu: Simplify unboosting checks rcu: Prevent early boot set_need_resched() from __rcu_pending() rcu: Dump local stack if cannot dump all CPUs' stacks rcu: Move __rcu_read_unlock()'s barrier() within if-statement rcu: Improve rcu_assign_pointer() and RCU_INIT_POINTER() documentation rcu: Make rcu_assign_pointer() unconditionally insert a memory barrier rcu: Make rcu_implicit_dynticks_qs() locals be correct size rcu: Eliminate in_irq() checks in rcu_enter_nohz() nohz: Remove nohz_cpu_mask rcu: Document interpretation of RCU-lockdep splats rcu: Allow rcutorture's stat_interval parameter to be changed at runtime ...
This commit is contained in:
Linus Torvalds
@@ -548,7 +548,7 @@ do { \
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_PROVE_RCU
|
||||
extern void lockdep_rcu_dereference(const char *file, const int line);
|
||||
void lockdep_rcu_suspicious(const char *file, const int line, const char *s);
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_LOCKDEP_H */
|
||||
|
||||
+139
-165
@@ -33,6 +33,7 @@
|
||||
#ifndef __LINUX_RCUPDATE_H
|
||||
#define __LINUX_RCUPDATE_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/cache.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/threads.h>
|
||||
@@ -64,32 +65,74 @@ static inline void rcutorture_record_progress(unsigned long vernum)
|
||||
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
|
||||
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
|
||||
|
||||
/**
|
||||
* struct rcu_head - callback structure for use with RCU
|
||||
* @next: next update requests in a list
|
||||
* @func: actual update function to call after the grace period.
|
||||
*/
|
||||
struct rcu_head {
|
||||
struct rcu_head *next;
|
||||
void (*func)(struct rcu_head *head);
|
||||
};
|
||||
|
||||
/* Exported common interfaces */
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RCU
|
||||
|
||||
/**
|
||||
* call_rcu() - Queue an RCU callback for invocation after a grace period.
|
||||
* @head: structure to be used for queueing the RCU updates.
|
||||
* @func: actual callback function to be invoked after the grace period
|
||||
*
|
||||
* The callback function will be invoked some time after a full grace
|
||||
* period elapses, in other words after all pre-existing RCU read-side
|
||||
* critical sections have completed. However, the callback function
|
||||
* might well execute concurrently with RCU read-side critical sections
|
||||
* that started after call_rcu() was invoked. RCU read-side critical
|
||||
* sections are delimited by rcu_read_lock() and rcu_read_unlock(),
|
||||
* and may be nested.
|
||||
*/
|
||||
extern void call_rcu(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *head));
|
||||
|
||||
#else /* #ifdef CONFIG_PREEMPT_RCU */
|
||||
|
||||
/* In classic RCU, call_rcu() is just call_rcu_sched(). */
|
||||
#define call_rcu call_rcu_sched
|
||||
|
||||
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
|
||||
|
||||
/**
|
||||
* call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
|
||||
* @head: structure to be used for queueing the RCU updates.
|
||||
* @func: actual callback function to be invoked after the grace period
|
||||
*
|
||||
* The callback function will be invoked some time after a full grace
|
||||
* period elapses, in other words after all currently executing RCU
|
||||
* read-side critical sections have completed. call_rcu_bh() assumes
|
||||
* that the read-side critical sections end on completion of a softirq
|
||||
* handler. This means that read-side critical sections in process
|
||||
* context must not be interrupted by softirqs. This interface is to be
|
||||
* used when most of the read-side critical sections are in softirq context.
|
||||
* RCU read-side critical sections are delimited by :
|
||||
* - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
|
||||
* OR
|
||||
* - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
|
||||
* These may be nested.
|
||||
*/
|
||||
extern void call_rcu_bh(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *head));
|
||||
|
||||
/**
|
||||
* call_rcu_sched() - Queue an RCU for invocation after sched grace period.
|
||||
* @head: structure to be used for queueing the RCU updates.
|
||||
* @func: actual callback function to be invoked after the grace period
|
||||
*
|
||||
* The callback function will be invoked some time after a full grace
|
||||
* period elapses, in other words after all currently executing RCU
|
||||
* read-side critical sections have completed. call_rcu_sched() assumes
|
||||
* that the read-side critical sections end on enabling of preemption
|
||||
* or on voluntary preemption.
|
||||
* RCU read-side critical sections are delimited by :
|
||||
* - rcu_read_lock_sched() and rcu_read_unlock_sched(),
|
||||
* OR
|
||||
* anything that disables preemption.
|
||||
* These may be nested.
|
||||
*/
|
||||
extern void call_rcu_sched(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *rcu));
|
||||
|
||||
extern void synchronize_sched(void);
|
||||
extern void rcu_barrier_bh(void);
|
||||
extern void rcu_barrier_sched(void);
|
||||
|
||||
static inline void __rcu_read_lock_bh(void)
|
||||
{
|
||||
local_bh_disable();
|
||||
}
|
||||
|
||||
static inline void __rcu_read_unlock_bh(void)
|
||||
{
|
||||
local_bh_enable();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RCU
|
||||
|
||||
@@ -152,6 +195,15 @@ static inline void rcu_exit_nohz(void)
|
||||
|
||||
#endif /* #else #ifdef CONFIG_NO_HZ */
|
||||
|
||||
/*
|
||||
* Infrastructure to implement the synchronize_() primitives in
|
||||
* TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
|
||||
*/
|
||||
|
||||
typedef void call_rcu_func_t(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *head));
|
||||
void wait_rcu_gp(call_rcu_func_t crf);
|
||||
|
||||
#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
|
||||
#include <linux/rcutree.h>
|
||||
#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
|
||||
@@ -297,19 +349,31 @@ extern int rcu_my_thread_group_empty(void);
|
||||
/**
|
||||
* rcu_lockdep_assert - emit lockdep splat if specified condition not met
|
||||
* @c: condition to check
|
||||
* @s: informative message
|
||||
*/
|
||||
#define rcu_lockdep_assert(c) \
|
||||
#define rcu_lockdep_assert(c, s) \
|
||||
do { \
|
||||
static bool __warned; \
|
||||
if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
|
||||
__warned = true; \
|
||||
lockdep_rcu_dereference(__FILE__, __LINE__); \
|
||||
lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define rcu_sleep_check() \
|
||||
do { \
|
||||
rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
|
||||
"Illegal context switch in RCU-bh" \
|
||||
" read-side critical section"); \
|
||||
rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
|
||||
"Illegal context switch in RCU-sched"\
|
||||
" read-side critical section"); \
|
||||
} while (0)
|
||||
|
||||
#else /* #ifdef CONFIG_PROVE_RCU */
|
||||
|
||||
#define rcu_lockdep_assert(c) do { } while (0)
|
||||
#define rcu_lockdep_assert(c, s) do { } while (0)
|
||||
#define rcu_sleep_check() do { } while (0)
|
||||
|
||||
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
||||
|
||||
@@ -338,14 +402,16 @@ extern int rcu_my_thread_group_empty(void);
|
||||
#define __rcu_dereference_check(p, c, space) \
|
||||
({ \
|
||||
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
|
||||
rcu_lockdep_assert(c); \
|
||||
rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
|
||||
" usage"); \
|
||||
rcu_dereference_sparse(p, space); \
|
||||
smp_read_barrier_depends(); \
|
||||
((typeof(*p) __force __kernel *)(_________p1)); \
|
||||
})
|
||||
#define __rcu_dereference_protected(p, c, space) \
|
||||
({ \
|
||||
rcu_lockdep_assert(c); \
|
||||
rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
|
||||
" usage"); \
|
||||
rcu_dereference_sparse(p, space); \
|
||||
((typeof(*p) __force __kernel *)(p)); \
|
||||
})
|
||||
@@ -359,15 +425,15 @@ extern int rcu_my_thread_group_empty(void);
|
||||
#define __rcu_dereference_index_check(p, c) \
|
||||
({ \
|
||||
typeof(p) _________p1 = ACCESS_ONCE(p); \
|
||||
rcu_lockdep_assert(c); \
|
||||
rcu_lockdep_assert(c, \
|
||||
"suspicious rcu_dereference_index_check()" \
|
||||
" usage"); \
|
||||
smp_read_barrier_depends(); \
|
||||
(_________p1); \
|
||||
})
|
||||
#define __rcu_assign_pointer(p, v, space) \
|
||||
({ \
|
||||
if (!__builtin_constant_p(v) || \
|
||||
((v) != NULL)) \
|
||||
smp_wmb(); \
|
||||
smp_wmb(); \
|
||||
(p) = (typeof(*v) __force space *)(v); \
|
||||
})
|
||||
|
||||
@@ -500,26 +566,6 @@ extern int rcu_my_thread_group_empty(void);
|
||||
#define rcu_dereference_protected(p, c) \
|
||||
__rcu_dereference_protected((p), (c), __rcu)
|
||||
|
||||
/**
|
||||
* rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented
|
||||
* @p: The pointer to read, prior to dereferencing
|
||||
* @c: The conditions under which the dereference will take place
|
||||
*
|
||||
* This is the RCU-bh counterpart to rcu_dereference_protected().
|
||||
*/
|
||||
#define rcu_dereference_bh_protected(p, c) \
|
||||
__rcu_dereference_protected((p), (c), __rcu)
|
||||
|
||||
/**
|
||||
* rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented
|
||||
* @p: The pointer to read, prior to dereferencing
|
||||
* @c: The conditions under which the dereference will take place
|
||||
*
|
||||
* This is the RCU-sched counterpart to rcu_dereference_protected().
|
||||
*/
|
||||
#define rcu_dereference_sched_protected(p, c) \
|
||||
__rcu_dereference_protected((p), (c), __rcu)
|
||||
|
||||
|
||||
/**
|
||||
* rcu_dereference() - fetch RCU-protected pointer for dereferencing
|
||||
@@ -630,7 +676,7 @@ static inline void rcu_read_unlock(void)
|
||||
*/
|
||||
static inline void rcu_read_lock_bh(void)
|
||||
{
|
||||
__rcu_read_lock_bh();
|
||||
local_bh_disable();
|
||||
__acquire(RCU_BH);
|
||||
rcu_read_acquire_bh();
|
||||
}
|
||||
@@ -644,7 +690,7 @@ static inline void rcu_read_unlock_bh(void)
|
||||
{
|
||||
rcu_read_release_bh();
|
||||
__release(RCU_BH);
|
||||
__rcu_read_unlock_bh();
|
||||
local_bh_enable();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -698,11 +744,18 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
|
||||
* any prior initialization. Returns the value assigned.
|
||||
*
|
||||
* Inserts memory barriers on architectures that require them
|
||||
* (pretty much all of them other than x86), and also prevents
|
||||
* the compiler from reordering the code that initializes the
|
||||
* structure after the pointer assignment. More importantly, this
|
||||
* call documents which pointers will be dereferenced by RCU read-side
|
||||
* code.
|
||||
* (which is most of them), and also prevents the compiler from
|
||||
* reordering the code that initializes the structure after the pointer
|
||||
* assignment. More importantly, this call documents which pointers
|
||||
* will be dereferenced by RCU read-side code.
|
||||
*
|
||||
* In some special cases, you may use RCU_INIT_POINTER() instead
|
||||
* of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
|
||||
* to the fact that it does not constrain either the CPU or the compiler.
|
||||
* That said, using RCU_INIT_POINTER() when you should have used
|
||||
* rcu_assign_pointer() is a very bad thing that results in
|
||||
* impossible-to-diagnose memory corruption. So please be careful.
|
||||
* See the RCU_INIT_POINTER() comment header for details.
|
||||
*/
|
||||
#define rcu_assign_pointer(p, v) \
|
||||
__rcu_assign_pointer((p), (v), __rcu)
|
||||
@@ -710,105 +763,38 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
|
||||
/**
|
||||
* RCU_INIT_POINTER() - initialize an RCU protected pointer
|
||||
*
|
||||
* Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep
|
||||
* splats.
|
||||
* Initialize an RCU-protected pointer in special cases where readers
|
||||
* do not need ordering constraints on the CPU or the compiler. These
|
||||
* special cases are:
|
||||
*
|
||||
* 1. This use of RCU_INIT_POINTER() is NULLing out the pointer -or-
|
||||
* 2. The caller has taken whatever steps are required to prevent
|
||||
* RCU readers from concurrently accessing this pointer -or-
|
||||
* 3. The referenced data structure has already been exposed to
|
||||
* readers either at compile time or via rcu_assign_pointer() -and-
|
||||
* a. You have not made -any- reader-visible changes to
|
||||
* this structure since then -or-
|
||||
* b. It is OK for readers accessing this structure from its
|
||||
* new location to see the old state of the structure. (For
|
||||
* example, the changes were to statistical counters or to
|
||||
* other state where exact synchronization is not required.)
|
||||
*
|
||||
* Failure to follow these rules governing use of RCU_INIT_POINTER() will
|
||||
* result in impossible-to-diagnose memory corruption. As in the structures
|
||||
* will look OK in crash dumps, but any concurrent RCU readers might
|
||||
* see pre-initialized values of the referenced data structure. So
|
||||
* please be very careful how you use RCU_INIT_POINTER()!!!
|
||||
*
|
||||
* If you are creating an RCU-protected linked structure that is accessed
|
||||
* by a single external-to-structure RCU-protected pointer, then you may
|
||||
* use RCU_INIT_POINTER() to initialize the internal RCU-protected
|
||||
* pointers, but you must use rcu_assign_pointer() to initialize the
|
||||
* external-to-structure pointer -after- you have completely initialized
|
||||
* the reader-accessible portions of the linked structure.
|
||||
*/
|
||||
#define RCU_INIT_POINTER(p, v) \
|
||||
p = (typeof(*v) __force __rcu *)(v)
|
||||
|
||||
/* Infrastructure to implement the synchronize_() primitives. */
|
||||
|
||||
struct rcu_synchronize {
|
||||
struct rcu_head head;
|
||||
struct completion completion;
|
||||
};
|
||||
|
||||
extern void wakeme_after_rcu(struct rcu_head *head);
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RCU
|
||||
|
||||
/**
|
||||
* call_rcu() - Queue an RCU callback for invocation after a grace period.
|
||||
* @head: structure to be used for queueing the RCU updates.
|
||||
* @func: actual callback function to be invoked after the grace period
|
||||
*
|
||||
* The callback function will be invoked some time after a full grace
|
||||
* period elapses, in other words after all pre-existing RCU read-side
|
||||
* critical sections have completed. However, the callback function
|
||||
* might well execute concurrently with RCU read-side critical sections
|
||||
* that started after call_rcu() was invoked. RCU read-side critical
|
||||
* sections are delimited by rcu_read_lock() and rcu_read_unlock(),
|
||||
* and may be nested.
|
||||
*/
|
||||
extern void call_rcu(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *head));
|
||||
|
||||
#else /* #ifdef CONFIG_PREEMPT_RCU */
|
||||
|
||||
/* In classic RCU, call_rcu() is just call_rcu_sched(). */
|
||||
#define call_rcu call_rcu_sched
|
||||
|
||||
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
|
||||
|
||||
/**
|
||||
* call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
|
||||
* @head: structure to be used for queueing the RCU updates.
|
||||
* @func: actual callback function to be invoked after the grace period
|
||||
*
|
||||
* The callback function will be invoked some time after a full grace
|
||||
* period elapses, in other words after all currently executing RCU
|
||||
* read-side critical sections have completed. call_rcu_bh() assumes
|
||||
* that the read-side critical sections end on completion of a softirq
|
||||
* handler. This means that read-side critical sections in process
|
||||
* context must not be interrupted by softirqs. This interface is to be
|
||||
* used when most of the read-side critical sections are in softirq context.
|
||||
* RCU read-side critical sections are delimited by :
|
||||
* - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
|
||||
* OR
|
||||
* - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
|
||||
* These may be nested.
|
||||
*/
|
||||
extern void call_rcu_bh(struct rcu_head *head,
|
||||
void (*func)(struct rcu_head *head));
|
||||
|
||||
/*
|
||||
* debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
|
||||
* by call_rcu() and rcu callback execution, and are therefore not part of the
|
||||
* RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
|
||||
*/
|
||||
|
||||
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
|
||||
# define STATE_RCU_HEAD_READY 0
|
||||
# define STATE_RCU_HEAD_QUEUED 1
|
||||
|
||||
extern struct debug_obj_descr rcuhead_debug_descr;
|
||||
|
||||
static inline void debug_rcu_head_queue(struct rcu_head *head)
|
||||
{
|
||||
WARN_ON_ONCE((unsigned long)head & 0x3);
|
||||
debug_object_activate(head, &rcuhead_debug_descr);
|
||||
debug_object_active_state(head, &rcuhead_debug_descr,
|
||||
STATE_RCU_HEAD_READY,
|
||||
STATE_RCU_HEAD_QUEUED);
|
||||
}
|
||||
|
||||
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
||||
{
|
||||
debug_object_active_state(head, &rcuhead_debug_descr,
|
||||
STATE_RCU_HEAD_QUEUED,
|
||||
STATE_RCU_HEAD_READY);
|
||||
debug_object_deactivate(head, &rcuhead_debug_descr);
|
||||
}
|
||||
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
||||
static inline void debug_rcu_head_queue(struct rcu_head *head)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void debug_rcu_head_unqueue(struct rcu_head *head)
|
||||
{
|
||||
}
|
||||
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
||||
|
||||
static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)
|
||||
{
|
||||
return offset < 4096;
|
||||
@@ -827,18 +813,6 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset)
|
||||
call_rcu(head, (rcu_callback)offset);
|
||||
}
|
||||
|
||||
extern void kfree(const void *);
|
||||
|
||||
static inline void __rcu_reclaim(struct rcu_head *head)
|
||||
{
|
||||
unsigned long offset = (unsigned long)head->func;
|
||||
|
||||
if (__is_kfree_rcu_offset(offset))
|
||||
kfree((void *)head - offset);
|
||||
else
|
||||
head->func(head);
|
||||
}
|
||||
|
||||
/**
|
||||
* kfree_rcu() - kfree an object after a grace period.
|
||||
* @ptr: pointer to kfree
|
||||
|
||||
+19
-1
@@ -27,9 +27,23 @@
|
||||
|
||||
#include <linux/cache.h>
|
||||
|
||||
#ifdef CONFIG_RCU_BOOST
|
||||
static inline void rcu_init(void)
|
||||
{
|
||||
}
|
||||
#else /* #ifdef CONFIG_RCU_BOOST */
|
||||
void rcu_init(void);
|
||||
#endif /* #else #ifdef CONFIG_RCU_BOOST */
|
||||
|
||||
static inline void rcu_barrier_bh(void)
|
||||
{
|
||||
wait_rcu_gp(call_rcu_bh);
|
||||
}
|
||||
|
||||
static inline void rcu_barrier_sched(void)
|
||||
{
|
||||
wait_rcu_gp(call_rcu_sched);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TINY_RCU
|
||||
|
||||
@@ -45,9 +59,13 @@ static inline void rcu_barrier(void)
|
||||
|
||||
#else /* #ifdef CONFIG_TINY_RCU */
|
||||
|
||||
void rcu_barrier(void);
|
||||
void synchronize_rcu_expedited(void);
|
||||
|
||||
static inline void rcu_barrier(void)
|
||||
{
|
||||
wait_rcu_gp(call_rcu);
|
||||
}
|
||||
|
||||
#endif /* #else #ifdef CONFIG_TINY_RCU */
|
||||
|
||||
static inline void synchronize_rcu_bh(void)
|
||||
|
||||
@@ -67,6 +67,8 @@ static inline void synchronize_rcu_bh_expedited(void)
|
||||
}
|
||||
|
||||
extern void rcu_barrier(void);
|
||||
extern void rcu_barrier_bh(void);
|
||||
extern void rcu_barrier_sched(void);
|
||||
|
||||
extern unsigned long rcutorture_testseq;
|
||||
extern unsigned long rcutorture_vernum;
|
||||
|
||||
@@ -270,7 +270,6 @@ extern void init_idle_bootup_task(struct task_struct *idle);
|
||||
|
||||
extern int runqueue_is_locked(int cpu);
|
||||
|
||||
extern cpumask_var_t nohz_cpu_mask;
|
||||
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ)
|
||||
extern void select_nohz_load_balancer(int stop_tick);
|
||||
extern int get_nohz_timer_target(void);
|
||||
@@ -1260,9 +1259,6 @@ struct task_struct {
|
||||
#ifdef CONFIG_PREEMPT_RCU
|
||||
int rcu_read_lock_nesting;
|
||||
char rcu_read_unlock_special;
|
||||
#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
|
||||
int rcu_boosted;
|
||||
#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
|
||||
struct list_head rcu_node_entry;
|
||||
#endif /* #ifdef CONFIG_PREEMPT_RCU */
|
||||
#ifdef CONFIG_TREE_PREEMPT_RCU
|
||||
|
||||
@@ -238,6 +238,16 @@ struct ustat {
|
||||
char f_fpack[6];
|
||||
};
|
||||
|
||||
/**
|
||||
* struct rcu_head - callback structure for use with RCU
|
||||
* @next: next update requests in a list
|
||||
* @func: actual update function to call after the grace period.
|
||||
*/
|
||||
struct rcu_head {
|
||||
struct rcu_head *next;
|
||||
void (*func)(struct rcu_head *head);
|
||||
};
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* _LINUX_TYPES_H */
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
#undef TRACE_SYSTEM
|
||||
#define TRACE_SYSTEM rcu
|
||||
|
||||
#if !defined(_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ)
|
||||
#define _TRACE_RCU_H
|
||||
|
||||
#include <linux/tracepoint.h>
|
||||
|
||||
/*
|
||||
* Tracepoint for start/end markers used for utilization calculations.
|
||||
* By convention, the string is of the following forms:
|
||||
*
|
||||
* "Start <activity>" -- Mark the start of the specified activity,
|
||||
* such as "context switch". Nesting is permitted.
|
||||
* "End <activity>" -- Mark the end of the specified activity.
|
||||
*
|
||||
* An "@" character within "<activity>" is a comment character: Data
|
||||
* reduction scripts will ignore the "@" and the remainder of the line.
|
||||
*/
|
||||
TRACE_EVENT(rcu_utilization,
|
||||
|
||||
TP_PROTO(char *s),
|
||||
|
||||
TP_ARGS(s),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, s)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->s = s;
|
||||
),
|
||||
|
||||
TP_printk("%s", __entry->s)
|
||||
);
|
||||
|
||||
#ifdef CONFIG_RCU_TRACE
|
||||
|
||||
#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
|
||||
|
||||
/*
|
||||
* Tracepoint for grace-period events: starting and ending a grace
|
||||
* period ("start" and "end", respectively), a CPU noting the start
|
||||
* of a new grace period or the end of an old grace period ("cpustart"
|
||||
* and "cpuend", respectively), a CPU passing through a quiescent
|
||||
* state ("cpuqs"), a CPU coming online or going offline ("cpuonl"
|
||||
* and "cpuofl", respectively), and a CPU being kicked for being too
|
||||
* long in dyntick-idle mode ("kick").
|
||||
*/
|
||||
TRACE_EVENT(rcu_grace_period,
|
||||
|
||||
TP_PROTO(char *rcuname, unsigned long gpnum, char *gpevent),
|
||||
|
||||
TP_ARGS(rcuname, gpnum, gpevent),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(char *, gpevent)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->gpevent = gpevent;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %s",
|
||||
__entry->rcuname, __entry->gpnum, __entry->gpevent)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for grace-period-initialization events. These are
|
||||
* distinguished by the type of RCU, the new grace-period number, the
|
||||
* rcu_node structure level, the starting and ending CPU covered by the
|
||||
* rcu_node structure, and the mask of CPUs that will be waited for.
|
||||
* All but the type of RCU are extracted from the rcu_node structure.
|
||||
*/
|
||||
TRACE_EVENT(rcu_grace_period_init,
|
||||
|
||||
TP_PROTO(char *rcuname, unsigned long gpnum, u8 level,
|
||||
int grplo, int grphi, unsigned long qsmask),
|
||||
|
||||
TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(u8, level)
|
||||
__field(int, grplo)
|
||||
__field(int, grphi)
|
||||
__field(unsigned long, qsmask)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->level = level;
|
||||
__entry->grplo = grplo;
|
||||
__entry->grphi = grphi;
|
||||
__entry->qsmask = qsmask;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %u %d %d %lx",
|
||||
__entry->rcuname, __entry->gpnum, __entry->level,
|
||||
__entry->grplo, __entry->grphi, __entry->qsmask)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for tasks blocking within preemptible-RCU read-side
|
||||
* critical sections. Track the type of RCU (which one day might
|
||||
* include SRCU), the grace-period number that the task is blocking
|
||||
* (the current or the next), and the task's PID.
|
||||
*/
|
||||
TRACE_EVENT(rcu_preempt_task,
|
||||
|
||||
TP_PROTO(char *rcuname, int pid, unsigned long gpnum),
|
||||
|
||||
TP_ARGS(rcuname, pid, gpnum),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(int, pid)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->pid = pid;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %d",
|
||||
__entry->rcuname, __entry->gpnum, __entry->pid)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for tasks that blocked within a given preemptible-RCU
|
||||
* read-side critical section exiting that critical section. Track the
|
||||
* type of RCU (which one day might include SRCU) and the task's PID.
|
||||
*/
|
||||
TRACE_EVENT(rcu_unlock_preempted_task,
|
||||
|
||||
TP_PROTO(char *rcuname, unsigned long gpnum, int pid),
|
||||
|
||||
TP_ARGS(rcuname, gpnum, pid),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(int, pid)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->pid = pid;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for quiescent-state-reporting events. These are
|
||||
* distinguished by the type of RCU, the grace-period number, the
|
||||
* mask of quiescent lower-level entities, the rcu_node structure level,
|
||||
* the starting and ending CPU covered by the rcu_node structure, and
|
||||
* whether there are any blocked tasks blocking the current grace period.
|
||||
* All but the type of RCU are extracted from the rcu_node structure.
|
||||
*/
|
||||
TRACE_EVENT(rcu_quiescent_state_report,
|
||||
|
||||
TP_PROTO(char *rcuname, unsigned long gpnum,
|
||||
unsigned long mask, unsigned long qsmask,
|
||||
u8 level, int grplo, int grphi, int gp_tasks),
|
||||
|
||||
TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(unsigned long, mask)
|
||||
__field(unsigned long, qsmask)
|
||||
__field(u8, level)
|
||||
__field(int, grplo)
|
||||
__field(int, grphi)
|
||||
__field(u8, gp_tasks)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->mask = mask;
|
||||
__entry->qsmask = qsmask;
|
||||
__entry->level = level;
|
||||
__entry->grplo = grplo;
|
||||
__entry->grphi = grphi;
|
||||
__entry->gp_tasks = gp_tasks;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %lx>%lx %u %d %d %u",
|
||||
__entry->rcuname, __entry->gpnum,
|
||||
__entry->mask, __entry->qsmask, __entry->level,
|
||||
__entry->grplo, __entry->grphi, __entry->gp_tasks)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for quiescent states detected by force_quiescent_state().
|
||||
* These trace events include the type of RCU, the grace-period number
|
||||
* that was blocked by the CPU, the CPU itself, and the type of quiescent
|
||||
* state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
|
||||
* or "kick" when kicking a CPU that has been in dyntick-idle mode for
|
||||
* too long.
|
||||
*/
|
||||
TRACE_EVENT(rcu_fqs,
|
||||
|
||||
TP_PROTO(char *rcuname, unsigned long gpnum, int cpu, char *qsevent),
|
||||
|
||||
TP_ARGS(rcuname, gpnum, cpu, qsevent),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(unsigned long, gpnum)
|
||||
__field(int, cpu)
|
||||
__field(char *, qsevent)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->gpnum = gpnum;
|
||||
__entry->cpu = cpu;
|
||||
__entry->qsevent = qsevent;
|
||||
),
|
||||
|
||||
TP_printk("%s %lu %d %s",
|
||||
__entry->rcuname, __entry->gpnum,
|
||||
__entry->cpu, __entry->qsevent)
|
||||
);
|
||||
|
||||
#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) */
|
||||
|
||||
/*
|
||||
* Tracepoint for dyntick-idle entry/exit events. These take a string
|
||||
* as argument: "Start" for entering dyntick-idle mode and "End" for
|
||||
* leaving it.
|
||||
*/
|
||||
TRACE_EVENT(rcu_dyntick,
|
||||
|
||||
TP_PROTO(char *polarity),
|
||||
|
||||
TP_ARGS(polarity),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, polarity)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->polarity = polarity;
|
||||
),
|
||||
|
||||
TP_printk("%s", __entry->polarity)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for the registration of a single RCU callback function.
|
||||
* The first argument is the type of RCU, the second argument is
|
||||
* a pointer to the RCU callback itself, and the third element is the
|
||||
* new RCU callback queue length for the current CPU.
|
||||
*/
|
||||
TRACE_EVENT(rcu_callback,
|
||||
|
||||
TP_PROTO(char *rcuname, struct rcu_head *rhp, long qlen),
|
||||
|
||||
TP_ARGS(rcuname, rhp, qlen),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(void *, rhp)
|
||||
__field(void *, func)
|
||||
__field(long, qlen)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->rhp = rhp;
|
||||
__entry->func = rhp->func;
|
||||
__entry->qlen = qlen;
|
||||
),
|
||||
|
||||
TP_printk("%s rhp=%p func=%pf %ld",
|
||||
__entry->rcuname, __entry->rhp, __entry->func, __entry->qlen)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for the registration of a single RCU callback of the special
|
||||
* kfree() form. The first argument is the RCU type, the second argument
|
||||
* is a pointer to the RCU callback, the third argument is the offset
|
||||
* of the callback within the enclosing RCU-protected data structure,
|
||||
* and the fourth argument is the new RCU callback queue length for the
|
||||
* current CPU.
|
||||
*/
|
||||
TRACE_EVENT(rcu_kfree_callback,
|
||||
|
||||
TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset,
|
||||
long qlen),
|
||||
|
||||
TP_ARGS(rcuname, rhp, offset, qlen),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(void *, rhp)
|
||||
__field(unsigned long, offset)
|
||||
__field(long, qlen)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->rhp = rhp;
|
||||
__entry->offset = offset;
|
||||
__entry->qlen = qlen;
|
||||
),
|
||||
|
||||
TP_printk("%s rhp=%p func=%ld %ld",
|
||||
__entry->rcuname, __entry->rhp, __entry->offset,
|
||||
__entry->qlen)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for marking the beginning rcu_do_batch, performed to start
|
||||
* RCU callback invocation. The first argument is the RCU flavor,
|
||||
* the second is the total number of callbacks (including those that
|
||||
* are not yet ready to be invoked), and the third argument is the
|
||||
* current RCU-callback batch limit.
|
||||
*/
|
||||
TRACE_EVENT(rcu_batch_start,
|
||||
|
||||
TP_PROTO(char *rcuname, long qlen, int blimit),
|
||||
|
||||
TP_ARGS(rcuname, qlen, blimit),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(long, qlen)
|
||||
__field(int, blimit)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->qlen = qlen;
|
||||
__entry->blimit = blimit;
|
||||
),
|
||||
|
||||
TP_printk("%s CBs=%ld bl=%d",
|
||||
__entry->rcuname, __entry->qlen, __entry->blimit)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for the invocation of a single RCU callback function.
|
||||
* The first argument is the type of RCU, and the second argument is
|
||||
* a pointer to the RCU callback itself.
|
||||
*/
|
||||
TRACE_EVENT(rcu_invoke_callback,
|
||||
|
||||
TP_PROTO(char *rcuname, struct rcu_head *rhp),
|
||||
|
||||
TP_ARGS(rcuname, rhp),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(void *, rhp)
|
||||
__field(void *, func)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->rhp = rhp;
|
||||
__entry->func = rhp->func;
|
||||
),
|
||||
|
||||
TP_printk("%s rhp=%p func=%pf",
|
||||
__entry->rcuname, __entry->rhp, __entry->func)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for the invocation of a single RCU callback of the special
|
||||
* kfree() form. The first argument is the RCU flavor, the second
|
||||
* argument is a pointer to the RCU callback, and the third argument
|
||||
* is the offset of the callback within the enclosing RCU-protected
|
||||
* data structure.
|
||||
*/
|
||||
TRACE_EVENT(rcu_invoke_kfree_callback,
|
||||
|
||||
TP_PROTO(char *rcuname, struct rcu_head *rhp, unsigned long offset),
|
||||
|
||||
TP_ARGS(rcuname, rhp, offset),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(void *, rhp)
|
||||
__field(unsigned long, offset)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->rhp = rhp;
|
||||
__entry->offset = offset;
|
||||
),
|
||||
|
||||
TP_printk("%s rhp=%p func=%ld",
|
||||
__entry->rcuname, __entry->rhp, __entry->offset)
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for exiting rcu_do_batch after RCU callbacks have been
|
||||
* invoked. The first argument is the name of the RCU flavor and
|
||||
* the second argument is number of callbacks actually invoked.
|
||||
*/
|
||||
TRACE_EVENT(rcu_batch_end,
|
||||
|
||||
TP_PROTO(char *rcuname, int callbacks_invoked),
|
||||
|
||||
TP_ARGS(rcuname, callbacks_invoked),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(char *, rcuname)
|
||||
__field(int, callbacks_invoked)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->rcuname = rcuname;
|
||||
__entry->callbacks_invoked = callbacks_invoked;
|
||||
),
|
||||
|
||||
TP_printk("%s CBs-invoked=%d",
|
||||
__entry->rcuname, __entry->callbacks_invoked)
|
||||
);
|
||||
|
||||
#else /* #ifdef CONFIG_RCU_TRACE */
|
||||
|
||||
#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
|
||||
#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, qsmask) do { } while (0)
|
||||
#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0)
|
||||
#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0)
|
||||
#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks) do { } while (0)
|
||||
#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0)
|
||||
#define trace_rcu_dyntick(polarity) do { } while (0)
|
||||
#define trace_rcu_callback(rcuname, rhp, qlen) do { } while (0)
|
||||
#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen) do { } while (0)
|
||||
#define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
|
||||
#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
|
||||
#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
|
||||
#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
|
||||
|
||||
#endif /* #else #ifdef CONFIG_RCU_TRACE */
|
||||
|
||||
#endif /* _TRACE_RCU_H */
|
||||
|
||||
/* This part must be outside protection */
|
||||
#include <trace/define_trace.h>
|
||||
Reference in New Issue
Block a user