From 78cf7a145c200d185e2370c712abd8235d1bd5aa Mon Sep 17 00:00:00 2001 From: JianMin Liu Date: Tue, 21 Jul 2020 13:36:22 +0800 Subject: [PATCH] ANDROID: rwsem: Add vendor hook to the rw-semaphore Add the hook to apply vendor's performance tune for owner of rwsem. Add the hook for the waiter list of rwsem to allow vendor perform waiting queue enhancement ANDROID_VENDOR_DATA added to rw_semaphore Bug: 319399115 Signed-off-by: JianMin Liu Signed-off-by: Guanwun Chen Change-Id: I007a5e26f3db2adaeaf4e5ccea414ce7abfa83b8 --- drivers/android/vendor_hooks.c | 5 +++++ include/linux/rwsem.h | 2 ++ include/trace/hooks/rwsem.h | 30 ++++++++++++++++++++++++++++++ kernel/locking/rwsem.c | 20 ++++++++++++++++++-- 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 include/trace/hooks/rwsem.h diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index c46ef08d2dd8..332cf53f5577 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -43,6 +43,7 @@ #include #include #include +#include /* * Export tracepoints that act as a bare tracehook (ie: have no trace event @@ -141,3 +142,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_test_clear_look_around_ref); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_arm64_serror_panic); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_sea); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_capacity_show); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_wake); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_finished); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_rwsem_list_add); diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index c8b543d428b0..611ac4ca1d78 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -32,6 +32,7 @@ #ifdef CONFIG_RWSEM_SPIN_ON_OWNER #include #endif +#include /* * For an uncontended rwsem, count and owner are the only fields a task @@ -64,6 +65,7 @@ struct rw_semaphore { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif + ANDROID_VENDOR_DATA(1); }; #define RWSEM_UNLOCKED_VALUE 0UL diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h new file mode 100644 index 000000000000..71215dfd5eff --- /dev/null +++ b/include/trace/hooks/rwsem.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rwsem +#define TRACE_INCLUDE_PATH trace/hooks +#if !defined(_TRACE_HOOK_RWSEM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HOOK_RWSEM_H +#include +/* + * Following tracepoints are not exported in tracefs and provide a + * mechanism for vendor modules to hook and extend functionality + */ +struct rw_semaphore; +struct rwsem_waiter; +DECLARE_HOOK(android_vh_rwsem_init, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_wake, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_write_finished, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_alter_rwsem_list_add, + TP_PROTO(struct rwsem_waiter *waiter, + struct rw_semaphore *sem, + bool *already_on_list), + TP_ARGS(waiter, sem, already_on_list)); +#endif /* _TRACE_HOOK_RWSEM_H */ +/* This part must be outside protection */ +#include diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 38847be89725..909b96da7802 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -32,6 +32,7 @@ #ifndef CONFIG_PREEMPT_RT #include "lock_events.h" #include +#include /* * The least significant 2 bits of the owner value has the following @@ -326,6 +327,7 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name, #ifdef CONFIG_RWSEM_SPIN_ON_OWNER osq_lock_init(&sem->osq); #endif + trace_android_vh_rwsem_init(sem); } EXPORT_SYMBOL(__init_rwsem); @@ -998,6 +1000,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat long rcnt = (count >> RWSEM_READER_SHIFT); struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); + bool already_on_list = false; /* * To prevent a constant stream of readers from starving a sleeping @@ -1054,12 +1057,17 @@ queue: } adjustment += RWSEM_FLAG_WAITERS; } - rwsem_add_waiter(sem, &waiter); + trace_android_vh_alter_rwsem_list_add( + &waiter, + sem, &already_on_list); + if (!already_on_list) + rwsem_add_waiter(sem, &waiter); /* we're now waiting on the lock, but no longer actively locking */ count = atomic_long_add_return(adjustment, &sem->count); rwsem_cond_wake_waiter(sem, count, &wake_q); + trace_android_vh_rwsem_wake(sem); raw_spin_unlock_irq(&sem->wait_lock); if (!wake_q_empty(&wake_q)) @@ -1110,6 +1118,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) { struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); + bool already_on_list = false; /* do optimistic spinning and steal lock if possible */ if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { @@ -1127,7 +1136,11 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) waiter.handoff_set = false; raw_spin_lock_irq(&sem->wait_lock); - rwsem_add_waiter(sem, &waiter); + trace_android_vh_alter_rwsem_list_add( + &waiter, + sem, &already_on_list); + if (!already_on_list) + rwsem_add_waiter(sem, &waiter); /* we're now waiting on the lock */ if (rwsem_first_waiter(sem) != &waiter) { @@ -1146,6 +1159,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); } + trace_android_vh_rwsem_wake(sem); /* wait until we successfully acquire the lock */ trace_android_vh_rwsem_write_wait_start(sem); set_current_state(state); @@ -1634,6 +1648,7 @@ EXPORT_SYMBOL(up_read); void up_write(struct rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); + trace_android_vh_rwsem_write_finished(sem); __up_write(sem); } EXPORT_SYMBOL(up_write); @@ -1644,6 +1659,7 @@ EXPORT_SYMBOL(up_write); void downgrade_write(struct rw_semaphore *sem) { lock_downgrade(&sem->dep_map, _RET_IP_); + trace_android_vh_rwsem_write_finished(sem); __downgrade_write(sem); } EXPORT_SYMBOL(downgrade_write);