From 44079bd16490c6b714c699cb2e7926f7806a525b Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Thu, 18 Apr 2024 00:57:11 +0800 Subject: [PATCH] ANDROID: vendor_hooks: add hooks in rwsem these hooks are required by the following features: 1.For rwsem readers, currently only the latest reader will be recorded in sem->owner. We add hooks to record all readers which have acquired the lock, once there are UX threads blocked in the rwsem waiting list, these read_owners will be given high priority in scheduling. 2.For rwsem writer, when a writer acquires the lock, we check whether there are UX threads blocked in the rwsem wait list. If so, we give this writer a high priority in scheduling so that it can release the lock as soon as possible. Both of these features can optimize the priority inversion problem caused by rwsem and improve system responsiveness and performance. Bug: 335408185 Change-Id: I82a6fbb6acd2ce05d049e686b61e34e4d3b39a5e Signed-off-by: zhujingpeng [jstultz: Rebased and resolved minor conflict] Signed-off-by: John Stultz (cherry picked from commit 188c41744ddcccef6daf6dcd4d6a444dabdc2f94) --- drivers/android/vendor_hooks.c | 4 ++++ include/trace/hooks/rwsem.h | 13 +++++++++++++ kernel/locking/rwsem.c | 6 ++++++ 3 files changed, 23 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 8b04ea142156..134b9dba88dc 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -491,3 +491,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_save_cmp_len); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_down_read); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_up_write); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_percpu_rwsem_wait_complete); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_writer_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_writer_owned); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index 77e34478a42f..6da44e7e28c6 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -31,6 +31,19 @@ DECLARE_HOOK(android_vh_rwsem_downgrade_wake_finish, DECLARE_HOOK(android_vh_rwsem_wake_finish, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); +DECLARE_HOOK(android_vh_record_rwsem_reader_owned, + TP_PROTO(struct rw_semaphore *sem, + struct list_head *wlist), + TP_ARGS(sem, wlist)); +DECLARE_HOOK(android_vh_clear_rwsem_reader_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_record_rwsem_writer_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_clear_rwsem_writer_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index d1ac321821aa..8b6df74640ea 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -143,12 +143,14 @@ static inline void rwsem_set_owner(struct rw_semaphore *sem) { lockdep_assert_preemption_disabled(); atomic_long_set(&sem->owner, (long)current); + trace_android_vh_record_rwsem_writer_owned(sem); } static inline void rwsem_clear_owner(struct rw_semaphore *sem) { lockdep_assert_preemption_disabled(); atomic_long_set(&sem->owner, 0); + trace_android_vh_clear_rwsem_writer_owned(sem); } /* @@ -181,6 +183,7 @@ static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { __rwsem_set_reader_owned(sem, current); + trace_android_vh_record_rwsem_reader_owned(sem, NULL); } #ifdef CONFIG_DEBUG_RWSEMS @@ -218,6 +221,7 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { unsigned long val = atomic_long_read(&sem->owner); + trace_android_vh_clear_rwsem_reader_owned(sem); while ((val & ~RWSEM_OWNER_FLAGS_MASK) == (unsigned long)current) { if (atomic_long_try_cmpxchg(&sem->owner, &val, val & RWSEM_OWNER_FLAGS_MASK)) @@ -227,6 +231,7 @@ static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) #else static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) { + trace_android_vh_clear_rwsem_reader_owned(sem); } #endif @@ -547,6 +552,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, if (adjustment) atomic_long_add(adjustment, &sem->count); + trace_android_vh_record_rwsem_reader_owned(sem, &wlist); /* 2nd pass */ list_for_each_entry_safe(waiter, tmp, &wlist, list) {