From 032603c88f6462c50a66c93f437954ecdb160e64 Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Tue, 23 Apr 2024 21:50:15 +0800 Subject: [PATCH] ANDROID: vendor_hooks: add hooks in rwsem read trylock When the lock is owned by readers and there is no RWSEM_FLAG_HANDOFF set, we allow some specific new readers to acquire the lock immediately in this hook, event if there are some writer tasks in the wait_list. This features can optimize the priority inversion problem caused by rwsem and improve system responsiveness and performance. Bug: 336506800 Change-Id: I7e8fded73579933d1f61faa9fb6e5f300ffd71bf Signed-off-by: zhujingpeng [jstultz: rebased, resolved collision] Signed-off-by: John Stultz (cherry picked from commit ace91cdf950654038d449745d8218b2b3eb91017) --- drivers/android/vendor_hooks.c | 1 + include/trace/hooks/rwsem.h | 3 +++ kernel/locking/rwsem.c | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 134b9dba88dc..c447840b88f5 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -495,3 +495,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_reader_owned); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_reader_owned); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_writer_owned); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_rwsem_writer_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_trylock_failed); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index 6da44e7e28c6..3eed572080e5 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -44,6 +44,9 @@ DECLARE_HOOK(android_vh_record_rwsem_writer_owned, DECLARE_HOOK(android_vh_clear_rwsem_writer_owned, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_read_trylock_failed, + TP_PROTO(struct rw_semaphore *sem, long *cntp, int *ret), + TP_ARGS(sem, cntp, ret)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 8b6df74640ea..067b6b08a399 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -254,6 +254,8 @@ static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) { + int ret = 0; + *cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); if (WARN_ON_ONCE(*cntp < 0)) @@ -265,6 +267,13 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) return true; } + trace_android_vh_rwsem_read_trylock_failed(sem, cntp, &ret); + if (ret) { + rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(sem, jiffies); + return true; + } + return false; } @@ -1337,6 +1346,15 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) break; } } + + if (!ret) { + trace_android_vh_rwsem_read_trylock_failed(sem, NULL, &ret); + if (ret) { + rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(sem, jiffies); + } + } + preempt_enable(); return ret; }