From be7f2eb0e8ee96aa6c84fd5ba8379d97d6214e51 Mon Sep 17 00:00:00 2001 From: zhujingpeng Date: Mon, 15 Apr 2024 21:22:50 +0800 Subject: [PATCH] ANDROID: GKI: add percpu_rwsem vendor hooks When a writer has set sem->block and is waiting for active readers to complete, we still allow some specific new readers to entry the critical section, which can help prevent priority inversion from impacting system responsiveness and performance. Bug: 334851707 Change-Id: I9e2a7df1efb326763487423d64bcf74d8dec23f8 Signed-off-by: zhujingpeng (cherry picked from commit 458cdb59f7719f81504d392daa95a8c99c30c276) --- drivers/android/vendor_hooks.c | 3 +++ include/trace/hooks/dtask.h | 9 +++++++++ kernel/locking/percpu-rwsem.c | 13 ++++++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 14e5dd6dc17b..8b04ea142156 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -488,3 +488,6 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lock_folio_drop_mmap_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_update_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernated_do_mem_alloc); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_save_cmp_len); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_down_read); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_percpu_rwsem_up_write); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_percpu_rwsem_wait_complete); diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index 942241510d03..52aae20f7136 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -106,6 +106,15 @@ DECLARE_HOOK(android_vh_sched_show_task, DECLARE_HOOK(android_vh_percpu_rwsem_wq_add, TP_PROTO(struct percpu_rw_semaphore *sem, bool reader), TP_ARGS(sem, reader)); +DECLARE_HOOK(android_vh_percpu_rwsem_down_read, + TP_PROTO(struct percpu_rw_semaphore *sem, bool try, bool *ret), + TP_ARGS(sem, try, ret)); +DECLARE_HOOK(android_vh_percpu_rwsem_up_write, + TP_PROTO(struct percpu_rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_RESTRICTED_HOOK(android_rvh_percpu_rwsem_wait_complete, + TP_PROTO(struct percpu_rw_semaphore *sem, long state, bool *complete), + TP_ARGS(sem, state, complete), 1); struct mutex_waiter; DECLARE_HOOK(android_vh_alter_mutex_list_add, diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 65731595483b..1ef86261f475 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -182,9 +182,15 @@ static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader) bool __sched __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) { + bool ret = false; + if (__percpu_down_read_trylock(sem)) return true; + trace_android_vh_percpu_rwsem_down_read(sem, try, &ret); + if (ret) + return true; + if (try) return false; @@ -240,6 +246,7 @@ static bool readers_active_check(struct percpu_rw_semaphore *sem) void __sched percpu_down_write(struct percpu_rw_semaphore *sem) { bool contended = false; + bool complete = false; might_sleep(); rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); @@ -266,7 +273,9 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) */ /* Wait for all active readers to complete. */ - rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); + trace_android_rvh_percpu_rwsem_wait_complete(sem, TASK_UNINTERRUPTIBLE, &complete); + if (!complete) + rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); if (contended) trace_contention_end(sem, 0); trace_android_vh_record_pcpu_rwsem_starttime(sem, jiffies); @@ -277,6 +286,8 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); + trace_android_vh_percpu_rwsem_up_write(sem); + /* * Signal the writer is done, no fast path yet. *