From 4f0fccc35a9fb82f2e25badcb24bca171512444d Mon Sep 17 00:00:00 2001 From: xieliujie Date: Thu, 28 Mar 2024 18:36:46 +0800 Subject: [PATCH] ANDROID: vendor_hook: Add hooks to support reader optimistic spin in rwsem Since upstream commit 617f3ef95177 ("locking/rwsem: Remove reader optimistic spinning"), vendors have seen increased contention and blocking on rwsems. There are attempts to actively fix this upstream: https://lore.kernel.org/lkml/20240406081126.8030-1-bongkyu7.kim@samsung.com/ But in the meantime, provide vendorhooks so that vendors can implement their own optimistic spin routine. In doing so, vendors see improvements in cold launch times on important apps. Bug: 433638448 Bug: 331742151 Change-Id: I7466413de9ee1293e86f73880931235d7a9142ac Signed-off-by: xieliujie [jstultz: Rewrote commit message] Signed-off-by: John Stultz (cherry picked from commit abe0af2abe51fae4fb072c8d7124aaa37f1da574) --- drivers/android/vendor_hooks.c | 2 ++ include/trace/hooks/rwsem.h | 6 ++++++ kernel/locking/osq_lock.c | 2 ++ kernel/locking/rwsem.c | 14 ++++++++++++-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 590fc921b89f..1d2a6f9a13e9 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -106,6 +106,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_opt_spin_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_can_spin_on_owner); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_direct_rsteal); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_optimistic_rspin); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mpam_set); diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index 3eed572080e5..bd3030f12ba1 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -47,6 +47,12 @@ DECLARE_HOOK(android_vh_clear_rwsem_writer_owned, DECLARE_HOOK(android_vh_rwsem_read_trylock_failed, TP_PROTO(struct rw_semaphore *sem, long *cntp, int *ret), TP_ARGS(sem, cntp, ret)); +DECLARE_HOOK(android_vh_rwsem_direct_rsteal, + TP_PROTO(struct rw_semaphore *sem, bool *steal), + TP_ARGS(sem, steal)); +DECLARE_HOOK(android_vh_rwsem_optimistic_rspin, + TP_PROTO(struct rw_semaphore *sem, long *adjustment, bool *rspin), + TP_ARGS(sem, adjustment, rspin)); #endif /* _TRACE_HOOK_RWSEM_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c index 75a6f6133866..c7db04198178 100644 --- a/kernel/locking/osq_lock.c +++ b/kernel/locking/osq_lock.c @@ -206,6 +206,7 @@ bool osq_lock(struct optimistic_spin_queue *lock) return false; } +EXPORT_SYMBOL_GPL(osq_lock); void osq_unlock(struct optimistic_spin_queue *lock) { @@ -233,3 +234,4 @@ void osq_unlock(struct optimistic_spin_queue *lock) if (next) WRITE_ONCE(next->locked, 1); } +EXPORT_SYMBOL_GPL(osq_unlock); diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 8efb11b8b6d2..b65d9ce469ac 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -1032,6 +1032,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat struct rwsem_waiter waiter; DEFINE_WAKE_Q(wake_q); bool already_on_list = false; + bool steal = true; + bool rspin = false; /* * To prevent a constant stream of readers from starving a sleeping @@ -1045,7 +1047,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat /* * Reader optimistic lock stealing. */ - if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { + trace_android_vh_rwsem_direct_rsteal(sem, &steal); + if (steal && !(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { rwsem_set_reader_owned(sem); lockevent_inc(rwsem_rlock_steal); @@ -1053,7 +1056,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat * Wake up other readers in the wait queue if it is * the first reader. */ - if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { +wake_readers: + if ((rcnt == 1 || rspin) && (count & RWSEM_FLAG_WAITERS)) { raw_spin_lock_irq(&sem->wait_lock); if (!list_empty(&sem->wait_list)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, @@ -1064,6 +1068,12 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat trace_android_vh_record_rwsem_lock_starttime(sem, jiffies); return sem; } + /* + * Reader optimistic spinning and stealing. + */ + trace_android_vh_rwsem_optimistic_rspin(sem, &adjustment, &rspin); + if (rspin) + goto wake_readers; queue: waiter.task = current;