tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ANDROID: vendor_hook: Add hooks to support reader optimistic spin in rwsem

Browse Source 
Since upstream commit 617f3ef951 ("locking/rwsem: Remove
reader optimistic spinning"), vendors have seen increased
contention and blocking on rwsems.

There are attempts to actively fix this upstream:
  https://lore.kernel.org/lkml/20240406081126.8030-1-bongkyu7.kim@samsung.com/

But in the meantime, provide vendorhooks so that vendors can
implement their own optimistic spin routine. In doing so,
vendors see improvements in cold launch times on important apps.

Bug: 433638448
Bug: 331742151

Change-Id: I7466413de9ee1293e86f73880931235d7a9142ac
Signed-off-by: xieliujie <xieliujie@oppo.com>
[jstultz: Rewrote commit message]
Signed-off-by: John Stultz <jstultz@google.com>
(cherry picked from commit abe0af2abe51fae4fb072c8d7124aaa37f1da574)
This commit is contained in:
xieliujie
2024-03-28 18:36:46 +08:00
committed by Todd Kjos
parent 379b99745e
commit 4f0fccc35a
4 changed files with 22 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/locking/osq_lock.c
+2
View File
@@ -206,6 +206,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
return false;
}
EXPORT_SYMBOL_GPL(osq_lock);
void osq_unlock(struct optimistic_spin_queue *lock)
{
@@ -233,3 +234,4 @@ void osq_unlock(struct optimistic_spin_queue *lock)
if (next)
WRITE_ONCE(next->locked, 1);
}
EXPORT_SYMBOL_GPL(osq_unlock);
kernel/locking/rwsem.c
+12 -2
View File
@@ -1032,6 +1032,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
struct rwsem_waiter waiter;
DEFINE_WAKE_Q(wake_q);
bool already_on_list = false;
bool steal = true;
bool rspin = false;
/*
* To prevent a constant stream of readers from starving a sleeping
@@ -1045,7 +1047,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
/*
* Reader optimistic lock stealing.
*/
if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) {
trace_android_vh_rwsem_direct_rsteal(sem, &steal);
if (steal && !(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) {
rwsem_set_reader_owned(sem);
lockevent_inc(rwsem_rlock_steal);
@@ -1053,7 +1056,8 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
* Wake up other readers in the wait queue if it is
* the first reader.
*/
if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) {
wake_readers:
if ((rcnt == 1 || rspin) && (count & RWSEM_FLAG_WAITERS)) {
raw_spin_lock_irq(&sem->wait_lock);
if (!list_empty(&sem->wait_list))
rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,
@@ -1064,6 +1068,12 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
trace_android_vh_record_rwsem_lock_starttime(sem, jiffies);
return sem;
}
/*
* Reader optimistic spinning and stealing.
*/
trace_android_vh_rwsem_optimistic_rspin(sem, &adjustment, &rspin);
if (rspin)
goto wake_readers;
queue:
waiter.task = current;