From 3d98011e2a257b447c684763f3ba12eefd6971f3 Mon Sep 17 00:00:00 2001 From: xieliujie Date: Wed, 12 Feb 2025 16:17:09 +0800 Subject: [PATCH] ANDROID: vendor_hook: add hooks to protect locking-tsk in cpu scheduler Providing vendor hooks to record the start time of holding the lock, which protects rwsem/mutex locking-process from being preemptedfor a short time in some cases. - android_vh_record_mutex_lock_starttime - android_vh_record_rtmutex_lock_starttime - android_vh_record_rwsem_lock_starttime - android_vh_record_pcpu_rwsem_starttime This commit is a combination of the following three commits. https://android-review.googlesource.com/c/kernel/common/+/2579595 https://android-review.googlesource.com/c/kernel/common/+/2617070 https://android-review.googlesource.com/c/kernel/common/+/2694147 Bug: 241191475 Change-Id: If91859f558bf20c3297353bea8aadaa3ab4765e9 Signed-off-by: xieliujie --- drivers/android/vendor_hooks.c | 4 ++++ include/linux/percpu-rwsem.h | 9 ++++++++- include/trace/hooks/dtask.h | 12 ++++++++++++ kernel/locking/mutex.c | 15 ++++++++++++--- kernel/locking/percpu-rwsem.c | 17 +++++++++++++++++ kernel/locking/rtmutex_api.c | 7 ++++++- kernel/locking/rwsem.c | 9 +++++++++ 7 files changed, 68 insertions(+), 5 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index a275cae115fc..5442af200773 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -171,6 +171,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_delete); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_replace); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_lookup); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_is_initialized); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_mutex_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rtmutex_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_lock_starttime); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_pcpu_rwsem_starttime); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds); diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index c012df33a9f0..186664513fb2 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -9,6 +9,9 @@ #include #include +void _trace_android_vh_record_pcpu_rwsem_starttime( + struct task_struct *tsk, unsigned long settime); + struct percpu_rw_semaphore { struct rcu_sync rss; unsigned int __percpu *read_count; @@ -67,6 +70,7 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) * The preempt_enable() prevents the compiler from * bleeding the critical section out. */ + _trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); preempt_enable(); } @@ -88,8 +92,10 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) * bleeding the critical section out. */ - if (ret) + if (ret) { + _trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); + } return ret; } @@ -118,6 +124,7 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) this_cpu_dec(*sem->read_count); rcuwait_wake_up(&sem->writer); } + _trace_android_vh_record_pcpu_rwsem_starttime(current, 0); preempt_enable(); } diff --git a/include/trace/hooks/dtask.h b/include/trace/hooks/dtask.h index ade106752a68..ff7939ad07f9 100644 --- a/include/trace/hooks/dtask.h +++ b/include/trace/hooks/dtask.h @@ -113,6 +113,18 @@ DECLARE_HOOK(android_vh_mutex_unlock_slowpath, TP_PROTO(struct mutex *lock), TP_ARGS(lock)); +DECLARE_HOOK(android_vh_record_mutex_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_rtmutex_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_rwsem_lock_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); +DECLARE_HOOK(android_vh_record_pcpu_rwsem_starttime, + TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies), + TP_ARGS(tsk, settime_jiffies)); #endif /* _TRACE_HOOK_DTASK_H */ /* This part must be outside protection */ #include diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index e3e2f47f74c6..5ae44fd88dde 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -147,8 +147,10 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock) unsigned long curr = (unsigned long)current; unsigned long zero = 0UL; - if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) + if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) { + trace_android_vh_record_mutex_lock_starttime(current, jiffies); return true; + } return false; } @@ -534,10 +536,13 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne void __sched mutex_unlock(struct mutex *lock) { #ifndef CONFIG_DEBUG_LOCK_ALLOC - if (__mutex_unlock_fast(lock)) + if (__mutex_unlock_fast(lock)) { + trace_android_vh_record_mutex_lock_starttime(current, 0); return; + } #endif __mutex_unlock_slowpath(lock, _RET_IP_); + trace_android_vh_record_mutex_lock_starttime(current, 0); } EXPORT_SYMBOL(mutex_unlock); @@ -609,6 +614,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas if (ww_ctx) ww_mutex_set_context_fastpath(ww, ww_ctx); trace_contention_end(lock, 0); + trace_android_vh_record_mutex_lock_starttime(current, jiffies); preempt_enable(); return 0; } @@ -752,6 +758,7 @@ skip_wait: raw_spin_unlock(¤t->blocked_lock); raw_spin_unlock_irqrestore(&lock->wait_lock, flags); wake_up_q(&wake_q); + trace_android_vh_record_mutex_lock_starttime(current, jiffies); preempt_enable(); return 0; @@ -1159,8 +1166,10 @@ int __sched mutex_trylock(struct mutex *lock) MUTEX_WARN_ON(lock->magic != lock); locked = __mutex_trylock(lock); - if (locked) + if (locked) { + trace_android_vh_record_mutex_lock_starttime(current, jiffies); mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); + } return locked; } diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 6083883c4fe0..0b8c24d8d2f9 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -11,6 +11,21 @@ #include #include +#include + +/* + * trace_android_vh_record_pcpu_rwsem_starttime is called in + * include/linux/percpu-rwsem.h by including include/hooks/dtask.h, which + * will result to build-err. So we create + * func:_trace_android_vh_record_pcpu_rwsem_starttime for percpu-rwsem.h to call. + */ +void _trace_android_vh_record_pcpu_rwsem_starttime(struct task_struct *tsk, + unsigned long settime) +{ + trace_android_vh_record_pcpu_rwsem_starttime(tsk, settime); +} +EXPORT_SYMBOL_GPL(_trace_android_vh_record_pcpu_rwsem_starttime); + int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, const char *name, struct lock_class_key *key) { @@ -253,6 +268,7 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem) rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); if (contended) trace_contention_end(sem, 0); + trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies); } EXPORT_SYMBOL_GPL(percpu_down_write); @@ -283,5 +299,6 @@ void percpu_up_write(struct percpu_rw_semaphore *sem) * exclusive write lock because its counting. */ rcu_sync_exit(&sem->rss); + trace_android_vh_record_pcpu_rwsem_starttime(current, 0); } EXPORT_SYMBOL_GPL(percpu_up_write); diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 924d1ad22d74..cf50e000092e 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -31,6 +31,8 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, ret = __rt_mutex_lock(&lock->rtmutex, state); if (ret) mutex_release(&lock->dep_map, _RET_IP_); + else + trace_android_vh_record_rtmutex_lock_starttime(current, jiffies); return ret; } @@ -123,8 +125,10 @@ int __sched rt_mutex_trylock(struct rt_mutex *lock) return 0; ret = __rt_mutex_trylock(&lock->rtmutex); - if (ret) + if (ret) { + trace_android_vh_record_rtmutex_lock_starttime(current, jiffies); mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); + } return ret; } @@ -137,6 +141,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_trylock); */ void __sched rt_mutex_unlock(struct rt_mutex *lock) { + trace_android_vh_record_rtmutex_lock_starttime(current, 0); mutex_release(&lock->dep_map, _RET_IP_); __rt_mutex_unlock(&lock->rtmutex); } diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index bdd296a9090a..2b24645f185a 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -256,6 +256,7 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) if (!(*cntp & RWSEM_READ_FAILED_MASK)) { rwsem_set_reader_owned(sem); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return true; } @@ -267,6 +268,7 @@ static inline bool rwsem_write_trylock(struct rw_semaphore *sem) long tmp = RWSEM_UNLOCKED_VALUE; if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); rwsem_set_owner(sem); return true; } @@ -1042,6 +1044,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat raw_spin_unlock_irq(&sem->wait_lock); wake_up_q(&wake_q); } + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } @@ -1111,6 +1114,7 @@ queue: trace_android_vh_rwsem_read_wait_finish(sem); lockevent_inc(rwsem_rlock); trace_contention_end(sem, 0); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; out_nolock: @@ -1135,6 +1139,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) /* do optimistic spinning and steal lock if possible */ if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { /* rwsem_optimistic_spin() implies ACQUIRE on success */ + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; } @@ -1215,6 +1220,7 @@ trylock_again: raw_spin_unlock_irq(&sem->wait_lock); lockevent_inc(rwsem_wlock); trace_contention_end(sem, 0); + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); return sem; out_nolock: @@ -1321,6 +1327,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) tmp + RWSEM_READER_BIAS)) { rwsem_set_reader_owned(sem); ret = 1; + trace_android_vh_record_rwsem_lock_starttime(current, jiffies); break; } } @@ -1385,6 +1392,7 @@ static inline void __up_read(struct rw_semaphore *sem) clear_nonspinnable(sem); rwsem_wake(sem); } + trace_android_vh_record_rwsem_lock_starttime(current, 0); preempt_enable(); } @@ -1408,6 +1416,7 @@ static inline void __up_write(struct rw_semaphore *sem) tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); if (unlikely(tmp & RWSEM_FLAG_WAITERS)) rwsem_wake(sem); + trace_android_vh_record_rwsem_lock_starttime(current, 0); preempt_enable(); }