From 28b35bdaea49d8d7737889bd9068908c2a241108 Mon Sep 17 00:00:00 2001 From: Bo Lan Date: Thu, 6 Mar 2025 14:25:47 +0800 Subject: [PATCH] ANDROID: writeback: add vendor hooks to do a hierarchical writeback control For short-life or high-IO-freq files we marked, the writeback tasks will act on an additional writeback dispatching list, achieving hierarchical control of dirty writeback without affecting other original writeback tasks. Bug: 400883625 Change-Id: I7973cedeb15896d2cd2224443bf857111b2adac7 Signed-off-by: Bo Lan --- drivers/android/vendor_hooks.c | 8 ++++++++ fs/fcntl.c | 2 ++ fs/fs-writeback.c | 14 +++++++++++--- fs/inode.c | 4 ++++ fs/sync.c | 8 +++++++- include/trace/hooks/fs.h | 33 +++++++++++++++++++++++++++++++++ mm/page-writeback.c | 3 +++ 7 files changed, 68 insertions(+), 4 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 7372a3154837..b4507b61cf12 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -392,3 +392,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_reply); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_trans); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_proc_transaction); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_thread_read); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_inode_io_list_del); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_redirty_tail_locked); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_queue_io); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mark_inode_dirty); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_wb_dirty_limits); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_vfs_fsync_range); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_fcntl); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_evict); diff --git a/fs/fcntl.c b/fs/fcntl.c index 3d89de31066a..c351c39ae423 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "internal.h" @@ -551,6 +552,7 @@ static long do_fcntl(int fd, unsigned int cmd, unsigned long arg, err = fcntl_set_rw_hint(filp, cmd, arg); break; default: + trace_android_rvh_do_fcntl(filp, cmd, arg, &err); break; } return err; diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 2391b09f4ced..240878bfbc28 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -79,6 +79,8 @@ static inline struct inode *wb_inode(struct list_head *head) */ #define CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage); @@ -1289,6 +1291,7 @@ void inode_io_list_del(struct inode *inode) spin_lock(&inode->i_lock); inode->i_state &= ~I_SYNC_QUEUED; + trace_android_vh_inode_io_list_del(inode, wb); list_del_init(&inode->i_io_list); wb_io_lists_depopulated(wb); @@ -1344,6 +1347,7 @@ void sb_clear_inode_writeback(struct inode *inode) */ static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb) { + struct list_head *target_list = &wb->b_dirty; assert_spin_locked(&inode->i_lock); inode->i_state &= ~I_SYNC_QUEUED; @@ -1357,14 +1361,15 @@ static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb) wb_io_lists_depopulated(wb); return; } - if (!list_empty(&wb->b_dirty)) { + trace_android_vh_redirty_tail_locked(&target_list, inode, wb); + if (!list_empty(target_list)) { struct inode *tail; - tail = wb_inode(wb->b_dirty.next); + tail = wb_inode(target_list->next); if (time_before(inode->dirtied_when, tail->dirtied_when)) inode->dirtied_when = jiffies; } - inode_io_list_move_locked(inode, wb, &wb->b_dirty); + inode_io_list_move_locked(inode, wb, target_list); } static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) @@ -1483,6 +1488,7 @@ static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work, assert_spin_locked(&wb->list_lock); list_splice_init(&wb->b_more_io, &wb->b_io); moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before); + trace_android_vh_queue_io(wb, work->for_kupdate, dirtied_before, &moved); if (!work->for_sync) time_expire_jif = jiffies - dirtytime_expire_interval * HZ; moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, @@ -2569,6 +2575,8 @@ void __mark_inode_dirty(struct inode *inode, int flags) else dirty_list = &wb->b_dirty_time; + trace_android_vh_mark_inode_dirty(inode, wb, + &dirty_list); wakeup_bdi = inode_io_list_move_locked(inode, wb, dirty_list); diff --git a/fs/inode.c b/fs/inode.c index 8dabb224f941..bfd5d7929979 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -24,6 +24,9 @@ #include #include "internal.h" +#undef CREATE_TRACE_POINTS +#include + /* * Inode locking rules: * @@ -707,6 +710,7 @@ static void evict(struct inode *inode) if (!list_empty(&inode->i_io_list)) inode_io_list_del(inode); + trace_android_vh_evict(inode); inode_sb_list_del(inode); spin_lock(&inode->i_lock); diff --git a/fs/sync.c b/fs/sync.c index e0e2214b9310..c37e548b2b3b 100644 --- a/fs/sync.c +++ b/fs/sync.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "internal.h" #define VALID_FLAGS (SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| \ @@ -183,8 +184,13 @@ int vfs_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) if (!file->f_op->fsync) return -EINVAL; - if (!datasync && (inode->i_state & I_DIRTY_TIME)) + if (!datasync && (inode->i_state & I_DIRTY_TIME)) { + unsigned long cut_off = 0; mark_inode_dirty_sync(inode); + trace_android_vh_vfs_fsync_range(inode, &cut_off); + if (cut_off) + return 0; + } return file->f_op->fsync(file, start, end, datasync); } EXPORT_SYMBOL(vfs_fsync_range); diff --git a/include/trace/hooks/fs.h b/include/trace/hooks/fs.h index 016f951e92a3..f5498f044b4f 100644 --- a/include/trace/hooks/fs.h +++ b/include/trace/hooks/fs.h @@ -27,6 +27,39 @@ DECLARE_HOOK(android_vh_f2fs_printk, TP_PROTO(struct f2fs_sb_info *sbi, struct va_format *vaf, int level, bool limit_rate), TP_ARGS(sbi, vaf, level, limit_rate)); +DECLARE_HOOK(android_vh_wb_dirty_limits, + TP_PROTO(unsigned long *thresh, struct bdi_writeback *wb), + TP_ARGS(thresh, wb)); + +DECLARE_HOOK(android_vh_evict, + TP_PROTO(struct inode *inode), + TP_ARGS(inode)); + +DECLARE_HOOK(android_vh_inode_io_list_del, + TP_PROTO(struct inode *inode, struct bdi_writeback *wb), + TP_ARGS(inode, wb)); + +DECLARE_HOOK(android_vh_redirty_tail_locked, + TP_PROTO(struct list_head **target_list, struct inode *inode, + struct bdi_writeback *wb), + TP_ARGS(target_list, inode, wb)); + +DECLARE_HOOK(android_vh_queue_io, + TP_PROTO(struct bdi_writeback *wb, unsigned int for_kupdate, + unsigned long dirtied_before, int *moved), + TP_ARGS(wb, for_kupdate, dirtied_before, moved)); + +DECLARE_HOOK(android_vh_mark_inode_dirty, + TP_PROTO(struct inode *inode, struct bdi_writeback *wb, struct list_head **dirty_list), + TP_ARGS(inode, wb, dirty_list)); + +DECLARE_HOOK(android_vh_vfs_fsync_range, + TP_PROTO(struct inode *inode, unsigned long *cut_off), + TP_ARGS(inode, cut_off)); + +DECLARE_RESTRICTED_HOOK(android_rvh_do_fcntl, + TP_PROTO(struct file *filp, unsigned int cmd, unsigned long arg, long *err), + TP_ARGS(filp, cmd, arg, err), 1); #endif /* _TRACE_HOOK_FS_H */ /* This part must be outside protection */ diff --git a/mm/page-writeback.c b/mm/page-writeback.c index f476bade77be..d058900383e8 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "internal.h" @@ -1725,6 +1726,7 @@ static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE); dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); } + trace_android_vh_wb_dirty_limits(&dtc->wb_dirty, wb); } static unsigned long domain_poll_intv(struct dirty_throttle_control *dtc, @@ -2163,6 +2165,7 @@ static void wb_bg_dirty_limits(struct dirty_throttle_control *dtc) dtc->wb_dirty = wb_stat_sum(wb, WB_RECLAIMABLE); else dtc->wb_dirty = wb_stat(wb, WB_RECLAIMABLE); + trace_android_vh_wb_dirty_limits(&dtc->wb_dirty, wb); } static bool domain_over_bg_thresh(struct dirty_throttle_control *dtc)