From 9e8a1bb6fba02fe8951ded70bfa4ddd8ac5110ed Mon Sep 17 00:00:00 2001 From: Hailong Liu Date: Mon, 7 Apr 2025 09:34:11 +0800 Subject: [PATCH] ANDROID: mm: add vendor hooks for file folio reclaim. Add hooks for file folio reclaim. the vendor hook keep reclaimed file folios and help direct && kswapd reclaim. Fix Conflicts: drivers/android/vendor_hooks.c include/trace/hooks/mm.h include/trace/hooks/vmscan.h mm/vmscan.c remove the change of mm/memcontrol.c because https://android-review.googlesource.com/c/kernel/common/+/3603374 Bug: 407947362 Bug: 449066551 Bug: 451083029 Change-Id: I4364ae2d269e4b4b88c12823ebf66abaa4650585 Signed-off-by: Hailong Liu (cherry picked from commit 43bf81d175f5480a5537205f92b632ec8e5f1787) (cherry picked from commit 6bae5d457e35a9e3ebefdf45e8e9e8f957fd6c84) --- drivers/android/vendor_hooks.c | 8 ++++++++ include/linux/mm_inline.h | 10 ++++++++++ include/trace/hooks/mm.h | 13 +++++++++++++ include/trace/hooks/vmscan.h | 13 +++++++++++++ mm/filemap.c | 1 + mm/page_alloc.c | 8 +++++++- mm/vmscan.c | 15 ++++++++++++++- 7 files changed, 66 insertions(+), 2 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 093329c34333..8e763cefbbce 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -623,3 +623,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_fiq_dump); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_looper_state_registered); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_thread_release); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_filemap_pages); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lru_gen_add_folio_skip); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_lru_gen_del_folio_skip); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_perform_reclaim); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_kswapd_shrink_node); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_keep_reclaimed_folio); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_clear_reclaimed_folio); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_evict_folios_bypass); diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index ecd2a17c3b79..35b50f0981d1 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -261,6 +261,11 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); struct lru_gen_folio *lrugen = &lruvec->lrugen; + bool skip = false; + + trace_android_vh_lru_gen_add_folio_skip(lruvec, folio, &skip); + if (skip) + return true; VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); @@ -287,6 +292,11 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, { unsigned long flags; int gen = folio_lru_gen(folio); + bool skip = false; + + trace_android_vh_lru_gen_del_folio_skip(lruvec, folio, &skip); + if (skip) + return true; if (gen < 0) return false; diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index eed10d8666f0..77f98a31cad0 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -20,6 +20,10 @@ struct compact_control; DECLARE_RESTRICTED_HOOK(android_rvh_shmem_get_folio, TP_PROTO(struct shmem_inode_info *info, struct folio **folio, int order), TP_ARGS(info, folio, order), 3); +DECLARE_RESTRICTED_HOOK(android_rvh_perform_reclaim, + TP_PROTO(int order, gfp_t gfp_mask, nodemask_t *nodemask, + unsigned long *progress, bool *skip), + TP_ARGS(order, gfp_mask, nodemask, progress, skip), 4); DECLARE_HOOK(android_vh_shmem_mod_shmem, TP_PROTO(struct address_space *mapping, long nr_pages), TP_ARGS(mapping, nr_pages)); @@ -318,6 +322,15 @@ DECLARE_HOOK(android_vh_alloc_pages_slowpath_end, unsigned long pages_reclaimed, int retry_loop_count), TP_ARGS(gfp_mask, order, alloc_start, stime, did_some_progress, pages_reclaimed, retry_loop_count)); +DECLARE_HOOK(android_vh_filemap_pages, + TP_PROTO(struct folio *folio), + TP_ARGS(folio)); +DECLARE_HOOK(android_vh_lru_gen_add_folio_skip, + TP_PROTO(struct lruvec *lruvec, struct folio *folio, bool *skip), + TP_ARGS(lruvec, folio, skip)); +DECLARE_HOOK(android_vh_lru_gen_del_folio_skip, + TP_PROTO(struct lruvec *lruvec, struct folio *folio, bool *skip), + TP_ARGS(lruvec, folio, skip)); DECLARE_HOOK(android_vh_add_lazyfree_bypass, TP_PROTO(struct lruvec *lruvec, struct folio *folio, bool *bypass), TP_ARGS(lruvec, folio, bypass)); diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index bd545f8444df..e95911530f16 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -15,6 +15,9 @@ struct lruvec; DECLARE_RESTRICTED_HOOK(android_rvh_set_balance_anon_file_reclaim, TP_PROTO(bool *balance_anon_file_reclaim), TP_ARGS(balance_anon_file_reclaim), 1); +DECLARE_RESTRICTED_HOOK(android_rvh_kswapd_shrink_node, + TP_PROTO(unsigned long *nr_reclaimed), + TP_ARGS(nr_reclaimed), 1); DECLARE_HOOK(android_vh_check_folio_look_around_ref, TP_PROTO(struct folio *folio, int *skip), TP_ARGS(folio, skip)); @@ -31,6 +34,16 @@ DECLARE_HOOK(android_vh_inode_lru_isolate, DECLARE_HOOK(android_vh_invalidate_mapping_pagevec, TP_PROTO(struct address_space *mapping, bool *skip), TP_ARGS(mapping, skip)); +DECLARE_HOOK(android_vh_keep_reclaimed_folio, + TP_PROTO(struct folio *folio, int refcount, bool *keep), + TP_ARGS(folio, refcount, keep)); +DECLARE_HOOK(android_vh_clear_reclaimed_folio, + TP_PROTO(struct folio *folio, bool reclaimed), + TP_ARGS(folio, reclaimed)); +DECLARE_HOOK(android_vh_evict_folios_bypass, + TP_PROTO(struct folio *folio, bool *bypass), + TP_ARGS(folio, bypass)); + DECLARE_HOOK(android_vh_modify_scan_control, TP_PROTO(u64 *ext, unsigned long *nr_to_reclaim, struct mem_cgroup *target_mem_cgroup, diff --git a/mm/filemap.c b/mm/filemap.c index 378136064103..d89f31a49e8e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3746,6 +3746,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, last_pgoff = xas.xa_index; end = folio_next_index(folio) - 1; nr_pages = min(end, end_pgoff) - xas.xa_index + 1; + trace_android_vh_filemap_pages(folio); if (!folio_test_large(folio)) ret |= filemap_map_order0_folio(vmf, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c2cfe2a1311a..92436bb4bc0c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -4363,6 +4363,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, { unsigned int noreclaim_flag; unsigned long progress; + bool skip = false; cond_resched(); @@ -4371,9 +4372,14 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); + trace_android_rvh_perform_reclaim(order, gfp_mask, ac->nodemask, + &progress, &skip); + if (skip) + goto out; + progress = try_to_free_pages(ac->zonelist, order, gfp_mask, ac->nodemask); - +out: memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); diff --git a/mm/vmscan.c b/mm/vmscan.c index 2526acf3278f..4762fbe86d0c 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -806,8 +806,15 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, * same address_space. */ if (reclaimed && folio_is_file_lru(folio) && - !mapping_exiting(mapping) && !dax_mapping(mapping)) + !mapping_exiting(mapping) && !dax_mapping(mapping)) { + bool keep = false; + + trace_android_vh_keep_reclaimed_folio(folio, refcount, &keep); + if (keep) + goto cannot_free; shadow = workingset_eviction(folio, target_memcg); + } + trace_android_vh_clear_reclaimed_folio(folio, reclaimed); __filemap_remove_folio(folio, shadow); xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) @@ -4777,6 +4784,11 @@ retry: list_for_each_entry_safe_reverse(folio, next, &list, lru) { DEFINE_MIN_SEQ(lruvec); + bool bypass = false; + + trace_android_vh_evict_folios_bypass(folio, &bypass); + if (bypass) + continue; if (!folio_evictable(folio)) { list_del(&folio->lru); @@ -6990,6 +7002,7 @@ static bool kswapd_shrink_node(pg_data_t *pgdat, sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); } + trace_android_rvh_kswapd_shrink_node(&sc->nr_to_reclaim); /* * Historically care was taken to put equal pressure on all zones but