From 792718bb234ac19db3efdbc3215485e874c41584 Mon Sep 17 00:00:00 2001 From: Pengfei Li Date: Thu, 17 Jul 2025 10:04:42 +0800 Subject: [PATCH] ANDROID: vendor_hook: Added hook to tune reclaimed huge page android_vh_mm_customize_longterm_pinnable - This hook allows vendors to customize some special folios that are not migrated. They should not be migrated to avoid application lags. android_vh_mm_do_madvise_bypass - This hook allows vendors to customize some madvise behaviors so that some madvise requests can be completed quickly in certain scenarios. android_vh_mm_migrate_one_page - This hook allows vendors to track certain folios that are being migrated. android_vh_mm_remove_migration_pte_bypass - This hook is used to properly handle the migration of the aforementioned special folio. android_vh_mm_split_huge_page_bypass - This hook is used in certain scenarios where large folios should not be split to avoid the overhead caused by splitting. android_vh_mm_try_split_folio_bypass - This hook is used to skip splitting large folios during migration in certain scenarios. Bug: 431672372 Change-Id: I3c3580651e330dc8cfa6fa6f7a4f85fba1ba9807 Signed-off-by: Pengfei Li --- drivers/android/vendor_hooks.c | 6 ++++++ include/linux/mm.h | 9 +++++++++ include/trace/hooks/madvise.h | 4 ++++ include/trace/hooks/mm.h | 16 ++++++++++++++++ mm/gup.c | 11 +++++++++++ mm/huge_memory.c | 5 +++++ mm/madvise.c | 6 ++++++ mm/migrate.c | 11 +++++++++++ mm/rmap.c | 1 + 9 files changed, 69 insertions(+) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 82af4b916234..a9f7a2c5fa1e 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -604,3 +604,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_file_is_tiny); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_pgdat_balanced); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_reclaim_idx); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_can_compact); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_longterm_pinnable); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_do_madvise_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_migrate_one_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_remove_migration_pte_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_split_huge_page_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_try_split_folio_bypass); diff --git a/include/linux/mm.h b/include/linux/mm.h index 62b8f0403498..354d337ab05e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2099,8 +2099,17 @@ static inline bool is_zero_folio(const struct folio *folio) /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ #ifdef CONFIG_MIGRATION +extern void _trace_android_vh_mm_customize_longterm_pinnable(struct folio *folio, + bool *is_longterm_pinnable); + static inline bool folio_is_longterm_pinnable(struct folio *folio) { + bool is_longterm_pinnable = false; + + _trace_android_vh_mm_customize_longterm_pinnable(folio, &is_longterm_pinnable); + if (is_longterm_pinnable) + return true; + #ifdef CONFIG_CMA int mt = folio_migratetype(folio); diff --git a/include/trace/hooks/madvise.h b/include/trace/hooks/madvise.h index 0dbf30712a9d..3dace49e3753 100644 --- a/include/trace/hooks/madvise.h +++ b/include/trace/hooks/madvise.h @@ -20,6 +20,10 @@ DECLARE_HOOK(android_vh_process_madvise_return_error, DECLARE_HOOK(android_vh_madvise_pageout_bypass, TP_PROTO(struct mm_struct *mm, bool pageout, int *ret), TP_ARGS(mm, pageout, ret)); +DECLARE_HOOK(android_vh_mm_do_madvise_bypass, + TP_PROTO(struct mm_struct *mm, unsigned long start, size_t len, + int behavior, int *error, bool *bypass), + TP_ARGS(mm, start, len, behavior, error, bypass)); #endif diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index db37abab2da8..8edee8a675b1 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -581,6 +581,22 @@ DECLARE_HOOK(android_vh_mm_customize_zone_max_order, DECLARE_HOOK(android_vh_mm_customize_zone_pageset, TP_PROTO(struct zone *zone, int *new_high_min, int *new_high_max, int *new_batch), TP_ARGS(zone, new_high_min, new_high_max, new_batch)); +DECLARE_HOOK(android_vh_mm_customize_longterm_pinnable, + TP_PROTO(struct folio *folio, bool *is_longterm_pinnable), + TP_ARGS(folio, is_longterm_pinnable)); +DECLARE_HOOK(android_vh_mm_migrate_one_page, + TP_PROTO(struct page *page, const vm_flags_t vm_flags), + TP_ARGS(page, vm_flags)); +DECLARE_HOOK(android_vh_mm_remove_migration_pte_bypass, + TP_PROTO(struct folio *dst, struct vm_area_struct *vma, unsigned long addr, + struct folio *src, bool *bypass), + TP_ARGS(dst, vma, addr, src, bypass)); +DECLARE_HOOK(android_vh_mm_split_huge_page_bypass, + TP_PROTO(struct page *page, struct list_head *list, int *ret, bool *bypass), + TP_ARGS(page, list, ret, bypass)); +DECLARE_HOOK(android_vh_mm_try_split_folio_bypass, + TP_PROTO(struct folio *folio, bool *bypass), + TP_ARGS(folio, bypass)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/gup.c b/mm/gup.c index e323843cc5dd..bcd81fdda84b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -471,6 +471,17 @@ void unpin_folios(struct folio **folios, unsigned long nfolios) } EXPORT_SYMBOL_GPL(unpin_folios); +/* + * trace_android_vh_mm_customize_longterm_pinnable is called in include/linux/mm.h + * by including include/trace/hooks/mm.h, which will result to build-err. + * So we create func: _trace_android_vh_mm_customize_longterm_pinnable. + */ +void _trace_android_vh_mm_customize_longterm_pinnable(struct folio *folio, + bool *is_longterm_pinnable) +{ + trace_android_vh_mm_customize_longterm_pinnable(folio, is_longterm_pinnable); +} + /* * Set the MMF_HAS_PINNED if not set yet; after set it'll be there for the mm's * lifecycle. Avoid setting the bit unless necessary, or it might cause write diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 04fd908a074c..2113225974dc 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3380,6 +3380,7 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, int extra_pins, ret; pgoff_t end; bool is_hzp; + bool bypass = false; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); @@ -3493,6 +3494,10 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, end = shmem_fallocend(mapping->host, end); } + trace_android_vh_mm_split_huge_page_bypass(page, list, &ret, &bypass); + if (bypass) + goto out_unlock; + /* * Racy check if we can split the page, before unmap_folio() will * split PMDs diff --git a/mm/madvise.c b/mm/madvise.c index 26ccf76a6355..a1b236d256d6 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1687,6 +1687,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh int write; size_t len; struct blk_plug plug; + bool bypass = false; if (!madvise_behavior_valid(behavior)) return -EINVAL; @@ -1711,6 +1712,11 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh return madvise_inject_error(behavior, start, start + len_in); #endif + trace_android_vh_mm_do_madvise_bypass(mm, start, len, behavior, + &error, &bypass); + if (bypass) + return error; + write = madvise_need_mmap_write(behavior); if (write) { if (mmap_write_lock_killable(mm)) diff --git a/mm/migrate.c b/mm/migrate.c index 972ee53353cb..8fd84722cb67 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -254,6 +254,12 @@ static bool remove_migration_pte(struct folio *folio, { struct rmap_walk_arg *rmap_walk_arg = arg; DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + bool bypass = false; + + trace_android_vh_mm_remove_migration_pte_bypass(folio, vma, addr, + rmap_walk_arg->folio, &bypass); + if (bypass) + return true; while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -1568,6 +1574,11 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f enum migrate_mode mode) { int rc; + bool bypass = false; + + trace_android_vh_mm_try_split_folio_bypass(folio, &bypass); + if (bypass) + return -EBUSY; if (mode == MIGRATE_ASYNC) { if (!folio_trylock(folio)) diff --git a/mm/rmap.c b/mm/rmap.c index 0c4ffc185326..40cf0a1a8296 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2306,6 +2306,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, set_pte_at(mm, address, pvmw.pte, swp_pte); trace_set_migration_pte(address, pte_val(swp_pte), folio_order(folio)); + trace_android_vh_mm_migrate_one_page(subpage, vma->vm_flags); /* * No need to invalidate here it will synchronize on * against the special swap migration pte.