From 73c426b11f28690a937e5293fad0b22bd042733c Mon Sep 17 00:00:00 2001 From: Pengfei Li Date: Mon, 14 Jul 2025 16:02:10 +0800 Subject: [PATCH] ANDROID: vendor_hook: Added hook for memory allocation tuning The following hooks are added for memory allocation tuning: android_vh_mm_customize_alloc_anon_thp - When allocating anonymous folios, this hook will set different orders based on the importance of the processes. In particular, some critical processes will allocate folios directly in this hook to speed up memory allocation. android_vh_mm_customize_ac - In __alloc_pages_noprof, this hook provides the ability to modify alloc_context and alloc_flags to accommodate processes of different importance. android_vh_mm_customize_rmqueue - In rmqueue, this hook provides the ability to modify alloc_flags and migratetype, which will be used by vendors to customize alloc_flags and migratetype to speed up memory allocation. android_vh_mm_customize_suitable_zone - For some specific gfp_mask and order requests, this hook is used to skip zones that should not be allocated, and for some critical processes, the wmark limit is skipped and jumps to try_this_zone for allocation to prevent it from being stuck. android_vh_mm_customize_wmark_ok - Vendors use this hook to customize the return value of wmark_ok based on different payloads. android_vh_mm_customize_zone_max_order - This hook is used to limit the maximum order allowed for special zone. It reduces a lot of unnecessary repeated merges and splits (because large orders are rarely allocated). android_vh_mm_customize_zone_pageset - This hook allows vendors to customize the high and batch of pagesets. Bug: 431730501 Bug: 431672372 Change-Id: I65d6b2cd63688291e6360597bc2a4157fa228df7 Signed-off-by: Pengfei Li --- drivers/android/vendor_hooks.c | 7 +++++++ include/trace/hooks/mm.h | 26 +++++++++++++++++++++++ mm/memory.c | 8 ++++++- mm/page_alloc.c | 38 +++++++++++++++++++++++++++++++++- 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index bd180cdeab90..8472847cbedd 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -592,3 +592,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_gzvm_destroy_vm_post_process); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_usb_dev_suspend); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swap_bio_charge); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_alloc_anon_thp); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_ac); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_rmqueue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_suitable_zone); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_wmark_ok); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_max_order); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_pageset); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index dfd69fa48a3c..db37abab2da8 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -555,6 +555,32 @@ DECLARE_HOOK(android_vh_gcma_cc_store_page_bypass, DECLARE_HOOK(android_vh_swap_bio_charge, TP_PROTO(struct bio *bio), TP_ARGS(bio)); +DECLARE_HOOK(android_vh_mm_customize_alloc_anon_thp, + TP_PROTO(gfp_t *gfp_mask, unsigned long *orders, int *order, struct folio **folio), + TP_ARGS(gfp_mask, orders, order, folio)); +DECLARE_HOOK(android_vh_mm_customize_ac, + TP_PROTO(gfp_t gfp, unsigned int order, struct zonelist **zonelist, + struct zoneref **preferred_zoneref, enum zone_type *highest_zoneidx, + unsigned int *alloc_flags), + TP_ARGS(gfp, order, zonelist, preferred_zoneref, highest_zoneidx, alloc_flags)); +DECLARE_HOOK(android_vh_mm_customize_rmqueue, + TP_PROTO(struct zone *zone, unsigned int order, unsigned int *alloc_flags, + int *migratetype), + TP_ARGS(zone, order, alloc_flags, migratetype)); +DECLARE_HOOK(android_vh_mm_customize_suitable_zone, + TP_PROTO(struct zone *zone, gfp_t gfp, int order, enum zone_type highest_zoneidx, + bool *use_this_zone, bool *suitable), + TP_ARGS(zone, gfp, order, highest_zoneidx, use_this_zone, suitable)); +DECLARE_HOOK(android_vh_mm_customize_wmark_ok, + TP_PROTO(struct zone *zone, unsigned int order, enum zone_type highest_zoneidx, + bool *wmark_ok, bool *customized), + TP_ARGS(zone, order, highest_zoneidx, wmark_ok, customized)); +DECLARE_HOOK(android_vh_mm_customize_zone_max_order, + TP_PROTO(struct zone *zone, int *max_order), + TP_ARGS(zone, max_order)); +DECLARE_HOOK(android_vh_mm_customize_zone_pageset, + TP_PROTO(struct zone *zone, int *new_high_min, int *new_high_max, int *new_batch), + TP_ARGS(zone, new_high_min, new_high_max, new_batch)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/memory.c b/mm/memory.c index caf01a420197..5ed0097a6128 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4690,7 +4690,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; #ifdef CONFIG_TRANSPARENT_HUGEPAGE unsigned long orders; - struct folio *folio; + struct folio *folio = NULL; unsigned long addr; pte_t *pte; gfp_t gfp; @@ -4739,10 +4739,16 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) /* Try allocating the highest of the remaining orders. */ gfp = vma_thp_gfp_mask(vma); + + trace_android_vh_mm_customize_alloc_anon_thp(&gfp, &orders, &order, &folio); + if (folio) + goto allocated; + while (orders) { addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); folio = vma_alloc_folio(gfp, order, vma, addr, true); if (folio) { +allocated: if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); folio_put(folio); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 01e7586a7e11..18e78a14626e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -871,6 +871,14 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, NULL) != NULL; } +static int zone_max_order(struct zone *zone) +{ + int max_order = MAX_PAGE_ORDER; + + trace_android_vh_mm_customize_zone_max_order(zone, &max_order); + return max_order; +} + /* * Freeing function for a buddy system allocator. * @@ -906,6 +914,7 @@ static inline void __free_one_page(struct page *page, struct page *buddy; bool to_tail; bool bypass = false; + int max_order = zone_max_order(zone); trace_android_vh_free_one_page_bypass(page, zone, order, migratetype, (int)fpi_flags, &bypass); @@ -922,7 +931,7 @@ static inline void __free_one_page(struct page *page, account_freepages(zone, 1 << order, migratetype); - while (order < MAX_PAGE_ORDER) { + while (order < max_order) { int buddy_mt = migratetype; if (compaction_capture(capc, page, order, migratetype)) { @@ -980,6 +989,8 @@ done_merging: to_tail = true; else if (is_shuffle_order(order)) to_tail = shuffle_pick_tail(); + else if (max_order != MAX_PAGE_ORDER) + to_tail = false; else to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); @@ -3305,6 +3316,8 @@ struct page *rmqueue(struct zone *preferred_zone, { struct page *page; + trace_android_vh_mm_customize_rmqueue(zone, order, &alloc_flags, &migratetype); + if (likely(pcp_allowed_order(order))) { page = rmqueue_pcplist(preferred_zone, zone, order, migratetype, alloc_flags); @@ -3507,6 +3520,13 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, { long min = mark; int o; + bool customized = false; + bool wmark_ok = false; + + trace_android_vh_mm_customize_wmark_ok(z, order, highest_zoneidx, + &wmark_ok, &customized); + if (customized) + return wmark_ok; /* free_pages may go negative - that's OK */ free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); @@ -3756,9 +3776,19 @@ retry: z = ac->preferred_zoneref; for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, ac->nodemask) { + bool use_this_zone = false; + bool suitable = true; struct page *page; unsigned long mark; + trace_android_vh_mm_customize_suitable_zone(zone, gfp_mask, order, ac->highest_zoneidx, + &use_this_zone, &suitable); + if (!suitable) + continue; + + if (use_this_zone) + goto try_this_zone; + if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && !__cpuset_zone_allowed(zone, gfp_mask)) @@ -5209,6 +5239,9 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, &alloc_gfp, &alloc_flags)) return NULL; + trace_android_vh_mm_customize_ac(gfp, order, &ac.zonelist, &ac.preferred_zoneref, + &ac.highest_zoneidx, &alloc_flags); + trace_android_rvh_try_alloc_pages_gfp(&page, order, gfp, gfp_zone(gfp)); if (page) goto out; @@ -6177,6 +6210,9 @@ static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) zone->pageset_batch == new_batch) return; + trace_android_vh_mm_customize_zone_pageset(zone, &new_high_min, + &new_high_max, &new_batch); + zone->pageset_high_min = new_high_min; zone->pageset_high_max = new_high_max; zone->pageset_batch = new_batch;