diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index bd180cdeab90..8472847cbedd 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -592,3 +592,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_gzvm_destroy_vm_post_process); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_usb_dev_suspend); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_usb_dev_resume); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swap_bio_charge); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_alloc_anon_thp); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_ac); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_rmqueue); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_suitable_zone); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_wmark_ok); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_max_order); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_pageset); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index dfd69fa48a3c..db37abab2da8 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -555,6 +555,32 @@ DECLARE_HOOK(android_vh_gcma_cc_store_page_bypass, DECLARE_HOOK(android_vh_swap_bio_charge, TP_PROTO(struct bio *bio), TP_ARGS(bio)); +DECLARE_HOOK(android_vh_mm_customize_alloc_anon_thp, + TP_PROTO(gfp_t *gfp_mask, unsigned long *orders, int *order, struct folio **folio), + TP_ARGS(gfp_mask, orders, order, folio)); +DECLARE_HOOK(android_vh_mm_customize_ac, + TP_PROTO(gfp_t gfp, unsigned int order, struct zonelist **zonelist, + struct zoneref **preferred_zoneref, enum zone_type *highest_zoneidx, + unsigned int *alloc_flags), + TP_ARGS(gfp, order, zonelist, preferred_zoneref, highest_zoneidx, alloc_flags)); +DECLARE_HOOK(android_vh_mm_customize_rmqueue, + TP_PROTO(struct zone *zone, unsigned int order, unsigned int *alloc_flags, + int *migratetype), + TP_ARGS(zone, order, alloc_flags, migratetype)); +DECLARE_HOOK(android_vh_mm_customize_suitable_zone, + TP_PROTO(struct zone *zone, gfp_t gfp, int order, enum zone_type highest_zoneidx, + bool *use_this_zone, bool *suitable), + TP_ARGS(zone, gfp, order, highest_zoneidx, use_this_zone, suitable)); +DECLARE_HOOK(android_vh_mm_customize_wmark_ok, + TP_PROTO(struct zone *zone, unsigned int order, enum zone_type highest_zoneidx, + bool *wmark_ok, bool *customized), + TP_ARGS(zone, order, highest_zoneidx, wmark_ok, customized)); +DECLARE_HOOK(android_vh_mm_customize_zone_max_order, + TP_PROTO(struct zone *zone, int *max_order), + TP_ARGS(zone, max_order)); +DECLARE_HOOK(android_vh_mm_customize_zone_pageset, + TP_PROTO(struct zone *zone, int *new_high_min, int *new_high_max, int *new_batch), + TP_ARGS(zone, new_high_min, new_high_max, new_batch)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/memory.c b/mm/memory.c index caf01a420197..5ed0097a6128 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4690,7 +4690,7 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; #ifdef CONFIG_TRANSPARENT_HUGEPAGE unsigned long orders; - struct folio *folio; + struct folio *folio = NULL; unsigned long addr; pte_t *pte; gfp_t gfp; @@ -4739,10 +4739,16 @@ static struct folio *alloc_anon_folio(struct vm_fault *vmf) /* Try allocating the highest of the remaining orders. */ gfp = vma_thp_gfp_mask(vma); + + trace_android_vh_mm_customize_alloc_anon_thp(&gfp, &orders, &order, &folio); + if (folio) + goto allocated; + while (orders) { addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order); folio = vma_alloc_folio(gfp, order, vma, addr, true); if (folio) { +allocated: if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); folio_put(folio); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 01e7586a7e11..18e78a14626e 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -871,6 +871,14 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, NULL) != NULL; } +static int zone_max_order(struct zone *zone) +{ + int max_order = MAX_PAGE_ORDER; + + trace_android_vh_mm_customize_zone_max_order(zone, &max_order); + return max_order; +} + /* * Freeing function for a buddy system allocator. * @@ -906,6 +914,7 @@ static inline void __free_one_page(struct page *page, struct page *buddy; bool to_tail; bool bypass = false; + int max_order = zone_max_order(zone); trace_android_vh_free_one_page_bypass(page, zone, order, migratetype, (int)fpi_flags, &bypass); @@ -922,7 +931,7 @@ static inline void __free_one_page(struct page *page, account_freepages(zone, 1 << order, migratetype); - while (order < MAX_PAGE_ORDER) { + while (order < max_order) { int buddy_mt = migratetype; if (compaction_capture(capc, page, order, migratetype)) { @@ -980,6 +989,8 @@ done_merging: to_tail = true; else if (is_shuffle_order(order)) to_tail = shuffle_pick_tail(); + else if (max_order != MAX_PAGE_ORDER) + to_tail = false; else to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order); @@ -3305,6 +3316,8 @@ struct page *rmqueue(struct zone *preferred_zone, { struct page *page; + trace_android_vh_mm_customize_rmqueue(zone, order, &alloc_flags, &migratetype); + if (likely(pcp_allowed_order(order))) { page = rmqueue_pcplist(preferred_zone, zone, order, migratetype, alloc_flags); @@ -3507,6 +3520,13 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, { long min = mark; int o; + bool customized = false; + bool wmark_ok = false; + + trace_android_vh_mm_customize_wmark_ok(z, order, highest_zoneidx, + &wmark_ok, &customized); + if (customized) + return wmark_ok; /* free_pages may go negative - that's OK */ free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags); @@ -3756,9 +3776,19 @@ retry: z = ac->preferred_zoneref; for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx, ac->nodemask) { + bool use_this_zone = false; + bool suitable = true; struct page *page; unsigned long mark; + trace_android_vh_mm_customize_suitable_zone(zone, gfp_mask, order, ac->highest_zoneidx, + &use_this_zone, &suitable); + if (!suitable) + continue; + + if (use_this_zone) + goto try_this_zone; + if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) && !__cpuset_zone_allowed(zone, gfp_mask)) @@ -5209,6 +5239,9 @@ struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, &alloc_gfp, &alloc_flags)) return NULL; + trace_android_vh_mm_customize_ac(gfp, order, &ac.zonelist, &ac.preferred_zoneref, + &ac.highest_zoneidx, &alloc_flags); + trace_android_rvh_try_alloc_pages_gfp(&page, order, gfp, gfp_zone(gfp)); if (page) goto out; @@ -6177,6 +6210,9 @@ static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online) zone->pageset_batch == new_batch) return; + trace_android_vh_mm_customize_zone_pageset(zone, &new_high_min, + &new_high_max, &new_batch); + zone->pageset_high_min = new_high_min; zone->pageset_high_max = new_high_max; zone->pageset_batch = new_batch;