From 1de7425d75a65ab0d10387c92eb9cd97fd3de7e6 Mon Sep 17 00:00:00 2001 From: Oven Date: Fri, 30 Jun 2023 10:57:12 +0800 Subject: [PATCH] ANDROID: vendor_hooks: Add hooks to avoid key threads stalled in memory allocations We add these hooks to avoid key threads blocked in memory allocation path. -android_vh_free_unref_page_bypass ----We create a memory pool for the key threads. This hook determines whether a page should be free to the pool or to buddy freelist. It works with a existing hook `android_vh_alloc_pages_reclaim_bypass`, which takes pages out of the pool. -android_vh_kvmalloc_node_use_vmalloc ----For key threads, we perfer not to run into direct reclaim. So we clear __GFP_DIRECT_RECLAIM flag. For threads which are not that important, we perfer use vmalloc. -android_vh_should_alloc_pages_retry ----Before key threads run into direct reclaim, we want to retry with a lower watermark. -android_vh_unreserve_highatomic_bypass ----We want to keep more highatomic pages when unreserve them to avoid highatomic allocation failures. -android_vh_rmqueue_bulk_bypass ----We found sometimes when key threads run into rmqueue_bulk, it took several milliseconds spinning at zone->lock or filling per-cpu pages. We use this hook to take pages from the mempool mentioned above, rather than grab zone->lock and fill a batch of pages to per-cpu. Bug: 288216516 Bug: 395197442 Change-Id: I1656032d6819ca627723341987b6094775bc345f Signed-off-by: Oven Signed-off-by: huzhanyuan Signed-off-by: wengle --- drivers/android/vendor_hooks.c | 5 +++++ include/trace/hooks/mm.h | 18 ++++++++++++++++++ mm/page_alloc.c | 31 +++++++++++++++++++++++++++++++ mm/util.c | 8 +++++++- 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index b4507b61cf12..49896e8b9cc4 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -169,6 +169,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_balance_anon_file_reclaim); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_show_max_freq); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_slab_alloc_node); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_slab_free); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_unref_page_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_kvmalloc_node_use_vmalloc); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_should_alloc_pages_retry); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_unreserve_highatomic_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rmqueue_bulk_bypass); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_hw_protection_shutdown); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_insert); EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_delete); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 282bad224730..6d541b31ea42 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -149,6 +149,24 @@ struct slabinfo; DECLARE_HOOK(android_vh_cache_show, TP_PROTO(struct seq_file *m, struct slabinfo *sinfo, struct kmem_cache *s), TP_ARGS(m, sinfo, s)); +DECLARE_HOOK(android_vh_free_unref_page_bypass, + TP_PROTO(struct page *page, int order, int migratetype, bool *bypass), + TP_ARGS(page, order, migratetype, bypass)); +DECLARE_HOOK(android_vh_kvmalloc_node_use_vmalloc, + TP_PROTO(size_t size, gfp_t *kmalloc_flags, bool *use_vmalloc), + TP_ARGS(size, kmalloc_flags, use_vmalloc)); +DECLARE_HOOK(android_vh_should_alloc_pages_retry, + TP_PROTO(gfp_t gfp_mask, int order, int *alloc_flags, + int migratetype, struct zone *preferred_zone, struct page **page, bool *should_alloc_retry), + TP_ARGS(gfp_mask, order, alloc_flags, + migratetype, preferred_zone, page, should_alloc_retry)); +DECLARE_HOOK(android_vh_unreserve_highatomic_bypass, + TP_PROTO(bool force, struct zone *zone, bool *skip_unreserve_highatomic), + TP_ARGS(force, zone, skip_unreserve_highatomic)); +DECLARE_HOOK(android_vh_rmqueue_bulk_bypass, + TP_PROTO(unsigned int order, struct per_cpu_pages *pcp, int migratetype, + struct list_head *list), + TP_ARGS(order, pcp, migratetype, list)); DECLARE_HOOK(android_vh_madvise_swapin_walk_pmd_entry, TP_PROTO(swp_entry_t entry), TP_ARGS(entry)); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6f85a1b2801e..947f0d98617b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1413,10 +1413,12 @@ static void free_one_page(struct zone *zone, struct page *page, static void __free_pages_ok(struct page *page, unsigned int order, fpi_t fpi_flags) { + int migratetype; unsigned long pfn = page_to_pfn(page); struct zone *zone = page_zone(page); bool skip_free_pages_prepare = false; bool skip_free_pages_ok = false; + bool skip_free_unref_page = false; trace_android_vh_free_pages_prepare_bypass(page, order, fpi_flags, &skip_free_pages_prepare); @@ -1430,6 +1432,15 @@ skip_prepare: fpi_flags, &skip_free_pages_ok); if (skip_free_pages_ok) return; + /* + * Calling get_pfnblock_migratetype() without spin_lock_irqsave() here + * is used to avoid calling get_pfnblock_migratetype() under the lock. + * This will reduce the lock holding time. + */ + migratetype = get_pfnblock_migratetype(page, pfn); + trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page); + if (skip_free_unref_page) + return; free_one_page(zone, page, pfn, order, fpi_flags); } @@ -2276,6 +2287,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, struct page *page; int order; int ret; + bool skip_unreserve_highatomic = false; for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx, ac->nodemask) { @@ -2287,6 +2299,11 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac, pageblock_nr_pages) continue; + trace_android_vh_unreserve_highatomic_bypass(force, zone, + &skip_unreserve_highatomic); + if (skip_unreserve_highatomic) + continue; + spin_lock_irqsave(&zone->lock, flags); for (order = 0; order < NR_PAGE_ORDERS; order++) { struct free_area *area = &(zone->free_area[order]); @@ -2847,6 +2864,7 @@ void free_unref_page(struct page *page, unsigned int order) struct zone *zone; unsigned long pfn = page_to_pfn(page); int migratetype; + bool skip_free_unref_page = false; if (!pcp_allowed_order(order)) { __free_pages_ok(page, order, FPI_NONE); @@ -2865,6 +2883,9 @@ void free_unref_page(struct page *page, unsigned int order) * excessively into the page allocator */ migratetype = get_pfnblock_migratetype(page, pfn); + trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page); + if (skip_free_unref_page) + return; if (unlikely(migratetype > MIGRATE_RECLAIMABLE)) { if (unlikely(is_migrate_isolate(migratetype))) { free_one_page(page_zone(page), page, pfn, order, FPI_NONE); @@ -3216,6 +3237,9 @@ struct page *___rmqueue_pcplist(struct zone *zone, unsigned int order, int batch = nr_pcp_alloc(pcp, zone, order); int alloced; + trace_android_vh_rmqueue_bulk_bypass(order, pcp, migratetype, list); + if (!list_empty(list)) + goto get_list; alloced = rmqueue_bulk(zone, order, batch, list, migratetype, alloc_flags); @@ -3225,6 +3249,7 @@ struct page *___rmqueue_pcplist(struct zone *zone, unsigned int order, return NULL; } +get_list: page = list_first_entry(list, struct page, pcp_list); list_del(&page->pcp_list); pcp->count -= 1 << order; @@ -4495,6 +4520,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, unsigned long pages_reclaimed = 0; int retry_loop_count = 0; u64 stime = 0; + bool should_alloc_retry = false; if (unlikely(nofail)) { /* @@ -4655,6 +4681,11 @@ retry: if (current->flags & PF_MEMALLOC) goto nopage; + trace_android_vh_should_alloc_pages_retry(gfp_mask, order, &alloc_flags, + ac->migratetype, ac->preferred_zoneref->zone, &page, &should_alloc_retry); + if (should_alloc_retry) + goto retry; + /* Try direct reclaim and then allocating */ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, &did_some_progress); diff --git a/mm/util.c b/mm/util.c index 06d16ce9a3c1..d8c35f15593f 100644 --- a/mm/util.c +++ b/mm/util.c @@ -31,8 +31,9 @@ #include "internal.h" #include "swap.h" -#ifndef __GENSYMS__ +#ifndef __GENKSYMS__ #include +#include #endif /** @@ -655,7 +656,11 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) { void *ret; + bool use_vmalloc = false; + trace_android_vh_kvmalloc_node_use_vmalloc(size, &flags, &use_vmalloc); + if (use_vmalloc) + goto use_vmalloc_node; /* * It doesn't really make sense to fallback to vmalloc for sub page * requests @@ -682,6 +687,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) * about the resulting pointer, and cannot play * protection games. */ +use_vmalloc_node: return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0));