tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ANDROID: vendor_hooks: Add hooks to avoid key threads stalled in memory allocations

Browse Source 
We add these hooks to avoid key threads blocked in memory allocation
path.
-android_vh_free_unref_page_bypass  ----We create a memory pool for the
key threads. This hook determines whether a page should be free to the
pool or to buddy freelist. It works with a existing hook
`android_vh_alloc_pages_reclaim_bypass`, which takes pages out of the
pool.

-android_vh_kvmalloc_node_use_vmalloc  ----For key threads, we perfer
not to run into direct reclaim. So we clear __GFP_DIRECT_RECLAIM flag.
For threads which are not that important, we perfer use vmalloc.

-android_vh_should_alloc_pages_retry  ----Before key threads run into
direct reclaim, we want to retry with a lower watermark.

-android_vh_unreserve_highatomic_bypass  ----We want to keep more
highatomic pages when unreserve them to avoid highatomic allocation
failures.

-android_vh_rmqueue_bulk_bypass  ----We found sometimes when key threads
run into rmqueue_bulk,  it took several milliseconds spinning at
zone->lock or filling per-cpu pages. We use this hook to take pages from
the mempool mentioned above,  rather than grab zone->lock and fill a
batch of pages to per-cpu.

Bug: 288216516
Bug: 395197442
Change-Id: I1656032d6819ca627723341987b6094775bc345f
Signed-off-by: Oven <liyangouwen1@oppo.com>
Signed-off-by: huzhanyuan  <huzhanyuan@oppo.com>
Signed-off-by: wengle <wengle@oppo.com>
This commit is contained in:
Oven
2023-06-30 10:57:12 +08:00
committed by Carlos Llamas
parent cc40b85c50
commit 1de7425d75
4 changed files with 61 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drivers/android/vendor_hooks.c
+5
View File
@@ -169,6 +169,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_balance_anon_file_reclaim);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_show_max_freq);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_slab_alloc_node);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_slab_free);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_free_unref_page_bypass);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_kvmalloc_node_use_vmalloc);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_should_alloc_pages_retry);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_unreserve_highatomic_bypass);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rmqueue_bulk_bypass);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_hw_protection_shutdown);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_insert);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_selinux_avc_node_delete);
include/trace/hooks/mm.h
+18
View File
@@ -149,6 +149,24 @@ struct slabinfo;
DECLARE_HOOK(android_vh_cache_show,
TP_PROTO(struct seq_file *m, struct slabinfo *sinfo, struct kmem_cache *s),
TP_ARGS(m, sinfo, s));
DECLARE_HOOK(android_vh_free_unref_page_bypass,
TP_PROTO(struct page *page, int order, int migratetype, bool *bypass),
TP_ARGS(page, order, migratetype, bypass));
DECLARE_HOOK(android_vh_kvmalloc_node_use_vmalloc,
TP_PROTO(size_t size, gfp_t *kmalloc_flags, bool *use_vmalloc),
TP_ARGS(size, kmalloc_flags, use_vmalloc));
DECLARE_HOOK(android_vh_should_alloc_pages_retry,
TP_PROTO(gfp_t gfp_mask, int order, int *alloc_flags,
int migratetype, struct zone *preferred_zone, struct page **page, bool *should_alloc_retry),
TP_ARGS(gfp_mask, order, alloc_flags,
migratetype, preferred_zone, page, should_alloc_retry));
DECLARE_HOOK(android_vh_unreserve_highatomic_bypass,
TP_PROTO(bool force, struct zone *zone, bool *skip_unreserve_highatomic),
TP_ARGS(force, zone, skip_unreserve_highatomic));
DECLARE_HOOK(android_vh_rmqueue_bulk_bypass,
TP_PROTO(unsigned int order, struct per_cpu_pages *pcp, int migratetype,
struct list_head *list),
TP_ARGS(order, pcp, migratetype, list));
DECLARE_HOOK(android_vh_madvise_swapin_walk_pmd_entry,
TP_PROTO(swp_entry_t entry),
TP_ARGS(entry));
mm/page_alloc.c
+31
View File
@@ -1413,10 +1413,12 @@ static void free_one_page(struct zone *zone, struct page *page,
static void __free_pages_ok(struct page *page, unsigned int order,
fpi_t fpi_flags)
{
int migratetype;
unsigned long pfn = page_to_pfn(page);
struct zone *zone = page_zone(page);
bool skip_free_pages_prepare = false;
bool skip_free_pages_ok = false;
bool skip_free_unref_page = false;
trace_android_vh_free_pages_prepare_bypass(page, order,
fpi_flags, &skip_free_pages_prepare);
@@ -1430,6 +1432,15 @@ skip_prepare:
fpi_flags, &skip_free_pages_ok);
if (skip_free_pages_ok)
return;
/*
* Calling get_pfnblock_migratetype() without spin_lock_irqsave() here
* is used to avoid calling get_pfnblock_migratetype() under the lock.
* This will reduce the lock holding time.
*/
migratetype = get_pfnblock_migratetype(page, pfn);
trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
if (skip_free_unref_page)
return;
free_one_page(zone, page, pfn, order, fpi_flags);
}
@@ -2276,6 +2287,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
struct page *page;
int order;
int ret;
bool skip_unreserve_highatomic = false;
for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
ac->nodemask) {
@@ -2287,6 +2299,11 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
pageblock_nr_pages)
continue;
trace_android_vh_unreserve_highatomic_bypass(force, zone,
&skip_unreserve_highatomic);
if (skip_unreserve_highatomic)
continue;
spin_lock_irqsave(&zone->lock, flags);
for (order = 0; order < NR_PAGE_ORDERS; order++) {
struct free_area *area = &(zone->free_area[order]);
@@ -2847,6 +2864,7 @@ void free_unref_page(struct page *page, unsigned int order)
struct zone *zone;
unsigned long pfn = page_to_pfn(page);
int migratetype;
bool skip_free_unref_page = false;
if (!pcp_allowed_order(order)) {
__free_pages_ok(page, order, FPI_NONE);
@@ -2865,6 +2883,9 @@ void free_unref_page(struct page *page, unsigned int order)
* excessively into the page allocator
*/
migratetype = get_pfnblock_migratetype(page, pfn);
trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
if (skip_free_unref_page)
return;
if (unlikely(migratetype > MIGRATE_RECLAIMABLE)) {
if (unlikely(is_migrate_isolate(migratetype))) {
free_one_page(page_zone(page), page, pfn, order, FPI_NONE);
@@ -3216,6 +3237,9 @@ struct page *___rmqueue_pcplist(struct zone *zone, unsigned int order,
int batch = nr_pcp_alloc(pcp, zone, order);
int alloced;
trace_android_vh_rmqueue_bulk_bypass(order, pcp, migratetype, list);
if (!list_empty(list))
goto get_list;
alloced = rmqueue_bulk(zone, order,
batch, list,
migratetype, alloc_flags);
@@ -3225,6 +3249,7 @@ struct page *___rmqueue_pcplist(struct zone *zone, unsigned int order,
return NULL;
}
get_list:
page = list_first_entry(list, struct page, pcp_list);
list_del(&page->pcp_list);
pcp->count -= 1 << order;
@@ -4495,6 +4520,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
unsigned long pages_reclaimed = 0;
int retry_loop_count = 0;
u64 stime = 0;
bool should_alloc_retry = false;
if (unlikely(nofail)) {
/*
@@ -4655,6 +4681,11 @@ retry:
if (current->flags & PF_MEMALLOC)
goto nopage;
trace_android_vh_should_alloc_pages_retry(gfp_mask, order, &alloc_flags,
ac->migratetype, ac->preferred_zoneref->zone, &page, &should_alloc_retry);
if (should_alloc_retry)
goto retry;
/* Try direct reclaim and then allocating */
page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
&did_some_progress);
mm/util.c
+7 -1
View File
@@ -31,8 +31,9 @@
#include "internal.h"
#include "swap.h"
#ifndef __GENSYMS__
#ifndef __GENKSYMS__
#include <trace/hooks/syscall_check.h>
#include <trace/hooks/mm.h>
#endif
/**
@@ -655,7 +656,11 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
{
void *ret;
bool use_vmalloc = false;
trace_android_vh_kvmalloc_node_use_vmalloc(size, &flags, &use_vmalloc);
if (use_vmalloc)
goto use_vmalloc_node;
/*
* It doesn't really make sense to fallback to vmalloc for sub page
* requests
@@ -682,6 +687,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
* about the resulting pointer, and cannot play
* protection games.
*/
use_vmalloc_node:
return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
node, __builtin_return_address(0));