diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 8b96ceb6655a..3364a94d1c63 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -537,3 +537,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_calculate_totalreserve_pages); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_madvise_cold_pageout_skip); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rmqueue_pcplist_override_batch); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_group_exit); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_add_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_swapmem_gather_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_swapmem_gather_finish); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 642360f0bd13..7ec6c57bf1b5 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -487,6 +487,21 @@ DECLARE_HOOK(android_vh_rmqueue_pcplist_override_batch, DECLARE_HOOK(android_vh_do_group_exit, TP_PROTO(struct task_struct *tsk), TP_ARGS(tsk)); +DECLARE_HOOK(android_vh_swapmem_gather_init, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_swapmem_gather_add_bypass, + TP_PROTO(struct mm_struct *mm, swp_entry_t entry, int nr, bool *bypass), + TP_ARGS(mm, entry, nr, bypass)); +DECLARE_HOOK(android_vh_swapmem_gather_finish, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_oom_swapmem_gather_init, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_oom_swapmem_gather_finish, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/memory.c b/mm/memory.c index 83dff501b03f..fa0d9ad17bd3 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1604,6 +1604,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t *pte; swp_entry_t entry; int nr; + bool bypass = false; tlb_change_page_size(tlb, PAGE_SIZE); init_rss_vec(rss); @@ -1663,6 +1664,9 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (!should_zap_cows(details)) continue; rss[MM_SWAPENTS] -= nr; + trace_android_vh_swapmem_gather_add_bypass(mm, entry, nr, &bypass); + if (bypass) + goto skip; free_swap_and_cache_nr(entry, nr); } else if (is_migration_entry(entry)) { folio = pfn_swap_entry_folio(entry); @@ -1694,6 +1698,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pr_alert("unrecognized swap entry 0x%lx\n", entry.val); WARN_ON_ONCE(1); } +skip: clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm); zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent); } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); diff --git a/mm/mmap.c b/mm/mmap.c index e3f0298e8eff..2c9f981cde59 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1952,9 +1952,11 @@ void exit_mmap(struct mm_struct *mm) lru_add_drain(); flush_cache_mm(mm); tlb_gather_mmu_fullmm(&tlb, mm); + trace_android_vh_swapmem_gather_init(mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ unmap_vmas(&tlb, &vmi.mas, vma, 0, ULONG_MAX, ULONG_MAX, false); + trace_android_vh_swapmem_gather_finish(mm); mmap_read_unlock(mm); /* diff --git a/mm/oom_kill.c b/mm/oom_kill.c index a4bade089121..d828a51fb123 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -53,6 +53,8 @@ #define CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include #undef CREATE_TRACE_POINTS #include @@ -531,6 +533,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) */ set_bit(MMF_UNSTABLE, &mm->flags); + trace_android_vh_oom_swapmem_gather_init(mm); for_each_vma(vmi, vma) { if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) continue; @@ -563,6 +566,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) tlb_finish_mmu(&tlb); } } + trace_android_vh_oom_swapmem_gather_finish(mm); return ret; }