ANDROID: vendor_hook: Added hook to tune reclaimed huge page

android_vh_mm_customize_longterm_pinnable - This hook allows vendors to
customize some special folios that are not migrated. They should not be
migrated to avoid application lags.

android_vh_mm_do_madvise_bypass - This hook allows vendors to customize
some madvise behaviors so that some madvise requests can be completed
quickly in certain scenarios.

android_vh_mm_migrate_one_page - This hook allows vendors to track
certain folios that are being migrated.

android_vh_mm_remove_migration_pte_bypass - This hook is used to
properly handle the migration of the aforementioned special folio.

android_vh_mm_split_huge_page_bypass - This hook is used in certain
scenarios where large folios should not be split to avoid the overhead
caused by splitting.

android_vh_mm_try_split_folio_bypass - This hook is used to skip
splitting large folios during migration in certain scenarios.

Bug: 431672372
Change-Id: I3c3580651e330dc8cfa6fa6f7a4f85fba1ba9807
Signed-off-by: Pengfei Li <pengfei.kernel@vivo.corp-partner.google.com>

drivers/android/vendor_hooks.c

@@ -604,3 +604,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_file_is_tiny);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_pgdat_balanced);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_reclaim_idx);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_zone_can_compact);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_customize_longterm_pinnable);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_do_madvise_bypass);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_migrate_one_page);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_remove_migration_pte_bypass);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_split_huge_page_bypass);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mm_try_split_folio_bypass);

include/linux/mm.h

@@ -2099,8 +2099,17 @@ static inline bool is_zero_folio(const struct folio *folio)
 
 /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */
 #ifdef CONFIG_MIGRATION
+extern void _trace_android_vh_mm_customize_longterm_pinnable(struct folio *folio,
+		bool *is_longterm_pinnable);
+
 static inline bool folio_is_longterm_pinnable(struct folio *folio)
 {
+	bool is_longterm_pinnable = false;
+
+	_trace_android_vh_mm_customize_longterm_pinnable(folio, &is_longterm_pinnable);
+	if (is_longterm_pinnable)
+		return true;
+
 #ifdef CONFIG_CMA
 	int mt = folio_migratetype(folio);
 

include/trace/hooks/madvise.h

@@ -20,6 +20,10 @@ DECLARE_HOOK(android_vh_process_madvise_return_error,
 DECLARE_HOOK(android_vh_madvise_pageout_bypass,
 	TP_PROTO(struct mm_struct *mm, bool pageout, int *ret),
 	TP_ARGS(mm, pageout, ret));
+DECLARE_HOOK(android_vh_mm_do_madvise_bypass,
+	TP_PROTO(struct mm_struct *mm, unsigned long start, size_t len,
+		 int behavior, int *error, bool *bypass),
+	TP_ARGS(mm, start, len, behavior, error, bypass));
 
 #endif
 

include/trace/hooks/mm.h

@@ -581,6 +581,22 @@ DECLARE_HOOK(android_vh_mm_customize_zone_max_order,
 DECLARE_HOOK(android_vh_mm_customize_zone_pageset,
 	TP_PROTO(struct zone *zone, int *new_high_min, int *new_high_max, int *new_batch),
 	TP_ARGS(zone, new_high_min, new_high_max, new_batch));
+DECLARE_HOOK(android_vh_mm_customize_longterm_pinnable,
+	TP_PROTO(struct folio *folio, bool *is_longterm_pinnable),
+	TP_ARGS(folio, is_longterm_pinnable));
+DECLARE_HOOK(android_vh_mm_migrate_one_page,
+	TP_PROTO(struct page *page, const vm_flags_t vm_flags),
+	TP_ARGS(page, vm_flags));
+DECLARE_HOOK(android_vh_mm_remove_migration_pte_bypass,
+	TP_PROTO(struct folio *dst, struct vm_area_struct *vma, unsigned long addr,
+		 struct folio *src, bool *bypass),
+	TP_ARGS(dst, vma, addr, src, bypass));
+DECLARE_HOOK(android_vh_mm_split_huge_page_bypass,
+	TP_PROTO(struct page *page, struct list_head *list, int *ret, bool *bypass),
+	TP_ARGS(page, list, ret, bypass));
+DECLARE_HOOK(android_vh_mm_try_split_folio_bypass,
+	TP_PROTO(struct folio *folio, bool *bypass),
+	TP_ARGS(folio, bypass));
 #endif /* _TRACE_HOOK_MM_H */
 
 /* This part must be outside protection */

mm/gup.c

@@ -471,6 +471,17 @@ void unpin_folios(struct folio **folios, unsigned long nfolios)
 }
 EXPORT_SYMBOL_GPL(unpin_folios);
 
+/*
+ * trace_android_vh_mm_customize_longterm_pinnable is called in include/linux/mm.h
+ * by including include/trace/hooks/mm.h, which will result to build-err.
+ * So we create func: _trace_android_vh_mm_customize_longterm_pinnable.
+ */
+void _trace_android_vh_mm_customize_longterm_pinnable(struct folio *folio,
+		bool *is_longterm_pinnable)
+{
+	trace_android_vh_mm_customize_longterm_pinnable(folio, is_longterm_pinnable);
+}
+
 /*
  * Set the MMF_HAS_PINNED if not set yet; after set it'll be there for the mm's
  * lifecycle.  Avoid setting the bit unless necessary, or it might cause write

mm/huge_memory.c

@@ -3380,6 +3380,7 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
 	int extra_pins, ret;
 	pgoff_t end;
 	bool is_hzp;
+	bool bypass = false;
 
 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
 	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
@@ -3493,6 +3494,10 @@ int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
 			end = shmem_fallocend(mapping->host, end);
 	}
 
+	trace_android_vh_mm_split_huge_page_bypass(page, list, &ret, &bypass);
+	if (bypass)
+		goto out_unlock;
+
 	/*
 	 * Racy check if we can split the page, before unmap_folio() will
 	 * split PMDs

mm/madvise.c

@@ -1687,6 +1687,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh
 	int write;
 	size_t len;
 	struct blk_plug plug;
+	bool bypass = false;
 
 	if (!madvise_behavior_valid(behavior))
 		return -EINVAL;
@@ -1711,6 +1712,11 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh
 		return madvise_inject_error(behavior, start, start + len_in);
 #endif
 
+	trace_android_vh_mm_do_madvise_bypass(mm, start, len, behavior,
+					      &error, &bypass);
+	if (bypass)
+		return error;
+
 	write = madvise_need_mmap_write(behavior);
 	if (write) {
 		if (mmap_write_lock_killable(mm))

mm/migrate.c

@@ -254,6 +254,12 @@ static bool remove_migration_pte(struct folio *folio,
 {
 	struct rmap_walk_arg *rmap_walk_arg = arg;
 	DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION);
+	bool bypass = false;
+
+	trace_android_vh_mm_remove_migration_pte_bypass(folio, vma, addr,
+							rmap_walk_arg->folio, &bypass);
+	if (bypass)
+		return true;
 
 	while (page_vma_mapped_walk(&pvmw)) {
 		rmap_t rmap_flags = RMAP_NONE;
@@ -1568,6 +1574,11 @@ static inline int try_split_folio(struct folio *folio, struct list_head *split_f
 				  enum migrate_mode mode)
 {
 	int rc;
+	bool bypass = false;
+
+	trace_android_vh_mm_try_split_folio_bypass(folio, &bypass);
+	if (bypass)
+		return -EBUSY;
 
 	if (mode == MIGRATE_ASYNC) {
 		if (!folio_trylock(folio))

mm/rmap.c

@@ -2306,6 +2306,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 				set_pte_at(mm, address, pvmw.pte, swp_pte);
 			trace_set_migration_pte(address, pte_val(swp_pte),
 						folio_order(folio));
+			trace_android_vh_mm_migrate_one_page(subpage, vma->vm_flags);
 			/*
 			 * No need to invalidate here it will synchronize on
 			 * against the special swap migration pte.