diff --git a/include/linux/mm.h b/include/linux/mm.h index 0e210f861766..089916efe119 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1231,8 +1232,13 @@ static inline unsigned int folio_order(const struct folio *folio) */ static inline int put_page_testzero(struct page *page) { + int ret; + VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); - return page_ref_dec_and_test(page); + ret = page_ref_dec_and_test(page); + page_pinner_put_page(page); + + return ret; } static inline int folio_put_testzero(struct folio *folio) diff --git a/include/linux/page_pinner.h b/include/linux/page_pinner.h index cb29f5c3d68e..d87706cc3317 100644 --- a/include/linux/page_pinner.h +++ b/include/linux/page_pinner.h @@ -24,9 +24,6 @@ static inline void page_pinner_put_page(struct page *page) if (!static_branch_unlikely(&page_pinner_inited)) return; - if (!static_branch_unlikely(&failure_tracking)) - return; - __page_pinner_put_page(page); } @@ -35,9 +32,6 @@ static inline void page_pinner_failure_detect(struct page *page) if (!static_branch_unlikely(&page_pinner_inited)) return; - if (!static_branch_unlikely(&failure_tracking)) - return; - __page_pinner_failure_detect(page); } #else diff --git a/mm/page_pinner.c b/mm/page_pinner.c index 7ef84314c295..052a220e488a 100644 --- a/mm/page_pinner.c +++ b/mm/page_pinner.c @@ -57,9 +57,9 @@ static struct page_pinner_buffer pp_buffer; static bool page_pinner_enabled; DEFINE_STATIC_KEY_FALSE(page_pinner_inited); +EXPORT_SYMBOL_GPL(page_pinner_inited); DEFINE_STATIC_KEY_TRUE(failure_tracking); -EXPORT_SYMBOL_GPL(failure_tracking); static depot_stack_handle_t failure_handle; @@ -255,11 +255,15 @@ err: void __page_pinner_failure_detect(struct page *page) { - struct page_ext *page_ext = page_ext_get(page); + struct page_ext *page_ext; struct page_pinner *page_pinner; struct captured_pinner record; u64 now; + if (!static_branch_unlikely(&failure_tracking)) + return; + + page_ext = page_ext_get(page); if (unlikely(!page_ext)) return; @@ -285,11 +289,15 @@ EXPORT_SYMBOL_GPL(__page_pinner_failure_detect); void __page_pinner_put_page(struct page *page) { - struct page_ext *page_ext = page_ext_get(page); + struct page_ext *page_ext; struct page_pinner *page_pinner; struct captured_pinner record; u64 now, ts_usec; + if (!static_branch_unlikely(&failure_tracking)) + return; + + page_ext = page_ext_get(page); if (unlikely(!page_ext)) return;