From 1036072ad0856d62612c3bfd545849b9b36479f3 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 30 Jun 2023 18:02:44 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Remove locking from EL2 allocation fast-paths hyp_{get,put}_page() are called extensively from the page-table code to adjust reference counts on page-table pages. As a small step towards removing reader serialisation on these paths, drop the 'hyp_pool' lock in the case where the refcount remains positive, only taking the lock if the page is to be freed back to the allocator. Remove a misleading comment at the same time, which implies that a page with a refcount of zero and which is not attached to a freelist is unsafe whereas in practice it's the other way around which can lead to problems. Bug: 357781595 Change-Id: I249636d3bc993fd0c01038fc885e110849d0c414 Signed-off-by: Will Deacon Signed-off-by: Fuad Tabba --- arch/arm64/kvm/hyp/include/nvhe/gfp.h | 6 +----- arch/arm64/kvm/hyp/nvhe/page_alloc.c | 16 ++++------------ 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h index 97c527ef53c2..24eb7840d98e 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/gfp.h +++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h @@ -10,11 +10,7 @@ #define HYP_NO_ORDER USHRT_MAX struct hyp_pool { - /* - * Spinlock protecting concurrent changes to the memory pool as well as - * the struct hyp_page of the pool's pages until we have a proper atomic - * API at EL2. - */ + /* lock protecting concurrent changes to the memory pool. */ hyp_spinlock_t lock; struct list_head free_area[NR_PAGE_ORDERS]; phys_addr_t range_start; diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index 169cdb43b4b8..96b52f545af0 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -155,33 +155,25 @@ static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p) { - if (hyp_page_ref_dec_and_test(p)) + if (hyp_page_ref_dec_and_test(p)) { + hyp_spin_lock(&pool->lock); __hyp_attach_page(pool, p); + hyp_spin_unlock(&pool->lock); + } } -/* - * Changes to the buddy tree and page refcounts must be done with the hyp_pool - * lock held. If a refcount change requires an update to the buddy tree (e.g. - * hyp_put_page()), both operations must be done within the same critical - * section to guarantee transient states (e.g. a page with null refcount but - * not yet attached to a free list) can't be observed by well-behaved readers. - */ void hyp_put_page(struct hyp_pool *pool, void *addr) { struct hyp_page *p = hyp_virt_to_page(addr); - hyp_spin_lock(&pool->lock); __hyp_put_page(pool, p); - hyp_spin_unlock(&pool->lock); } void hyp_get_page(struct hyp_pool *pool, void *addr) { struct hyp_page *p = hyp_virt_to_page(addr); - hyp_spin_lock(&pool->lock); hyp_page_ref_inc(p); - hyp_spin_unlock(&pool->lock); } void hyp_split_page(struct hyp_page *p)