From ab910ce16b5db551711655c66f2ab73e0a4ed3ed Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Thu, 30 Jan 2025 10:54:52 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Re-enable THP support for np-guests There was a temporary tweak to user_mem_abort() to force all stage-2 mappings to page granularity as the upstream NP guest code didn't support THP yet. Fix this situation, and remove the tweak from user_mem_abort(). Bug: 357781595 Bug: 235345873 Change-Id: I64ed361ad9b6b8b8d4d23a1a5bac490559292ccb Signed-off-by: Quentin Perret --- arch/arm64/include/asm/kvm_pkvm.h | 1 + arch/arm64/kvm/mmu.c | 2 +- arch/arm64/kvm/pkvm.c | 33 +++++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 9c645c47c1f7..31b22c3f02df 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -476,6 +476,7 @@ struct pkvm_mapping { struct rb_node node; u64 gfn; u64 pfn; + u64 nr_pages; u64 __subtree_last; /* Internal member for interval tree */ }; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6e42a2cc1c8f..36f67d548e13 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1873,7 +1873,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * logging_active is guaranteed to never be true for VM_PFNMAP * memslots. */ - if (logging_active || is_protected_kvm_enabled()) { + if (logging_active) { force_pte = true; vma_shift = PAGE_SHIFT; } else { diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index fd493f233176..c9da73845605 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -1552,7 +1552,7 @@ static u64 __pkvm_mapping_start(struct pkvm_mapping *m) static u64 __pkvm_mapping_end(struct pkvm_mapping *m) { - return (m->gfn + 1) * PAGE_SIZE - 1; + return (m->gfn + m->nr_pages) * PAGE_SIZE - 1; } INTERVAL_TREE_DEFINE(struct pkvm_mapping, node, u64, __subtree_last, @@ -1589,7 +1589,8 @@ static int __pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 start, u64 e return 0; for_each_mapping_in_range_safe(pgt, start, end, mapping) { - ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_guest, handle, mapping->gfn, 1); + ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_guest, handle, mapping->gfn, + mapping->nr_pages); if (WARN_ON(ret)) return ret; pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); @@ -1615,20 +1616,36 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 pfn = phys >> PAGE_SHIFT; int ret; - if (size != PAGE_SIZE) + if (size != PAGE_SIZE && size != PMD_SIZE) return -EINVAL; lockdep_assert_held_write(&kvm->mmu_lock); - ret = kvm_call_hyp_nvhe(__pkvm_host_share_guest, pfn, gfn, prot, 1); - if (ret) { - /* Is the gfn already mapped due to a racing vCPU? */ - if (ret == -EPERM) + + /* + * Calling stage2_map() on top of existing mappings is either happening because of a race + * with another vCPU, or because we're changing between page and block mappings. As per + * user_mem_abort(), same-size permission faults are handled in the relax_perms() path. + */ + mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, addr + size - 1); + if (mapping) { + if (size == (mapping->nr_pages * PAGE_SIZE)) return -EAGAIN; + + /* Remove _any_ pkvm_mapping overlapping with the range, bigger or smaller. */ + ret = __pkvm_pgtable_stage2_unmap(pgt, addr, addr + size); + if (ret) + return ret; + mapping = NULL; } + ret = kvm_call_hyp_nvhe(__pkvm_host_share_guest, pfn, gfn, prot, size / PAGE_SIZE); + if (WARN_ON(ret)) + return ret; + swap(mapping, cache->mapping); mapping->gfn = gfn; mapping->pfn = pfn; + mapping->nr_pages = size / PAGE_SIZE; pkvm_mapping_insert(mapping, &pgt->pkvm_mappings); return ret; @@ -1656,7 +1673,7 @@ int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) lockdep_assert_held(&kvm->mmu_lock); for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) - __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), PAGE_SIZE); + __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), PAGE_SIZE * mapping->nr_pages); return 0; }