diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 0bd30e0c950c..b375c70bbb84 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -201,7 +201,7 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) static bool guest_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot) { - return true; + return false; } static bool guest_stage2_pte_is_counted(kvm_pte_t pte, u32 level) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index c522f642b9d6..e7fcb0b782b4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1647,7 +1647,10 @@ static int pkvm_host_map_guest(u64 pfn, u64 gfn, u64 nr_pages, /* * Getting -EPERM at this point implies that the pfn has already been * mapped. This should only ever happen when two vCPUs faulted on the - * same page, and the current one lost the race to do the mapping. + * same page, and the current one lost the race to do the mapping... + * + * ...or if we've tried to map a region containing an already mapped + * entry. */ return (ret == -EPERM) ? -EAGAIN : ret; } @@ -1713,17 +1716,18 @@ static int pkvm_relax_perms(struct kvm_vcpu *vcpu, u64 pfn, u64 gfn, u8 order, (void *)prot, false); } -static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, - struct kvm_memory_slot *memslot) +static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t *fault_ipa, + struct kvm_memory_slot *memslot, size_t *size) { - struct mm_struct *mm = current->mm; unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; struct kvm_hyp_memcache *hyp_memcache = &vcpu->arch.stage2_mc; + unsigned long pmd_offset, page_size; + struct mm_struct *mm = current->mm; struct kvm_pinned_page *ppage; struct kvm *kvm = vcpu->kvm; struct kvm_s2_mmu *mmu = &kvm->arch.mmu; struct page *page; - gfn_t gfn = fault_ipa >> PAGE_SHIFT; + gfn_t gfn = *fault_ipa >> PAGE_SHIFT; unsigned long hva = gfn_to_hva_memslot_prot(memslot, gfn, NULL); int ret, nr_pages; u64 pfn; @@ -1740,10 +1744,6 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (!ppage) return -ENOMEM; - ret = account_locked_vm(mm, 1, true); - if (ret) - goto free_ppage; - mmap_read_lock(mm); ret = pin_user_pages(hva, 1, flags, &page); mmap_read_unlock(mm); @@ -1751,10 +1751,10 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (ret == -EHWPOISON) { kvm_send_hwpoison_signal(hva, PAGE_SHIFT); ret = 0; - goto dec_account; + goto free_ppage; } else if (ret != 1) { ret = -EFAULT; - goto dec_account; + goto free_ppage; } else if (kvm->arch.pkvm.enabled && !folio_test_swapbacked(page_folio(page))) { /* @@ -1775,18 +1775,50 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, goto unpin; } - write_lock(&kvm->mmu_lock); pfn = page_to_pfn(page); - ret = pkvm_host_map_guest(pfn, fault_ipa >> PAGE_SHIFT, 1, KVM_PGTABLE_PROT_R); + pmd_offset = *fault_ipa & (PMD_SIZE - 1); + page_size = transparent_hugepage_adjust(kvm, memslot, + hva, &pfn, + fault_ipa); + page = pfn_to_page(pfn); + +retry: + if (size) + *size = page_size; + + ret = account_locked_vm(mm, page_size >> PAGE_SHIFT, true); + if (ret) + goto unpin; + + write_lock(&kvm->mmu_lock); + /* + * If we already have a mapping in the middle of the THP, we have no + * other choice than enforcing PAGE_SIZE for pkvm_host_map_guest() to + * succeed. + */ + if (page_size > PAGE_SIZE && + find_ppage(kvm, *fault_ipa)) { + write_unlock(&kvm->mmu_lock); + *fault_ipa += pmd_offset; + pfn += pmd_offset >> PAGE_SHIFT; + page = pfn_to_page(pfn); + account_locked_vm(mm, page_size >> PAGE_SHIFT, false); + page_size = PAGE_SIZE; + goto retry; + } + + ret = pkvm_host_map_guest(pfn, *fault_ipa >> PAGE_SHIFT, + page_size >> PAGE_SHIFT, KVM_PGTABLE_PROT_R); if (ret) { if (ret == -EAGAIN) ret = 0; - goto unlock; + + goto dec_account; } ppage->page = page; - ppage->ipa = fault_ipa; - ppage->order = 0; + ppage->ipa = *fault_ipa; + ppage->order = get_order(page_size); ppage->pins = 1 << ppage->order; ppage->dirty = kvm->arch.pkvm.enabled; WARN_ON(insert_ppage(kvm, ppage)); @@ -1794,12 +1826,11 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return 0; -unlock: +dec_account: write_unlock(&kvm->mmu_lock); + account_locked_vm(mm, page_size >> PAGE_SHIFT, false); unpin: unpin_user_pages(&page, 1); -dec_account: - account_locked_vm(mm, 1, false); free_ppage: kfree(ppage); @@ -1810,6 +1841,7 @@ int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t si { phys_addr_t ipa_end = fault_ipa + size - 1; struct kvm_pinned_page *ppage; + unsigned long page_size; int err = 0, idx; if (!PAGE_ALIGNED(size) || !PAGE_ALIGNED(fault_ipa)) @@ -1827,6 +1859,7 @@ int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t si fault_ipa, ipa_end); while (fault_ipa < ipa_end) { if (ppage && ppage->ipa == fault_ipa) { + page_size = PAGE_SIZE << ppage->order; ppage = kvm_pinned_pages_iter_next(ppage, fault_ipa, ipa_end); } else { gfn_t gfn = gpa_to_gfn(fault_ipa); @@ -1842,7 +1875,7 @@ int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t si } read_unlock(&vcpu->kvm->mmu_lock); - err = pkvm_mem_abort(vcpu, fault_ipa, memslot); + err = pkvm_mem_abort(vcpu, &fault_ipa, memslot, &page_size); read_lock(&vcpu->kvm->mmu_lock); if (err) goto end; @@ -1852,10 +1885,10 @@ int pkvm_mem_abort_range(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, size_t si * reference. */ ppage = kvm_pinned_pages_iter_first(&vcpu->kvm->arch.pkvm.pinned_pages, - fault_ipa + PAGE_SIZE, ipa_end); + fault_ipa + page_size, ipa_end); } - fault_ipa += PAGE_SIZE; + fault_ipa += page_size; } end: read_unlock(&vcpu->kvm->mmu_lock); @@ -1928,7 +1961,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * logging_active is guaranteed to never be true for VM_PFNMAP * memslots. */ - if (logging_active || is_protected_kvm_enabled()) { + if (logging_active) { force_pte = true; vma_shift = PAGE_SHIFT; } else { @@ -2351,7 +2384,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } if (is_protected_kvm_enabled() && !esr_fsc_is_permission_fault(esr)) - ret = pkvm_mem_abort(vcpu, fault_ipa, memslot); + ret = pkvm_mem_abort(vcpu, &fault_ipa, memslot, NULL); else ret = user_mem_abort(vcpu, fault_ipa, nested, memslot,