diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 1d6de7c09695..d36bef0d06d1 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -71,8 +71,8 @@ int __pkvm_host_lazy_pte(u64 pfn, u64 nr_pages, bool enable); u64 __pkvm_ptdump_get_config(pkvm_handle_t handle, enum pkvm_ptdump_ops op); u64 __pkvm_ptdump_walk_range(pkvm_handle_t handle, struct pkvm_ptdump_log_hdr *log_hva); -int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn); int hyp_check_range_owned(u64 addr, u64 size); +int __pkvm_install_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, diff --git a/arch/arm64/kvm/hyp/nvhe/device/device.c b/arch/arm64/kvm/hyp/nvhe/device/device.c index 1a4891e1d77d..89433ab54ae7 100644 --- a/arch/arm64/kvm/hyp/nvhe/device/device.c +++ b/arch/arm64/kvm/hyp/nvhe/device/device.c @@ -228,7 +228,7 @@ int pkvm_host_map_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn) if (ret) goto out_ret; - ret = pkvm_hyp_donate_guest(hyp_vcpu, pfn, gfn); + ret = __pkvm_install_guest_mmio(hyp_vcpu, pfn, gfn); out_ret: hyp_spin_unlock(&device_spinlock); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index a138018ded79..efddb796c20a 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -1411,7 +1411,7 @@ int ___pkvm_host_donate_hyp(u64 pfn, u64 nr_pages, bool accept_mmio) default_hyp_prot(hyp_pfn_to_phys(pfn))); } -int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn) +static int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn) { struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); u64 phys = hyp_pfn_to_phys(pfn); @@ -1421,26 +1421,20 @@ int pkvm_hyp_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 pfn, u64 gfn) enum kvm_pgtable_prot prot; int ret; - hyp_lock_component(); - guest_lock_component(vm); + hyp_assert_lock_held(&pkvm_pgd_lock); + hyp_assert_lock_held(&vm->pgtable_lock); ret = __hyp_check_page_state_range(hyp_addr, size, PKVM_PAGE_OWNED); if (ret) - goto unlock; + return ret;; ret = __guest_check_page_state_range(vcpu, ipa, size, PKVM_NOPAGE); if (ret) - goto unlock; + return ret; WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, hyp_addr, size) != size); prot = pkvm_mkstate(default_guest_prot(addr_is_memory(phys)), PKVM_PAGE_OWNED); - WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot, - &vcpu->vcpu.arch.stage2_mc, 0)); - -unlock: - guest_unlock_component(vm); - hyp_unlock_component(); - - return ret; + return WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot, + &vcpu->vcpu.arch.stage2_mc, 0)); } int __pkvm_host_donate_hyp_locked(u64 pfn, u64 nr_pages, enum kvm_pgtable_prot prot) @@ -2172,6 +2166,49 @@ bool __pkvm_check_ioguard_page(struct pkvm_hyp_vcpu *hyp_vcpu) return ret; } +static int __pkvm_remove_ioguard_page(struct pkvm_hyp_vm *vm, u64 ipa) +{ + int ret; + kvm_pte_t pte; + s8 level; + + hyp_assert_lock_held(&vm->pgtable_lock); + + if (!test_bit(KVM_ARCH_FLAG_MMIO_GUARD, &vm->kvm.arch.flags)) + return -EINVAL; + + if (!PAGE_ALIGNED(ipa)) + return -EINVAL; + + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + if (ret) + return ret; + + if (BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)) == PAGE_SIZE && + pte == KVM_INVALID_PTE_MMIO_NOTE) + return kvm_pgtable_stage2_unmap(&vm->pgt, ipa, PAGE_SIZE); + + return kvm_pte_valid(pte) ? -EEXIST : -EINVAL; +} + +int __pkvm_install_guest_mmio(struct pkvm_hyp_vcpu *hyp_vcpu, u64 pfn, u64 gfn) +{ + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + u64 ipa = gfn << PAGE_SHIFT; + int ret; + + hyp_lock_component(); + guest_lock_component(vm); + ret = __pkvm_remove_ioguard_page(vm, ipa); + if (ret) + goto out_unlock; + ret = pkvm_hyp_donate_guest(hyp_vcpu, pfn, gfn); +out_unlock: + guest_unlock_component(vm); + hyp_unlock_component(); + return ret; +} + int host_stage2_get_leaf(phys_addr_t phys, kvm_pte_t *ptep, s8 *level) { int ret;