diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index b62b94b42f46..6b38dc74dfa2 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -47,6 +47,8 @@ int kvm_iommu_snapshot_host_stage2(struct kvm_hyp_iommu_domain *domain); int kvm_iommu_dev_block_dma(pkvm_handle_t iommu_id, u32 endpoint_id, bool host_to_guest); +int kvm_iommu_force_free_domain(pkvm_handle_t domain_id, struct pkvm_hyp_vm *vm); + struct kvm_iommu_ops { int (*init)(void); int (*alloc_domain)(struct kvm_hyp_iommu_domain *domain, int type); diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index eb6855151295..231abb8711bf 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -68,7 +68,7 @@ struct pkvm_hyp_vm { /* pvIOMMUs attached. */ struct list_head pviommus; struct hyp_pool iommu_pool; - + struct list_head domains; /* Primary vCPU pending entry to the pvmfw */ struct pkvm_hyp_vcpu *pvmfw_entry_vcpu; diff --git a/arch/arm64/kvm/hyp/include/nvhe/pviommu.h b/arch/arm64/kvm/hyp/include/nvhe/pviommu.h index c0c51547c5a2..268b16049c41 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pviommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pviommu.h @@ -6,6 +6,9 @@ #ifndef __ARM64_KVM_NVHE_PVIOMMU_H__ #define __ARM64_KVM_NVHE_PVIOMMU_H__ +#include + bool kvm_handle_pviommu_hvc(struct kvm_vcpu *vcpu, u64 *exit_code); +void kvm_iommu_teardown_guest_domains(struct pkvm_hyp_vm *hyp_vm); #endif /* __ARM64_KVM_NVHE_PVIOMMU_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index 8f10b698fd86..06fc99b35a22 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -368,6 +368,23 @@ out_unlock: return ret; } +int kvm_iommu_force_free_domain(pkvm_handle_t domain_id, struct pkvm_hyp_vm *vm) +{ + struct kvm_hyp_iommu_domain *domain = handle_to_domain(domain_id); + + BUG_ON(!domain); + cur_context = vm->vcpus[0]; + + hyp_spin_lock(&kvm_iommu_domain_lock); + atomic_set(&domain->refs, 0); + kvm_iommu_ops->free_domain(domain); + memset(domain, 0, sizeof(*domain)); + hyp_spin_unlock(&kvm_iommu_domain_lock); + cur_context = NULL; + + return 0; +} + int kvm_iommu_attach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id, u32 endpoint_id, u32 pasid, u32 pasid_bits) { diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/pviommu-host.c b/arch/arm64/kvm/hyp/nvhe/iommu/pviommu-host.c index 64838324ccda..ed29350167aa 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/pviommu-host.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/pviommu-host.c @@ -3,6 +3,7 @@ * Copyright (C) 2023 Google LLC * Author: Mostafa Saleh */ +#include #include struct pviommu_host pviommus[MAX_NR_PVIOMMU]; @@ -106,6 +107,7 @@ int pkvm_pviommu_finalise(struct pkvm_hyp_vm *hyp_vm) hyp_spin_lock(&host_pviommu_lock); INIT_LIST_HEAD(&hyp_vm->pviommus); + INIT_LIST_HEAD(&hyp_vm->domains); for (i = 0; i < MAX_NR_PVIOMMU ; ++i) { struct pviommu_host *ph = &pviommus[i]; @@ -133,6 +135,7 @@ void pkvm_pviommu_teardown(struct pkvm_hyp_vm *hyp_vm) ph->nr_entries = 0; ph->finalized = false; } + kvm_iommu_teardown_guest_domains(hyp_vm); hyp_spin_unlock(&host_pviommu_lock); } diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/pviommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/pviommu.c index b02f6b4cb221..33db3917f643 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/pviommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/pviommu.c @@ -6,12 +6,18 @@ #include +#include #include #include #include #include #include +struct pviommu_guest_domain { + pkvm_handle_t id; + struct list_head list; +}; + static DEFINE_HYP_SPINLOCK(pviommu_guest_domain_lock); #define KVM_IOMMU_MAX_GUEST_DOMAINS (KVM_IOMMU_MAX_DOMAINS >> 1) @@ -132,6 +138,19 @@ static bool pkvm_guest_iommu_alloc_domain(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *e int ret; int domain_id = 0; struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + struct pviommu_guest_domain *guest_domain; + struct kvm_hyp_req *req; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + + guest_domain = hyp_alloc(sizeof(*guest_domain)); + if (!guest_domain) { + BUG_ON(hyp_alloc_errno() != -ENOMEM); + req = pkvm_hyp_req_reserve(hyp_vcpu, REQ_MEM_DEST_HYP_ALLOC); + req->mem.nr_pages = hyp_alloc_missing_donations(); + req->mem.sz_alloc = PAGE_SIZE; + pkvm_pviommu_hyp_req(exit_code); + return false; + } /* MBZ */ if (smccc_get_arg2(vcpu) || smccc_get_arg3(vcpu) || smccc_get_arg4(vcpu) || @@ -147,6 +166,7 @@ static bool pkvm_guest_iommu_alloc_domain(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *e if (ret == -ENOMEM) { pkvm_guest_iommu_free_id(domain_id); hyp_spin_unlock(&pviommu_guest_domain_lock); + hyp_free(guest_domain); pkvm_pviommu_hyp_req(exit_code); return false; } else if (ret) { @@ -154,12 +174,15 @@ static bool pkvm_guest_iommu_alloc_domain(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *e goto out_inval; } + guest_domain->id = domain_id; + list_add_tail(&guest_domain->list, &vm->domains); hyp_spin_unlock(&pviommu_guest_domain_lock); smccc_set_retval(vcpu, SMCCC_RET_SUCCESS, domain_id, 0, 0); return true; out_inval: hyp_spin_unlock(&pviommu_guest_domain_lock); + hyp_free(guest_domain); smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0); return true; } @@ -169,6 +192,8 @@ static bool pkvm_guest_iommu_free_domain(struct pkvm_hyp_vcpu *hyp_vcpu) int ret; struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; u64 domain_id = smccc_get_arg2(vcpu); + struct pviommu_guest_domain *guest_domain, *temp; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); if (smccc_get_arg3(vcpu) || smccc_get_arg4(vcpu) || smccc_get_arg5(vcpu) || smccc_get_arg6(vcpu)) { @@ -178,8 +203,18 @@ static bool pkvm_guest_iommu_free_domain(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_spin_lock(&pviommu_guest_domain_lock); ret = kvm_iommu_free_domain(domain_id); - if (!ret) - pkvm_guest_iommu_free_id(domain_id); + if (ret) + goto out_unlock; + list_for_each_entry_safe(guest_domain, temp, &vm->domains, list) { + if (guest_domain->id == domain_id) { + pkvm_guest_iommu_free_id(domain_id); + list_del(&guest_domain->list); + hyp_free(guest_domain); + break; + } + } + +out_unlock: hyp_spin_unlock(&pviommu_guest_domain_lock); out_ret: @@ -313,6 +348,20 @@ static bool pkvm_guest_iommu_unmap(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_cod return true; } +void kvm_iommu_teardown_guest_domains(struct pkvm_hyp_vm *hyp_vm) +{ + struct pviommu_guest_domain *guest_domain, *temp; + + hyp_spin_lock(&pviommu_guest_domain_lock); + list_for_each_entry_safe(guest_domain, temp, &hyp_vm->domains, list) { + kvm_iommu_force_free_domain(guest_domain->id, hyp_vm); + pkvm_guest_iommu_free_id(guest_domain->id); + list_del(&guest_domain->list); + hyp_free(guest_domain); + } + hyp_spin_unlock(&pviommu_guest_domain_lock); +} + bool kvm_handle_pviommu_hvc(struct kvm_vcpu *vcpu, u64 *exit_code) { u64 iommu_op = smccc_get_arg1(vcpu);