diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index dff9f2c40c8d..1fad502b8c48 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -98,6 +98,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_start_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_reclaim_dying_guest_page, + __KVM_HOST_SMCCC_FUNC___pkvm_reclaim_dying_guest_ffa_resources, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, diff --git a/arch/arm64/kvm/hyp/include/nvhe/ffa.h b/arch/arm64/kvm/hyp/include/nvhe/ffa.h index 37261934c021..9e26634c7d49 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/ffa.h +++ b/arch/arm64/kvm/hyp/include/nvhe/ffa.h @@ -28,5 +28,6 @@ int hyp_ffa_init(void *pages); bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id); bool kvm_guest_ffa_handler(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code); struct ffa_mem_transfer *find_transfer_by_handle(u64 ffa_handle, struct kvm_ffa_buffers *buf); +int kvm_dying_guest_reclaim_ffa_resources(struct pkvm_hyp_vm *vm); #endif /* __KVM_HYP_FFA_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 7da9402f004e..ba430c324127 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -117,6 +117,7 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu); int __pkvm_start_teardown_vm(pkvm_handle_t handle); int __pkvm_finalize_teardown_vm(pkvm_handle_t handle); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 pfn, u64 gfn, u8 order); +int __pkvm_reclaim_dying_guest_ffa_resources(pkvm_handle_t handle); struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx); diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 34f30c0f4cec..5ed2c9baccee 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -1264,6 +1264,60 @@ out_guest: return true; } +static void kvm_guest_try_reclaim_transfer(struct ffa_mem_transfer *transfer, + struct pkvm_hyp_vm *vm) +{ + struct ffa_translation *translation, *tmp; + struct arm_smccc_res res; + + ffa_mem_reclaim(&res, HANDLE_LOW(transfer->ffa_handle), + HANDLE_HIGH(transfer->ffa_handle), 0); + if (res.a0 != FFA_SUCCESS) + return; + + list_for_each_entry_safe(translation, tmp, &transfer->translations, node) { + WARN_ON(__pkvm_guest_unshare_ffa_page(vm->vcpus[0], translation->ipa)); + list_del(&translation->node); + hyp_free(translation); + } + + list_del(&transfer->node); + hyp_free(transfer); +} + +int kvm_dying_guest_reclaim_ffa_resources(struct pkvm_hyp_vm *vm) +{ + struct kvm_ffa_buffers *ffa_buf = &vm->ffa_buf; + struct ffa_mem_transfer *transfer; + int ret = 0; + + hyp_spin_lock(&kvm_ffa_hyp_lock); + if (!ffa_buf->tx && !ffa_buf->rx) + goto unlock; + + if (list_empty(&ffa_buf->xfer_list)) { + /* XXX - needs an explicit rxtx unmap call ? */ + if (ffa_buf->tx) { + WARN_ON(__pkvm_guest_unshare_hyp_page(vm->vcpus[0], ffa_buf->tx_ipa)); + ffa_buf->tx = NULL; + } + if (ffa_buf->rx) { + WARN_ON(__pkvm_guest_unshare_hyp_page(vm->vcpus[0], ffa_buf->rx_ipa)); + ffa_buf->rx = NULL; + } + goto unlock; + } + + transfer = list_first_entry(&ffa_buf->xfer_list, typeof(*transfer), node); + kvm_guest_try_reclaim_transfer(transfer, vm); + ret = -EAGAIN; + +unlock: + hyp_spin_unlock(&kvm_ffa_hyp_lock); + + return ret; +} + int hyp_ffa_init(void *pages) { struct arm_smccc_res res; diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index cdac60ad59ba..642a8a8c05e1 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1473,6 +1473,13 @@ static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ __pkvm_reclaim_dying_guest_page(handle, pfn, gfn, order); } +static void handle___pkvm_reclaim_dying_guest_ffa_resources(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_ffa_resources(handle); +} + static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); @@ -1922,6 +1929,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_start_teardown_vm), HANDLE_FUNC(__pkvm_finalize_teardown_vm), HANDLE_FUNC(__pkvm_reclaim_dying_guest_page), + HANDLE_FUNC(__pkvm_reclaim_dying_guest_ffa_resources), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), HANDLE_FUNC(__pkvm_vcpu_sync_state), diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index efd9638527f9..d1876f108276 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -2389,12 +2389,10 @@ int __pkvm_host_reclaim_page(struct pkvm_hyp_vm *vm, u64 pfn, u64 ipa, u8 order) goto unlock; } - /* We could avoid TLB inval, it is done per VMID on the finalize path */ - WARN_ON(kvm_pgtable_stage2_unmap(&vm->pgt, ipa, page_size)); - switch ((int)guest_get_page_state(pte, ipa)) { case PKVM_PAGE_OWNED: WARN_ON(__host_check_page_state_range(phys, page_size, PKVM_NOPAGE)); + /* No vCPUs of the guest can run, doing this prior to stage-2 unmap is OK */ hyp_poison_page(phys, page_size); psci_mem_protect_dec(1 << order); break; @@ -2403,12 +2401,18 @@ int __pkvm_host_reclaim_page(struct pkvm_hyp_vm *vm, u64 pfn, u64 ipa, u8 order) WARN_ON(__host_check_page_state_range(phys, page_size, PKVM_PAGE_SHARED_OWNED)); break; case PKVM_PAGE_SHARED_OWNED: - WARN_ON(__host_check_page_state_range(phys, page_size, PKVM_PAGE_SHARED_BORROWED)); + if (__host_check_page_state_range(phys, page_size, PKVM_PAGE_SHARED_BORROWED)) { + /* Presumably a page shared via FF-A, will be handled separately */ + ret = -EBUSY; + goto unlock; + } break; default: BUG_ON(1); } + /* We could avoid TLB inval, it is done per VMID on the finalize path */ + WARN_ON(kvm_pgtable_stage2_unmap(&vm->pgt, ipa, page_size)); WARN_ON(host_stage2_set_owner_locked(phys, page_size, PKVM_ID_HOST)); unlock: diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index beeba0abf00c..239ad9281dca 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -379,6 +379,20 @@ unlock: return ret; } +int __pkvm_reclaim_dying_guest_ffa_resources(pkvm_handle_t handle) +{ + struct pkvm_hyp_vm *hyp_vm; + int ret = -EINVAL; + + hyp_read_lock(&vm_table_lock); + hyp_vm = get_vm_by_handle(handle); + if (hyp_vm && hyp_vm->is_dying) + ret = kvm_dying_guest_reclaim_ffa_resources(hyp_vm); + hyp_read_unlock(&vm_table_lock); + + return ret; +} + struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx) { @@ -981,6 +995,12 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) remove_vm_table_entry(handle); hyp_write_unlock(&vm_table_lock); + /* A well-behaved host will have reclaimed all FF-A resources already */ + do { + err = kvm_dying_guest_reclaim_ffa_resources(hyp_vm); + } while (err == -EAGAIN); + WARN_ON(err); + pkvm_devices_teardown(hyp_vm); /* diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 75c8a16fbf84..d09d72d2ab4e 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -366,23 +366,35 @@ static void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) struct mm_struct *mm = current->mm; struct kvm_pinned_page *ppage; struct kvm_vcpu *host_vcpu; - unsigned long pages = 0; + unsigned long nr_busy; + unsigned long pages; unsigned long idx; + int ret; if (!pkvm_is_hyp_created(host_kvm)) goto out_free; WARN_ON(kvm_call_hyp_nvhe(__pkvm_start_teardown_vm, host_kvm->arch.pkvm.handle)); +retry: + pages = 0; + nr_busy = 0; ppage = kvm_pinned_pages_iter_first(&host_kvm->arch.pkvm.pinned_pages, 0, ~(0UL)); while (ppage) { struct kvm_pinned_page *next; u16 pins = ppage->pins; - WARN_ON(pkvm_call_hyp_nvhe_ppage(ppage, + ret = pkvm_call_hyp_nvhe_ppage(ppage, __reclaim_dying_guest_page_call, - host_kvm, true)); + host_kvm, true); cond_resched(); + if (ret == -EBUSY) { + nr_busy++; + next = kvm_pinned_pages_iter_next(ppage, 0, ~(0UL)); + ppage = next; + continue; + } + WARN_ON(ret); unpin_user_pages_dirty_lock(&ppage->page, 1, true); next = kvm_pinned_pages_iter_next(ppage, 0, ~(0UL)); @@ -394,6 +406,16 @@ static void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) account_locked_vm(mm, pages, false); + if (nr_busy) { + do { + ret = kvm_call_hyp_nvhe(__pkvm_reclaim_dying_guest_ffa_resources, + host_kvm->arch.pkvm.handle); + WARN_ON(ret && ret != -EAGAIN); + cond_resched(); + } while (ret == -EAGAIN); + goto retry; + } + WARN_ON(kvm_call_hyp_nvhe(__pkvm_finalize_teardown_vm, host_kvm->arch.pkvm.handle)); out_free: