From e365f69928fd6496aa661b62d6b0768efdfba340 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Mon, 30 Sep 2024 14:51:38 +0000 Subject: [PATCH] ANDROID: KVM: Send VM availability FF-A direct messages to Trustzone Store the secure partitions interested in receiving VM availability messages in an array when the caller invokes the FFA_PARTITIONS_INFO_GET interface. Notify these secure partitions about VM creation and teardown and keep track whether a VM creation message has been sent successfully from pKVM to the TEE. If we get a later destruction message for the same VM, make sure to complete the creation message first before sending out the destruction one. The FF-A spec only allows destruction messages for VMs in the available state, and SPs should return INVALID_PARAMETERS when receiving this message in any other state. Bug: 269285339 Bug: 278749606 Change-Id: Ic812ad8b6fb3f30c821085141900c4e22555af57 Signed-off-by: Sebastian Ene Signed-off-by: Andrei Homescu --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/kvm/hyp/include/nvhe/ffa.h | 3 + arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 4 + arch/arm64/kvm/hyp/nvhe/ffa.c | 343 ++++++++++++++++++++++--- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 8 + arch/arm64/kvm/hyp/nvhe/pkvm.c | 41 ++- arch/arm64/kvm/pkvm.c | 69 ++++- include/linux/arm_ffa.h | 7 + 8 files changed, 442 insertions(+), 34 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 393959580bd2..6ed81277e36e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -100,6 +100,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_reclaim_dying_guest_page, __KVM_HOST_SMCCC_FUNC___pkvm_reclaim_dying_guest_ffa_resources, + __KVM_HOST_SMCCC_FUNC___pkvm_notify_guest_vm_avail, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, diff --git a/arch/arm64/kvm/hyp/include/nvhe/ffa.h b/arch/arm64/kvm/hyp/include/nvhe/ffa.h index ab010dfd1717..46bc01d0af91 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/ffa.h +++ b/arch/arm64/kvm/hyp/include/nvhe/ffa.h @@ -12,6 +12,8 @@ #define FFA_MIN_FUNC_NUM 0x60 #define FFA_MAX_FUNC_NUM 0xFF +#define FFA_INVALID_HANDLE (-1LL) + /* * "ID value 0 must be returned at the Non-secure physical FF-A instance" * We share this ID with the host. @@ -29,6 +31,7 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id); bool kvm_guest_ffa_handler(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code); struct ffa_mem_transfer *find_transfer_by_handle(u64 ffa_handle, struct kvm_ffa_buffers *buf); int kvm_dying_guest_reclaim_ffa_resources(struct pkvm_hyp_vm *vm); +int kvm_guest_notify_availability(u32 ffa_handle, struct kvm_ffa_buffers *ffa_buf, bool is_dying); u32 ffa_get_hypervisor_version(void); static inline bool is_ffa_call(u64 func_id) diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 0eb558db312a..f79c7e3470fa 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -48,6 +48,8 @@ struct kvm_ffa_buffers { void *rx; u64 rx_ipa; struct list_head xfer_list; + u64 vm_avail_bitmap; + u64 vm_creating_bitmap; }; /* @@ -122,6 +124,7 @@ int __pkvm_start_teardown_vm(pkvm_handle_t handle); int __pkvm_finalize_teardown_vm(pkvm_handle_t handle); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 pfn, u64 gfn, u8 order); int __pkvm_reclaim_dying_guest_ffa_resources(pkvm_handle_t handle); +int __pkvm_notify_guest_vm_avail(pkvm_handle_t handle); struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx); @@ -218,5 +221,6 @@ int pkvm_device_register_reset(u64 phys, void *cookie, int (*cb)(void *cookie, bool host_to_guest)); int pkvm_handle_empty_memcache(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code); u32 hyp_vcpu_to_ffa_handle(struct pkvm_hyp_vcpu *hyp_vcpu); +u32 vm_handle_to_ffa_handle(pkvm_handle_t vm_handle); #endif /* __ARM64_KVM_NVHE_PKVM_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index 1a87da5ea77a..a8fa431f0109 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -44,6 +44,15 @@ #define VM_FFA_SUPPORTED(vcpu) ((vcpu)->kvm->arch.pkvm.ffa_support) #define FFA_INVALID_SPM_HANDLE (BIT(63) - 1) +/* The maximum number of secure partitions that can register for VM availability */ +#define FFA_MAX_VM_AVAIL_SPS (8) +#define FFA_VM_AVAIL_SPS_OOM (-2) + +#define FFA_PART_VM_AVAIL_MASK (FFA_PARTITION_DIRECT_RECV |\ + FFA_PARTITION_HYP_CREATE_VM |\ + FFA_PARTITION_HYP_DESTROY_VM) +#define FFA_PART_SUPPORTS_VM_AVAIL (FFA_PART_VM_AVAIL_MASK) + /* * A buffer to hold the maximum descriptor size we can see from the host, * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP @@ -75,6 +84,8 @@ static struct kvm_ffa_buffers hyp_buffers; static struct kvm_ffa_buffers host_buffers; static u32 hyp_ffa_version; static bool has_version_negotiated; +static bool has_hyp_ffa_buffer_mapped; +static bool has_host_signalled; static struct ffa_handle *spm_handles, *spm_free_handle; static u32 num_spm_handles; @@ -82,6 +93,16 @@ static u32 num_spm_handles; static DEFINE_HYP_SPINLOCK(version_lock); static DEFINE_HYP_SPINLOCK(kvm_ffa_hyp_lock); +/* Secure partitions that can receive VM availability messages */ +struct kvm_ffa_vm_avail_sp { + u16 sp_id; + bool wants_create; + bool wants_destroy; +}; + +static struct kvm_ffa_vm_avail_sp vm_avail_sps[FFA_MAX_VM_AVAIL_SPS]; +static int num_vm_avail_sps = -1; + static struct kvm_ffa_buffers *ffa_get_buffers(struct pkvm_hyp_vcpu *hyp_vcpu) { if (!hyp_vcpu) @@ -177,12 +198,27 @@ static int ffa_map_hyp_buffers(u64 ffa_page_count) { struct arm_smccc_res res; + /* + * Ensure that the read of `has_hyp_ffa_buffer_mapped` is visible + * to other CPUs before proceeding. + */ + if (smp_load_acquire(&has_hyp_ffa_buffer_mapped)) + return 0; + arm_smccc_1_1_smc(FFA_FN64_RXTX_MAP, hyp_virt_to_phys(hyp_buffers.tx), hyp_virt_to_phys(hyp_buffers.rx), ffa_page_count, 0, 0, 0, 0, &res); + if (res.a0 != FFA_SUCCESS) + return res.a2; + + /* + * Ensure that the write to `has_hyp_ffa_buffer_mapped` is visible + * to other CPUs after the previous operations. + */ + smp_store_release(&has_hyp_ffa_buffer_mapped, true); return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; } @@ -191,12 +227,27 @@ static int ffa_unmap_hyp_buffers(void) { struct arm_smccc_res res; + /* + * Ensure that the read of `has_hyp_ffa_buffer_mapped` is visible + * to other CPUs before proceeding. + */ + if (!smp_load_acquire(&has_hyp_ffa_buffer_mapped)) + return 0; + arm_smccc_1_1_smc(FFA_RXTX_UNMAP, HOST_FFA_ID, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 != FFA_SUCCESS) + return res.a2; - return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; + /* + * Ensure that the write to `has_hyp_ffa_buffer_mapped` is visible + * to other CPUs after the previous operations. + */ + smp_store_release(&has_hyp_ffa_buffer_mapped, false); + + return FFA_RET_SUCCESS; } static void ffa_mem_frag_tx(struct arm_smccc_res *res, u32 handle_lo, @@ -250,6 +301,156 @@ static void ffa_rx_release(struct arm_smccc_res *res) res); } +static int parse_vm_availability_resp(u32 partition_sz, u32 count) +{ + struct ffa_partition_info *part; + u32 i, j, off; + bool supports_direct_recv, wants_create, wants_destroy; + + if (num_vm_avail_sps >= 0) + return FFA_RET_SUCCESS; + if (num_vm_avail_sps == FFA_VM_AVAIL_SPS_OOM) + return FFA_RET_NO_MEMORY; + + num_vm_avail_sps = 0; + for (i = 0; i < count; i++) { + if (check_mul_overflow(i, partition_sz, &off)) + return FFA_RET_INVALID_PARAMETERS; + + if (off >= KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) + return FFA_RET_INVALID_PARAMETERS; + + part = hyp_buffers.rx + off; + supports_direct_recv = part->properties & FFA_PARTITION_DIRECT_RECV; + wants_create = part->properties & FFA_PARTITION_HYP_CREATE_VM; + wants_destroy = part->properties & FFA_PARTITION_HYP_DESTROY_VM; + + if (supports_direct_recv && (wants_create || wants_destroy)) { + /* Check for duplicate SP IDs */ + for (j = 0; j < num_vm_avail_sps; j++) + if (vm_avail_sps[j].sp_id == part->id) + break; + + if (j == num_vm_avail_sps) { + if (num_vm_avail_sps >= FFA_MAX_VM_AVAIL_SPS) { + /* We ran out of space in the array */ + num_vm_avail_sps = FFA_VM_AVAIL_SPS_OOM; + return FFA_RET_NO_MEMORY; + } + + vm_avail_sps[num_vm_avail_sps].sp_id = part->id; + vm_avail_sps[num_vm_avail_sps].wants_create = wants_create; + vm_avail_sps[num_vm_avail_sps].wants_destroy = wants_destroy; + num_vm_avail_sps++; + } + } + } + + return FFA_RET_SUCCESS; +} + +static int kvm_notify_vm_availability(uint16_t vm_handle, struct kvm_ffa_buffers *ffa_buf, + u32 availability_msg) +{ + int i; + struct arm_smccc_res res; + u64 avail_bit = availability_msg != FFA_VM_DESTRUCTION_MSG; + + for (i = 0; i < num_vm_avail_sps; i++) { + u64 sp_mask = 1UL << i; + u64 avail_value = avail_bit << i; + uint32_t dest = ((uint32_t)vm_avail_sps[i].sp_id << 16) | hyp_smp_processor_id(); + + if ((ffa_buf->vm_avail_bitmap & sp_mask) == avail_value && + !(ffa_buf->vm_creating_bitmap & sp_mask)) + continue; + + if (avail_bit && !vm_avail_sps[i].wants_create) { + /* + * The SP did not ask for creation messages, + * so just mark this VM as available and + * continue + */ + ffa_buf->vm_avail_bitmap |= avail_value; + continue; + } else if (!avail_bit && !vm_avail_sps[i].wants_destroy) { + /* + * The SP did not ask for destruction messages, + * so just mark this VM as not available and + * continue + */ + ffa_buf->vm_avail_bitmap &= ~sp_mask; + continue; + } + + /* + * Give the SP some cycles in advance, + * in case it got interrupted the last time. + * + * Some TEEs return NOT_SUPPORTED instead. + * If that happens, ignore the error and continue. + */ + arm_smccc_1_1_smc(FFA_RUN, dest, 0, 0, 0, 0, 0, 0, &res); + if (res.a0 == FFA_ERROR && (int)res.a2 != FFA_RET_NOT_SUPPORTED) + return ffa_to_linux_errno(res.a2); + else if (res.a0 == FFA_INTERRUPT) + return -EINTR; + + if (availability_msg == FFA_VM_DESTRUCTION_MSG && + (ffa_buf->vm_creating_bitmap & sp_mask)) { + /* + * If we sent the initial creation message for this VM + * but never got the success response from the TEE, we + * need to keep trying to create it until it works. + * Otherwise we cannot destroy it. + * + * TODO: this is not triggered for SPs that requested only + * creation messages (but not destruction). In that case, + * we will never retry the creation message, and the SP + * will probably leak its state for the pending VM. + */ + arm_smccc_1_1_smc(FFA_MSG_SEND_DIRECT_REQ, vm_avail_sps[i].sp_id, + FFA_VM_CREATION_MSG, HANDLE_LOW(FFA_INVALID_HANDLE), + HANDLE_HIGH(FFA_INVALID_HANDLE), vm_handle, 0, 0, + &res); + + if (res.a0 != FFA_MSG_SEND_DIRECT_RESP) + return -EINVAL; + if (res.a3 != FFA_RET_SUCCESS) + return ffa_to_linux_errno(res.a3); + + /* Creation completed successfully, clear the flag */ + ffa_buf->vm_creating_bitmap &= ~sp_mask; + } + + arm_smccc_1_1_smc(FFA_MSG_SEND_DIRECT_REQ, vm_avail_sps[i].sp_id, + availability_msg, HANDLE_LOW(FFA_INVALID_HANDLE), + HANDLE_HIGH(FFA_INVALID_HANDLE), vm_handle, 0, 0, + &res); + if (res.a0 != FFA_MSG_SEND_DIRECT_RESP) + return -EINVAL; + + switch ((int)res.a3) { + case FFA_RET_SUCCESS: + ffa_buf->vm_avail_bitmap &= ~sp_mask; + ffa_buf->vm_avail_bitmap |= avail_value; + ffa_buf->vm_creating_bitmap &= ~sp_mask; + break; + + case FFA_RET_INTERRUPTED: + case FFA_RET_RETRY: + if (availability_msg == FFA_VM_CREATION_MSG) + ffa_buf->vm_creating_bitmap |= sp_mask; + + fallthrough; + default: + return ffa_to_linux_errno(res.a3); + } + } + + return 0; +} + static void do_ffa_rxtx_map(struct arm_smccc_res *res, struct kvm_cpu_context *ctxt, struct pkvm_hyp_vcpu *hyp_vcpu) @@ -1075,6 +1276,48 @@ out_handled: ffa_to_smccc_res_prop(res, ret, prop); } +static void do_ffa_part_get_response(struct arm_smccc_res *res, + u32 uuid0, u32 uuid1, u32 uuid2, + u32 uuid3, u32 flags, struct kvm_ffa_buffers *ffa_buf) +{ + int ret; + u32 count, partition_sz, copy_sz; + + arm_smccc_1_1_smc(FFA_PARTITION_INFO_GET, uuid0, uuid1, + uuid2, uuid3, flags, 0, 0, + res); + + if (res->a0 != FFA_SUCCESS) + return; + + count = res->a2; + if (!count) + return; + + if (hyp_ffa_version > FFA_VERSION_1_0) { + /* Get the number of partitions deployed in the system */ + if (flags & 0x1) + return; + + partition_sz = res->a3; + } else + /* FFA_VERSION_1_0 lacks the size in the response */ + partition_sz = FFA_1_0_PARTITON_INFO_SZ; + + copy_sz = partition_sz * count; + if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { + ffa_to_smccc_res(res, FFA_RET_ABORTED); + return; + } + + ret = parse_vm_availability_resp(partition_sz, count); + if (ret) + ffa_to_smccc_res(res, ret); + + if (ffa_buf) + memcpy(ffa_buf->rx, hyp_buffers.rx, copy_sz); +} + static int hyp_ffa_post_init(void) { size_t min_rxtx_sz; @@ -1145,6 +1388,10 @@ static void do_ffa_version(struct arm_smccc_res *res, if (hyp_ffa_post_init()) { res->a0 = FFA_RET_NOT_SUPPORTED; } else { + /* + * Ensure that the write to `has_version_negotiated` is visible + * to other CPUs after the previous operations. + */ smp_store_release(&has_version_negotiated, true); res->a0 = hyp_ffa_version; } @@ -1179,7 +1426,6 @@ static void do_ffa_part_get(struct arm_smccc_res *res, DECLARE_REG(u32, uuid2, ctxt, 3); DECLARE_REG(u32, uuid3, ctxt, 4); DECLARE_REG(u32, flags, ctxt, 5); - u32 count, partition_sz, copy_sz; struct kvm_ffa_buffers *ffa_buf; hyp_spin_lock(&kvm_ffa_hyp_lock); @@ -1189,35 +1435,7 @@ static void do_ffa_part_get(struct arm_smccc_res *res, goto out_unlock; } - arm_smccc_1_1_smc(FFA_PARTITION_INFO_GET, uuid0, uuid1, - uuid2, uuid3, flags, 0, 0, - res); - - if (res->a0 != FFA_SUCCESS) - goto out_unlock; - - count = res->a2; - if (!count) - goto out_unlock; - - if (hyp_ffa_version > FFA_VERSION_1_0) { - /* Get the number of partitions deployed in the system */ - if (flags & 0x1) - goto out_unlock; - - partition_sz = res->a3; - } else { - /* FFA_VERSION_1_0 lacks the size in the response */ - partition_sz = FFA_1_0_PARTITON_INFO_SZ; - } - - copy_sz = partition_sz * count; - if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { - ffa_to_smccc_res(res, FFA_RET_ABORTED); - goto out_unlock; - } - - memcpy(ffa_buf->rx, hyp_buffers.rx, copy_sz); + do_ffa_part_get_response(res, uuid0, uuid1, uuid2, uuid3, flags, ffa_buf); out_unlock: hyp_spin_unlock(&kvm_ffa_hyp_lock); } @@ -1242,9 +1460,32 @@ static void do_ffa_direct_msg(struct kvm_cpu_context *ctxt, __hyp_enter(); } +static int kvm_host_ffa_signal_availability(void) +{ + int ret; + struct arm_smccc_res res; + + /* + * Map our hypervisor buffers into the SPMD before mapping and + * pinning the host buffers in our own address space. + */ + ret = ffa_map_hyp_buffers((KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE); + if (ret) + return ffa_to_linux_errno(ret); + + do_ffa_part_get_response(&res, 0, 0, 0, 0, 0, NULL); + if (res.a0 != FFA_SUCCESS) + return ffa_to_linux_errno(ret); + + ffa_rx_release(&res); + + return kvm_notify_vm_availability(HOST_FFA_ID, &host_buffers, FFA_VM_CREATION_MSG); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_res res; + int ret; /* * There's no way we can tell what a non-standard SMC call might @@ -1268,6 +1509,34 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) goto out_handled; } + /* + * Notify TZ of host VM creation immediately + * before handling the first non-version SMC/HVC + */ + if (func_id != FFA_VERSION && !has_host_signalled) { + ret = kvm_host_ffa_signal_availability(); + if (!ret) + /* + * Ensure that the write to `has_host_signalled` is visible + * to other CPUs after the previous operations. + */ + has_host_signalled = true; + else if (ret == -EAGAIN || ret == -EINTR) { + /* + * Don't retry with interrupts masked as we will spin + * forever. + */ + if (host_ctxt->regs.pstate & PSR_I_BIT) { + ffa_to_smccc_error(&res, FFA_RET_DENIED); + goto out_handled; + } + + /* Go back to the host and replay the last instruction */ + write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR); + return true; + } + } + switch (func_id) { case FFA_FEATURES: if (!do_ffa_features(&res, host_ctxt)) @@ -1484,6 +1753,18 @@ unlock: return ret; } +int kvm_guest_notify_availability(u32 ffa_handle, struct kvm_ffa_buffers *ffa_buf, bool is_dying) +{ + int ret; + + hyp_spin_lock(&kvm_ffa_hyp_lock); + ret = kvm_notify_vm_availability(ffa_handle, ffa_buf, + is_dying ? FFA_VM_DESTRUCTION_MSG : FFA_VM_CREATION_MSG); + hyp_spin_unlock(&kvm_ffa_hyp_lock); + + return ret; +} + u32 ffa_get_hypervisor_version(void) { u32 version = 0; diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 24151339cf5a..49ef169eac87 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1469,6 +1469,13 @@ static void handle___pkvm_reclaim_dying_guest_ffa_resources(struct kvm_cpu_conte cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_ffa_resources(handle); } +static void handle___pkvm_notify_guest_vm_avail(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_notify_guest_vm_avail(handle); +} + static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); @@ -1947,6 +1954,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_finalize_teardown_vm), HANDLE_FUNC(__pkvm_reclaim_dying_guest_page), HANDLE_FUNC(__pkvm_reclaim_dying_guest_ffa_resources), + HANDLE_FUNC(__pkvm_notify_guest_vm_avail), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), HANDLE_FUNC(__pkvm_vcpu_sync_state), diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index ce2ba16f536e..41222f0a76f9 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -396,6 +396,25 @@ int __pkvm_reclaim_dying_guest_ffa_resources(pkvm_handle_t handle) return ret; } +int __pkvm_notify_guest_vm_avail(pkvm_handle_t handle) +{ + struct pkvm_hyp_vm *hyp_vm; + int ret = 0; + + hyp_read_lock(&vm_table_lock); + hyp_vm = get_vm_by_handle(handle); + if (!hyp_vm || !hyp_vm->kvm.arch.pkvm.ffa_support) { + ret = -EBUSY; + goto unlock; + } + + ret = kvm_guest_notify_availability(vm_handle_to_ffa_handle(handle), &hyp_vm->ffa_buf, + hyp_vm->is_dying); +unlock: + hyp_read_unlock(&vm_table_lock); + return ret; +} + struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx) { @@ -761,6 +780,18 @@ static void remove_vm_table_entry(pkvm_handle_t handle) hyp_assert_write_lock_held(&vm_table_lock); hyp_vm = vm_table[vm_handle_to_idx(handle)]; + + /* + * If we didn't send the destruction message leak the vmid to + * prevent others from using it. + */ + if (hyp_vm->kvm.arch.pkvm.ffa_support && + hyp_vm->ffa_buf.vm_avail_bitmap) { + vm_table[vm_handle_to_idx(handle)] = (void *)0xdeadbeef; + list_del(&hyp_vm->vm_list); + return; + } + vm_table[vm_handle_to_idx(handle)] = NULL; list_del(&hyp_vm->vm_list); } @@ -1804,6 +1835,14 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) return true; } +u32 vm_handle_to_ffa_handle(pkvm_handle_t vm_handle) +{ + if (!vm_handle) + return HOST_FFA_ID; + else + return vm_handle_to_idx(vm_handle) + 1; +} + u32 hyp_vcpu_to_ffa_handle(struct pkvm_hyp_vcpu *hyp_vcpu) { pkvm_handle_t vm_handle; @@ -1812,5 +1851,5 @@ u32 hyp_vcpu_to_ffa_handle(struct pkvm_hyp_vcpu *hyp_vcpu) return HOST_FFA_ID; vm_handle = hyp_vcpu->vcpu.kvm->arch.pkvm.handle; - return vm_handle_to_idx(vm_handle) + 1; + return vm_handle_to_ffa_handle(vm_handle); } diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index f7c3b747ee2f..5b1bb04def8e 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -4,6 +4,8 @@ * Author: Quentin Perret */ +#include +#include #include #include #include @@ -39,6 +41,15 @@ #define PKVM_DEVICE_ASSIGN_COMPAT "pkvm,device-assignment" +/* + * Retry the VM creation message for the host for a maximul total + * amount of times, with sleeps in between. For the first few attempts, + * do a faster reschedule instead of a full sleep. + */ +#define VM_AVAILABILITY_FAST_RETRIES 5 +#define VM_AVAILABILITY_TOTAL_RETRIES 500 +#define VM_AVAILABILITY_RETRY_SLEEP_MS 10 + DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); static phys_addr_t pvmfw_base; @@ -367,6 +378,52 @@ static int __reclaim_dying_guest_page_call(u64 pfn, u64 gfn, u8 order, void *arg pfn, gfn, order); } +/* __pkvm_notify_guest_vm_avail_retry - notify secure of the VM state change + * @host_kvm: the kvm structure + * @availability_msg: the VM state that will be notified + * + * Returns: 0 when the notification is sent with success, -EINTR or -EAGAIN if + * the destruction notification is interrupted and retries exceeded and + * a positive value indicating the remaining jiffies when the creation + * notification is sent but interrupted. + */ +static int __pkvm_notify_guest_vm_avail_retry(struct kvm *host_kvm, u32 availability_msg) +{ + int ret, retries; + long timeout; + + if (!host_kvm->arch.pkvm.ffa_support) + return 0; + + for (retries = 0; retries < VM_AVAILABILITY_TOTAL_RETRIES; retries++) { + ret = kvm_call_hyp_nvhe(__pkvm_notify_guest_vm_avail, + host_kvm->arch.pkvm.handle); + if (!ret) + return 0; + else if (ret != -EINTR && ret != -EAGAIN) + return ret; + + if (retries < VM_AVAILABILITY_FAST_RETRIES) { + cond_resched(); + } else if (availability_msg == FFA_VM_DESTRUCTION_MSG) { + msleep(VM_AVAILABILITY_RETRY_SLEEP_MS); + } else { + timeout = msecs_to_jiffies(VM_AVAILABILITY_RETRY_SLEEP_MS); + timeout = schedule_timeout_killable(timeout); + if (timeout) { + /* + * The timer did not expire, + * most likely because the + * process was killed. + */ + return ret; + } + } + } + + return ret; +} + static void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) { struct mm_struct *mm = current->mm; @@ -375,7 +432,7 @@ static void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) unsigned long nr_busy; unsigned long pages; unsigned long idx; - int ret; + int ret, notify_status; if (!pkvm_is_hyp_created(host_kvm)) goto out_free; @@ -412,11 +469,16 @@ retry: account_locked_vm(mm, pages, false); + notify_status = __pkvm_notify_guest_vm_avail_retry(host_kvm, FFA_VM_DESTRUCTION_MSG); if (nr_busy) { do { ret = kvm_call_hyp_nvhe(__pkvm_reclaim_dying_guest_ffa_resources, host_kvm->arch.pkvm.handle); WARN_ON(ret && ret != -EAGAIN); + + if (notify_status == -EINTR || notify_status == -EAGAIN) + notify_status = __pkvm_notify_guest_vm_avail_retry( + host_kvm, FFA_VM_DESTRUCTION_MSG); cond_resched(); } while (ret == -EAGAIN); goto retry; @@ -488,8 +550,11 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) WRITE_ONCE(host_kvm->arch.pkvm.handle, ret); kvm_account_pgtable_pages(pgd, pgd_sz >> PAGE_SHIFT); + ret = __pkvm_notify_guest_vm_avail_retry(host_kvm, FFA_VM_CREATION_MSG); + if (ret) + goto free_pgd; - return 0; + return ret; free_pgd: free_pages_exact(pgd, pgd_sz); atomic64_sub(pgd_sz, &host_kvm->stat.protected_hyp_mem); diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index ef69167cb301..0f9782e836cf 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -239,12 +239,19 @@ struct ffa_partition_info { #define FFA_PARTITION_INDIRECT_MSG BIT(2) /* partition can receive notifications */ #define FFA_PARTITION_NOTIFICATION_RECV BIT(3) +/* partition must be informed about each VM that is created by the Hypervisor */ +#define FFA_PARTITION_HYP_CREATE_VM BIT(6) +/* partition must be informed about each VM that is destroyed by the Hypervisor */ +#define FFA_PARTITION_HYP_DESTROY_VM BIT(7) /* partition runs in the AArch64 execution state. */ #define FFA_PARTITION_AARCH64_EXEC BIT(8) u32 properties; u32 uuid[4]; }; +#define FFA_VM_CREATION_MSG (BIT(31) | (BIT(2))) +#define FFA_VM_DESTRUCTION_MSG (FFA_VM_CREATION_MSG | BIT(1)) + static inline bool ffa_partition_check_property(struct ffa_device *dev, u32 property) {