diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 064f3a951c64..c2d483a3350b 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -973,6 +973,8 @@ struct kvm_vcpu_arch { #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0)) /* SVE config completed */ #define VCPU_SVE_FINALIZED __vcpu_single_flag(cflags, BIT(1)) +/* pKVM VCPU setup completed */ +#define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2)) /* Exception pending */ #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0)) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index b1ba85e304d8..bfb94caf7bd0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -24,6 +24,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); int pkvm_create_hyp_vm(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); bool pkvm_is_hyp_created(struct kvm *kvm); +int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu); void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa); /* diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8354e2511467..41a9f05b08f5 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -871,9 +871,14 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) /* Start with the vcpu in a dirty state */ if (!kvm_vm_is_protected(vcpu->kvm)) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + ret = pkvm_create_hyp_vm(kvm); if (ret) return ret; + + ret = pkvm_create_hyp_vcpu(vcpu); + if (ret) + return ret; } mutex_lock(&kvm->arch.config_lock); diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index f03dc558c811..fea6d1815614 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -61,10 +61,6 @@ struct pkvm_hyp_vm { unsigned short refcount; - /* - * The number of vcpus initialized and ready to run. - */ - unsigned int nr_vcpus; hyp_spinlock_t vcpus_lock; /* diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index d7d1aecfbe50..1ff4677ca07a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -386,10 +386,12 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, hyp_read_lock(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); - if (!hyp_vm || hyp_vm->is_dying || READ_ONCE(hyp_vm->nr_vcpus) <= vcpu_idx) + if (!hyp_vm || hyp_vm->is_dying || hyp_vm->kvm.created_vcpus <= vcpu_idx) goto unlock; hyp_vcpu = hyp_vm->vcpus[vcpu_idx]; + if (!hyp_vcpu) + goto unlock; /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */ if (unlikely(cmpxchg_relaxed(&hyp_vcpu->loaded_hyp_vcpu, NULL, @@ -553,6 +555,9 @@ static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], for (i = 0; i < nr_vcpus; i++) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i]; + if (!hyp_vcpu) + continue; + unpin_host_vcpu(hyp_vcpu); if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) @@ -631,8 +636,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct pkvm_hyp_vm *hyp_vm, - struct kvm_vcpu *host_vcpu, - unsigned int vcpu_idx) + struct kvm_vcpu *host_vcpu) { int ret = 0; u32 mp_state; @@ -647,11 +651,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, return -EBUSY; } - if (host_vcpu->vcpu_idx != vcpu_idx) { - ret = -EINVAL; - goto done; - } - mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) { ret = -EINVAL; @@ -662,7 +661,7 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); - hyp_vcpu->vcpu.vcpu_idx = vcpu_idx; + hyp_vcpu->vcpu.vcpu_idx = READ_ONCE(host_vcpu->vcpu_idx); hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); @@ -867,18 +866,22 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) } hyp_spin_lock(&hyp_vm->vcpus_lock); - idx = hyp_vm->nr_vcpus; + ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); + if (ret) + goto unlock_vcpus; + + idx = hyp_vcpu->vcpu.vcpu_idx; if (idx >= hyp_vm->kvm.created_vcpus) { ret = -EINVAL; goto unlock_vcpus; } - ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx); - if (ret) + if (hyp_vm->vcpus[idx]) { + ret = -EINVAL; goto unlock_vcpus; + } hyp_vm->vcpus[idx] = hyp_vcpu; - hyp_vm->nr_vcpus++; unlock_vcpus: hyp_spin_unlock(&hyp_vm->vcpus_lock); @@ -954,14 +957,17 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) mc = &host_kvm->arch.pkvm.stage2_teardown_mc; destroy_hyp_vm_pgt(hyp_vm); drain_hyp_pool(hyp_vm, mc); - unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus); + unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); /* Push the metadata pages to the teardown memcache */ - for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) { + for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; struct kvm_hyp_memcache *vcpu_mc; void *addr; + if (!hyp_vcpu) + continue; + vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; while (vcpu_mc->nr_pages) { unsigned long order; @@ -1120,8 +1126,10 @@ struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, mpidr &= MPIDR_HWID_BITMASK; hyp_spin_lock(&hyp_vm->vcpus_lock); - for (i = 0; i < hyp_vm->nr_vcpus; i++) { + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { hyp_vcpu = hyp_vm->vcpus[i]; + if (!hyp_vcpu) + continue; if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) goto unlock; @@ -1229,9 +1237,12 @@ static bool pvm_psci_vcpu_affinity_info(struct pkvm_hyp_vcpu *hyp_vcpu) * Otherwise, return OFF. */ hyp_spin_lock(&hyp_vm->vcpus_lock); - for (i = 0; i < hyp_vm->nr_vcpus; i++) { + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { struct pkvm_hyp_vcpu *target = hyp_vm->vcpus[i]; + if (!target) + continue; + mpidr = kvm_vcpu_get_mpidr_aff(&target->vcpu); if ((mpidr & target_affinity_mask) == target_affinity) { diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 091ed259905c..ab74515ca6fa 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -244,6 +244,15 @@ void __init kvm_hyp_reserve(void) hyp_mem_base); } + +static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) +{ + if (kvm_vm_is_protected(vcpu->kvm)) + vcpu->arch.sve_state = NULL; + + vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); +} + static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *host_vcpu) { pkvm_handle_t handle = host_vcpu->kvm->arch.pkvm.handle; @@ -262,8 +271,10 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *host_vcpu) host_vcpu->arch.hyp_reqs = hyp_reqs; ret = kvm_call_refill_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu); - if (!ret) + if (!ret) { + __pkvm_vcpu_hyp_created(host_vcpu); return 0; + } kvm_unshare_hyp(hyp_reqs, hyp_reqs + 1); err_free_reqs: @@ -400,12 +411,6 @@ out_free: } } -static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) -{ - if (kvm_vm_is_protected(vcpu->kvm)) - vcpu->arch.sve_state = NULL; -} - /* * Allocates and donates memory for hypervisor VM structs at EL2. * @@ -418,8 +423,6 @@ static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) */ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) { - struct kvm_vcpu *host_vcpu; - unsigned long idx; size_t pgd_sz; void *pgd; int ret; @@ -448,21 +451,9 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) WRITE_ONCE(host_kvm->arch.pkvm.handle, ret); - /* Donate memory for the vcpus at hyp and initialize it. */ - kvm_for_each_vcpu(idx, host_vcpu, host_kvm) { - ret = __pkvm_create_hyp_vcpu(host_vcpu); - if (ret) - goto destroy_vm; - __pkvm_vcpu_hyp_created(host_vcpu); - } - kvm_account_pgtable_pages(pgd, pgd_sz >> PAGE_SHIFT); return 0; - -destroy_vm: - __pkvm_destroy_hyp_vm(host_kvm); - return ret; free_pgd: free_pages_exact(pgd, pgd_sz); atomic64_sub(pgd_sz, &host_kvm->stat.protected_hyp_mem); @@ -487,6 +478,18 @@ int pkvm_create_hyp_vm(struct kvm *host_kvm) return ret; } +int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) +{ + int ret = 0; + + mutex_lock(&vcpu->kvm->arch.config_lock); + if (!vcpu_get_flag(vcpu, VCPU_PKVM_FINALIZED)) + ret = __pkvm_create_hyp_vcpu(vcpu); + mutex_unlock(&vcpu->kvm->arch.config_lock); + + return ret; +} + void pkvm_destroy_hyp_vm(struct kvm *host_kvm) { mutex_lock(&host_kvm->arch.config_lock);