From 9bcaa7a08d77ea098a0f158c3ad2097fce9522da Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Thu, 13 Feb 2025 10:07:03 +0000 Subject: [PATCH] BACKPORT: FROMLIST: KVM: arm64: Create each pKVM hyp vcpu after its corresponding host vcpu Instead of creating and initializing _all_ hyp vcpus in pKVM when the first host vcpu runs for the first time, initialize _each_ hyp vcpu in conjunction with its corresponding host vcpu. Some of the host vcpu state (e.g., system registers and traps values) are not initialized until the first time the host vcpu is run. Therefore, initializing a hyp vcpu before its corresponding host vcpu has run for the first time might not view the complete host state of these vcpus. Additionally, this behavior is inline with non-protected modes. Bug: 394097954 Bug: 357781595 Link: https://lore.kernel.org/all/20250214150258.464798-4-tabba@google.com/ [tabba@: minor conflicts and added checks for hyp_vcpu in code not upstream yet] Change-Id: I4c2b4a9f1272e70e1c81db41345a6950ee6e911a Signed-off-by: Fuad Tabba --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/include/asm/kvm_pkvm.h | 1 + arch/arm64/kvm/arm.c | 5 +++ arch/arm64/kvm/hyp/include/nvhe/pkvm.h | 4 --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 45 ++++++++++++++++---------- arch/arm64/kvm/pkvm.c | 45 ++++++++++++++------------ 6 files changed, 60 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 064f3a951c64..c2d483a3350b 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -973,6 +973,8 @@ struct kvm_vcpu_arch { #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0)) /* SVE config completed */ #define VCPU_SVE_FINALIZED __vcpu_single_flag(cflags, BIT(1)) +/* pKVM VCPU setup completed */ +#define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2)) /* Exception pending */ #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0)) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index b1ba85e304d8..bfb94caf7bd0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -24,6 +24,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); int pkvm_create_hyp_vm(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); bool pkvm_is_hyp_created(struct kvm *kvm); +int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu); void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa); /* diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8354e2511467..41a9f05b08f5 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -871,9 +871,14 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) /* Start with the vcpu in a dirty state */ if (!kvm_vm_is_protected(vcpu->kvm)) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + ret = pkvm_create_hyp_vm(kvm); if (ret) return ret; + + ret = pkvm_create_hyp_vcpu(vcpu); + if (ret) + return ret; } mutex_lock(&kvm->arch.config_lock); diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index f03dc558c811..fea6d1815614 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -61,10 +61,6 @@ struct pkvm_hyp_vm { unsigned short refcount; - /* - * The number of vcpus initialized and ready to run. - */ - unsigned int nr_vcpus; hyp_spinlock_t vcpus_lock; /* diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index d7d1aecfbe50..1ff4677ca07a 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -386,10 +386,12 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, hyp_read_lock(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); - if (!hyp_vm || hyp_vm->is_dying || READ_ONCE(hyp_vm->nr_vcpus) <= vcpu_idx) + if (!hyp_vm || hyp_vm->is_dying || hyp_vm->kvm.created_vcpus <= vcpu_idx) goto unlock; hyp_vcpu = hyp_vm->vcpus[vcpu_idx]; + if (!hyp_vcpu) + goto unlock; /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */ if (unlikely(cmpxchg_relaxed(&hyp_vcpu->loaded_hyp_vcpu, NULL, @@ -553,6 +555,9 @@ static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], for (i = 0; i < nr_vcpus; i++) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i]; + if (!hyp_vcpu) + continue; + unpin_host_vcpu(hyp_vcpu); if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) @@ -631,8 +636,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct pkvm_hyp_vm *hyp_vm, - struct kvm_vcpu *host_vcpu, - unsigned int vcpu_idx) + struct kvm_vcpu *host_vcpu) { int ret = 0; u32 mp_state; @@ -647,11 +651,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, return -EBUSY; } - if (host_vcpu->vcpu_idx != vcpu_idx) { - ret = -EINVAL; - goto done; - } - mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) { ret = -EINVAL; @@ -662,7 +661,7 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); - hyp_vcpu->vcpu.vcpu_idx = vcpu_idx; + hyp_vcpu->vcpu.vcpu_idx = READ_ONCE(host_vcpu->vcpu_idx); hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); @@ -867,18 +866,22 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) } hyp_spin_lock(&hyp_vm->vcpus_lock); - idx = hyp_vm->nr_vcpus; + ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); + if (ret) + goto unlock_vcpus; + + idx = hyp_vcpu->vcpu.vcpu_idx; if (idx >= hyp_vm->kvm.created_vcpus) { ret = -EINVAL; goto unlock_vcpus; } - ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx); - if (ret) + if (hyp_vm->vcpus[idx]) { + ret = -EINVAL; goto unlock_vcpus; + } hyp_vm->vcpus[idx] = hyp_vcpu; - hyp_vm->nr_vcpus++; unlock_vcpus: hyp_spin_unlock(&hyp_vm->vcpus_lock); @@ -954,14 +957,17 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) mc = &host_kvm->arch.pkvm.stage2_teardown_mc; destroy_hyp_vm_pgt(hyp_vm); drain_hyp_pool(hyp_vm, mc); - unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus); + unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); /* Push the metadata pages to the teardown memcache */ - for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) { + for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; struct kvm_hyp_memcache *vcpu_mc; void *addr; + if (!hyp_vcpu) + continue; + vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; while (vcpu_mc->nr_pages) { unsigned long order; @@ -1120,8 +1126,10 @@ struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, mpidr &= MPIDR_HWID_BITMASK; hyp_spin_lock(&hyp_vm->vcpus_lock); - for (i = 0; i < hyp_vm->nr_vcpus; i++) { + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { hyp_vcpu = hyp_vm->vcpus[i]; + if (!hyp_vcpu) + continue; if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) goto unlock; @@ -1229,9 +1237,12 @@ static bool pvm_psci_vcpu_affinity_info(struct pkvm_hyp_vcpu *hyp_vcpu) * Otherwise, return OFF. */ hyp_spin_lock(&hyp_vm->vcpus_lock); - for (i = 0; i < hyp_vm->nr_vcpus; i++) { + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { struct pkvm_hyp_vcpu *target = hyp_vm->vcpus[i]; + if (!target) + continue; + mpidr = kvm_vcpu_get_mpidr_aff(&target->vcpu); if ((mpidr & target_affinity_mask) == target_affinity) { diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 091ed259905c..ab74515ca6fa 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -244,6 +244,15 @@ void __init kvm_hyp_reserve(void) hyp_mem_base); } + +static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) +{ + if (kvm_vm_is_protected(vcpu->kvm)) + vcpu->arch.sve_state = NULL; + + vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); +} + static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *host_vcpu) { pkvm_handle_t handle = host_vcpu->kvm->arch.pkvm.handle; @@ -262,8 +271,10 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *host_vcpu) host_vcpu->arch.hyp_reqs = hyp_reqs; ret = kvm_call_refill_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu); - if (!ret) + if (!ret) { + __pkvm_vcpu_hyp_created(host_vcpu); return 0; + } kvm_unshare_hyp(hyp_reqs, hyp_reqs + 1); err_free_reqs: @@ -400,12 +411,6 @@ out_free: } } -static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) -{ - if (kvm_vm_is_protected(vcpu->kvm)) - vcpu->arch.sve_state = NULL; -} - /* * Allocates and donates memory for hypervisor VM structs at EL2. * @@ -418,8 +423,6 @@ static void __pkvm_vcpu_hyp_created(struct kvm_vcpu *vcpu) */ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) { - struct kvm_vcpu *host_vcpu; - unsigned long idx; size_t pgd_sz; void *pgd; int ret; @@ -448,21 +451,9 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) WRITE_ONCE(host_kvm->arch.pkvm.handle, ret); - /* Donate memory for the vcpus at hyp and initialize it. */ - kvm_for_each_vcpu(idx, host_vcpu, host_kvm) { - ret = __pkvm_create_hyp_vcpu(host_vcpu); - if (ret) - goto destroy_vm; - __pkvm_vcpu_hyp_created(host_vcpu); - } - kvm_account_pgtable_pages(pgd, pgd_sz >> PAGE_SHIFT); return 0; - -destroy_vm: - __pkvm_destroy_hyp_vm(host_kvm); - return ret; free_pgd: free_pages_exact(pgd, pgd_sz); atomic64_sub(pgd_sz, &host_kvm->stat.protected_hyp_mem); @@ -487,6 +478,18 @@ int pkvm_create_hyp_vm(struct kvm *host_kvm) return ret; } +int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) +{ + int ret = 0; + + mutex_lock(&vcpu->kvm->arch.config_lock); + if (!vcpu_get_flag(vcpu, VCPU_PKVM_FINALIZED)) + ret = __pkvm_create_hyp_vcpu(vcpu); + mutex_unlock(&vcpu->kvm->arch.config_lock); + + return ret; +} + void pkvm_destroy_hyp_vm(struct kvm *host_kvm) { mutex_lock(&host_kvm->arch.config_lock);