From c8303029c094149edc7c852f0595e67fe4ff2285 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 22 May 2025 18:08:25 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add host_split_guest for pKVM In preparation for supporting relinquish of memory mapped at the stage-2 by a huge mapping, add a HVC to break a block mapping into last level mappings. Bug: 419548963 Bug: 357781595 Bug: 278011447 Change-Id: If2c4e375b26d8015b770f7710ecccbb7c6ed284a Signed-off-by: Vincent Donnefort --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_pgtable.h | 3 +- arch/arm64/include/asm/kvm_pkvm.h | 3 +- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-main.c | 24 +++++++++++ arch/arm64/kvm/hyp/nvhe/mem_protect.c | 24 +++++++++++ arch/arm64/kvm/hyp/pgtable.c | 43 +++++++++++++++++-- arch/arm64/kvm/pkvm.c | 3 +- 8 files changed, 92 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 64d0a9d5da86..393959580bd2 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -88,6 +88,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_host_wrprotect_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_test_clear_young_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_mkyoung_guest, + __KVM_HOST_SMCCC_FUNC___pkvm_host_split_guest, __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc, __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run, __KVM_HOST_SMCCC_FUNC___kvm_timer_set_cntvoff, diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index fbb2ca8d2edd..1f654b5b9d10 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -866,8 +866,7 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size); * kvm_pgtable_stage2_split() is best effort: it tries to break as many * blocks in the input range as allowed by @mc_capacity. */ -int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc); +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc); /** * kvm_pgtable_walk() - Walk a page-table. diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index d3584eb32ba3..5aa7e637ef6a 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -534,8 +534,7 @@ int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_ enum kvm_pgtable_walk_flags flags); kvm_pte_t pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_walk_flags flags); -int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc); +int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc); void pkvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, struct kvm_pgtable_pte_ops *pte_ops, void *pgtable, s8 level); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index cbd6b7f8e64f..d6cedba5f2a5 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -64,6 +64,7 @@ int __pkvm_host_relax_perms_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu, enum kvm_ int __pkvm_host_wrprotect_guest(u64 gfn, struct pkvm_hyp_vm *hyp_vm, u64 size); int __pkvm_host_test_clear_young_guest(u64 gfn, u64 size, bool mkold, struct pkvm_hyp_vm *vm); kvm_pte_t __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu); +int __pkvm_host_split_guest(u64 gfn, u64 size, struct pkvm_hyp_vcpu *vcpu); int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa, u64 nr_pages, u64 *nr_shared); int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d4347a7cd3e2..df0035c048a6 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1217,6 +1217,29 @@ out: cpu_reg(host_ctxt, 1) = pte; } +static void handle___pkvm_host_split_guest(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(u64, gfn, host_ctxt, 1); + DECLARE_REG(u64, size, host_ctxt, 2); + struct pkvm_hyp_vcpu *hyp_vcpu; + int ret = -EINVAL; + + if (!is_protected_kvm_enabled()) + goto out; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu) + goto out; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + goto out; + + ret = __pkvm_host_split_guest(gfn, size, hyp_vcpu); + +out: + cpu_reg(host_ctxt, 1) = ret; +} + static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -1912,6 +1935,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_host_wrprotect_guest), HANDLE_FUNC(__pkvm_host_test_clear_young_guest), HANDLE_FUNC(__pkvm_host_mkyoung_guest), + HANDLE_FUNC(__pkvm_host_split_guest), HANDLE_FUNC(__kvm_adjust_pc), HANDLE_FUNC(__kvm_vcpu_run), HANDLE_FUNC(__kvm_timer_set_cntvoff), diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 35e73405c313..4c3504ae77ca 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -2192,6 +2192,30 @@ kvm_pte_t __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu) return pte; } +int __pkvm_host_split_guest(u64 gfn, u64 size, struct pkvm_hyp_vcpu *vcpu) +{ + struct kvm_hyp_memcache *mc = &vcpu->vcpu.arch.stage2_mc; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + u64 ipa = hyp_pfn_to_phys(gfn); + int ret; + + if (size != PMD_SIZE) + return -EINVAL; + + guest_lock_component(vm); + + /* + * stage2_split() already checks the existing mapping is valid and PMD-level. + * No other check is necessary. + */ + + ret = kvm_pgtable_stage2_split(&vm->pgt, ipa, size, mc); + + guest_unlock_component(vm); + + return ret; +} + static int __host_set_owner_guest(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa, size_t size, bool is_memory) { diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 2cda40681c8c..be3417d0d9d4 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -1786,13 +1786,48 @@ static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, return 0; } -int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc) +static int pkvm_stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) { + struct stage2_map_data *data = ctx->arg; + struct kvm_pgtable *pgt = data->mmu->pgt; + struct kvm_hyp_memcache *mc = data->memcache; + enum kvm_pgtable_prot prot; + kvm_pte_t pte = ctx->old; + kvm_pte_t *childp; + + if (ctx->level == KVM_PGTABLE_LAST_LEVEL) + return 0; + + /* We can only split PMD-level blocks */ + if (!kvm_pte_valid(pte) || ctx->level != KVM_PGTABLE_LAST_LEVEL - 1) + return -EINVAL; + + prot = kvm_pgtable_stage2_pte_prot(pte); + childp = kvm_pgtable_stage2_create_unlinked(pgt, kvm_pte_to_phys(pte), + ctx->level, prot, mc, true); + if (IS_ERR(childp)) + return PTR_ERR(childp); + + WARN_ON(!stage2_try_break_pte(ctx, data->mmu)); + + stage2_make_pte(ctx, kvm_init_table_pte(childp, ctx->mm_ops)); + + return 0; +} + +int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc) +{ + struct stage2_map_data data = { + .mmu = pgt->mmu, + .memcache = mc, + }; struct kvm_pgtable_walker walker = { - .cb = stage2_split_walker, + .cb = static_branch_unlikely(&kvm_protected_mode_initialized) ? + pkvm_stage2_split_walker : stage2_split_walker, + .arg = static_branch_unlikely(&kvm_protected_mode_initialized) ? + &data : mc, .flags = KVM_PGTABLE_WALK_LEAF, - .arg = mc, }; int ret; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index b5ac47c69612..278a53040c29 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -1841,8 +1841,7 @@ kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys return NULL; } -int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, - struct kvm_mmu_memory_cache *mc) +int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc) { WARN_ON_ONCE(1); return -EINVAL;