From 748cec4e28f480d1f751e675de4e3172968390e0 Mon Sep 17 00:00:00 2001 From: Sebastian Ene Date: Tue, 13 Jun 2023 11:29:11 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Introduce PTE management callbacks In preparation for supporting a different reference counting mechanism for the host, introduce a structure that will hold all the page table management callbacks. This currently wraps the force_pte callback and the pte_is_counted. Pass the address of this ops structure as part of the pagetable initialization. Store the address of the ops structure in the pagetable structure as well as in the visiting context. Change-Id: I6d112664fd3da5d98b068edef2ca42b6c3aa56fb Bug: 357781595 Signed-off-by: Sebastian Ene --- arch/arm64/include/asm/kvm_pgtable.h | 36 ++++++++--- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 41 ++++++++++-- arch/arm64/kvm/hyp/pgtable.c | 91 +++++++++++++++------------ arch/arm64/kvm/mmu.c | 26 +++++++- 4 files changed, 138 insertions(+), 56 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index c3dcb3a54b47..ad91c5d2ae01 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -302,6 +302,22 @@ enum kvm_pgtable_prot { typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end, enum kvm_pgtable_prot prot); +typedef bool (*kvm_pgtable_pte_is_counted_cb_t)(kvm_pte_t pte, u32 level); + +/** + * struct kvm_pgtable_pte_ops - PTE callbacks. + * @force_pte_cb: Force the mapping granularity to pages and + * return true if we support this instead of + * block mappings. + * @pte_is_counted_cb Verify the attributes of the @pte argument + * and return true if the descriptor needs to be + * refcounted, otherwise return false. + */ +struct kvm_pgtable_pte_ops { + kvm_pgtable_force_pte_cb_t force_pte_cb; + kvm_pgtable_pte_is_counted_cb_t pte_is_counted_cb; +}; + /** * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk. * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid @@ -337,6 +353,7 @@ struct kvm_pgtable_visit_ctx { void *arg; struct kvm_pgtable_mm_ops *mm_ops; u64 start; + struct kvm_pgtable_pte_ops *pte_ops; u64 addr; u64 end; s8 level; @@ -437,8 +454,7 @@ static inline bool kvm_pgtable_walk_lock_held(void) * @mm_ops: Memory management callbacks. * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables. * @flags: Stage-2 page-table flags. - * @force_pte_cb: Function that returns true if page level mappings must - * be used instead of block mappings. + * @pte_ops: PTE callbacks. */ struct kvm_pgtable { u32 ia_bits; @@ -449,7 +465,7 @@ struct kvm_pgtable { /* Stage-2 only */ struct kvm_s2_mmu *mmu; enum kvm_pgtable_stage2_flags flags; - kvm_pgtable_force_pte_cb_t force_pte_cb; + struct kvm_pgtable_pte_ops *pte_ops; }; /** @@ -545,18 +561,17 @@ size_t kvm_pgtable_stage2_pgd_size(u64 vtcr); * @mmu: S2 MMU context for this S2 translation * @mm_ops: Memory management callbacks. * @flags: Stage-2 configuration flags. - * @force_pte_cb: Function that returns true if page level mappings must - * be used instead of block mappings. + * @pte_ops: PTE callbacks. * * Return: 0 on success, negative error code on failure. */ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops, enum kvm_pgtable_stage2_flags flags, - kvm_pgtable_force_pte_cb_t force_pte_cb); + struct kvm_pgtable_pte_ops *pte_ops); -#define kvm_pgtable_stage2_init(pgt, mmu, mm_ops) \ - __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL) +#define kvm_pgtable_stage2_init(pgt, mmu, mm_ops, pte_ops) \ + __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, pte_ops) /** * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table. @@ -570,13 +585,16 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); /** * kvm_pgtable_stage2_free_unlinked() - Free an unlinked stage-2 paging structure. * @mm_ops: Memory management callbacks. + * @pte_ops: Pagetable entries management callbacks. * @pgtable: Unlinked stage-2 paging structure to be freed. * @level: Level of the stage-2 paging structure to be freed. * * The page-table is assumed to be unreachable by any hardware walkers prior to * freeing and therefore no TLB invalidation is performed. */ -void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level); +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, + struct kvm_pgtable_pte_ops *pte_ops, + void *pgtable, s8 level); /** * kvm_pgtable_stage2_create_unlinked() - Create an unlinked stage-2 paging structure. diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index aeb8909941d1..a2f7df6d641e 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -33,6 +33,18 @@ static struct hyp_pool host_s2_pool; static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm); #define current_vm (*this_cpu_ptr(&__current_vm)) +static struct kvm_pgtable_pte_ops host_s2_pte_ops; +static bool host_stage2_force_pte(u64 addr, u64 end, enum kvm_pgtable_prot prot); +static bool host_stage2_pte_is_counted(kvm_pte_t pte, u32 level); +static bool guest_stage2_force_pte_cb(u64 addr, u64 end, + enum kvm_pgtable_prot prot); +static bool guest_stage2_pte_is_counted(kvm_pte_t pte, u32 level); + +static struct kvm_pgtable_pte_ops guest_s2_pte_ops = { + .force_pte_cb = guest_stage2_force_pte_cb, + .pte_is_counted_cb = guest_stage2_pte_is_counted +}; + static void guest_lock_component(struct pkvm_hyp_vm *vm) { hyp_spin_lock(&vm->pgtable_lock); @@ -98,7 +110,8 @@ static void host_s2_put_page(void *addr) static void host_s2_free_unlinked_table(void *addr, s8 level) { - kvm_pgtable_stage2_free_unlinked(&host_mmu.mm_ops, addr, level); + kvm_pgtable_stage2_free_unlinked(&host_mmu.mm_ops, host_mmu.pgt.pte_ops, + addr, level); } static int prepare_s2_pool(void *pgt_pool_base) @@ -158,8 +171,6 @@ static int prepopulate_host_stage2(void) return host_stage2_idmap_locked(addr, BIT(host_mmu.pgt.ia_bits) - addr, PKVM_HOST_MMIO_PROT); } -static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot); - int kvm_host_prepare_stage2(void *pgt_pool_base) { struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu; @@ -173,9 +184,12 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) if (ret) return ret; + host_s2_pte_ops.force_pte_cb = host_stage2_force_pte; + host_s2_pte_ops.pte_is_counted_cb = host_stage2_pte_is_counted; + ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu, &host_mmu.mm_ops, KVM_HOST_S2_FLAGS, - host_stage2_force_pte_cb); + &host_s2_pte_ops); if (ret) return ret; @@ -192,6 +206,11 @@ static bool guest_stage2_force_pte_cb(u64 addr, u64 end, return true; } +static bool guest_stage2_pte_is_counted(kvm_pte_t pte, u32 level) +{ + return host_stage2_pte_is_counted(pte, level); +} + static void *guest_s2_zalloc_pages_exact(size_t size) { void *addr = hyp_alloc_pages(¤t_vm->pool, get_order(size)); @@ -281,7 +300,7 @@ int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd) guest_lock_component(vm); ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0, - guest_stage2_force_pte_cb); + &guest_s2_pte_ops); guest_unlock_component(vm); if (ret) return ret; @@ -602,7 +621,7 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id) addr, size, &host_s2_pool, annotation); } -static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot) +static bool host_stage2_force_pte(u64 addr, u64 end, enum kvm_pgtable_prot prot) { /* * Block mappings must be used with care in the host stage-2 as a @@ -621,6 +640,16 @@ static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot pr return prot != default_host_prot(range_is_memory(addr, end)); } +static bool host_stage2_pte_is_counted(kvm_pte_t pte, u32 level) +{ + /* + * The refcount tracks valid entries as well as invalid entries if they + * encode ownership of a page to another entity than the page-table + * owner, whose id is 0. + */ + return !!pte; +} + static int host_stage2_idmap(u64 addr) { struct kvm_mem_range range; diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b1dfd9832e37..b7da798af554 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -154,10 +154,13 @@ static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker, } static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, - struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level); + struct kvm_pgtable_mm_ops *mm_ops, + struct kvm_pgtable_pte_ops *pte_ops, + kvm_pteref_t pgtable, s8 level); static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, struct kvm_pgtable_mm_ops *mm_ops, + struct kvm_pgtable_pte_ops *pte_ops, kvm_pteref_t pteref, s8 level) { enum kvm_pgtable_walk_flags flags = data->walker->flags; @@ -168,6 +171,7 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, .arg = data->walker->arg, .mm_ops = mm_ops, .start = data->start, + .pte_ops = pte_ops, .addr = data->addr, .end = data->end, .level = level, @@ -208,7 +212,7 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, } childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops); - ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1); + ret = __kvm_pgtable_walk(data, mm_ops, pte_ops, childp, level + 1); if (!kvm_pgtable_walk_continue(data->walker, ret)) goto out; @@ -223,7 +227,9 @@ out: } static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, - struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level) + struct kvm_pgtable_mm_ops *mm_ops, + struct kvm_pgtable_pte_ops *pte_ops, + kvm_pteref_t pgtable, s8 level) { u32 idx; int ret = 0; @@ -238,7 +244,7 @@ static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, if (data->addr >= data->end) break; - ret = __kvm_pgtable_visit(data, mm_ops, pteref, level); + ret = __kvm_pgtable_visit(data, mm_ops, pte_ops, pteref, level); if (ret) break; } @@ -261,7 +267,8 @@ static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_da for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) { kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE]; - ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level); + ret = __kvm_pgtable_walk(data, pgt->mm_ops, pgt->pte_ops, + pteref, pgt->start_level); if (ret) break; } @@ -532,7 +539,7 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, pgt->start_level = start_level; pgt->mm_ops = mm_ops; pgt->mmu = NULL; - pgt->force_pte_cb = NULL; + pgt->pte_ops = NULL; return 0; } @@ -761,16 +768,6 @@ static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new) return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)); } -static bool stage2_pte_is_counted(kvm_pte_t pte) -{ - /* - * The refcount tracks valid entries as well as invalid entries if they - * encode ownership of a page to another entity than the page-table - * owner, whose id is 0. - */ - return !!pte; -} - static bool stage2_pte_is_locked(kvm_pte_t pte) { return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED); @@ -804,6 +801,7 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, struct kvm_s2_mmu *mmu) { struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; + struct kvm_pgtable_pte_ops *pte_ops = ctx->pte_ops; if (stage2_pte_is_locked(ctx->old)) { /* @@ -833,7 +831,7 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, } } - if (stage2_pte_is_counted(ctx->old)) + if (pte_ops->pte_is_counted_cb(ctx->old, ctx->level)) mm_ops->put_page(ctx->ptep); return true; @@ -842,10 +840,11 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx, static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new) { struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; + struct kvm_pgtable_pte_ops *pte_ops = ctx->pte_ops; WARN_ON(!stage2_pte_is_locked(*ctx->ptep)); - if (stage2_pte_is_counted(new)) + if (pte_ops->pte_is_counted_cb(new, ctx->level)) mm_ops->get_page(ctx->ptep); smp_store_release(ctx->ptep, new); @@ -946,6 +945,8 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, u64 granule = kvm_granule_size(ctx->level); struct kvm_pgtable *pgt = data->mmu->pgt; struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; + struct kvm_pgtable_pte_ops *pte_ops = pgt->pte_ops; + bool old_is_counted; if (!stage2_leaf_mapping_allowed(ctx, data)) return -E2BIG; @@ -955,21 +956,22 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx, else new = data->annotation; - /* - * Skip updating the PTE if we are trying to recreate the exact - * same mapping or only change the access permissions. Instead, - * the vCPU will exit one more time from guest if still needed - * and then go through the path of relaxing permissions. - */ - if (!stage2_pte_needs_update(ctx->old, new)) - return -EAGAIN; + old_is_counted = pte_ops->pte_is_counted_cb(ctx->old, ctx->level); + if (old_is_counted) { + /* + * Skip updating the PTE if we are trying to recreate the exact + * same mapping or only change the access permissions. Instead, + * the vCPU will exit one more time from guest if still needed + * and then go through the path of relaxing permissions. + */ + if (!stage2_pte_needs_update(ctx->old, new)) + return -EAGAIN; + } /* If we're only changing software bits, then store them and go! */ if (!kvm_pgtable_walk_shared(ctx) && !((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW)) { - bool old_is_counted = stage2_pte_is_counted(ctx->old); - - if (old_is_counted != stage2_pte_is_counted(new)) { + if (old_is_counted != pte_ops->pte_is_counted_cb(new, ctx->level)) { if (old_is_counted) mm_ops->put_page(ctx->ptep); else @@ -1081,11 +1083,13 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, void *mc, enum kvm_pgtable_walk_flags flags) { int ret; + struct kvm_pgtable_pte_ops *pte_ops = pgt->pte_ops; struct stage2_map_data map_data = { .phys = ALIGN_DOWN(phys, PAGE_SIZE), .mmu = pgt->mmu, .memcache = mc, - .force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot), + .force_pte = pte_ops->force_pte_cb && + pte_ops->force_pte_cb(addr, addr + size, prot), }; struct kvm_pgtable_walker walker = { .cb = stage2_map_walker, @@ -1095,6 +1099,9 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, .arg = &map_data, }; + if (pte_ops->force_pte_cb) + map_data.force_pte = pte_ops->force_pte_cb(addr, addr + size, prot); + if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys))) return -EINVAL; @@ -1137,12 +1144,13 @@ static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx, { struct kvm_pgtable *pgt = ctx->arg; struct kvm_s2_mmu *mmu = pgt->mmu; + struct kvm_pgtable_pte_ops *pte_ops = ctx->pte_ops; struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; kvm_pte_t *childp = NULL; bool need_flush = false; if (!kvm_pte_valid(ctx->old)) { - if (stage2_pte_is_counted(ctx->old)) { + if (pte_ops->pte_is_counted_cb(ctx->old, ctx->level)) { kvm_clear_pte(ctx->ptep); mm_ops->put_page(ctx->ptep); } @@ -1447,6 +1455,7 @@ kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, .end = kvm_granule_size(level), }; struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; + struct kvm_pgtable_pte_ops *pte_ops = pgt->pte_ops; kvm_pte_t *pgtable; int ret; @@ -1461,10 +1470,10 @@ kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, if (!pgtable) return ERR_PTR(-ENOMEM); - ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable, + ret = __kvm_pgtable_walk(&data, mm_ops, pte_ops, (kvm_pteref_t)pgtable, level + 1); if (ret) { - kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level); + kvm_pgtable_stage2_free_unlinked(mm_ops, pte_ops, pgtable, level); return ERR_PTR(ret); } @@ -1501,6 +1510,7 @@ static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t pte = ctx->old, new, *childp; enum kvm_pgtable_prot prot; s8 level = ctx->level; + struct kvm_pgtable_pte_ops *pte_ops = ctx->pte_ops; bool force_pte; int nr_pages; u64 phys; @@ -1546,7 +1556,7 @@ static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx, return PTR_ERR(childp); if (!stage2_try_break_pte(ctx, mmu)) { - kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level); + kvm_pgtable_stage2_free_unlinked(mm_ops, pte_ops, childp, level); return -EAGAIN; } @@ -1578,7 +1588,7 @@ int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops, enum kvm_pgtable_stage2_flags flags, - kvm_pgtable_force_pte_cb_t force_pte_cb) + struct kvm_pgtable_pte_ops *pte_ops) { size_t pgd_sz; u64 vtcr = mmu->vtcr; @@ -1596,7 +1606,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, pgt->mm_ops = mm_ops; pgt->mmu = mmu; pgt->flags = flags; - pgt->force_pte_cb = force_pte_cb; + pgt->pte_ops = pte_ops; /* Ensure zeroed PGD pages are visible to the hardware walker */ dsb(ishst); @@ -1616,8 +1626,9 @@ static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx, enum kvm_pgtable_walk_flags visit) { struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops; + struct kvm_pgtable_pte_ops *pte_ops = ctx->pte_ops; - if (!stage2_pte_is_counted(ctx->old)) + if (!pte_ops->pte_is_counted_cb(ctx->old, ctx->level)) return 0; mm_ops->put_page(ctx->ptep); @@ -1643,7 +1654,9 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) pgt->pgd = NULL; } -void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level) +void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, + struct kvm_pgtable_pte_ops *pte_ops, + void *pgtable, s8 level) { kvm_pteref_t ptep = (kvm_pteref_t)pgtable; struct kvm_pgtable_walker walker = { @@ -1663,7 +1676,7 @@ void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *p .end = kvm_granule_size(level), }; - WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1)); + WARN_ON(__kvm_pgtable_walk(&data, mm_ops, pte_ops, ptep, level + 1)); WARN_ON(mm_ops->page_count(pgtable) != 1); mm_ops->put_page(pgtable); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index ee0a5d1adc7b..78bde0bcf5c5 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -32,6 +32,15 @@ static phys_addr_t __ro_after_init hyp_idmap_vector; static unsigned long __ro_after_init io_map_base; +static bool stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot); +static bool stage2_pte_is_counted(kvm_pte_t pte, u32 level); + +static struct kvm_pgtable_pte_ops kvm_s2_pte_ops = { + .force_pte_cb = stage2_force_pte_cb, + .pte_is_counted_cb = stage2_pte_is_counted + +}; + static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, phys_addr_t size) { @@ -224,7 +233,8 @@ static void stage2_free_unlinked_table_rcu_cb(struct rcu_head *head) void *pgtable = page_to_virt(page); s8 level = page_private(page); - kvm_pgtable_stage2_free_unlinked(&kvm_s2_mm_ops, pgtable, level); + kvm_pgtable_stage2_free_unlinked(&kvm_s2_mm_ops, &kvm_s2_pte_ops, + pgtable, level); } static void stage2_free_unlinked_table(void *addr, s8 level) @@ -947,6 +957,17 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); } +static bool stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot prot) +{ + return false; +} + +static bool stage2_pte_is_counted(kvm_pte_t pte, u32 level) + +{ + return !!pte; +} + static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { .zalloc_page = stage2_memcache_zalloc_page, .zalloc_pages_exact = kvm_s2_zalloc_pages_exact, @@ -1043,7 +1064,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t return -ENOMEM; mmu->arch = &kvm->arch; - err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops); + err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops, + &kvm_s2_pte_ops); if (err) goto out_free_pgtable;