diff --git a/arch/arm64/kvm/hyp/include/nvhe/modules.h b/arch/arm64/kvm/hyp/include/nvhe/modules.h index f5d39e733ba6..e53d8def3dd7 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/modules.h +++ b/arch/arm64/kvm/hyp/include/nvhe/modules.h @@ -13,7 +13,7 @@ enum pkvm_psci_notification; int __pkvm_register_psci_notifier(void (*cb)(enum pkvm_psci_notification, struct user_pt_regs *)); #ifdef CONFIG_MODULES -int __pkvm_init_module(void *module_init); +int __pkvm_init_module(void *host_mod); int __pkvm_register_hcall(unsigned long hfn_hyp_va); int handle_host_dynamic_hcall(struct user_pt_regs *regs, int id); void __pkvm_close_module_registration(void); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index f9de985e5137..41a531bf0943 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1571,9 +1571,9 @@ static void handle___pkvm_unmap_module_page(struct kvm_cpu_context *host_ctxt) static void handle___pkvm_init_module(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(void *, ptr, host_ctxt, 1); + DECLARE_REG(void *, host_mod, host_ctxt, 1); - cpu_reg(host_ctxt, 1) = __pkvm_init_module(ptr); + cpu_reg(host_ctxt, 1) = __pkvm_init_module(host_mod); } static void handle___pkvm_register_hcall(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index a3b16031d3ca..09bff67706e8 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -267,9 +268,17 @@ const struct pkvm_module_ops module_ops = { .hyp_smp_processor_id = __hyp_smp_processor_id, }; -int __pkvm_init_module(void *module_init) +static void *pkvm_module_hyp_va(struct pkvm_el2_module *mod, void *kern_va) { - int (*do_module_init)(const struct pkvm_module_ops *ops) = module_init; + return kern_va - mod->sections.start + mod->hyp_va; +} + +int __pkvm_init_module(void *host_mod) +{ + int (*do_module_init)(const struct pkvm_module_ops *ops); + struct pkvm_el2_module *mod = kern_hyp_va(host_mod); + + do_module_init = pkvm_module_hyp_va(mod, (void *)mod->init); return do_module_init(&module_ops); } diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index f4ee8e6a2ec5..891bebdbe0da 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -1015,6 +1015,7 @@ static int pkvm_map_module_sections(struct pkvm_mod_sec_mapping *secs_map, return 0; } + static int __pkvm_cmp_mod_sec(const void *p1, const void *p2) { struct pkvm_mod_sec_mapping const *s1 = p1; @@ -1023,6 +1024,33 @@ static int __pkvm_cmp_mod_sec(const void *p1, const void *p2) return s1->sec->start < s2->sec->start ? -1 : s1->sec->start > s2->sec->start; } +static void *pkvm_map_module_struct(struct pkvm_el2_module *mod) +{ + void *addr = (void *)__get_free_page(GFP_KERNEL); + + if (!addr) + return NULL; + + if (kvm_share_hyp(addr, addr + PAGE_SIZE)) { + free_page((unsigned long)addr); + return NULL; + } + + /* + * pkvm_el2_module being stored in vmalloc we can't guarantee a + * linear map for the hypervisor to rely on. Copy the struct instead. + */ + memcpy(addr, mod, sizeof(*mod)); + + return addr; +} + +static void pkvm_unmap_module_struct(void *addr) +{ + kvm_unshare_hyp(addr, addr + PAGE_SIZE); + free_page((unsigned long)addr); +} + int __pkvm_load_el2_module(struct module *this, unsigned long *token) { struct pkvm_el2_module *mod = &this->arch.hyp; @@ -1033,11 +1061,11 @@ int __pkvm_load_el2_module(struct module *this, unsigned long *token) { &mod->event_ids, KVM_PGTABLE_PROT_R }, { &mod->data, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W }, }; - void *start, *end, *hyp_va; + void *start, *end, *hyp_va, *mod_remap; struct arm_smccc_res res; kvm_nvhe_reloc_t *endrel; int ret, i, secs_first; - size_t offset, size; + size_t size; /* The pKVM hyp only allows loading before it is fully initialized */ if (!is_protected_kvm_enabled() || is_pkvm_initialized()) @@ -1115,18 +1143,25 @@ int __pkvm_load_el2_module(struct module *this, unsigned long *token) if (ret) kvm_err("Failed to init module events: %d\n", ret); + mod_remap = pkvm_map_module_struct(mod); + if (!mod_remap) { + module_put(this); + return -ENOMEM; + } + ret = pkvm_map_module_sections(secs_map + secs_first, hyp_va, ARRAY_SIZE(secs_map) - secs_first); if (ret) { kvm_err("Failed to map EL2 module page: %d\n", ret); + pkvm_unmap_module_struct(mod_remap); module_put(this); return ret; } pkvm_el2_mod_add(mod); - offset = (size_t)((void *)mod->init - start); - ret = kvm_call_hyp_nvhe(__pkvm_init_module, hyp_va + offset); + ret = kvm_call_hyp_nvhe(__pkvm_init_module, mod_remap); + pkvm_unmap_module_struct(mod_remap); if (ret) { kvm_err("Failed to init EL2 module: %d\n", ret); list_del(&mod->node);