diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index f93a2852e6a7..8c8f7768ed52 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -105,9 +105,6 @@ void kvm_update_va_mask(struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst); void kvm_compute_layout(void); void kvm_apply_hyp_relocations(void); -void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va, - kvm_nvhe_reloc_t *begin, - kvm_nvhe_reloc_t *end); #define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index e8b1df800b85..ff279ace64ed 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -8,6 +8,7 @@ typedef void (*dyn_hcall_t)(struct user_pt_regs *); +#ifdef CONFIG_MODULES struct pkvm_module_ops { int (*create_private_mapping)(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, @@ -18,66 +19,36 @@ struct pkvm_module_ops { void (*putx64)(u64 x); }; -struct pkvm_module_section { - void *start; - void *end; -}; - -typedef s32 kvm_nvhe_reloc_t; - -struct pkvm_el2_module { - struct pkvm_module_section text; - struct pkvm_module_section bss; - struct pkvm_module_section rodata; - struct pkvm_module_section data; - kvm_nvhe_reloc_t *relocs; - unsigned int nr_relocs; - int (*init)(const struct pkvm_module_ops *ops); -}; - -int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, - unsigned long *token); +int __pkvm_load_el2_module(struct module *this, unsigned long *token); int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, unsigned long hyp_text_kern_va); +#else +static inline int __pkvm_load_el2_module(struct module *this, + unsigned long *token) +{ + return -ENOSYS; +} + +static inline int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va) +{ + return -ENOSYS; +} +#endif /* CONFIG_MODULES */ + #ifdef MODULE #define pkvm_load_el2_module(init_fn, token) \ ({ \ - extern char __kvm_nvhe___hypmod_text_start[]; \ - extern char __kvm_nvhe___hypmod_text_end[]; \ - extern char __kvm_nvhe___hypmod_bss_start[]; \ - extern char __kvm_nvhe___hypmod_bss_end[]; \ - extern char __kvm_nvhe___hypmod_rodata_start[]; \ - extern char __kvm_nvhe___hypmod_rodata_end[]; \ - extern char __kvm_nvhe___hypmod_data_start[]; \ - extern char __kvm_nvhe___hypmod_data_end[]; \ - extern char __kvm_nvhe___hyprel_start[]; \ - extern char __kvm_nvhe___hyprel_end[]; \ - struct pkvm_el2_module mod; \ - \ - mod.text.start = __kvm_nvhe___hypmod_text_start; \ - mod.text.end = __kvm_nvhe___hypmod_text_end; \ - mod.bss.start = __kvm_nvhe___hypmod_bss_start; \ - mod.bss.end = __kvm_nvhe___hypmod_bss_end; \ - mod.rodata.start = __kvm_nvhe___hypmod_rodata_start; \ - mod.rodata.end = __kvm_nvhe___hypmod_rodata_end; \ - mod.data.start = __kvm_nvhe___hypmod_data_start; \ - mod.data.end = __kvm_nvhe___hypmod_data_end; \ - mod.relocs = (kvm_nvhe_reloc_t *)__kvm_nvhe___hyprel_start; \ - mod.nr_relocs = (__kvm_nvhe___hyprel_end - __kvm_nvhe___hyprel_start) / \ - sizeof(*mod.relocs); \ - mod.init = init_fn; \ - \ - __pkvm_load_el2_module(&mod, THIS_MODULE, token); \ + THIS_MODULE->arch.hyp.init = init_fn; \ + __pkvm_load_el2_module(THIS_MODULE, token); \ }) #define pkvm_register_el2_mod_call(hfn, token) \ ({ \ - extern char __kvm_nvhe___hypmod_text_start[]; \ - unsigned long hyp_text_kern_va = \ - (unsigned long)__kvm_nvhe___hypmod_text_start; \ - __pkvm_register_el2_call(hfn, token, \ - hyp_text_kern_va); \ + unsigned long hyp_text_kern_va; \ + hyp_text_kern_va = THIS_MODULE->arch.hyp.text.start; \ + __pkvm_register_el2_call(hfn, token, hyp_text_kern_va); \ }) #define pkvm_el2_mod_call(id, ...) \ diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 79550b22ba19..0c190546fcb8 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -7,6 +7,36 @@ #include +#ifdef CONFIG_KVM +struct pkvm_module_section { + void *start; + void *end; +}; + +typedef s32 kvm_nvhe_reloc_t; +struct pkvm_module_ops; + +struct pkvm_el2_module { + struct pkvm_module_section text; + struct pkvm_module_section bss; + struct pkvm_module_section rodata; + struct pkvm_module_section data; + kvm_nvhe_reloc_t *relocs; + unsigned int nr_relocs; + int (*init)(const struct pkvm_module_ops *ops); +}; + +void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va, + kvm_nvhe_reloc_t *begin, + kvm_nvhe_reloc_t *end); + +#define ARM64_MODULE_KVM_ARCHDATA \ + /* For pKVM hypervisor modules */ \ + struct pkvm_el2_module hyp; +#else +#define ARM64_MODULE_KVM_ARCHDATA +#endif + struct mod_plt_sec { int plt_shndx; int plt_num_entries; @@ -19,6 +49,8 @@ struct mod_arch_specific { /* for CONFIG_DYNAMIC_FTRACE */ struct plt_entry *ftrace_trampolines; + + ARM64_MODULE_KVM_ARCHDATA }; u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 36b25af56324..a178989c4063 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -457,11 +457,69 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr, return 0; } +static int module_init_hyp(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod) +{ +#ifdef CONFIG_KVM + const Elf_Shdr *s; + + /* + * If the .hyp.text is missing or empty, this is not a hypervisor + * module so ignore the rest of it. + */ + s = find_section(hdr, sechdrs, ".hyp.text"); + if (!s || !s->sh_size) + return 0; + + mod->arch.hyp.text = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.bss"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.bss = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.rodata"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.rodata = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.data"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.data = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.reloc"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.relocs = (void *)s->sh_addr; + mod->arch.hyp.nr_relocs = s->sh_size / sizeof(*mod->arch.hyp.relocs); +#endif + return 0; +} + int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) { + int err; const Elf_Shdr *s; + s = find_section(hdr, sechdrs, ".altinstructions"); if (s) apply_alternatives_module((void *)s->sh_addr, s->sh_size); @@ -472,5 +530,9 @@ int module_finalize(const Elf_Ehdr *hdr, __pi_scs_patch((void *)s->sh_addr, s->sh_size); } - return module_init_ftrace_plt(hdr, sechdrs, me); + err = module_init_ftrace_plt(hdr, sechdrs, me); + if (err) + return err; + + return module_init_hyp(hdr, sechdrs, me); } diff --git a/arch/arm64/kvm/hyp/nvhe/module.lds.S b/arch/arm64/kvm/hyp/nvhe/module.lds.S index 696ab5408265..645080c681cd 100644 --- a/arch/arm64/kvm/hyp/nvhe/module.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/module.lds.S @@ -6,32 +6,21 @@ SECTIONS { .hyp.text : { HYP_SECTION_SYMBOL_NAME(.text) = .; - __hypmod_text_start = .; *(.text .text.*) - __hypmod_text_end = .; } .hyp.bss : { HYP_SECTION_SYMBOL_NAME(.bss) = .; - __hypmod_bss_start = .; *(.bss .bss.*) - FILL(0) - __hypmod_bss_end = .; } .hyp.rodata : { HYP_SECTION_SYMBOL_NAME(.rodata) = .; - __hypmod_rodata_start = .; - *(.rodata .rodata.* .note.gnu.property) - BYTE(0) - __hypmod_rodata_end = .; + *(.rodata .rodata.*) } .hyp.data : { HYP_SECTION_SYMBOL_NAME(.data) = .; - __hypmod_data_start = .; *(.data .data.*) - BYTE(0) - __hypmod_data_end = .; } } diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4a66795845..316f8b2e52ca 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -489,7 +489,6 @@ static int __init early_pkvm_enable_modules(char *arg) return 0; } early_param("kvm-arm.protected_modules", early_pkvm_enable_modules); -#endif struct pkvm_mod_sec_mapping { struct pkvm_module_section *sec; @@ -566,9 +565,9 @@ static int __pkvm_cmp_mod_sec(const void *p1, const void *p2) return s1->sec->start < s2->sec->start ? -1 : s1->sec->start > s2->sec->start; } -int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, - unsigned long *token) +int __pkvm_load_el2_module(struct module *this, unsigned long *token) { + struct pkvm_el2_module *mod = &this->arch.hyp; struct pkvm_mod_sec_mapping secs_map[] = { { &mod->text, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_X }, { &mod->bss, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W }, @@ -598,7 +597,7 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, sort(secs_map, ARRAY_SIZE(secs_map), sizeof(secs_map[0]), __pkvm_cmp_mod_sec, NULL); start = secs_map[0].sec->start; end = secs_map[ARRAY_SIZE(secs_map) - 1].sec->end; - size = PAGE_ALIGN(end - start); + size = end - start; hyp_va = (void *)kvm_call_hyp_nvhe(__pkvm_alloc_module_va, size >> PAGE_SHIFT); if (!hyp_va) { @@ -663,3 +662,4 @@ int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, return ret; } EXPORT_SYMBOL(__pkvm_register_el2_call); +#endif /* CONFIG_MODULES */