diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 6e9d6b984574..41d9b1fe9519 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -97,6 +97,13 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_swap_reader_tracing, __KVM_HOST_SMCCC_FUNC___pkvm_enable_event, __KVM_HOST_SMCCC_FUNC___pkvm_selftest_event, + __KVM_HOST_SMCCC_FUNC___pkvm_register_hcall, + + /* + * Start of the dynamically registered hypercalls. Start a bit + * further, just in case some modules... + */ + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls = 128, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index 280524a59251..e8b1df800b85 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -6,6 +6,8 @@ #include #include +typedef void (*dyn_hcall_t)(struct user_pt_regs *); + struct pkvm_module_ops { int (*create_private_mapping)(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, @@ -33,10 +35,12 @@ struct pkvm_el2_module { int (*init)(const struct pkvm_module_ops *ops); }; -#ifdef MODULE int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, unsigned long *token); +int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va); +#ifdef MODULE #define pkvm_load_el2_module(init_fn, token) \ ({ \ extern char __kvm_nvhe___hypmod_text_start[]; \ @@ -66,5 +70,25 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, \ __pkvm_load_el2_module(&mod, THIS_MODULE, token); \ }) + +#define pkvm_register_el2_mod_call(hfn, token) \ +({ \ + extern char __kvm_nvhe___hypmod_text_start[]; \ + unsigned long hyp_text_kern_va = \ + (unsigned long)__kvm_nvhe___hypmod_text_start; \ + __pkvm_register_el2_call(hfn, token, \ + hyp_text_kern_va); \ +}) + +#define pkvm_el2_mod_call(id, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_ID(id), \ + ##__VA_ARGS__, &res); \ + WARN_ON(res.a0 != SMCCC_RET_SUCCESS); \ + \ + res.a1; \ + }) #endif #endif diff --git a/arch/arm64/kvm/hyp/include/nvhe/modules.h b/arch/arm64/kvm/hyp/include/nvhe/modules.h index 493ee91c519a..f7c03cbb5a4b 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/modules.h +++ b/arch/arm64/kvm/hyp/include/nvhe/modules.h @@ -1,5 +1,18 @@ +#include + +#define HCALL_HANDLED 0 +#define HCALL_UNHANDLED -1 + #ifdef CONFIG_MODULES int __pkvm_init_module(void *module_init); +int __pkvm_register_hcall(unsigned long hfn_hyp_va); +int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt); #else static inline int __pkvm_init_module(void *module_init) { return -EOPNOTSUPP; } +static inline int +__pkvm_register_hcall(unsigned long hfn_hyp_va) { return -EOPNOTSUPP; } +static inline int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt) +{ + return HCALL_UNHANDLED; +} #endif diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index b1cf4544550d..dd961eb76083 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1311,6 +1311,13 @@ static void handle___pkvm_init_module(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_init_module(ptr); } +static void handle___pkvm_register_hcall(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, hfn_hyp_va, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_register_hcall(hfn_hyp_va); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1360,6 +1367,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_map_module_page), HANDLE_FUNC(__pkvm_unmap_module_page), HANDLE_FUNC(__pkvm_init_module), + HANDLE_FUNC(__pkvm_register_hcall), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) @@ -1368,6 +1376,9 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) unsigned long hcall_min = 0; hcall_t hfn; + if (handle_host_dynamic_hcall(host_ctxt) == HCALL_HANDLED) + return; + /* * If pKVM has been initialised then reject any calls to the * early "privileged" hypercalls. Note that we cannot reject diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index 3c54f2fe1300..e867e6d9a877 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include const struct pkvm_module_ops module_ops = { .create_private_mapping = __pkvm_create_private_mapping, @@ -23,6 +25,73 @@ int __pkvm_init_module(void *module_init) int ret; ret = do_module_init(&module_ops); - return ret; } + +#define MAX_DYNAMIC_HCALLS 128 + +atomic_t num_dynamic_hcalls = ATOMIC_INIT(0); +DEFINE_HYP_SPINLOCK(dyn_hcall_lock); + +static dyn_hcall_t host_dynamic_hcalls[MAX_DYNAMIC_HCALLS]; + +int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, id, host_ctxt, 0); + dyn_hcall_t hfn; + int dyn_id; + + /* + * TODO: static key to protect when no dynamic hcall is registered? + */ + + dyn_id = (int)(id - KVM_HOST_SMCCC_ID(0)) - + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls; + if (dyn_id < 0) + return HCALL_UNHANDLED; + + cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; + + /* + * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired + * with __pkvm_register_hcall(). + */ + if (dyn_id >= atomic_read_acquire(&num_dynamic_hcalls)) + goto end; + + hfn = READ_ONCE(host_dynamic_hcalls[dyn_id]); + if (!hfn) + goto end; + + cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS; + hfn(&host_ctxt->regs); +end: + return HCALL_HANDLED; +} + +int __pkvm_register_hcall(unsigned long hvn_hyp_va) +{ + dyn_hcall_t hfn = (void *)hvn_hyp_va; + int reserved_id; + + hyp_spin_lock(&dyn_hcall_lock); + + reserved_id = atomic_read(&num_dynamic_hcalls); + + if (reserved_id >= MAX_DYNAMIC_HCALLS) { + hyp_spin_unlock(&dyn_hcall_lock); + return -ENOMEM; + } + + WRITE_ONCE(host_dynamic_hcalls[reserved_id], hfn); + + /* + * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired + * with handle_host_dynamic_hcall. + */ + atomic_set_release(&num_dynamic_hcalls, reserved_id + 1); + + hyp_spin_unlock(&dyn_hcall_lock); + + return reserved_id + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls; +}; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index b34177cfbf32..574c9253e63c 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -637,3 +637,18 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, return 0; } EXPORT_SYMBOL(__pkvm_load_el2_module); + +int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va) +{ + unsigned long hfn_hyp_va, offset, text_hyp_va = token; + int ret; + + offset = (unsigned long)hfn - hyp_text_kern_va; + hfn_hyp_va = text_hyp_va + offset; + + ret = kvm_call_hyp_nvhe(__pkvm_register_hcall, + (unsigned long)hfn_hyp_va); + return ret; +} +EXPORT_SYMBOL(__pkvm_register_el2_call);