diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index e9c3d4469d85..cfa5d913fb04 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -78,7 +78,6 @@ int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); -int hyp_register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)); int hyp_pin_shared_mem(void *from, void *to); void hyp_unpin_shared_mem(void *from, void *to); int host_stage2_get_leaf(phys_addr_t phys, kvm_pte_t *ptep, s8 *level); diff --git a/arch/arm64/kvm/hyp/include/nvhe/modules.h b/arch/arm64/kvm/hyp/include/nvhe/modules.h index 4dd6c1ceed05..57674285609d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/modules.h +++ b/arch/arm64/kvm/hyp/include/nvhe/modules.h @@ -17,6 +17,7 @@ int __pkvm_init_module(void *module_init); int __pkvm_register_hcall(unsigned long hfn_hyp_va); int handle_host_dynamic_hcall(struct user_pt_regs *regs, int id); void __pkvm_close_module_registration(void); +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr); #else static inline int __pkvm_init_module(void *module_init) { return -EOPNOTSUPP; } static inline int @@ -27,4 +28,5 @@ handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt, int id) return HCALL_UNHANDLED; } static inline void __pkvm_close_module_registration(void) { } +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr) { return false; } #endif diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index f63c07f89ebc..ce8bbe0e4ff9 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -792,53 +792,6 @@ static void host_inject_abort(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(spsr, SYS_SPSR); } -#define MAX_HOST_FAULT_HANDLERS 16 - -static int -(*perm_fault_handlers[MAX_HOST_FAULT_HANDLERS])(struct user_pt_regs *regs, u64 esr, u64 addr); - -int hyp_register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)) -{ - static DEFINE_HYP_SPINLOCK(handlers_lock); - int i; - - hyp_spin_lock(&handlers_lock); - - for (i = 0; i < MAX_HOST_FAULT_HANDLERS; i++) { - if (!perm_fault_handlers[i]) { - WRITE_ONCE(perm_fault_handlers[i], cb); - break; - } - } - - hyp_spin_unlock(&handlers_lock); - - return i >= MAX_HOST_FAULT_HANDLERS ? -EBUSY : 0; -} - -static int handle_host_perm_fault(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr) -{ - int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr); - bool handled = false; - int i; - - for (i = 0; i < MAX_HOST_FAULT_HANDLERS; i++) { - int err; - - cb = READ_ONCE(perm_fault_handlers[i]); - if (!cb) - break; - - handled = true; - - err = cb(&host_ctxt->regs, esr, addr); - if (err) - return err; - } - - return handled ? 0 : -EPERM; -} - static bool is_dabt(u64 esr) { return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW; @@ -848,7 +801,7 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) { struct kvm_vcpu_fault_info fault; u64 esr, addr; - int ret = -EPERM; + int ret; esr = read_sysreg_el2(SYS_ESR); if (!__get_fault_info(esr, &fault)) { @@ -868,12 +821,18 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) kvm_iommu_host_dabt_handler(host_ctxt, esr, addr)) goto return_to_host; - /* If not handled, attempt to map the page. */ - if (ret == -EPERM) + switch (esr & ESR_ELx_FSC_TYPE) { + case ESR_ELx_FSC_FAULT: ret = host_stage2_idmap(addr); - - if ((esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM) - ret = handle_host_perm_fault(host_ctxt, esr, addr); + break; + case ESR_ELx_FSC_PERM: + ret = module_handle_host_perm_fault(&host_ctxt->regs, esr, addr); + ret = ret ? 0 /* handled */ : -EPERM; + break; + default: + ret = -EPERM; + break; + } if (ret == -EPERM) host_inject_abort(host_ctxt); diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index 12535e109866..53f0b8da357c 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -128,6 +128,64 @@ static int __hyp_smp_processor_id(void) return hyp_smp_processor_id(); } +#define MAX_MOD_HANDLERS 16 + +enum mod_handler_type { + HOST_FAULT_HANDLER = 0, + NUM_MOD_HANDLER_TYPES, +}; + +static void *mod_handlers[NUM_MOD_HANDLER_TYPES][MAX_MOD_HANDLERS]; + +static int mod_handler_register(enum mod_handler_type type, void *handler) +{ + int i; + + for (i = 0; i < MAX_MOD_HANDLERS; i++) { + if (!cmpxchg64_release(&mod_handlers[type][i], handler, NULL)) + return 0; + } + + return -EBUSY; +} + +static void *__get_mod_handler(enum mod_handler_type type, int i) +{ + if (WARN_ON(type >= NUM_MOD_HANDLER_TYPES)) + return NULL; + + if (i >= MAX_MOD_HANDLERS) + return NULL; + + i = array_index_nospec(i, MAX_MOD_HANDLERS); + + return smp_load_acquire(&mod_handlers[type][i]); +} + +#define for_each_mod_handler(type, handler, i) \ + for ((i) = 0, handler = (typeof(handler))__get_mod_handler(type, 0); \ + handler; \ + handler = (typeof(handler))__get_mod_handler(type, ++(i))) + +static int +__register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)) +{ + return mod_handler_register(HOST_FAULT_HANDLER, cb); +} + +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr) +{ + int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr); + int i; + + for_each_mod_handler(HOST_FAULT_HANDLER, cb, i) { + if (!cb(regs, esr, addr)) + return true; + } + + return false; +} + const struct pkvm_module_ops module_ops = { .create_private_mapping = __pkvm_create_private_mapping, .alloc_module_va = __pkvm_alloc_module_va, @@ -143,7 +201,7 @@ const struct pkvm_module_ops module_ops = { .flush_dcache_to_poc = __kvm_flush_dcache_to_poc, .update_hcr_el2 = __update_hcr_el2, .update_hfgwtr_el2 = __update_hfgwtr_el2, - .register_host_perm_fault_handler = hyp_register_host_perm_fault_handler, + .register_host_perm_fault_handler = __register_host_perm_fault_handler, .host_stage2_mod_prot = module_change_host_page_prot, .host_stage2_get_leaf = host_stage2_get_leaf, .host_stage2_enable_lazy_pte = host_stage2_enable_lazy_pte,