From 0a901ecd7f9ffbc4169811c90a0f26ed1ee1df7a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Fri, 6 Dec 2024 17:13:52 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Rework pKVM module fault handler pKVM modules can register up to 16 handlers. In preparation for supporting multiple handlers for the host SMCs, generalize and move over that support into modules.c. Bug: 357781595 Bug: 382397533 Change-Id: Iea9dea628dc5b7b75e46c7e581f3e507d039a06b Signed-off-by: Vincent Donnefort --- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 - arch/arm64/kvm/hyp/include/nvhe/modules.h | 2 + arch/arm64/kvm/hyp/nvhe/mem_protect.c | 65 ++++--------------- arch/arm64/kvm/hyp/nvhe/modules.c | 60 ++++++++++++++++- 4 files changed, 73 insertions(+), 55 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index e9c3d4469d85..cfa5d913fb04 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -78,7 +78,6 @@ int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); -int hyp_register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)); int hyp_pin_shared_mem(void *from, void *to); void hyp_unpin_shared_mem(void *from, void *to); int host_stage2_get_leaf(phys_addr_t phys, kvm_pte_t *ptep, s8 *level); diff --git a/arch/arm64/kvm/hyp/include/nvhe/modules.h b/arch/arm64/kvm/hyp/include/nvhe/modules.h index 4dd6c1ceed05..57674285609d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/modules.h +++ b/arch/arm64/kvm/hyp/include/nvhe/modules.h @@ -17,6 +17,7 @@ int __pkvm_init_module(void *module_init); int __pkvm_register_hcall(unsigned long hfn_hyp_va); int handle_host_dynamic_hcall(struct user_pt_regs *regs, int id); void __pkvm_close_module_registration(void); +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr); #else static inline int __pkvm_init_module(void *module_init) { return -EOPNOTSUPP; } static inline int @@ -27,4 +28,5 @@ handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt, int id) return HCALL_UNHANDLED; } static inline void __pkvm_close_module_registration(void) { } +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr) { return false; } #endif diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index f63c07f89ebc..ce8bbe0e4ff9 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -792,53 +792,6 @@ static void host_inject_abort(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(spsr, SYS_SPSR); } -#define MAX_HOST_FAULT_HANDLERS 16 - -static int -(*perm_fault_handlers[MAX_HOST_FAULT_HANDLERS])(struct user_pt_regs *regs, u64 esr, u64 addr); - -int hyp_register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)) -{ - static DEFINE_HYP_SPINLOCK(handlers_lock); - int i; - - hyp_spin_lock(&handlers_lock); - - for (i = 0; i < MAX_HOST_FAULT_HANDLERS; i++) { - if (!perm_fault_handlers[i]) { - WRITE_ONCE(perm_fault_handlers[i], cb); - break; - } - } - - hyp_spin_unlock(&handlers_lock); - - return i >= MAX_HOST_FAULT_HANDLERS ? -EBUSY : 0; -} - -static int handle_host_perm_fault(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr) -{ - int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr); - bool handled = false; - int i; - - for (i = 0; i < MAX_HOST_FAULT_HANDLERS; i++) { - int err; - - cb = READ_ONCE(perm_fault_handlers[i]); - if (!cb) - break; - - handled = true; - - err = cb(&host_ctxt->regs, esr, addr); - if (err) - return err; - } - - return handled ? 0 : -EPERM; -} - static bool is_dabt(u64 esr) { return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW; @@ -848,7 +801,7 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) { struct kvm_vcpu_fault_info fault; u64 esr, addr; - int ret = -EPERM; + int ret; esr = read_sysreg_el2(SYS_ESR); if (!__get_fault_info(esr, &fault)) { @@ -868,12 +821,18 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) kvm_iommu_host_dabt_handler(host_ctxt, esr, addr)) goto return_to_host; - /* If not handled, attempt to map the page. */ - if (ret == -EPERM) + switch (esr & ESR_ELx_FSC_TYPE) { + case ESR_ELx_FSC_FAULT: ret = host_stage2_idmap(addr); - - if ((esr & ESR_ELx_FSC_TYPE) == ESR_ELx_FSC_PERM) - ret = handle_host_perm_fault(host_ctxt, esr, addr); + break; + case ESR_ELx_FSC_PERM: + ret = module_handle_host_perm_fault(&host_ctxt->regs, esr, addr); + ret = ret ? 0 /* handled */ : -EPERM; + break; + default: + ret = -EPERM; + break; + } if (ret == -EPERM) host_inject_abort(host_ctxt); diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index 12535e109866..53f0b8da357c 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -128,6 +128,64 @@ static int __hyp_smp_processor_id(void) return hyp_smp_processor_id(); } +#define MAX_MOD_HANDLERS 16 + +enum mod_handler_type { + HOST_FAULT_HANDLER = 0, + NUM_MOD_HANDLER_TYPES, +}; + +static void *mod_handlers[NUM_MOD_HANDLER_TYPES][MAX_MOD_HANDLERS]; + +static int mod_handler_register(enum mod_handler_type type, void *handler) +{ + int i; + + for (i = 0; i < MAX_MOD_HANDLERS; i++) { + if (!cmpxchg64_release(&mod_handlers[type][i], handler, NULL)) + return 0; + } + + return -EBUSY; +} + +static void *__get_mod_handler(enum mod_handler_type type, int i) +{ + if (WARN_ON(type >= NUM_MOD_HANDLER_TYPES)) + return NULL; + + if (i >= MAX_MOD_HANDLERS) + return NULL; + + i = array_index_nospec(i, MAX_MOD_HANDLERS); + + return smp_load_acquire(&mod_handlers[type][i]); +} + +#define for_each_mod_handler(type, handler, i) \ + for ((i) = 0, handler = (typeof(handler))__get_mod_handler(type, 0); \ + handler; \ + handler = (typeof(handler))__get_mod_handler(type, ++(i))) + +static int +__register_host_perm_fault_handler(int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr)) +{ + return mod_handler_register(HOST_FAULT_HANDLER, cb); +} + +bool module_handle_host_perm_fault(struct user_pt_regs *regs, u64 esr, u64 addr) +{ + int (*cb)(struct user_pt_regs *regs, u64 esr, u64 addr); + int i; + + for_each_mod_handler(HOST_FAULT_HANDLER, cb, i) { + if (!cb(regs, esr, addr)) + return true; + } + + return false; +} + const struct pkvm_module_ops module_ops = { .create_private_mapping = __pkvm_create_private_mapping, .alloc_module_va = __pkvm_alloc_module_va, @@ -143,7 +201,7 @@ const struct pkvm_module_ops module_ops = { .flush_dcache_to_poc = __kvm_flush_dcache_to_poc, .update_hcr_el2 = __update_hcr_el2, .update_hfgwtr_el2 = __update_hfgwtr_el2, - .register_host_perm_fault_handler = hyp_register_host_perm_fault_handler, + .register_host_perm_fault_handler = __register_host_perm_fault_handler, .host_stage2_mod_prot = module_change_host_page_prot, .host_stage2_get_leaf = host_stage2_get_leaf, .host_stage2_enable_lazy_pte = host_stage2_enable_lazy_pte,