From db3f87c61c2da8d7ba39807e5692ed41e1e45af6 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Wed, 9 Aug 2023 12:24:49 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Add __pkvm_iommu_init HVC to initialize IOMMU driver IOMMU drivers so far have been initialized by directly assigning EL2 variables before __pkvm_init. That's not possible for out-of-tree drivers that can only be initialized from early modules running between __pkvm_init and __pkvm_prot_finalize. Add a new HVC that can only be called before __pkvm_prot_finalize to set the EL2 variables that way. Also add an argument for kvm_iommu_ops.init so as to allow passing dynamic arguments (such as location of config struct) to the driver. @smostafa Merge with new IOMMU EL1 code Apply new calls to SMMUv3 driver This commit changes the init flow of KVM IOMMU driver and it might differ between builtin and modules(not supported yet). For a builtin driver: - An early initcall as core_initcall would call kvm_iommu_register_driver which mainly registers a EL1 driver with KVM. - When pKVM is still privileged and before it drops it, it calls the driver Init callback, which then probes the driver and call kvm_iommu_init_hyp. - kvm_iommu_init_hyp would call the EL2 driver init, which expects the EL1 driver to be probed and IOMMU are existing in the driver specific shared structs. We need the extra step for the callback because we need to know when pKVM is ready to have the IOMMU hypercalls, and unlike EL2 modules builtin driver are not loaded by KVM. For modules in the module init it can both porbe and init the hyp HVC and ignore the drop privilege callback. Signed-off-by: David Brazdil Signed-off-by: Mostafa Saleh Bug: 277989609 Bug: 278749606 Change-Id: Iba683e7a26f2944407587661dc3ab1b0d098bdc7 --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_host.h | 4 +- arch/arm64/kvm/arm.c | 8 +--- arch/arm64/kvm/hyp/include/nvhe/iommu.h | 2 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 8 ++++ arch/arm64/kvm/hyp/nvhe/iommu/iommu.c | 14 ++++--- arch/arm64/kvm/hyp/nvhe/setup.c | 5 --- arch/arm64/kvm/iommu.c | 22 ++++++----- arch/arm64/kvm/pkvm.c | 8 +++- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c | 39 +++++++++---------- 10 files changed, 59 insertions(+), 52 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5bbeef180f97..4b41a0b39b6e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -73,6 +73,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_unmap_module_page, __KVM_HOST_SMCCC_FUNC___pkvm_init_module, __KVM_HOST_SMCCC_FUNC___pkvm_register_hcall, + __KVM_HOST_SMCCC_FUNC___pkvm_iommu_init, __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize, /* Hypercalls available after pKVM finalisation */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4cc1be7e97ba..94887579722d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1654,8 +1654,8 @@ struct kvm_iommu_driver { }; struct kvm_iommu_ops; -int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, - struct kvm_iommu_ops *el2_ops); +int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops); +int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops); int kvm_iommu_init_driver(void); void kvm_iommu_remove_driver(void); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 270fedcfc66a..c7bb66401755 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2495,15 +2495,9 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits) if (ret) return ret; - ret = kvm_iommu_init_driver(); - if (ret < 0) - return ret; - ret = do_pkvm_init(hyp_va_bits); - if (ret) { - kvm_iommu_remove_driver(); + if (ret) return ret; - } free_hyp_pgds(); diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index cff75d67d807..3d1ed95e257d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -53,7 +53,7 @@ struct kvm_iommu_ops { bool (*dabt_handler)(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr); }; -int kvm_iommu_init(void); +int kvm_iommu_init(struct kvm_iommu_ops *ops); int kvm_iommu_init_device(struct kvm_hyp_iommu *iommu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index cef5ca001e4e..b6d73d688abb 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1690,6 +1690,13 @@ static void handle___pkvm_host_hvc_pd(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = pkvm_host_hvc_pd(device_id, on); } +static void handle___pkvm_iommu_init(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_iommu_ops *, ops, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = kvm_iommu_init(ops); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1714,6 +1721,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_unmap_module_page), HANDLE_FUNC(__pkvm_init_module), HANDLE_FUNC(__pkvm_register_hcall), + HANDLE_FUNC(__pkvm_iommu_init), HANDLE_FUNC(__pkvm_prot_finalize), HANDLE_FUNC(__pkvm_host_share_hyp), diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index e7223694ac84..bb4869592a16 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -144,15 +144,15 @@ static void domain_put(struct kvm_hyp_iommu_domain *domain) BUG_ON(!atomic_dec_return_release(&domain->refs)); } -int kvm_iommu_init(void) +int kvm_iommu_init(struct kvm_iommu_ops *ops) { int ret; u64 domain_root_pfn = __hyp_pa(kvm_hyp_iommu_domains) >> PAGE_SHIFT; - if (!kvm_iommu_ops || - !kvm_iommu_ops->init || - !kvm_iommu_ops->alloc_domain || - !kvm_iommu_ops->free_domain) + if (!ops || + !ops->init || + !ops->alloc_domain || + !ops->free_domain) return 0; ret = hyp_pool_init_empty(&iommu_host_pool, 64); @@ -164,7 +164,9 @@ int kvm_iommu_init(void) if (ret) return ret; - ret = kvm_iommu_ops->init(); + kvm_iommu_ops = ops; + + ret = ops->init(); if (ret) goto out_reclaim_domain; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 7fbcebd2e5b0..64033e393a04 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -372,10 +371,6 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; - ret = kvm_iommu_init(); - if (ret) - goto out; - ret = fix_host_ownership(); if (ret) goto out; diff --git a/arch/arm64/kvm/iommu.c b/arch/arm64/kvm/iommu.c index 1e9694c8be9e..c6e884617bf1 100644 --- a/arch/arm64/kvm/iommu.c +++ b/arch/arm64/kvm/iommu.c @@ -13,11 +13,9 @@ struct kvm_iommu_driver *iommu_driver; extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops); -int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iommu_ops *el2_ops) +int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops) { - int ret; - - if (WARN_ON(!kern_ops || !el2_ops)) + if (WARN_ON(!kern_ops)) return -EINVAL; /* @@ -25,14 +23,18 @@ int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iomm * Ensure memory stores happening during a driver * init are observed before executing kvm iommu callbacks. */ - ret = cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0; - if (ret) - return ret; - - kvm_nvhe_sym(kvm_iommu_ops) = el2_ops; - return 0; + return cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0; } +int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops) +{ + if (!hyp_ops) + return -EINVAL; + + return kvm_call_hyp_nvhe(__pkvm_iommu_init, hyp_ops); +} +EXPORT_SYMBOL(kvm_iommu_init_hyp); + int kvm_iommu_init_driver(void) { if (!smp_load_acquire(&iommu_driver)) { diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 0c45acbbff6e..e8dce9673945 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -443,6 +443,12 @@ static int __init finalize_pkvm(void) if (pkvm_load_early_modules()) pkvm_firmware_rmem_clear(); + ret = kvm_iommu_init_driver(); + if (ret) { + pr_err("Failed to init KVM IOMMU driver: %d\n", ret); + pkvm_firmware_rmem_clear(); + } + /* * Exclude HYP sections from kmemleak so that they don't get peeked * at, which would end badly once inaccessible. @@ -455,7 +461,7 @@ static int __init finalize_pkvm(void) ret = pkvm_drop_host_privileges(); if (ret) { pr_err("Failed to finalize Hyp protection: %d\n", ret); - BUG(); + kvm_iommu_remove_driver(); } return 0; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c index 3fb5ab2d2a2f..f85ad155b418 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c @@ -924,6 +924,20 @@ static int smmu_put_device(struct device *dev, void *data) return 0; } +/* + * Drop the PM references of the SMMU taken at probe + * after it's guaranteed the hypervisor as initialized the SMMUs. + */ +static int kvm_arm_smmu_v3_post_init(void) +{ + if (!kvm_arm_smmu_count) + return 0; + + WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL, + NULL, smmu_put_device)); + return 0; +} + static int kvm_arm_smmu_v3_init_drv(void) { int ret; @@ -953,7 +967,10 @@ static int kvm_arm_smmu_v3_init_drv(void) kvm_hyp_arm_smmu_v3_smmus = kvm_arm_smmu_array; kvm_hyp_arm_smmu_v3_count = kvm_arm_smmu_count; - return 0; + ret = kvm_iommu_init_hyp(kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops)))); + if (ret) + return ret; + return kvm_arm_smmu_v3_post_init(); err_free: kvm_arm_smmu_array_free(); @@ -975,26 +992,8 @@ static int kvm_arm_smmu_v3_register(void) if (!is_protected_kvm_enabled()) return 0; - return kvm_iommu_register_driver(&kvm_smmu_v3_ops, - kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops)))); + return kvm_iommu_register_driver(&kvm_smmu_v3_ops); }; -/* - * KVM init hypervisor at device_sync init call, - * so we drop the PM references of the SMMU taken at probe - * at the late initcall where it's guaranteed the hypervisor - * has initialized the SMMUs. - */ -static int kvm_arm_smmu_v3_post_init(void) -{ - if (!kvm_arm_smmu_count) - return 0; - - WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL, - NULL, smmu_put_device)); - return 0; -} - core_initcall(kvm_arm_smmu_v3_register); -late_initcall(kvm_arm_smmu_v3_post_init); MODULE_LICENSE("GPL v2");