diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5bbeef180f97..4b41a0b39b6e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -73,6 +73,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_unmap_module_page, __KVM_HOST_SMCCC_FUNC___pkvm_init_module, __KVM_HOST_SMCCC_FUNC___pkvm_register_hcall, + __KVM_HOST_SMCCC_FUNC___pkvm_iommu_init, __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize, /* Hypercalls available after pKVM finalisation */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 4cc1be7e97ba..94887579722d 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1654,8 +1654,8 @@ struct kvm_iommu_driver { }; struct kvm_iommu_ops; -int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, - struct kvm_iommu_ops *el2_ops); +int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops); +int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops); int kvm_iommu_init_driver(void); void kvm_iommu_remove_driver(void); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 270fedcfc66a..c7bb66401755 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2495,15 +2495,9 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits) if (ret) return ret; - ret = kvm_iommu_init_driver(); - if (ret < 0) - return ret; - ret = do_pkvm_init(hyp_va_bits); - if (ret) { - kvm_iommu_remove_driver(); + if (ret) return ret; - } free_hyp_pgds(); diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index cff75d67d807..3d1ed95e257d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -53,7 +53,7 @@ struct kvm_iommu_ops { bool (*dabt_handler)(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr); }; -int kvm_iommu_init(void); +int kvm_iommu_init(struct kvm_iommu_ops *ops); int kvm_iommu_init_device(struct kvm_hyp_iommu *iommu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index cef5ca001e4e..b6d73d688abb 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1690,6 +1690,13 @@ static void handle___pkvm_host_hvc_pd(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = pkvm_host_hvc_pd(device_id, on); } +static void handle___pkvm_iommu_init(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_iommu_ops *, ops, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = kvm_iommu_init(ops); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1714,6 +1721,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_unmap_module_page), HANDLE_FUNC(__pkvm_init_module), HANDLE_FUNC(__pkvm_register_hcall), + HANDLE_FUNC(__pkvm_iommu_init), HANDLE_FUNC(__pkvm_prot_finalize), HANDLE_FUNC(__pkvm_host_share_hyp), diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index e7223694ac84..bb4869592a16 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -144,15 +144,15 @@ static void domain_put(struct kvm_hyp_iommu_domain *domain) BUG_ON(!atomic_dec_return_release(&domain->refs)); } -int kvm_iommu_init(void) +int kvm_iommu_init(struct kvm_iommu_ops *ops) { int ret; u64 domain_root_pfn = __hyp_pa(kvm_hyp_iommu_domains) >> PAGE_SHIFT; - if (!kvm_iommu_ops || - !kvm_iommu_ops->init || - !kvm_iommu_ops->alloc_domain || - !kvm_iommu_ops->free_domain) + if (!ops || + !ops->init || + !ops->alloc_domain || + !ops->free_domain) return 0; ret = hyp_pool_init_empty(&iommu_host_pool, 64); @@ -164,7 +164,9 @@ int kvm_iommu_init(void) if (ret) return ret; - ret = kvm_iommu_ops->init(); + kvm_iommu_ops = ops; + + ret = ops->init(); if (ret) goto out_reclaim_domain; diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 7fbcebd2e5b0..64033e393a04 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -372,10 +371,6 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; - ret = kvm_iommu_init(); - if (ret) - goto out; - ret = fix_host_ownership(); if (ret) goto out; diff --git a/arch/arm64/kvm/iommu.c b/arch/arm64/kvm/iommu.c index 1e9694c8be9e..c6e884617bf1 100644 --- a/arch/arm64/kvm/iommu.c +++ b/arch/arm64/kvm/iommu.c @@ -13,11 +13,9 @@ struct kvm_iommu_driver *iommu_driver; extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops); -int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iommu_ops *el2_ops) +int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops) { - int ret; - - if (WARN_ON(!kern_ops || !el2_ops)) + if (WARN_ON(!kern_ops)) return -EINVAL; /* @@ -25,14 +23,18 @@ int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iomm * Ensure memory stores happening during a driver * init are observed before executing kvm iommu callbacks. */ - ret = cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0; - if (ret) - return ret; - - kvm_nvhe_sym(kvm_iommu_ops) = el2_ops; - return 0; + return cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0; } +int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops) +{ + if (!hyp_ops) + return -EINVAL; + + return kvm_call_hyp_nvhe(__pkvm_iommu_init, hyp_ops); +} +EXPORT_SYMBOL(kvm_iommu_init_hyp); + int kvm_iommu_init_driver(void) { if (!smp_load_acquire(&iommu_driver)) { diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 0c45acbbff6e..e8dce9673945 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -443,6 +443,12 @@ static int __init finalize_pkvm(void) if (pkvm_load_early_modules()) pkvm_firmware_rmem_clear(); + ret = kvm_iommu_init_driver(); + if (ret) { + pr_err("Failed to init KVM IOMMU driver: %d\n", ret); + pkvm_firmware_rmem_clear(); + } + /* * Exclude HYP sections from kmemleak so that they don't get peeked * at, which would end badly once inaccessible. @@ -455,7 +461,7 @@ static int __init finalize_pkvm(void) ret = pkvm_drop_host_privileges(); if (ret) { pr_err("Failed to finalize Hyp protection: %d\n", ret); - BUG(); + kvm_iommu_remove_driver(); } return 0; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c index 3fb5ab2d2a2f..f85ad155b418 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c @@ -924,6 +924,20 @@ static int smmu_put_device(struct device *dev, void *data) return 0; } +/* + * Drop the PM references of the SMMU taken at probe + * after it's guaranteed the hypervisor as initialized the SMMUs. + */ +static int kvm_arm_smmu_v3_post_init(void) +{ + if (!kvm_arm_smmu_count) + return 0; + + WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL, + NULL, smmu_put_device)); + return 0; +} + static int kvm_arm_smmu_v3_init_drv(void) { int ret; @@ -953,7 +967,10 @@ static int kvm_arm_smmu_v3_init_drv(void) kvm_hyp_arm_smmu_v3_smmus = kvm_arm_smmu_array; kvm_hyp_arm_smmu_v3_count = kvm_arm_smmu_count; - return 0; + ret = kvm_iommu_init_hyp(kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops)))); + if (ret) + return ret; + return kvm_arm_smmu_v3_post_init(); err_free: kvm_arm_smmu_array_free(); @@ -975,26 +992,8 @@ static int kvm_arm_smmu_v3_register(void) if (!is_protected_kvm_enabled()) return 0; - return kvm_iommu_register_driver(&kvm_smmu_v3_ops, - kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops)))); + return kvm_iommu_register_driver(&kvm_smmu_v3_ops); }; -/* - * KVM init hypervisor at device_sync init call, - * so we drop the PM references of the SMMU taken at probe - * at the late initcall where it's guaranteed the hypervisor - * has initialized the SMMUs. - */ -static int kvm_arm_smmu_v3_post_init(void) -{ - if (!kvm_arm_smmu_count) - return 0; - - WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL, - NULL, smmu_put_device)); - return 0; -} - core_initcall(kvm_arm_smmu_v3_register); -late_initcall(kvm_arm_smmu_v3_post_init); MODULE_LICENSE("GPL v2");