tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ANDROID: KVM: arm64: Add __pkvm_iommu_init HVC to initialize IOMMU driver

Browse Source 
IOMMU drivers so far have been initialized by directly assigning EL2
variables before __pkvm_init. That's not possible for out-of-tree
drivers that can only be initialized from early modules running between
__pkvm_init and __pkvm_prot_finalize.

Add a new HVC that can only be called before __pkvm_prot_finalize to set
the EL2 variables that way.

Also add an argument for kvm_iommu_ops.init so as to allow passing
dynamic arguments (such as location of config struct) to the driver.

@smostafa
Merge with new IOMMU EL1 code
Apply new calls to SMMUv3 driver

This commit changes the init flow of KVM IOMMU driver and it might
differ between builtin and modules(not supported yet).

For a builtin driver:
- An early initcall as core_initcall would call
  kvm_iommu_register_driver which mainly registers a EL1 driver with
  KVM.

- When pKVM is still privileged and before it drops it, it calls
  the driver Init callback, which then probes the driver and
  call kvm_iommu_init_hyp.

- kvm_iommu_init_hyp would call the EL2 driver init, which expects
  the EL1 driver to be probed and IOMMU are existing in the driver
  specific shared structs.

We need the extra step for the callback because we need to know when
pKVM is ready to have the IOMMU hypercalls, and unlike EL2 modules
builtin driver are not loaded by KVM.

For modules in the module init it can both porbe and init the hyp HVC
and ignore the drop privilege callback.

Signed-off-by: David Brazdil <dbrazdil@google.com>
Signed-off-by: Mostafa Saleh <smostafa@google.com>
Bug: 277989609
Bug: 278749606
Change-Id: Iba683e7a26f2944407587661dc3ab1b0d098bdc7
This commit is contained in:
David Brazdil
2023-08-09 12:24:49 +00:00
committed by Mostafa Saleh
parent 99daab6948
commit db3f87c61c
10 changed files with 59 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/arm64/include/asm/kvm_asm.h
+1
View File
@@ -73,6 +73,7 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___pkvm_unmap_module_page,
__KVM_HOST_SMCCC_FUNC___pkvm_init_module,
__KVM_HOST_SMCCC_FUNC___pkvm_register_hcall,
__KVM_HOST_SMCCC_FUNC___pkvm_iommu_init,
__KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize,
/* Hypercalls available after pKVM finalisation */
arch/arm64/include/asm/kvm_host.h
+2 -2
View File
@@ -1654,8 +1654,8 @@ struct kvm_iommu_driver {
};
struct kvm_iommu_ops;
int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops,
struct kvm_iommu_ops *el2_ops);
int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops);
int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops);
int kvm_iommu_init_driver(void);
void kvm_iommu_remove_driver(void);
arch/arm64/kvm/arm.c
+1 -7
View File
@@ -2495,15 +2495,9 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
if (ret)
return ret;
ret = kvm_iommu_init_driver();
if (ret < 0)
return ret;
ret = do_pkvm_init(hyp_va_bits);
if (ret) {
kvm_iommu_remove_driver();
if (ret)
return ret;
}
free_hyp_pgds();
arch/arm64/kvm/hyp/include/nvhe/iommu.h
+1 -1
View File
@@ -53,7 +53,7 @@ struct kvm_iommu_ops {
bool (*dabt_handler)(struct kvm_cpu_context *host_ctxt, u64 esr, u64 addr);
};
int kvm_iommu_init(void);
int kvm_iommu_init(struct kvm_iommu_ops *ops);
int kvm_iommu_init_device(struct kvm_hyp_iommu *iommu);
arch/arm64/kvm/hyp/nvhe/hyp-main.c
+8
View File
@@ -1690,6 +1690,13 @@ static void handle___pkvm_host_hvc_pd(struct kvm_cpu_context *host_ctxt)
cpu_reg(host_ctxt, 1) = pkvm_host_hvc_pd(device_id, on);
}
static void handle___pkvm_iommu_init(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct kvm_iommu_ops *, ops, host_ctxt, 1);
cpu_reg(host_ctxt, 1) = kvm_iommu_init(ops);
}
typedef void (*hcall_t)(struct kvm_cpu_context *);
#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1714,6 +1721,7 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__pkvm_unmap_module_page),
HANDLE_FUNC(__pkvm_init_module),
HANDLE_FUNC(__pkvm_register_hcall),
HANDLE_FUNC(__pkvm_iommu_init),
HANDLE_FUNC(__pkvm_prot_finalize),
HANDLE_FUNC(__pkvm_host_share_hyp),
arch/arm64/kvm/hyp/nvhe/iommu/iommu.c
+8 -6
View File
@@ -144,15 +144,15 @@ static void domain_put(struct kvm_hyp_iommu_domain *domain)
BUG_ON(!atomic_dec_return_release(&domain->refs));
}
int kvm_iommu_init(void)
int kvm_iommu_init(struct kvm_iommu_ops *ops)
{
int ret;
u64 domain_root_pfn = __hyp_pa(kvm_hyp_iommu_domains) >> PAGE_SHIFT;
if (!kvm_iommu_ops ||
!kvm_iommu_ops->init ||
!kvm_iommu_ops->alloc_domain ||
!kvm_iommu_ops->free_domain)
if (!ops ||
!ops->init ||
!ops->alloc_domain ||
!ops->free_domain)
return 0;
ret = hyp_pool_init_empty(&iommu_host_pool, 64);
@@ -164,7 +164,9 @@ int kvm_iommu_init(void)
if (ret)
return ret;
ret = kvm_iommu_ops->init();
kvm_iommu_ops = ops;
ret = ops->init();
if (ret)
goto out_reclaim_domain;
arch/arm64/kvm/hyp/nvhe/setup.c
-5
View File
@@ -14,7 +14,6 @@
#include <nvhe/early_alloc.h>
#include <nvhe/ffa.h>
#include <nvhe/gfp.h>
#include <nvhe/iommu.h>
#include <nvhe/memory.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
@@ -372,10 +371,6 @@ void __noreturn __pkvm_init_finalise(void)
if (ret)
goto out;
ret = kvm_iommu_init();
if (ret)
goto out;
ret = fix_host_ownership();
if (ret)
goto out;
arch/arm64/kvm/iommu.c
+12 -10
View File
@@ -13,11 +13,9 @@
struct kvm_iommu_driver *iommu_driver;
extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops);
int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iommu_ops *el2_ops)
int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops)
{
int ret;
if (WARN_ON(!kern_ops || !el2_ops))
if (WARN_ON(!kern_ops))
return -EINVAL;
/*
@@ -25,14 +23,18 @@ int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops, struct kvm_iomm
* Ensure memory stores happening during a driver
* init are observed before executing kvm iommu callbacks.
*/
ret = cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0;
if (ret)
return ret;
kvm_nvhe_sym(kvm_iommu_ops) = el2_ops;
return 0;
return cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0;
}
int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops)
{
if (!hyp_ops)
return -EINVAL;
return kvm_call_hyp_nvhe(__pkvm_iommu_init, hyp_ops);
}
EXPORT_SYMBOL(kvm_iommu_init_hyp);
int kvm_iommu_init_driver(void)
{
if (!smp_load_acquire(&iommu_driver)) {
arch/arm64/kvm/pkvm.c
+7 -1
View File
@@ -443,6 +443,12 @@ static int __init finalize_pkvm(void)
if (pkvm_load_early_modules())
pkvm_firmware_rmem_clear();
ret = kvm_iommu_init_driver();
if (ret) {
pr_err("Failed to init KVM IOMMU driver: %d\n", ret);
pkvm_firmware_rmem_clear();
}
/*
* Exclude HYP sections from kmemleak so that they don't get peeked
* at, which would end badly once inaccessible.
@@ -455,7 +461,7 @@ static int __init finalize_pkvm(void)
ret = pkvm_drop_host_privileges();
if (ret) {
pr_err("Failed to finalize Hyp protection: %d\n", ret);
BUG();
kvm_iommu_remove_driver();
}
return 0;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c
+19 -20
View File
@@ -924,6 +924,20 @@ static int smmu_put_device(struct device *dev, void *data)
return 0;
}
/*
* Drop the PM references of the SMMU taken at probe
* after it's guaranteed the hypervisor as initialized the SMMUs.
*/
static int kvm_arm_smmu_v3_post_init(void)
{
if (!kvm_arm_smmu_count)
return 0;
WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL,
NULL, smmu_put_device));
return 0;
}
static int kvm_arm_smmu_v3_init_drv(void)
{
int ret;
@@ -953,7 +967,10 @@ static int kvm_arm_smmu_v3_init_drv(void)
kvm_hyp_arm_smmu_v3_smmus = kvm_arm_smmu_array;
kvm_hyp_arm_smmu_v3_count = kvm_arm_smmu_count;
return 0;
ret = kvm_iommu_init_hyp(kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops))));
if (ret)
return ret;
return kvm_arm_smmu_v3_post_init();
err_free:
kvm_arm_smmu_array_free();
@@ -975,26 +992,8 @@ static int kvm_arm_smmu_v3_register(void)
if (!is_protected_kvm_enabled())
return 0;
return kvm_iommu_register_driver(&kvm_smmu_v3_ops,
kern_hyp_va(lm_alias(&kvm_nvhe_sym(smmu_ops))));
return kvm_iommu_register_driver(&kvm_smmu_v3_ops);
};
/*
* KVM init hypervisor at device_sync init call,
* so we drop the PM references of the SMMU taken at probe
* at the late initcall where it's guaranteed the hypervisor
* has initialized the SMMUs.
*/
static int kvm_arm_smmu_v3_post_init(void)
{
if (!kvm_arm_smmu_count)
return 0;
WARN_ON(driver_for_each_device(&kvm_arm_smmu_driver.driver, NULL,
NULL, smmu_put_device));
return 0;
}
core_initcall(kvm_arm_smmu_v3_register);
late_initcall(kvm_arm_smmu_v3_post_init);
MODULE_LICENSE("GPL v2");