From 2ea2a15fa107ff9ed12b7efe9753bd8b7fa2a0ee Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Tue, 8 Apr 2025 09:53:08 +0000 Subject: [PATCH] ANDROID: KVM: iommu: Abstract hypercalls At the moment, the SMMUv3 driver calls KVM IOMMU hypercalls directly, that is not ideal as: - It can be built as a module, which means the HVCs become part of KMI - It duplicates some logic, for example, when mapping in the IOMMU, we want to keep calling the HVC and topping up the allocator in case of no memory, which is a common logic So, move those outside of the driver in the KVM IOMMU kernel part. Bug: 357781595 Change-Id: Ica2b682efe62429841642d4a947ac315770f7bb3 Signed-off-by: Mostafa Saleh Signed-off-by: Carlos Llamas --- arch/arm64/include/asm/kvm_host.h | 20 +++ arch/arm64/kvm/iommu.c | 168 ++++++++++++++++++ .../iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c | 131 ++------------ 3 files changed, 201 insertions(+), 118 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 549731b8709b..b1ebe2b4b408 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1763,6 +1763,26 @@ static inline void kvm_iommu_sg_free(struct kvm_iommu_sg *sg, unsigned int nents free_pages_exact(sg, PAGE_ALIGN(nents * sizeof(struct kvm_iommu_sg))); } + +#ifndef __KVM_NVHE_HYPERVISOR__ +int kvm_iommu_attach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id, + unsigned int endpoint, unsigned int pasid, + unsigned int ssid_bits); +int kvm_iommu_detach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id, + unsigned int endpoint, unsigned int pasid); +int kvm_iommu_alloc_domain(pkvm_handle_t domain_id, int type); +int kvm_iommu_free_domain(pkvm_handle_t domain_id); +int kvm_iommu_map_pages(pkvm_handle_t domain_id, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *total_mapped); +size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova, + size_t pgsize, size_t pgcount); +phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova); +size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, struct kvm_iommu_sg *sg, + unsigned long iova, unsigned int nent, + unsigned int prot, gfp_t gfp); +#endif + int kvm_iommu_share_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents); int kvm_iommu_unshare_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents); int kvm_iommu_device_num_ids(struct device *dev); diff --git a/arch/arm64/kvm/iommu.c b/arch/arm64/kvm/iommu.c index e018f5057923..55e513eae24f 100644 --- a/arch/arm64/kvm/iommu.c +++ b/arch/arm64/kvm/iommu.c @@ -5,11 +5,53 @@ */ #include +#include #include +#include #include +#define kvm_call_hyp_nvhe_mc(...) \ +({ \ + struct arm_smccc_res __res; \ + do { \ + __res = kvm_call_hyp_nvhe_smccc(__VA_ARGS__); \ + } while (__res.a1 && !kvm_iommu_topup_memcache(&__res, GFP_KERNEL));\ + __res.a1; \ +}) + + +static int kvm_iommu_topup_memcache(struct arm_smccc_res *res, gfp_t gfp) +{ + struct kvm_hyp_req req; + + hyp_reqs_smccc_decode(res, &req); + + if ((res->a1 == -ENOMEM) && (req.type != KVM_HYP_REQ_TYPE_MEM)) { + /* + * There is no way for drivers to populate hyp_alloc requests, + * so -ENOMEM + no request indicates that. + */ + return __pkvm_topup_hyp_alloc(1); + } else if (req.type != KVM_HYP_REQ_TYPE_MEM) { + return -EBADE; + } + + if (req.mem.dest == REQ_MEM_DEST_HYP_IOMMU) { + return __pkvm_topup_hyp_alloc_mgt_gfp(HYP_ALLOC_MGT_IOMMU_ID, + req.mem.nr_pages, + req.mem.sz_alloc, + gfp); + } else if (req.mem.dest == REQ_MEM_DEST_HYP_ALLOC) { + /* Fill hyp alloc*/ + return __pkvm_topup_hyp_alloc(req.mem.nr_pages); + } + + pr_err("Bogus mem request"); + return -EBADE; +} + struct kvm_iommu_driver *iommu_driver; extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops); @@ -163,3 +205,129 @@ void kvm_iommu_guest_free_mc(struct kvm_hyp_memcache *mc) else free_hyp_memcache(mc); } + +/* Hypercall abstractions exposed to kernel IOMMU drivers */ +int kvm_iommu_attach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id, + unsigned int endpoint, unsigned int pasid, + unsigned int ssid_bits) +{ + return kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_attach_dev, iommu_id, domain_id, + endpoint, pasid, ssid_bits); +} +EXPORT_SYMBOL(kvm_iommu_attach_dev); + +int kvm_iommu_detach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id, + unsigned int endpoint, unsigned int pasid) +{ + return kvm_call_hyp_nvhe(__pkvm_host_iommu_detach_dev, iommu_id, domain_id, + endpoint, pasid); +} +EXPORT_SYMBOL(kvm_iommu_detach_dev); + +int kvm_iommu_alloc_domain(pkvm_handle_t domain_id, int type) +{ + return kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain, + domain_id, type); +} +EXPORT_SYMBOL(kvm_iommu_alloc_domain); + +int kvm_iommu_free_domain(pkvm_handle_t domain_id) +{ + return kvm_call_hyp_nvhe(__pkvm_host_iommu_free_domain, domain_id); +} +EXPORT_SYMBOL(kvm_iommu_free_domain); + +int kvm_iommu_map_pages(pkvm_handle_t domain_id, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *total_mapped) +{ + size_t mapped; + size_t size = pgsize * pgcount; + struct arm_smccc_res res; + + do { + res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_pages, domain_id, + iova, paddr, pgsize, pgcount, prot); + mapped = res.a1; + iova += mapped; + paddr += mapped; + WARN_ON(mapped % pgsize); + WARN_ON(mapped > pgcount * pgsize); + pgcount -= mapped / pgsize; + *total_mapped += mapped; + } while (*total_mapped < size && !kvm_iommu_topup_memcache(&res, gfp)); + if (*total_mapped < size) + return -EINVAL; + return 0; +} +EXPORT_SYMBOL(kvm_iommu_map_pages); + +size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova, + size_t pgsize, size_t pgcount) +{ + size_t unmapped; + size_t total_unmapped = 0; + size_t size = pgsize * pgcount; + struct arm_smccc_res res; + + do { + res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_unmap_pages, + domain_id, iova, pgsize, pgcount); + unmapped = res.a1; + total_unmapped += unmapped; + iova += unmapped; + WARN_ON(unmapped % pgsize); + pgcount -= unmapped / pgsize; + + /* + * The page table driver can unmap less than we asked for. If it + * didn't unmap anything at all, then it either reached the end + * of the range, or it needs a page in the memcache to break a + * block mapping. + */ + } while (total_unmapped < size && + (unmapped || !kvm_iommu_topup_memcache(&res, GFP_ATOMIC))); + + return total_unmapped; + +} +EXPORT_SYMBOL(kvm_iommu_unmap_pages); + +phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova) +{ + return kvm_call_hyp_nvhe(__pkvm_host_iommu_iova_to_phys, domain_id, iova); +} +EXPORT_SYMBOL(kvm_iommu_iova_to_phys); + +size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, struct kvm_iommu_sg *sg, + unsigned long iova, unsigned int nent, + unsigned int prot, gfp_t gfp) +{ + size_t mapped, total_mapped = 0; + struct arm_smccc_res res; + + do { + res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_sg, + domain_id, iova, sg, nent, prot); + mapped = res.a1; + iova += mapped; + total_mapped += mapped; + /* Skip mapped */ + while (mapped) { + if (mapped < (sg->pgsize * sg->pgcount)) { + sg->phys += mapped; + sg->pgcount -= mapped / sg->pgsize; + mapped = 0; + } else { + mapped -= sg->pgsize * sg->pgcount; + sg++; + nent--; + } + } + + kvm_iommu_topup_memcache(&res, gfp); + } while (nent); + + return total_mapped; +} +EXPORT_SYMBOL(kvm_iommu_map_sg); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c index d112a2439b96..793c84361864 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -70,45 +69,6 @@ static DEFINE_IDA(kvm_arm_smmu_domain_ida); static int atomic_pages; module_param(atomic_pages, int, 0); -static int kvm_arm_smmu_topup_memcache(struct arm_smccc_res *res, gfp_t gfp) -{ - struct kvm_hyp_req req; - - hyp_reqs_smccc_decode(res, &req); - - if ((res->a1 == -ENOMEM) && (req.type != KVM_HYP_REQ_TYPE_MEM)) { - /* - * There is no way for drivers to populate hyp_alloc requests, - * so -ENOMEM + no request indicates that. - */ - return __pkvm_topup_hyp_alloc(1); - } else if (req.type != KVM_HYP_REQ_TYPE_MEM) { - return -EBADE; - } - - if (req.mem.dest == REQ_MEM_DEST_HYP_IOMMU) { - return __pkvm_topup_hyp_alloc_mgt_gfp(HYP_ALLOC_MGT_IOMMU_ID, - req.mem.nr_pages, - req.mem.sz_alloc, - gfp); - } else if (req.mem.dest == REQ_MEM_DEST_HYP_ALLOC) { - /* Fill hyp alloc*/ - return __pkvm_topup_hyp_alloc(req.mem.nr_pages); - } - - pr_err("Bogus mem request"); - return -EBADE; -} - -#define kvm_call_hyp_nvhe_mc(...) \ -({ \ - struct arm_smccc_res __res; \ - do { \ - __res = kvm_call_hyp_nvhe_smccc(__VA_ARGS__); \ - } while (__res.a1 && !kvm_arm_smmu_topup_memcache(&__res, GFP_KERNEL));\ - __res.a1; \ -}) - static struct platform_driver kvm_arm_smmu_driver; static struct arm_smmu_device * @@ -258,8 +218,7 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom kvm_smmu_domain->id = ret; - ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain, - kvm_smmu_domain->id, type); + ret = kvm_iommu_alloc_domain(kvm_smmu_domain->id, type); if (ret) { ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id); return ret; @@ -275,7 +234,7 @@ static void kvm_arm_smmu_domain_free(struct iommu_domain *domain) struct arm_smmu_device *smmu = kvm_smmu_domain->smmu; if (smmu && (kvm_smmu_domain->domain.type != IOMMU_DOMAIN_IDENTITY)) { - ret = kvm_call_hyp_nvhe(__pkvm_host_iommu_free_domain, kvm_smmu_domain->id); + ret = kvm_iommu_free_domain(kvm_smmu_domain->id); ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id); } kfree(kvm_smmu_domain); @@ -296,8 +255,7 @@ static int kvm_arm_smmu_detach_dev_pasid(struct host_arm_smmu_device *host_smmu, for (i = 0; i < fwspec->num_ids; i++) { int sid = fwspec->ids[i]; - ret = kvm_call_hyp_nvhe(__pkvm_host_iommu_detach_dev, - host_smmu->id, domain->id, sid, pasid); + ret = kvm_iommu_detach_dev(host_smmu->id, domain->id, sid, pasid); if (ret) { dev_err(smmu->dev, "cannot detach device %s (0x%x): %d\n", dev_name(master->dev), sid, ret); @@ -365,8 +323,7 @@ static int kvm_arm_smmu_set_dev_pasid(struct iommu_domain *domain, for (i = 0; i < fwspec->num_ids; i++) { int sid = fwspec->ids[i]; - ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_attach_dev, - host_smmu->id, kvm_smmu_domain->id, + ret = kvm_iommu_attach_dev(host_smmu->id, kvm_smmu_domain->id, sid, pasid, master->ssid_bits); if (ret) { dev_err(smmu->dev, "cannot attach device %s (0x%x): %d\n", @@ -422,27 +379,10 @@ static int kvm_arm_smmu_map_pages(struct iommu_domain *domain, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *total_mapped) { - size_t mapped; - size_t size = pgsize * pgcount; struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain); - struct arm_smccc_res res; - do { - res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_pages, - kvm_smmu_domain->id, - iova, paddr, pgsize, pgcount, prot); - mapped = res.a1; - iova += mapped; - paddr += mapped; - WARN_ON(mapped % pgsize); - WARN_ON(mapped > pgcount * pgsize); - pgcount -= mapped / pgsize; - *total_mapped += mapped; - } while (*total_mapped < size && !kvm_arm_smmu_topup_memcache(&res, gfp)); - if (*total_mapped < size) - return -EINVAL; - - return 0; + return kvm_iommu_map_pages(kvm_smmu_domain->id, iova, paddr, pgsize, + pgcount, prot, gfp, total_mapped); } static size_t kvm_arm_smmu_unmap_pages(struct iommu_domain *domain, @@ -450,32 +390,9 @@ static size_t kvm_arm_smmu_unmap_pages(struct iommu_domain *domain, size_t pgcount, struct iommu_iotlb_gather *iotlb_gather) { - size_t unmapped; - size_t total_unmapped = 0; - size_t size = pgsize * pgcount; struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain); - struct arm_smccc_res res; - do { - res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_unmap_pages, - kvm_smmu_domain->id, - iova, pgsize, pgcount); - unmapped = res.a1; - total_unmapped += unmapped; - iova += unmapped; - WARN_ON(unmapped % pgsize); - pgcount -= unmapped / pgsize; - - /* - * The page table driver can unmap less than we asked for. If it - * didn't unmap anything at all, then it either reached the end - * of the range, or it needs a page in the memcache to break a - * block mapping. - */ - } while (total_unmapped < size && - (unmapped || !kvm_arm_smmu_topup_memcache(&res, GFP_ATOMIC))); - - return total_unmapped; + return kvm_iommu_unmap_pages(kvm_smmu_domain->id, iova, pgsize, pgcount); } static phys_addr_t kvm_arm_smmu_iova_to_phys(struct iommu_domain *domain, @@ -483,7 +400,7 @@ static phys_addr_t kvm_arm_smmu_iova_to_phys(struct iommu_domain *domain, { struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain); - return kvm_call_hyp_nvhe(__pkvm_host_iommu_iova_to_phys, kvm_smmu_domain->id, iova); + return kvm_iommu_iova_to_phys(kvm_smmu_domain->id, iova); } struct kvm_arm_smmu_map_sg { @@ -543,32 +460,11 @@ static size_t kvm_arm_smmu_consume_deferred_map_sg(struct iommu_map_cookie_sg *c struct kvm_arm_smmu_map_sg *map_sg = container_of(cookie, struct kvm_arm_smmu_map_sg, cookie); struct kvm_iommu_sg *sg = map_sg->sg; - size_t mapped, total_mapped = 0; - struct arm_smccc_res res; struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(map_sg->cookie.domain); + size_t total_mapped; - do { - res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_sg, - kvm_smmu_domain->id, - map_sg->iova, sg, map_sg->ptr, map_sg->prot); - mapped = res.a1; - map_sg->iova += mapped; - total_mapped += mapped; - /* Skip mapped */ - while (mapped) { - if (mapped < (sg->pgsize * sg->pgcount)) { - sg->phys += mapped; - sg->pgcount -= mapped / sg->pgsize; - mapped = 0; - } else { - mapped -= sg->pgsize * sg->pgcount; - sg++; - map_sg->ptr--; - } - } - - kvm_arm_smmu_topup_memcache(&res, map_sg->gfp); - } while (map_sg->ptr); + total_mapped = kvm_iommu_map_sg(kvm_smmu_domain->id, sg, map_sg->iova, map_sg->ptr, + map_sg->prot, map_sg->gfp); kvm_iommu_unshare_hyp_sg(sg, map_sg->nents); kvm_iommu_sg_free(sg, map_sg->nents); @@ -1178,9 +1074,8 @@ static int kvm_arm_smmu_v3_init_drv(void) /* Preemptively allocate the identity domain. */ if (atomic_pages) { - ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain, - KVM_IOMMU_DOMAIN_IDMAP_ID, - KVM_IOMMU_DOMAIN_IDMAP_TYPE); + ret = kvm_iommu_alloc_domain(KVM_IOMMU_DOMAIN_IDMAP_ID, + KVM_IOMMU_DOMAIN_IDMAP_TYPE); if (ret) return ret; }