tmakin/ack-tegra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ANDROID: KVM: iommu: Abstract hypercalls

Browse Source 
At the moment, the SMMUv3 driver calls KVM IOMMU hypercalls directly,
that is not ideal as:
- It can be built as a module, which means the HVCs become part of KMI
- It duplicates some logic, for example, when mapping in the IOMMU,
  we want to keep calling the HVC and topping up the allocator in case
  of no memory, which is a common logic

So, move those outside of the driver in the KVM IOMMU kernel part.

Bug: 357781595
Change-Id: Ica2b682efe62429841642d4a947ac315770f7bb3
Signed-off-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Carlos Llamas <cmllamas@google.com>
This commit is contained in:
Mostafa Saleh
2025-04-08 09:53:08 +00:00
committed by Carlos Llamas
parent 37dc91cf2d
commit 2ea2a15fa1
3 changed files with 201 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
arch/arm64/include/asm/kvm_host.h
View File

@@ -1763,6 +1763,26 @@ static inline void kvm_iommu_sg_free(struct kvm_iommu_sg *sg, unsigned int nents
free_pages_exact(sg, PAGE_ALIGN(nents * sizeof(struct kvm_iommu_sg)));
}
#ifndef __KVM_NVHE_HYPERVISOR__
int kvm_iommu_attach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id,
unsigned int endpoint, unsigned int pasid,
unsigned int ssid_bits);
int kvm_iommu_detach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id,
unsigned int endpoint, unsigned int pasid);
int kvm_iommu_alloc_domain(pkvm_handle_t domain_id, int type);
int kvm_iommu_free_domain(pkvm_handle_t domain_id);
int kvm_iommu_map_pages(pkvm_handle_t domain_id, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *total_mapped);
size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova,
size_t pgsize, size_t pgcount);
phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova);
size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, struct kvm_iommu_sg *sg,
unsigned long iova, unsigned int nent,
unsigned int prot, gfp_t gfp);
#endif
int kvm_iommu_share_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents);
int kvm_iommu_unshare_hyp_sg(struct kvm_iommu_sg *sg, unsigned int nents);
int kvm_iommu_device_num_ids(struct device *dev);

168
arch/arm64/kvm/iommu.c
View File

@@ -5,11 +5,53 @@
*/
#include <asm/kvm_mmu.h>
#include <asm/kvm_pkvm.h>
#include <kvm/iommu.h>
#include <linux/arm-smccc.h>
#include <linux/kvm_host.h>
#define kvm_call_hyp_nvhe_mc(...) \
({ \
struct arm_smccc_res __res; \
do { \
__res = kvm_call_hyp_nvhe_smccc(__VA_ARGS__); \
} while (__res.a1 && !kvm_iommu_topup_memcache(&__res, GFP_KERNEL));\
__res.a1; \
})
static int kvm_iommu_topup_memcache(struct arm_smccc_res *res, gfp_t gfp)
{
struct kvm_hyp_req req;
hyp_reqs_smccc_decode(res, &req);
if ((res->a1 == -ENOMEM) && (req.type != KVM_HYP_REQ_TYPE_MEM)) {
/*
* There is no way for drivers to populate hyp_alloc requests,
* so -ENOMEM + no request indicates that.
*/
return __pkvm_topup_hyp_alloc(1);
} else if (req.type != KVM_HYP_REQ_TYPE_MEM) {
return -EBADE;
}
if (req.mem.dest == REQ_MEM_DEST_HYP_IOMMU) {
return __pkvm_topup_hyp_alloc_mgt_gfp(HYP_ALLOC_MGT_IOMMU_ID,
req.mem.nr_pages,
req.mem.sz_alloc,
gfp);
} else if (req.mem.dest == REQ_MEM_DEST_HYP_ALLOC) {
/* Fill hyp alloc*/
return __pkvm_topup_hyp_alloc(req.mem.nr_pages);
}
pr_err("Bogus mem request");
return -EBADE;
}
struct kvm_iommu_driver *iommu_driver;
extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops);
@@ -163,3 +205,129 @@ void kvm_iommu_guest_free_mc(struct kvm_hyp_memcache *mc)
else
free_hyp_memcache(mc);
}
/* Hypercall abstractions exposed to kernel IOMMU drivers */
int kvm_iommu_attach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id,
unsigned int endpoint, unsigned int pasid,
unsigned int ssid_bits)
{
return kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_attach_dev, iommu_id, domain_id,
endpoint, pasid, ssid_bits);
}
EXPORT_SYMBOL(kvm_iommu_attach_dev);
int kvm_iommu_detach_dev(pkvm_handle_t iommu_id, pkvm_handle_t domain_id,
unsigned int endpoint, unsigned int pasid)
{
return kvm_call_hyp_nvhe(__pkvm_host_iommu_detach_dev, iommu_id, domain_id,
endpoint, pasid);
}
EXPORT_SYMBOL(kvm_iommu_detach_dev);
int kvm_iommu_alloc_domain(pkvm_handle_t domain_id, int type)
{
return kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain,
domain_id, type);
}
EXPORT_SYMBOL(kvm_iommu_alloc_domain);
int kvm_iommu_free_domain(pkvm_handle_t domain_id)
{
return kvm_call_hyp_nvhe(__pkvm_host_iommu_free_domain, domain_id);
}
EXPORT_SYMBOL(kvm_iommu_free_domain);
int kvm_iommu_map_pages(pkvm_handle_t domain_id, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *total_mapped)
{
size_t mapped;
size_t size = pgsize * pgcount;
struct arm_smccc_res res;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_pages, domain_id,
iova, paddr, pgsize, pgcount, prot);
mapped = res.a1;
iova += mapped;
paddr += mapped;
WARN_ON(mapped % pgsize);
WARN_ON(mapped > pgcount * pgsize);
pgcount -= mapped / pgsize;
*total_mapped += mapped;
} while (*total_mapped < size && !kvm_iommu_topup_memcache(&res, gfp));
if (*total_mapped < size)
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(kvm_iommu_map_pages);
size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova,
size_t pgsize, size_t pgcount)
{
size_t unmapped;
size_t total_unmapped = 0;
size_t size = pgsize * pgcount;
struct arm_smccc_res res;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_unmap_pages,
domain_id, iova, pgsize, pgcount);
unmapped = res.a1;
total_unmapped += unmapped;
iova += unmapped;
WARN_ON(unmapped % pgsize);
pgcount -= unmapped / pgsize;
/*
* The page table driver can unmap less than we asked for. If it
* didn't unmap anything at all, then it either reached the end
* of the range, or it needs a page in the memcache to break a
* block mapping.
*/
} while (total_unmapped < size &&
(unmapped || !kvm_iommu_topup_memcache(&res, GFP_ATOMIC)));
return total_unmapped;
}
EXPORT_SYMBOL(kvm_iommu_unmap_pages);
phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova)
{
return kvm_call_hyp_nvhe(__pkvm_host_iommu_iova_to_phys, domain_id, iova);
}
EXPORT_SYMBOL(kvm_iommu_iova_to_phys);
size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, struct kvm_iommu_sg *sg,
unsigned long iova, unsigned int nent,
unsigned int prot, gfp_t gfp)
{
size_t mapped, total_mapped = 0;
struct arm_smccc_res res;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_sg,
domain_id, iova, sg, nent, prot);
mapped = res.a1;
iova += mapped;
total_mapped += mapped;
/* Skip mapped */
while (mapped) {
if (mapped < (sg->pgsize * sg->pgcount)) {
sg->phys += mapped;
sg->pgcount -= mapped / sg->pgsize;
mapped = 0;
} else {
mapped -= sg->pgsize * sg->pgcount;
sg++;
nent--;
}
}
kvm_iommu_topup_memcache(&res, gfp);
} while (nent);
return total_mapped;
}
EXPORT_SYMBOL(kvm_iommu_map_sg);

131
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c
View File

@@ -7,7 +7,6 @@
#include <asm/kvm_pkvm.h>
#include <asm/kvm_mmu.h>
#include <linux/arm-smccc.h>
#include <linux/moduleparam.h>
#include <linux/of_address.h>
#include <linux/of_platform.h>
@@ -70,45 +69,6 @@ static DEFINE_IDA(kvm_arm_smmu_domain_ida);
static int atomic_pages;
module_param(atomic_pages, int, 0);
static int kvm_arm_smmu_topup_memcache(struct arm_smccc_res *res, gfp_t gfp)
{
struct kvm_hyp_req req;
hyp_reqs_smccc_decode(res, &req);
if ((res->a1 == -ENOMEM) && (req.type != KVM_HYP_REQ_TYPE_MEM)) {
/*
* There is no way for drivers to populate hyp_alloc requests,
* so -ENOMEM + no request indicates that.
*/
return __pkvm_topup_hyp_alloc(1);
} else if (req.type != KVM_HYP_REQ_TYPE_MEM) {
return -EBADE;
}
if (req.mem.dest == REQ_MEM_DEST_HYP_IOMMU) {
return __pkvm_topup_hyp_alloc_mgt_gfp(HYP_ALLOC_MGT_IOMMU_ID,
req.mem.nr_pages,
req.mem.sz_alloc,
gfp);
} else if (req.mem.dest == REQ_MEM_DEST_HYP_ALLOC) {
/* Fill hyp alloc*/
return __pkvm_topup_hyp_alloc(req.mem.nr_pages);
}
pr_err("Bogus mem request");
return -EBADE;
}
#define kvm_call_hyp_nvhe_mc(...) \
({ \
struct arm_smccc_res __res; \
do { \
__res = kvm_call_hyp_nvhe_smccc(__VA_ARGS__); \
} while (__res.a1 && !kvm_arm_smmu_topup_memcache(&__res, GFP_KERNEL));\
__res.a1; \
})
static struct platform_driver kvm_arm_smmu_driver;
static struct arm_smmu_device *
@@ -258,8 +218,7 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom
kvm_smmu_domain->id = ret;
ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain,
kvm_smmu_domain->id, type);
ret = kvm_iommu_alloc_domain(kvm_smmu_domain->id, type);
if (ret) {
ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id);
return ret;
@@ -275,7 +234,7 @@ static void kvm_arm_smmu_domain_free(struct iommu_domain *domain)
struct arm_smmu_device *smmu = kvm_smmu_domain->smmu;
if (smmu && (kvm_smmu_domain->domain.type != IOMMU_DOMAIN_IDENTITY)) {
ret = kvm_call_hyp_nvhe(__pkvm_host_iommu_free_domain, kvm_smmu_domain->id);
ret = kvm_iommu_free_domain(kvm_smmu_domain->id);
ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id);
}
kfree(kvm_smmu_domain);
@@ -296,8 +255,7 @@ static int kvm_arm_smmu_detach_dev_pasid(struct host_arm_smmu_device *host_smmu,
for (i = 0; i < fwspec->num_ids; i++) {
int sid = fwspec->ids[i];
ret = kvm_call_hyp_nvhe(__pkvm_host_iommu_detach_dev,
host_smmu->id, domain->id, sid, pasid);
ret = kvm_iommu_detach_dev(host_smmu->id, domain->id, sid, pasid);
if (ret) {
dev_err(smmu->dev, "cannot detach device %s (0x%x): %d\n",
dev_name(master->dev), sid, ret);
@@ -365,8 +323,7 @@ static int kvm_arm_smmu_set_dev_pasid(struct iommu_domain *domain,
for (i = 0; i < fwspec->num_ids; i++) {
int sid = fwspec->ids[i];
ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_attach_dev,
host_smmu->id, kvm_smmu_domain->id,
ret = kvm_iommu_attach_dev(host_smmu->id, kvm_smmu_domain->id,
sid, pasid, master->ssid_bits);
if (ret) {
dev_err(smmu->dev, "cannot attach device %s (0x%x): %d\n",
@@ -422,27 +379,10 @@ static int kvm_arm_smmu_map_pages(struct iommu_domain *domain,
size_t pgsize, size_t pgcount, int prot,
gfp_t gfp, size_t *total_mapped)
{
size_t mapped;
size_t size = pgsize * pgcount;
struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain);
struct arm_smccc_res res;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_pages,
kvm_smmu_domain->id,
iova, paddr, pgsize, pgcount, prot);
mapped = res.a1;
iova += mapped;
paddr += mapped;
WARN_ON(mapped % pgsize);
WARN_ON(mapped > pgcount * pgsize);
pgcount -= mapped / pgsize;
*total_mapped += mapped;
} while (*total_mapped < size && !kvm_arm_smmu_topup_memcache(&res, gfp));
if (*total_mapped < size)
return -EINVAL;
return 0;
return kvm_iommu_map_pages(kvm_smmu_domain->id, iova, paddr, pgsize,
pgcount, prot, gfp, total_mapped);
}
static size_t kvm_arm_smmu_unmap_pages(struct iommu_domain *domain,
@@ -450,32 +390,9 @@ static size_t kvm_arm_smmu_unmap_pages(struct iommu_domain *domain,
size_t pgcount,
struct iommu_iotlb_gather *iotlb_gather)
{
size_t unmapped;
size_t total_unmapped = 0;
size_t size = pgsize * pgcount;
struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain);
struct arm_smccc_res res;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_unmap_pages,
kvm_smmu_domain->id,
iova, pgsize, pgcount);
unmapped = res.a1;
total_unmapped += unmapped;
iova += unmapped;
WARN_ON(unmapped % pgsize);
pgcount -= unmapped / pgsize;
/*
* The page table driver can unmap less than we asked for. If it
* didn't unmap anything at all, then it either reached the end
* of the range, or it needs a page in the memcache to break a
* block mapping.
*/
} while (total_unmapped < size &&
(unmapped || !kvm_arm_smmu_topup_memcache(&res, GFP_ATOMIC)));
return total_unmapped;
return kvm_iommu_unmap_pages(kvm_smmu_domain->id, iova, pgsize, pgcount);
}
static phys_addr_t kvm_arm_smmu_iova_to_phys(struct iommu_domain *domain,
@@ -483,7 +400,7 @@ static phys_addr_t kvm_arm_smmu_iova_to_phys(struct iommu_domain *domain,
{
struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain);
return kvm_call_hyp_nvhe(__pkvm_host_iommu_iova_to_phys, kvm_smmu_domain->id, iova);
return kvm_iommu_iova_to_phys(kvm_smmu_domain->id, iova);
}
struct kvm_arm_smmu_map_sg {
@@ -543,32 +460,11 @@ static size_t kvm_arm_smmu_consume_deferred_map_sg(struct iommu_map_cookie_sg *c
struct kvm_arm_smmu_map_sg *map_sg = container_of(cookie, struct kvm_arm_smmu_map_sg,
cookie);
struct kvm_iommu_sg *sg = map_sg->sg;
size_t mapped, total_mapped = 0;
struct arm_smccc_res res;
struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(map_sg->cookie.domain);
size_t total_mapped;
do {
res = kvm_call_hyp_nvhe_smccc(__pkvm_host_iommu_map_sg,
kvm_smmu_domain->id,
map_sg->iova, sg, map_sg->ptr, map_sg->prot);
mapped = res.a1;
map_sg->iova += mapped;
total_mapped += mapped;
/* Skip mapped */
while (mapped) {
if (mapped < (sg->pgsize * sg->pgcount)) {
sg->phys += mapped;
sg->pgcount -= mapped / sg->pgsize;
mapped = 0;
} else {
mapped -= sg->pgsize * sg->pgcount;
sg++;
map_sg->ptr--;
}
}
kvm_arm_smmu_topup_memcache(&res, map_sg->gfp);
} while (map_sg->ptr);
total_mapped = kvm_iommu_map_sg(kvm_smmu_domain->id, sg, map_sg->iova, map_sg->ptr,
map_sg->prot, map_sg->gfp);
kvm_iommu_unshare_hyp_sg(sg, map_sg->nents);
kvm_iommu_sg_free(sg, map_sg->nents);
@@ -1178,9 +1074,8 @@ static int kvm_arm_smmu_v3_init_drv(void)
/* Preemptively allocate the identity domain. */
if (atomic_pages) {
ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain,
KVM_IOMMU_DOMAIN_IDMAP_ID,
KVM_IOMMU_DOMAIN_IDMAP_TYPE);
ret = kvm_iommu_alloc_domain(KVM_IOMMU_DOMAIN_IDMAP_ID,
KVM_IOMMU_DOMAIN_IDMAP_TYPE);
if (ret)
return ret;
}