From 03ce7b849c19938b97790b0ee3cd5dbf43ebb667 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Sun, 10 Mar 2024 10:24:35 +0000 Subject: [PATCH] ANDROID: drivers/arm-smmu-v3-kvm: Support IOMMU_DOMAIN_IDENTITY Since all the hyp bits are there, we can use IOMMU_DOMAIN_IDENTITY with KVM_IOMMU_IDMAPPED_DOMAIN. This can be discovered per device from device tree using flag "iommu-idmapped" Bug: 277989609 Bug: 278749606 Change-Id: Icf003b4c2789f6025be77db54a670a5d0888e603 Signed-off-by: Mostafa Saleh --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c | 40 ++++++- .../arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h | 1 + .../iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c | 103 ++++++++++++++++++ .../iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h | 1 + 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c index 3b3b649762d5..469c9ab165cc 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-kvm.c @@ -34,6 +34,7 @@ struct kvm_arm_smmu_master { struct device *dev; struct xarray domains; u32 ssid_bits; + bool idmapped; /* Stage-2 is transparently identity mapped*/ }; struct kvm_arm_smmu_domain { @@ -145,6 +146,7 @@ static struct iommu_device *kvm_arm_smmu_probe_device(struct device *dev) device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits); master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); xa_init(&master->domains); + master->idmapped = device_property_read_bool(dev, "iommu-idmapped"); dev_iommu_priv_set(dev, master); if (!device_link_add(dev, smmu->dev, DL_FLAG_PM_RUNTIME | @@ -164,10 +166,10 @@ static struct iommu_domain *kvm_arm_smmu_domain_alloc(unsigned type) * We don't support * - IOMMU_DOMAIN_DMA_FQ because lazy unmap would clash with memory * donation to guests. - * - IOMMU_DOMAIN_IDENTITY: Requires a stage-2 only transparent domain. */ if (type != IOMMU_DOMAIN_DMA && - type != IOMMU_DOMAIN_UNMANAGED) + type != IOMMU_DOMAIN_UNMANAGED && + type != IOMMU_DOMAIN_IDENTITY) return ERR_PTR(-EOPNOTSUPP); kvm_smmu_domain = kzalloc(sizeof(*kvm_smmu_domain), GFP_KERNEL); @@ -194,6 +196,18 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom if (kvm_smmu_domain->smmu) return 0; + + kvm_smmu_domain->smmu = smmu; + + if (kvm_smmu_domain->domain.type == IOMMU_DOMAIN_IDENTITY) { + kvm_smmu_domain->id = KVM_IOMMU_DOMAIN_IDMAP_ID; + /* + * Identity domains doesn't use the DMA API, so no need to + * set the domain aperture. + */ + return 0; + } + /* Default to stage-1. */ if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) { ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; @@ -250,7 +264,6 @@ static int kvm_arm_smmu_domain_finalize(struct kvm_arm_smmu_domain *kvm_smmu_dom return ret; } - kvm_smmu_domain->smmu = smmu; return 0; } @@ -260,7 +273,7 @@ static void kvm_arm_smmu_domain_free(struct iommu_domain *domain) struct kvm_arm_smmu_domain *kvm_smmu_domain = to_kvm_smmu_domain(domain); struct arm_smmu_device *smmu = kvm_smmu_domain->smmu; - if (smmu) { + if (smmu && (kvm_smmu_domain->domain.type != IOMMU_DOMAIN_IDENTITY)) { ret = kvm_call_hyp_nvhe(__pkvm_host_iommu_free_domain, kvm_smmu_domain->id); ida_free(&kvm_arm_smmu_domain_ida, kvm_smmu_domain->id); } @@ -381,6 +394,15 @@ static int kvm_arm_smmu_attach_dev(struct iommu_domain *domain, return kvm_arm_smmu_set_dev_pasid(domain, dev, 0); } +static int kvm_arm_smmu_def_domain_type(struct device *dev) +{ + struct kvm_arm_smmu_master *master = dev_iommu_priv_get(dev); + + if (master->idmapped && atomic_pages) + return IOMMU_DOMAIN_IDENTITY; + return 0; +} + static bool kvm_arm_smmu_capable(struct device *dev, enum iommu_cap cap) { struct kvm_arm_smmu_master *master = dev_iommu_priv_get(dev); @@ -474,6 +496,7 @@ static struct iommu_ops kvm_arm_smmu_ops = { .pgsize_bitmap = -1UL, .remove_dev_pasid = kvm_arm_smmu_remove_dev_pasid, .owner = THIS_MODULE, + .def_domain_type = kvm_arm_smmu_def_domain_type, .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = kvm_arm_smmu_attach_dev, .free = kvm_arm_smmu_domain_free, @@ -1042,6 +1065,15 @@ static int kvm_arm_smmu_v3_init_drv(void) ret = kvm_iommu_init_hyp(ksym_ref_addr_nvhe(smmu_ops), &atomic_mc); if (ret) return ret; + + /* Preemptively allocate the identity domain. */ + if (atomic_pages) { + ret = kvm_call_hyp_nvhe_mc(__pkvm_host_iommu_alloc_domain, + KVM_IOMMU_DOMAIN_IDMAP_ID, + KVM_IOMMU_DOMAIN_IDMAP_TYPE); + if (ret) + return ret; + } return kvm_arm_smmu_v3_post_init(); err_free: diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h index a34e7a59566c..826f4e336b02 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h @@ -43,6 +43,7 @@ extern const struct pkvm_module_ops *mod_ops; #define __pkvm_host_unuse_dma(x, y) CALL_FROM_OPS(pkvm_host_unuse_dma, x, y) #define kvm_iommu_donate_pages_atomic(x) CALL_FROM_OPS(iommu_donate_pages_atomic, x) #define kvm_iommu_reclaim_pages_atomic(x, y) CALL_FROM_OPS(iommu_reclaim_pages_atomic, x, y) +#define kvm_iommu_snapshot_host_stage2(x) CALL_FROM_OPS(iommu_snapshot_host_stage2, x) #endif #endif /* __ARM_SMMU_V3_MODULE__ */ diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c index 1c7ff0ca343d..9ed61ed00f2b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c @@ -1069,6 +1069,7 @@ static int smmu_domain_finalise(struct hyp_arm_smmu_v3_device *smmu, int ret; struct io_pgtable_cfg cfg; struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + struct arm_lpae_io_pgtable *data; if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S1) { size_t ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; @@ -1097,6 +1098,11 @@ static int smmu_domain_finalise(struct hyp_arm_smmu_v3_device *smmu, hyp_spin_lock(&smmu_domain->pgt_lock); smmu_domain->pgtable = kvm_arm_io_pgtable_alloc(&cfg, domain, &ret); hyp_spin_unlock(&smmu_domain->pgt_lock); + if (ret) + return ret; + + data = io_pgtable_to_data(smmu_domain->pgtable); + data->idmapped = (domain->domain_id == KVM_IOMMU_DOMAIN_IDMAP_ID); return ret; } @@ -1194,6 +1200,7 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; struct domain_iommu_node *iommu_node = NULL; + bool init_idmap = false; hyp_write_lock(&smmu_domain->list_lock); kvm_iommu_lock(iommu); @@ -1203,6 +1210,21 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom goto out_unlock; } + + /* + * BYPASS domains only supported on stage-2 instances, that is over restrictive + * but for now as stage-1 is limited to VA_BITS to match the kernel, it might + * not cover the ia bits, we don't support it. + */ + if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_BYPASS) { + if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) { + smmu_domain->type = KVM_ARM_SMMU_DOMAIN_S2; + } else { + ret = -EINVAL; + goto out_unlock; + } + } + if (!smmu_existing_in_domain(smmu, smmu_domain)) { if (!smmu_domain_compat(smmu, smmu_domain)) { ret = -EBUSY; @@ -1223,6 +1245,8 @@ static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_dom ret = smmu_domain_finalise(smmu, domain); if (ret) goto out_unlock; + if (domain->domain_id == KVM_IOMMU_DOMAIN_IDMAP_ID) + init_idmap = true; } if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S2) { @@ -1266,6 +1290,10 @@ out_unlock: kvm_iommu_unlock(iommu); hyp_write_unlock(&smmu_domain->list_lock); + + if (init_idmap) + ret = kvm_iommu_snapshot_host_stage2(domain); + return ret; } @@ -1510,6 +1538,80 @@ int smmu_resume(struct kvm_hyp_iommu *iommu) return 0; } +/* + * Although SMMU can support multiple granules, it must at least support PAGE_SIZE + * as the CPU, and for the IDMAP domains, we only use this granule. + * As we optimize for memory usage and performance, we try to use block mappings + * when possible. + */ +static size_t smmu_pgsize_idmap(size_t size, u64 paddr) +{ + size_t pgsizes; + size_t pgsize_bitmask = 0; + + if (PAGE_SIZE == SZ_4K) { + pgsize_bitmask = SZ_4K | SZ_2M | SZ_1G; + } else if (PAGE_SIZE == SZ_16K) { + pgsize_bitmask = SZ_16K | SZ_32M; + } else if (PAGE_SIZE == SZ_64K){ + pgsize_bitmask = SZ_64K | SZ_512M; + } + + /* All page sizes that fit the size */ + pgsizes = pgsize_bitmask & GENMASK_ULL(__fls(size), 0); + + /* Address must be aligned to page size */ + if (likely(paddr)) + pgsizes &= GENMASK_ULL(__ffs(paddr), 0); + + WARN_ON(!pgsizes); + + return BIT(__fls(pgsizes)); +} + +static void smmu_host_stage2_idmap(struct kvm_hyp_iommu_domain *domain, + phys_addr_t start, phys_addr_t end, int prot) +{ + size_t size = end - start; + size_t pgsize, pgcount; + size_t mapped, unmapped; + int ret; + struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + struct io_pgtable *pgtable = smmu_domain->pgtable; + + end = min(end, BIT(pgtable->cfg.oas)); + if (start >= end) + return; + + if (prot) { + if (!(prot & IOMMU_MMIO)) + prot |= IOMMU_CACHE; + + while (size) { + mapped = 0; + pgsize = smmu_pgsize_idmap(size, start); + pgcount = size / pgsize; + ret = pgtable->ops.map_pages(&pgtable->ops, start, start, + pgsize, pgcount, prot, 0, &mapped); + size -= mapped; + start += mapped; + if (!mapped || ret) + return; + } + } else { + while (size) { + pgsize = smmu_pgsize_idmap(size, start); + pgcount = size / pgsize; + unmapped = pgtable->ops.unmap_pages(&pgtable->ops, start, + pgsize, pgcount, NULL); + size -= unmapped; + start += unmapped; + if (!unmapped) + return; + } + } +} + #ifdef MODULE int smmu_init_hyp_module(const struct pkvm_module_ops *ops) { @@ -1536,4 +1638,5 @@ struct kvm_iommu_ops smmu_ops = { .dabt_handler = smmu_dabt_handler, .suspend = smmu_suspend, .resume = smmu_resume, + .host_stage2_idmap = smmu_host_stage2_idmap, }; diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h index e8616ec5a048..9565a52ddced 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm_smmu_v3.h @@ -38,6 +38,7 @@ extern struct hyp_arm_smmu_v3_device *kvm_nvhe_sym(kvm_hyp_arm_smmu_v3_smmus); #define kvm_hyp_arm_smmu_v3_smmus kvm_nvhe_sym(kvm_hyp_arm_smmu_v3_smmus) enum kvm_arm_smmu_domain_type { + KVM_ARM_SMMU_DOMAIN_BYPASS = KVM_IOMMU_DOMAIN_IDMAP_TYPE, KVM_ARM_SMMU_DOMAIN_S1, KVM_ARM_SMMU_DOMAIN_S2, KVM_ARM_SMMU_DOMAIN_MAX,