diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/arm-smmu-v3.c b/arch/arm64/kvm/hyp/nvhe/iommu/arm-smmu-v3.c index 334cfc854246..7a945215c821 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/arm-smmu-v3.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/arm-smmu-v3.c @@ -68,6 +68,11 @@ struct hyp_arm_smmu_v3_domain { struct io_pgtable *pgtable; }; +static struct hyp_arm_smmu_v3_device *to_smmu(struct kvm_hyp_iommu *iommu) +{ + return container_of(iommu, struct hyp_arm_smmu_v3_device, iommu); +} + static int smmu_write_cr0(struct hyp_arm_smmu_v3_device *smmu, u32 val) { writel_relaxed(val, smmu->base + ARM_SMMU_CR0); @@ -203,7 +208,6 @@ static int smmu_send_cmd(struct hyp_arm_smmu_v3_device *smmu, return smmu_sync_cmd(smmu); } -__maybe_unused static int smmu_sync_ste(struct hyp_arm_smmu_v3_device *smmu, u32 sid) { struct arm_smmu_cmdq_ent cmd = { @@ -215,7 +219,6 @@ static int smmu_sync_ste(struct hyp_arm_smmu_v3_device *smmu, u32 sid) return smmu_send_cmd(smmu, &cmd); } -__maybe_unused static int smmu_sync_cd(struct hyp_arm_smmu_v3_device *smmu, u32 sid, u32 ssid) { struct arm_smmu_cmdq_ent cmd = { @@ -289,7 +292,6 @@ smmu_get_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) return &cfg->linear.table[sid]; } -__maybe_unused static struct arm_smmu_ste * smmu_get_alloc_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) { @@ -304,14 +306,12 @@ smmu_get_alloc_ste_ptr(struct hyp_arm_smmu_v3_device *smmu, u32 sid) return smmu_get_ste_ptr(smmu, sid); } -__maybe_unused static u64 *smmu_get_cd_ptr(u64 *cdtab, u32 ssid) { /* Only linear supported for now. */ return cdtab + ssid * CTXDESC_CD_DWORDS; } -__maybe_unused static u64 *smmu_alloc_cd(struct hyp_arm_smmu_v3_device *smmu, u32 pasid_bits) { u64 *cd_table; @@ -808,7 +808,6 @@ static void smmu_tlb_add_page(struct iommu_iotlb_gather *gather, smmu_tlb_inv_range(cookie, iova, granule, granule, true); } -__maybe_unused static const struct iommu_flush_ops smmu_tlb_ops = { .tlb_flush_all = smmu_tlb_flush_all, .tlb_flush_walk = smmu_tlb_flush_walk, @@ -826,6 +825,238 @@ static void smmu_iotlb_sync(struct kvm_hyp_iommu_domain *domain, smmu_tlb_inv_range(domain, gather->start, size, gather->pgsize, true); } +static int smmu_domain_config_s2(struct kvm_hyp_iommu_domain *domain, + struct arm_smmu_ste *ste) +{ + struct io_pgtable_cfg *cfg; + u64 ts, sl, ic, oc, sh, tg, ps; + struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + + cfg = &smmu_domain->pgtable->cfg; + ps = cfg->arm_lpae_s2_cfg.vtcr.ps; + tg = cfg->arm_lpae_s2_cfg.vtcr.tg; + sh = cfg->arm_lpae_s2_cfg.vtcr.sh; + oc = cfg->arm_lpae_s2_cfg.vtcr.orgn; + ic = cfg->arm_lpae_s2_cfg.vtcr.irgn; + sl = cfg->arm_lpae_s2_cfg.vtcr.sl; + ts = cfg->arm_lpae_s2_cfg.vtcr.tsz; + + ste->data[0] = STRTAB_STE_0_V | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); + ste->data[1] = FIELD_PREP(STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING); + ste->data[2] = FIELD_PREP(STRTAB_STE_2_VTCR, + FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, ps) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, tg) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, sh) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, oc) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, ic) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, sl) | + FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, ts)) | + FIELD_PREP(STRTAB_STE_2_S2VMID, domain->domain_id) | + STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2R; + ste->data[3] = cfg->arm_lpae_s2_cfg.vttbr & STRTAB_STE_3_S2TTB_MASK; + + return 0; +} + +static u64 *smmu_domain_config_s1_ste(struct hyp_arm_smmu_v3_device *smmu, + u32 pasid_bits, struct arm_smmu_ste *ste) +{ + u64 *cd_table; + + cd_table = smmu_alloc_cd(smmu, pasid_bits); + if (!cd_table) + return NULL; + + ste->data[1] = FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) | + FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH); + ste->data[0] = ((u64)cd_table & STRTAB_STE_0_S1CTXPTR_MASK) | + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) | + FIELD_PREP(STRTAB_STE_0_S1CDMAX, pasid_bits) | + FIELD_PREP(STRTAB_STE_0_S1FMT, STRTAB_STE_0_S1FMT_LINEAR) | + STRTAB_STE_0_V; + + return cd_table; +} + +/* + * This function handles configuration for pasid and non-pasid domains + * with the following assumptions: + * - pasid 0 always attached first, this should be the typicall flow + * for the kernel where attach_dev is always called before set_dev_pasid. + * In that case only pasid 0 is allowed to allocate memory for the CD, + * and other pasids would expect to find the tabel. + * - pasid 0 is detached last, also guaranteed from the kernel. + */ +static int smmu_domain_config_s1(struct hyp_arm_smmu_v3_device *smmu, + struct kvm_hyp_iommu_domain *domain, + u32 sid, u32 pasid, u32 pasid_bits, + struct arm_smmu_ste *ste) +{ + struct arm_smmu_ste *dst; + u64 val; + u64 *cd_entry, *cd_table; + struct io_pgtable_cfg *cfg; + struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + + cfg = &smmu_domain->pgtable->cfg; + dst = smmu_get_ste_ptr(smmu, sid); + val = dst->data[0]; + + if (FIELD_GET(STRTAB_STE_0_CFG, val) == STRTAB_STE_0_CFG_S2_TRANS) + return -EBUSY; + + if (pasid == 0) { + cd_table = smmu_domain_config_s1_ste(smmu, pasid_bits, ste); + if (!cd_table) + return -ENOMEM; + } else { + u32 nr_entries; + + cd_table = (u64 *)(FIELD_GET(STRTAB_STE_0_S1CTXPTR_MASK, val) << 6); + if (!cd_table) + return -EINVAL; + nr_entries = 1 << FIELD_GET(STRTAB_STE_0_S1CDMAX, val); + if (pasid >= nr_entries) + return -E2BIG; + } + + /* Write CD. */ + cd_entry = smmu_get_cd_ptr(hyp_phys_to_virt((u64)cd_table), pasid); + + /* CD already used by another device. */ + if (cd_entry[0]) + return -EBUSY; + + cd_entry[1] = cpu_to_le64(cfg->arm_lpae_s1_cfg.ttbr & CTXDESC_CD_1_TTB0_MASK); + cd_entry[2] = 0; + cd_entry[3] = cpu_to_le64(cfg->arm_lpae_s1_cfg.mair); + + /* STE is live. */ + if (pasid) + smmu_sync_cd(smmu, sid, pasid); + val = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) | + FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) | + FIELD_PREP(CTXDESC_CD_0_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) | + FIELD_PREP(CTXDESC_CD_0_TCR_IPS, cfg->arm_lpae_s1_cfg.tcr.ips) | + CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64 | + CTXDESC_CD_0_R | CTXDESC_CD_0_A | + CTXDESC_CD_0_ASET | + FIELD_PREP(CTXDESC_CD_0_ASID, domain->domain_id) | + CTXDESC_CD_0_V; + WRITE_ONCE(cd_entry[0], cpu_to_le64(val)); + /* STE is live. */ + if (pasid) + smmu_sync_cd(smmu, sid, pasid); + return 0; +} + +static int smmu_domain_finalise(struct hyp_arm_smmu_v3_device *smmu, + struct kvm_hyp_iommu_domain *domain) +{ + int ret; + struct io_pgtable_cfg cfg; + struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + + if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S1) { + size_t ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; + + cfg = (struct io_pgtable_cfg) { + .fmt = ARM_64_LPAE_S1, + .pgsize_bitmap = smmu->pgsize_bitmap, + .ias = min_t(unsigned long, ias, VA_BITS), + .oas = smmu->ias, + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, + .tlb = &smmu_tlb_ops, + }; + } else { + cfg = (struct io_pgtable_cfg) { + .fmt = ARM_64_LPAE_S2, + .pgsize_bitmap = smmu->pgsize_bitmap, + .ias = smmu->ias, + .oas = smmu->oas, + .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, + .tlb = &smmu_tlb_ops, + }; + } + + hyp_spin_lock(&smmu_domain->pgt_lock); + smmu_domain->pgtable = kvm_arm_io_pgtable_alloc(&cfg, domain, &ret); + hyp_spin_unlock(&smmu_domain->pgt_lock); + return ret; +} + +static int smmu_attach_dev(struct kvm_hyp_iommu *iommu, struct kvm_hyp_iommu_domain *domain, + u32 sid, u32 pasid, u32 pasid_bits) +{ + int i; + int ret; + struct arm_smmu_ste *dst; + struct arm_smmu_ste ste = {}; + struct hyp_arm_smmu_v3_device *smmu = to_smmu(iommu); + struct hyp_arm_smmu_v3_domain *smmu_domain = domain->priv; + + kvm_iommu_lock(iommu); + dst = smmu_get_alloc_ste_ptr(smmu, sid); + if (!dst) { + ret = -ENOMEM; + goto out_unlock; + } + + if (smmu_domain->smmu && (smmu != smmu_domain->smmu)) { + ret = -EINVAL; + goto out_unlock; + } + + if (!smmu_domain->pgtable) { + ret = smmu_domain_finalise(smmu, domain); + if (ret) + goto out_unlock; + } + + if (smmu_domain->type == KVM_ARM_SMMU_DOMAIN_S2) { + /* Device already attached or pasid for s2. */ + if (dst->data[0] || pasid) { + ret = -EBUSY; + goto out_unlock; + } + ret = smmu_domain_config_s2(domain, &ste); + } else { + /* + * Allocate and config CD, and update CD if possible. + */ + pasid_bits = min(pasid_bits, smmu->ssid_bits); + ret = smmu_domain_config_s1(smmu, domain, sid, pasid, + pasid_bits, &ste); + } + smmu_domain->smmu = smmu; + /* We don't update STEs for pasid domains. */ + if (ret || pasid) + goto out_unlock; + + /* + * The SMMU may cache a disabled STE. + * Initialize all fields, sync, then enable it. + */ + for (i = 1; i < STRTAB_STE_DWORDS; i++) + dst->data[i] = ste.data[i]; + + ret = smmu_sync_ste(smmu, sid); + if (ret) + goto out_unlock; + + WRITE_ONCE(dst->data[0], ste.data[0]); + ret = smmu_sync_ste(smmu, sid); + WARN_ON(ret); +out_unlock: + kvm_iommu_unlock(iommu); + return ret; +} + /* Shared with the kernel driver in EL1 */ struct kvm_iommu_ops smmu_ops = { .init = smmu_init, @@ -833,4 +1064,5 @@ struct kvm_iommu_ops smmu_ops = { .alloc_domain = smmu_alloc_domain, .free_domain = smmu_free_domain, .iotlb_sync = smmu_iotlb_sync, + .attach_dev = smmu_attach_dev, }; diff --git a/include/kvm/arm_smmu_v3.h b/include/kvm/arm_smmu_v3.h index ded98cbaebc1..e8616ec5a048 100644 --- a/include/kvm/arm_smmu_v3.h +++ b/include/kvm/arm_smmu_v3.h @@ -25,6 +25,10 @@ struct hyp_arm_smmu_v3_device { size_t cmdq_log2size; /* strtab_cfg.l2.l2ptrs is not used, instead computed from L1 */ struct arm_smmu_strtab_cfg strtab_cfg; + size_t oas; + size_t ias; + size_t pgsize_bitmap; + size_t ssid_bits; }; extern size_t kvm_nvhe_sym(kvm_hyp_arm_smmu_v3_count);