diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 17f24a8eb1b9..06d12b35fa3e 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -44,15 +44,22 @@ struct kvm_iommu_ops { phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, size_t *total_mapped); size_t (*unmap_pages)(struct kvm_hyp_iommu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount); + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather); phys_addr_t (*iova_to_phys)(struct kvm_hyp_iommu_domain *domain, unsigned long iova); - + void (*iotlb_sync)(struct kvm_hyp_iommu_domain *domain, + struct iommu_iotlb_gather *gather); }; int kvm_iommu_init(void); int kvm_iommu_init_device(struct kvm_hyp_iommu *iommu); +void kvm_iommu_iotlb_gather_add_page(struct kvm_hyp_iommu_domain *domain, + struct iommu_iotlb_gather *gather, + unsigned long iova, + size_t size); + static inline hyp_spinlock_t *kvm_iommu_get_lock(struct kvm_hyp_iommu *iommu) { /* See struct kvm_hyp_iommu */ diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index 83321cc5f466..a6e0f3634756 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -305,12 +305,30 @@ size_t kvm_iommu_map_pages(pkvm_handle_t domain_id, return total_mapped; } +static inline void kvm_iommu_iotlb_sync(struct kvm_hyp_iommu_domain *domain, + struct iommu_iotlb_gather *iotlb_gather) +{ + if (kvm_iommu_ops->iotlb_sync) + kvm_iommu_ops->iotlb_sync(domain, iotlb_gather); + + iommu_iotlb_gather_init(iotlb_gather); +} + +void kvm_iommu_iotlb_gather_add_page(struct kvm_hyp_iommu_domain *domain, + struct iommu_iotlb_gather *gather, + unsigned long iova, + size_t size) +{ + _iommu_iotlb_add_page(domain, gather, iova, size, kvm_iommu_iotlb_sync); +} + size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova, size_t pgsize, size_t pgcount) { size_t size; size_t unmapped; struct kvm_hyp_iommu_domain *domain; + struct iommu_iotlb_gather iotlb_gather; if (!pgsize || !pgcount) return 0; @@ -323,6 +341,7 @@ size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova, if (!domain || domain_get(domain)) return 0; + iommu_iotlb_gather_init(&iotlb_gather); /* * Unlike map, the common code doesn't call the __pkvm_host_unuse_dma, * because this means that we need either walk the table using iova_to_phys @@ -334,7 +353,8 @@ size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova, * standardized, we leave that to the driver. */ unmapped = kvm_iommu_ops->unmap_pages(domain, iova, pgsize, - pgcount); + pgcount, &iotlb_gather); + kvm_iommu_iotlb_sync(domain, &iotlb_gather); domain_put(domain); return unmapped; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index bd722f473635..c75877044185 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -911,6 +911,18 @@ static inline void iommu_iotlb_gather_add_range(struct iommu_iotlb_gather *gathe gather->end = end; } +/* + * If the new page is disjoint from the current range or is mapped at + * a different granularity, then sync the TLB so that the gather + * structure can be rewritten. + */ +#define _iommu_iotlb_add_page(domain, gather, iova, size, sync) \ + if (((gather)->pgsize && (gather)->pgsize != (size)) || \ + iommu_iotlb_gather_is_disjoint((gather), (iova), (size))) \ + sync((domain), (gather)); \ + (gather)->pgsize = (size); \ + iommu_iotlb_gather_add_range((gather), (iova), (size)) + /** * iommu_iotlb_gather_add_page - Gather for page-based TLB invalidation * @domain: IOMMU domain to be invalidated @@ -926,17 +938,7 @@ static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size) { - /* - * If the new page is disjoint from the current range or is mapped at - * a different granularity, then sync the TLB so that the gather - * structure can be rewritten. - */ - if ((gather->pgsize && gather->pgsize != size) || - iommu_iotlb_gather_is_disjoint(gather, iova, size)) - iommu_iotlb_sync(domain, gather); - - gather->pgsize = size; - iommu_iotlb_gather_add_range(gather, iova, size); + _iommu_iotlb_add_page(domain, gather, iova, size, iommu_iotlb_sync); } static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather)