From 6b871d2a3c0967bdc327330f2c3af970c9490c12 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Mon, 10 Apr 2023 21:46:53 +0000 Subject: [PATCH] ANDROID: KVM: arm64: iommu: Use guest ctxt for DMA pages Make use_dma logic aware of guests by using guest context if exists. Rename dma_use/unuse to remove host from it's name as it is now used by guests also. Bug: 357781595 Bug: 348382247 Bug: 236685427 Change-Id: Ic049cdea8ce5956975992314daaf64c69d3fde80 Signed-off-by: Mostafa Saleh --- arch/arm64/include/asm/kvm_pkvm_module.h | 4 +- arch/arm64/kvm/hyp/include/nvhe/iommu.h | 2 + arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 4 +- arch/arm64/kvm/hyp/nvhe/iommu/iommu.c | 13 ++++-- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 40 ++++++++++++++----- arch/arm64/kvm/hyp/nvhe/modules.c | 2 +- .../arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h | 2 +- .../iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c | 2 +- 8 files changed, 47 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index ccb577f06d77..1da077a911ff 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -158,7 +158,7 @@ enum pkvm_psci_notification { * Missing donations if allocator returns NULL * @iommu_iotlb_gather_add_page: * Add an IOVA range to an iommu_iotlb_gather. - * @pkvm_host_unuse_dma: Decrement the refcount for pages used for DMA, + * @pkvm_unuse_dma: Decrement the refcount for pages used for DMA, * this is typically called from the module after a * successful unmap() operation, so the hypervisor * can track the page state. @@ -240,7 +240,7 @@ struct pkvm_module_ops { struct iommu_iotlb_gather *gather, unsigned long iova, size_t size); - int (*pkvm_host_unuse_dma)(phys_addr_t phys_addr, size_t size); + int (*pkvm_unuse_dma)(phys_addr_t phys_addr, size_t size); #ifdef CONFIG_LIST_HARDENED /* These 2 functions change calling convention based on CONFIG_DEBUG_LIST. */ typeof(__list_add_valid_or_report) *list_add_valid_or_report; diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 361c046a456a..b62b94b42f46 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -87,6 +87,8 @@ int kvm_iommu_init(struct kvm_iommu_ops *ops, struct kvm_hyp_memcache *atomic_mc); int kvm_iommu_init_device(struct kvm_hyp_iommu *iommu); +int iommu_pkvm_unuse_dma(u64 phys_addr, size_t size); + void kvm_iommu_iotlb_gather_add_page(struct kvm_hyp_iommu_domain *domain, struct iommu_iotlb_gather *gather, unsigned long iova, diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 043e37994386..ae14492be415 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -71,8 +71,8 @@ int __pkvm_install_ioguard_page(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa, bool __pkvm_check_ioguard_page(struct pkvm_hyp_vcpu *hyp_vcpu); int __pkvm_guest_relinquish_to_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa, u64 *ppa); -int __pkvm_host_use_dma(u64 phys_addr, size_t size); -int __pkvm_host_unuse_dma(u64 phys_addr, size_t size); +int __pkvm_use_dma(u64 phys_addr, size_t size, struct pkvm_hyp_vcpu *hyp_vcpu); +int __pkvm_unuse_dma(u64 phys_addr, size_t size, struct pkvm_hyp_vcpu *hyp_vcpu); int __pkvm_host_lazy_pte(u64 pfn, u64 nr_pages, bool enable); u64 __pkvm_ptdump_get_config(pkvm_handle_t handle, enum pkvm_ptdump_ops op); u64 __pkvm_ptdump_walk_range(pkvm_handle_t handle, struct pkvm_ptdump_log_hdr *log_hva); diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c index e389f81dbf4d..2a018ef17ba0 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c @@ -83,6 +83,11 @@ struct pkvm_hyp_vcpu *__get_vcpu(void) return NULL; } +int iommu_pkvm_unuse_dma(u64 phys_addr, size_t size) +{ + return __pkvm_unuse_dma(phys_addr, size, __get_vcpu()); +} + static void *__kvm_iommu_donate_pages(struct hyp_pool *pool, u8 order, int flags) { void *p; @@ -415,7 +420,7 @@ size_t kvm_iommu_map_pages(pkvm_handle_t domain_id, if (!domain || domain_get(domain)) return 0; - ret = __pkvm_host_use_dma(paddr, size); + ret = __pkvm_use_dma(paddr, size, __get_vcpu()); if (ret) return 0; @@ -428,7 +433,7 @@ size_t kvm_iommu_map_pages(pkvm_handle_t domain_id, * so far. */ if (pgcount) - __pkvm_host_unuse_dma(paddr + total_mapped, pgcount * pgsize); + __pkvm_unuse_dma(paddr + total_mapped, pgcount * pgsize, __get_vcpu()); domain_put(domain); return total_mapped; @@ -551,7 +556,7 @@ size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, unsigned long iova, struct kvm_ iova + size < iova) goto out_unpin_sg; - ret = __pkvm_host_use_dma(phys, size); + ret = __pkvm_use_dma(phys, size, __get_vcpu()); if (ret) goto out_unpin_sg; @@ -562,7 +567,7 @@ size_t kvm_iommu_map_sg(pkvm_handle_t domain_id, unsigned long iova, struct kvm_ iova += mapped; /* Might need memory */ if (mapped != size) { - __pkvm_host_unuse_dma(phys, size - mapped); + __pkvm_unuse_dma(phys, size - mapped, __get_vcpu()); break; } sg++; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 18c49c0512de..e5470cc98d68 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -419,6 +419,12 @@ int __pkvm_guest_relinquish_to_host(struct pkvm_hyp_vcpu *vcpu, addr = ALIGN_DOWN(ipa, kvm_granule_size(level)); phys = kvm_pte_to_phys(pte); phys += ipa - addr; + /* page might be used for DMA! */ + if (hyp_page_count(hyp_phys_to_virt(phys))) { + ret = -EBUSY; + goto end; + } + hyp_poison_page(phys, PAGE_SIZE); psci_mem_protect_dec(1); @@ -1832,14 +1838,14 @@ int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages) return ret; } -static void __pkvm_host_use_dma_page(phys_addr_t phys_addr) +static void __pkvm_use_dma_page(phys_addr_t phys_addr) { struct hyp_page *p = hyp_phys_to_page(phys_addr); hyp_page_ref_inc(p); } -static void __pkvm_host_unuse_dma_page(phys_addr_t phys_addr) +static void __pkvm_unuse_dma_page(phys_addr_t phys_addr) { struct hyp_page *p = hyp_phys_to_page(phys_addr); @@ -1847,7 +1853,7 @@ static void __pkvm_host_unuse_dma_page(phys_addr_t phys_addr) } /* - * __pkvm_host_use_dma - Mark host memory as used for DMA + * __pkvm_use_dma - Mark memory as used for DMA * @phys_addr: physical address of the DMA region * @size: size of the DMA region * When a page is mapped in an IOMMU page table for DMA, it must @@ -1862,7 +1868,7 @@ static void __pkvm_host_unuse_dma_page(phys_addr_t phys_addr) * similar checks are needed in host_request_unshare() and * host_ack_unshare() */ -int __pkvm_host_use_dma(phys_addr_t phys_addr, size_t size) +int __pkvm_use_dma(phys_addr_t phys_addr, size_t size, struct pkvm_hyp_vcpu *hyp_vcpu) { int i; int ret = 0; @@ -1874,6 +1880,7 @@ int __pkvm_host_use_dma(phys_addr_t phys_addr, size_t size) return -EINVAL; host_lock_component(); + /* * Some differences between handling of RAM and device memory: * - The hyp vmemmap area for device memory is not backed by physical @@ -1885,6 +1892,11 @@ int __pkvm_host_use_dma(phys_addr_t phys_addr, size_t size) if (!reg) { enum kvm_pgtable_prot prot; + if (hyp_vcpu) { + ret = -EINVAL; + goto out_ret; + } + ret = ___host_check_page_state_range(phys_addr, size, PKVM_PAGE_TAINTED, reg, false); @@ -1898,12 +1910,16 @@ int __pkvm_host_use_dma(phys_addr_t phys_addr, size_t size) prot = pkvm_mkstate(PKVM_HOST_MMIO_PROT, PKVM_PAGE_TAINTED); ret = host_stage2_idmap_locked(phys_addr, size, prot, false); } else { - ret = ___host_check_page_state_range(phys_addr, size, PKVM_PAGE_OWNED, reg, false); - if (ret) - goto out_ret; + /* For VMs, we know if we reach this point the VM has access to the page. */ + if (!hyp_vcpu) { + ret = ___host_check_page_state_range(phys_addr, size, + PKVM_PAGE_OWNED, reg, false); + if (ret) + goto out_ret; + } for (i = 0; i < nr_pages; i++) - __pkvm_host_use_dma_page(phys_addr + i * PAGE_SIZE); + __pkvm_use_dma_page(phys_addr + i * PAGE_SIZE); } out_ret: @@ -1911,15 +1927,17 @@ out_ret: return ret; } -int __pkvm_host_unuse_dma(phys_addr_t phys_addr, size_t size) +int __pkvm_unuse_dma(phys_addr_t phys_addr, size_t size, struct pkvm_hyp_vcpu *hyp_vcpu) { int i; size_t nr_pages = size >> PAGE_SHIFT; if (WARN_ON(!PAGE_ALIGNED(phys_addr | size))) return -EINVAL; - if (!range_is_memory(phys_addr, phys_addr + size)) + if (!range_is_memory(phys_addr, phys_addr + size)) { + WARN_ON(hyp_vcpu); return 0; + } host_lock_component(); /* @@ -1928,7 +1946,7 @@ int __pkvm_host_unuse_dma(phys_addr_t phys_addr, size_t size) * in the host s2, there can be no failure. */ for (i = 0; i < nr_pages; i++) - __pkvm_host_unuse_dma_page(phys_addr + i * PAGE_SIZE); + __pkvm_unuse_dma_page(phys_addr + i * PAGE_SIZE); host_unlock_component(); return 0; diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index c28d49126ffb..7d5ec6ba1f95 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -256,7 +256,7 @@ const struct pkvm_module_ops module_ops = { .iommu_init_device = kvm_iommu_init_device, .udelay = pkvm_udelay, .iommu_iotlb_gather_add_page = kvm_iommu_iotlb_gather_add_page, - .pkvm_host_unuse_dma = __pkvm_host_unuse_dma, + .pkvm_unuse_dma = iommu_pkvm_unuse_dma, #ifdef CONFIG_LIST_HARDENED .list_add_valid_or_report = __list_add_valid_or_report, .list_del_entry_valid_or_report = __list_del_entry_valid_or_report, diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h index 826f4e336b02..1db0ca54e556 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3-module.h @@ -40,7 +40,7 @@ extern const struct pkvm_module_ops *mod_ops; #define hyp_pin_shared_mem(x, y) CALL_FROM_OPS(pin_shared_mem, x, y) #define kvm_iommu_iotlb_gather_add_page(x, y, z, w) \ CALL_FROM_OPS(iommu_iotlb_gather_add_page, x, y, z, w) -#define __pkvm_host_unuse_dma(x, y) CALL_FROM_OPS(pkvm_host_unuse_dma, x, y) +#define iommu_pkvm_unuse_dma(x, y) CALL_FROM_OPS(pkvm_unuse_dma, x, y) #define kvm_iommu_donate_pages_atomic(x) CALL_FROM_OPS(iommu_donate_pages_atomic, x) #define kvm_iommu_reclaim_pages_atomic(x, y) CALL_FROM_OPS(iommu_reclaim_pages_atomic, x, y) #define kvm_iommu_snapshot_host_stage2(x) CALL_FROM_OPS(iommu_snapshot_host_stage2, x) diff --git a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c index b4e035991de3..0e2d24135252 100644 --- a/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/pkvm/arm-smmu-v3.c @@ -900,7 +900,7 @@ static void smmu_unmap_visit_leaf(phys_addr_t addr, size_t size, /* Might be a cleared table. */ if (!pte) return; - WARN_ON(__pkvm_host_unuse_dma(addr, size)); + WARN_ON(iommu_pkvm_unuse_dma(addr, size)); *ptep = 0; }