dma-mapping: direct calls for dma-iommu
Directly call into dma-iommu just like we have been doing for dma-direct for a while. This avoids the indirect call overhead for IOMMU ops and removes the need to have DMA ops entirely for many common configurations. Signed-off-by: Leon Romanovsky <leonro@nvidia.com> Signed-off-by: Leon Romanovsky <leon@kernel.org> Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Acked-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Christoph Hellwig <hch@lst.de>
This commit is contained in:
Leon Romanovsky
committed by
Christoph Hellwig
Christoph Hellwig
parent
f69e342eec
commit
b5c58b2fdc
@@ -8,8 +8,12 @@ config HAS_DMA
|
||||
depends on !NO_DMA
|
||||
default y
|
||||
|
||||
config DMA_OPS_HELPERS
|
||||
bool
|
||||
|
||||
config DMA_OPS
|
||||
depends on HAS_DMA
|
||||
select DMA_OPS_HELPERS
|
||||
bool
|
||||
|
||||
#
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
obj-$(CONFIG_HAS_DMA) += mapping.o direct.o
|
||||
obj-$(CONFIG_DMA_OPS) += ops_helpers.o
|
||||
obj-$(CONFIG_DMA_OPS_HELPERS) += ops_helpers.o
|
||||
obj-$(CONFIG_DMA_OPS) += dummy.o
|
||||
obj-$(CONFIG_DMA_CMA) += contiguous.o
|
||||
obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o
|
||||
|
||||
+74
-9
@@ -10,6 +10,7 @@
|
||||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/export.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/iommu-dma.h>
|
||||
#include <linux/kmsan.h>
|
||||
#include <linux/of_device.h>
|
||||
#include <linux/slab.h>
|
||||
@@ -113,11 +114,27 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
|
||||
}
|
||||
EXPORT_SYMBOL(dmam_alloc_attrs);
|
||||
|
||||
#ifdef CONFIG_IOMMU_DMA
|
||||
static bool use_dma_iommu(struct device *dev)
|
||||
{
|
||||
return dev->dma_iommu;
|
||||
}
|
||||
#else
|
||||
static bool use_dma_iommu(struct device *dev)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool dma_go_direct(struct device *dev, dma_addr_t mask,
|
||||
const struct dma_map_ops *ops)
|
||||
{
|
||||
if (use_dma_iommu(dev))
|
||||
return false;
|
||||
|
||||
if (likely(!ops))
|
||||
return true;
|
||||
|
||||
#ifdef CONFIG_DMA_OPS_BYPASS
|
||||
if (dev->dma_ops_bypass)
|
||||
return min_not_zero(mask, dev->bus_dma_limit) >=
|
||||
@@ -159,6 +176,8 @@ dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
|
||||
if (dma_map_direct(dev, ops) ||
|
||||
arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size))
|
||||
addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
addr = iommu_dma_map_page(dev, page, offset, size, dir, attrs);
|
||||
else
|
||||
addr = ops->map_page(dev, page, offset, size, dir, attrs);
|
||||
kmsan_handle_dma(page, offset, size, dir);
|
||||
@@ -177,6 +196,8 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
|
||||
if (dma_map_direct(dev, ops) ||
|
||||
arch_dma_unmap_page_direct(dev, addr + size))
|
||||
dma_direct_unmap_page(dev, addr, size, dir, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_unmap_page(dev, addr, size, dir, attrs);
|
||||
else
|
||||
ops->unmap_page(dev, addr, size, dir, attrs);
|
||||
debug_dma_unmap_page(dev, addr, size, dir);
|
||||
@@ -197,6 +218,8 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
|
||||
if (dma_map_direct(dev, ops) ||
|
||||
arch_dma_map_sg_direct(dev, sg, nents))
|
||||
ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs);
|
||||
else
|
||||
ents = ops->map_sg(dev, sg, nents, dir, attrs);
|
||||
|
||||
@@ -291,7 +314,9 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
|
||||
if (dma_map_direct(dev, ops) ||
|
||||
arch_dma_unmap_sg_direct(dev, sg, nents))
|
||||
dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
|
||||
else
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_unmap_sg(dev, sg, nents, dir, attrs);
|
||||
else if (ops->unmap_sg)
|
||||
ops->unmap_sg(dev, sg, nents, dir, attrs);
|
||||
}
|
||||
EXPORT_SYMBOL(dma_unmap_sg_attrs);
|
||||
@@ -309,6 +334,8 @@ dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
|
||||
|
||||
if (dma_map_direct(dev, ops))
|
||||
addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
addr = iommu_dma_map_resource(dev, phys_addr, size, dir, attrs);
|
||||
else if (ops->map_resource)
|
||||
addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
|
||||
|
||||
@@ -323,7 +350,11 @@ void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
|
||||
BUG_ON(!valid_dma_direction(dir));
|
||||
if (!dma_map_direct(dev, ops) && ops->unmap_resource)
|
||||
if (dma_map_direct(dev, ops))
|
||||
; /* nothing to do: uncached and no swiotlb */
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_unmap_resource(dev, addr, size, dir, attrs);
|
||||
else if (ops->unmap_resource)
|
||||
ops->unmap_resource(dev, addr, size, dir, attrs);
|
||||
debug_dma_unmap_resource(dev, addr, size, dir);
|
||||
}
|
||||
@@ -338,6 +369,8 @@ void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
|
||||
BUG_ON(!valid_dma_direction(dir));
|
||||
if (dma_map_direct(dev, ops))
|
||||
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_sync_single_for_cpu(dev, addr, size, dir);
|
||||
else if (ops->sync_single_for_cpu)
|
||||
ops->sync_single_for_cpu(dev, addr, size, dir);
|
||||
debug_dma_sync_single_for_cpu(dev, addr, size, dir);
|
||||
@@ -352,6 +385,8 @@ void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
|
||||
BUG_ON(!valid_dma_direction(dir));
|
||||
if (dma_map_direct(dev, ops))
|
||||
dma_direct_sync_single_for_device(dev, addr, size, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_sync_single_for_device(dev, addr, size, dir);
|
||||
else if (ops->sync_single_for_device)
|
||||
ops->sync_single_for_device(dev, addr, size, dir);
|
||||
debug_dma_sync_single_for_device(dev, addr, size, dir);
|
||||
@@ -366,6 +401,8 @@ void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
|
||||
BUG_ON(!valid_dma_direction(dir));
|
||||
if (dma_map_direct(dev, ops))
|
||||
dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
|
||||
else if (ops->sync_sg_for_cpu)
|
||||
ops->sync_sg_for_cpu(dev, sg, nelems, dir);
|
||||
debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
|
||||
@@ -380,6 +417,8 @@ void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
|
||||
BUG_ON(!valid_dma_direction(dir));
|
||||
if (dma_map_direct(dev, ops))
|
||||
dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_sync_sg_for_device(dev, sg, nelems, dir);
|
||||
else if (ops->sync_sg_for_device)
|
||||
ops->sync_sg_for_device(dev, sg, nelems, dir);
|
||||
debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
|
||||
@@ -405,7 +444,7 @@ static void dma_setup_need_sync(struct device *dev)
|
||||
{
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
|
||||
if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC))
|
||||
if (dma_map_direct(dev, ops) || use_dma_iommu(dev))
|
||||
/*
|
||||
* dma_skip_sync will be reset to %false on first SWIOTLB buffer
|
||||
* mapping, if any. During the device initialization, it's
|
||||
@@ -446,6 +485,9 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
|
||||
size, attrs);
|
||||
if (use_dma_iommu(dev))
|
||||
return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr,
|
||||
size, attrs);
|
||||
if (!ops->get_sgtable)
|
||||
return -ENXIO;
|
||||
return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);
|
||||
@@ -482,6 +524,8 @@ bool dma_can_mmap(struct device *dev)
|
||||
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
return dma_direct_can_mmap(dev);
|
||||
if (use_dma_iommu(dev))
|
||||
return true;
|
||||
return ops->mmap != NULL;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dma_can_mmap);
|
||||
@@ -508,6 +552,9 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
|
||||
attrs);
|
||||
if (use_dma_iommu(dev))
|
||||
return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size,
|
||||
attrs);
|
||||
if (!ops->mmap)
|
||||
return -ENXIO;
|
||||
return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
|
||||
@@ -559,6 +606,8 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
|
||||
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);
|
||||
else if (ops->alloc)
|
||||
cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
|
||||
else
|
||||
@@ -591,6 +640,8 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
|
||||
debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs);
|
||||
else if (ops->free)
|
||||
ops->free(dev, size, cpu_addr, dma_handle, attrs);
|
||||
}
|
||||
@@ -611,6 +662,8 @@ static struct page *__dma_alloc_pages(struct device *dev, size_t size,
|
||||
size = PAGE_ALIGN(size);
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);
|
||||
if (use_dma_iommu(dev))
|
||||
return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp);
|
||||
if (!ops->alloc_pages_op)
|
||||
return NULL;
|
||||
return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp);
|
||||
@@ -635,6 +688,8 @@ static void __dma_free_pages(struct device *dev, size_t size, struct page *page,
|
||||
size = PAGE_ALIGN(size);
|
||||
if (dma_alloc_direct(dev, ops))
|
||||
dma_direct_free_pages(dev, size, page, dma_handle, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
dma_common_free_pages(dev, size, page, dma_handle, dir);
|
||||
else if (ops->free_pages)
|
||||
ops->free_pages(dev, size, page, dma_handle, dir);
|
||||
}
|
||||
@@ -697,6 +752,8 @@ struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
|
||||
|
||||
if (ops && ops->alloc_noncontiguous)
|
||||
sgt = ops->alloc_noncontiguous(dev, size, dir, gfp, attrs);
|
||||
else if (use_dma_iommu(dev))
|
||||
sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs);
|
||||
else
|
||||
sgt = alloc_single_sgt(dev, size, dir, gfp);
|
||||
|
||||
@@ -725,6 +782,8 @@ void dma_free_noncontiguous(struct device *dev, size_t size,
|
||||
debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
|
||||
if (ops && ops->free_noncontiguous)
|
||||
ops->free_noncontiguous(dev, size, sgt, dir);
|
||||
else if (use_dma_iommu(dev))
|
||||
iommu_dma_free_noncontiguous(dev, size, sgt, dir);
|
||||
else
|
||||
free_single_sgt(dev, size, sgt, dir);
|
||||
}
|
||||
@@ -772,6 +831,8 @@ static int dma_supported(struct device *dev, u64 mask)
|
||||
{
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
|
||||
if (WARN_ON(ops && use_dma_iommu(dev)))
|
||||
return false;
|
||||
/*
|
||||
* ->dma_supported sets the bypass flag, so we must always call
|
||||
* into the method here unless the device is truly direct mapped.
|
||||
@@ -787,17 +848,14 @@ bool dma_pci_p2pdma_supported(struct device *dev)
|
||||
{
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
|
||||
/* if ops is not set, dma direct will be used which supports P2PDMA */
|
||||
if (!ops)
|
||||
return true;
|
||||
|
||||
/*
|
||||
* Note: dma_ops_bypass is not checked here because P2PDMA should
|
||||
* not be used with dma mapping ops that do not have support even
|
||||
* if the specific device is bypassing them.
|
||||
*/
|
||||
|
||||
return ops->flags & DMA_F_PCI_P2PDMA_SUPPORTED;
|
||||
/* if ops is not set, dma direct and default IOMMU support P2PDMA */
|
||||
return !ops;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported);
|
||||
|
||||
@@ -865,6 +923,8 @@ size_t dma_max_mapping_size(struct device *dev)
|
||||
|
||||
if (dma_map_direct(dev, ops))
|
||||
size = dma_direct_max_mapping_size(dev);
|
||||
else if (use_dma_iommu(dev))
|
||||
size = iommu_dma_max_mapping_size(dev);
|
||||
else if (ops && ops->max_mapping_size)
|
||||
size = ops->max_mapping_size(dev);
|
||||
|
||||
@@ -877,7 +937,9 @@ size_t dma_opt_mapping_size(struct device *dev)
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
size_t size = SIZE_MAX;
|
||||
|
||||
if (ops && ops->opt_mapping_size)
|
||||
if (use_dma_iommu(dev))
|
||||
size = iommu_dma_opt_mapping_size();
|
||||
else if (ops && ops->opt_mapping_size)
|
||||
size = ops->opt_mapping_size();
|
||||
|
||||
return min(dma_max_mapping_size(dev), size);
|
||||
@@ -888,6 +950,9 @@ unsigned long dma_get_merge_boundary(struct device *dev)
|
||||
{
|
||||
const struct dma_map_ops *ops = get_dma_ops(dev);
|
||||
|
||||
if (use_dma_iommu(dev))
|
||||
return iommu_dma_get_merge_boundary(dev);
|
||||
|
||||
if (!ops || !ops->get_merge_boundary)
|
||||
return 0; /* can't merge */
|
||||
|
||||
|
||||
Reference in New Issue
Block a user