--- v11
+++ v7
@@ -1,100 +1,173 @@
-Propagate the swiotlb_force setting into io_tlb_default_mem->force and
-use it to determine whether to bounce the data or not. This will be
-useful later to allow for different pools.
+Update is_swiotlb_buffer to add a struct device argument. This will be
+useful later to allow for restricted DMA pool.
Signed-off-by: Claire Chang <tientzu@chromium.org>
-Reviewed-by: Christoph Hellwig <hch@lst.de>
---
- include/linux/swiotlb.h | 11 +++++++++++
- kernel/dma/direct.c | 2 +-
- kernel/dma/direct.h | 2 +-
- kernel/dma/swiotlb.c | 4 ++++
- 4 files changed, 17 insertions(+), 2 deletions(-)
+ drivers/iommu/dma-iommu.c | 12 ++++++------
+ drivers/xen/swiotlb-xen.c | 2 +-
+ include/linux/swiotlb.h | 6 +++---
+ kernel/dma/direct.c | 6 +++---
+ kernel/dma/direct.h | 6 +++---
+ 5 files changed, 16 insertions(+), 16 deletions(-)
+diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
+index 7bcdd1205535..a5df35bfd150 100644
+--- a/drivers/iommu/dma-iommu.c
++++ b/drivers/iommu/dma-iommu.c
+@@ -504,7 +504,7 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr,
+
+ __iommu_dma_unmap(dev, dma_addr, size);
+
+- if (unlikely(is_swiotlb_buffer(phys)))
++ if (unlikely(is_swiotlb_buffer(dev, phys)))
+ swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
+ }
+
+@@ -575,7 +575,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
+ }
+
+ iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask);
+- if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(phys))
++ if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
+ swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs);
+ return iova;
+ }
+@@ -781,7 +781,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev,
+ if (!dev_is_dma_coherent(dev))
+ arch_sync_dma_for_cpu(phys, size, dir);
+
+- if (is_swiotlb_buffer(phys))
++ if (is_swiotlb_buffer(dev, phys))
+ swiotlb_sync_single_for_cpu(dev, phys, size, dir);
+ }
+
+@@ -794,7 +794,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev,
+ return;
+
+ phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
+- if (is_swiotlb_buffer(phys))
++ if (is_swiotlb_buffer(dev, phys))
+ swiotlb_sync_single_for_device(dev, phys, size, dir);
+
+ if (!dev_is_dma_coherent(dev))
+@@ -815,7 +815,7 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev,
+ if (!dev_is_dma_coherent(dev))
+ arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
+
+- if (is_swiotlb_buffer(sg_phys(sg)))
++ if (is_swiotlb_buffer(dev, sg_phys(sg)))
+ swiotlb_sync_single_for_cpu(dev, sg_phys(sg),
+ sg->length, dir);
+ }
+@@ -832,7 +832,7 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,
+ return;
+
+ for_each_sg(sgl, sg, nelems, i) {
+- if (is_swiotlb_buffer(sg_phys(sg)))
++ if (is_swiotlb_buffer(dev, sg_phys(sg)))
+ swiotlb_sync_single_for_device(dev, sg_phys(sg),
+ sg->length, dir);
+
+diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
+index 4c89afc0df62..0c6ed09f8513 100644
+--- a/drivers/xen/swiotlb-xen.c
++++ b/drivers/xen/swiotlb-xen.c
+@@ -100,7 +100,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
+ * in our domain. Therefore _only_ check address within our domain.
+ */
+ if (pfn_valid(PFN_DOWN(paddr)))
+- return is_swiotlb_buffer(paddr);
++ return is_swiotlb_buffer(dev, paddr);
+ return 0;
+ }
+
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
-index dd1c30a83058..efcd56e3a16c 100644
+index b469f04cca26..2a6cca07540b 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
-@@ -84,6 +84,7 @@ extern enum swiotlb_force swiotlb_force;
- * unmap calls.
- * @debugfs: The dentry to debugfs.
- * @late_alloc: %true if allocated using the page allocator
-+ * @force: %true if swiotlb is forced
- */
- struct io_tlb_mem {
- phys_addr_t start;
-@@ -94,6 +95,7 @@ struct io_tlb_mem {
- spinlock_t lock;
- struct dentry *debugfs;
- bool late_alloc;
-+ bool force;
- struct io_tlb_slot {
- phys_addr_t orig_addr;
- size_t alloc_size;
-@@ -109,6 +111,11 @@ static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
+@@ -113,9 +113,9 @@ static inline struct io_tlb_mem *get_io_tlb_mem(struct device *dev)
+ return io_tlb_default_mem;
+ }
+
+-static inline bool is_swiotlb_buffer(phys_addr_t paddr)
++static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
+ {
+- struct io_tlb_mem *mem = io_tlb_default_mem;
++ struct io_tlb_mem *mem = get_io_tlb_mem(dev);
+
return mem && paddr >= mem->start && paddr < mem->end;
}
-
-+static inline bool is_dev_swiotlb_force(struct device *dev)
-+{
-+ return dev->dma_io_tlb_mem->force;
-+}
-+
- void __init swiotlb_exit(void);
- unsigned int swiotlb_max_segment(void);
- size_t swiotlb_max_mapping_size(struct device *dev);
-@@ -120,6 +127,10 @@ static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
+@@ -127,7 +127,7 @@ bool is_swiotlb_active(void);
+ void __init swiotlb_adjust_size(unsigned long size);
+ #else
+ #define swiotlb_force SWIOTLB_NO_FORCE
+-static inline bool is_swiotlb_buffer(phys_addr_t paddr)
++static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
return false;
}
-+static inline bool is_dev_swiotlb_force(struct device *dev)
-+{
-+ return false;
-+}
- static inline void swiotlb_exit(void)
- {
- }
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
-index 7a88c34d0867..3713461d6fe0 100644
+index f737e3347059..84c9feb5474a 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
-@@ -496,7 +496,7 @@ size_t dma_direct_max_mapping_size(struct device *dev)
+@@ -343,7 +343,7 @@ void dma_direct_sync_sg_for_device(struct device *dev,
+ for_each_sg(sgl, sg, nents, i) {
+ phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
+
+- if (unlikely(is_swiotlb_buffer(paddr)))
++ if (unlikely(is_swiotlb_buffer(dev, paddr)))
+ swiotlb_sync_single_for_device(dev, paddr, sg->length,
+ dir);
+
+@@ -369,7 +369,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
+ if (!dev_is_dma_coherent(dev))
+ arch_sync_dma_for_cpu(paddr, sg->length, dir);
+
+- if (unlikely(is_swiotlb_buffer(paddr)))
++ if (unlikely(is_swiotlb_buffer(dev, paddr)))
+ swiotlb_sync_single_for_cpu(dev, paddr, sg->length,
+ dir);
+
+@@ -504,7 +504,7 @@ size_t dma_direct_max_mapping_size(struct device *dev)
+ bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr)
{
- /* If SWIOTLB is active, use its maximum mapping size */
- if (is_swiotlb_active(dev) &&
-- (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
-+ (dma_addressing_limited(dev) || is_dev_swiotlb_force(dev)))
- return swiotlb_max_mapping_size(dev);
- return SIZE_MAX;
+ return !dev_is_dma_coherent(dev) ||
+- is_swiotlb_buffer(dma_to_phys(dev, dma_addr));
++ is_swiotlb_buffer(dev, dma_to_phys(dev, dma_addr));
}
+
+ /**
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
-index 13e9e7158d94..6c4d13caceb1 100644
+index 50afc05b6f1d..13e9e7158d94 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
-@@ -87,7 +87,7 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev,
- phys_addr_t phys = page_to_phys(page) + offset;
- dma_addr_t dma_addr = phys_to_dma(dev, phys);
+@@ -56,7 +56,7 @@ static inline void dma_direct_sync_single_for_device(struct device *dev,
+ {
+ phys_addr_t paddr = dma_to_phys(dev, addr);
-- if (unlikely(swiotlb_force == SWIOTLB_FORCE))
-+ if (is_dev_swiotlb_force(dev))
- return swiotlb_map(dev, phys, size, dir, attrs);
+- if (unlikely(is_swiotlb_buffer(paddr)))
++ if (unlikely(is_swiotlb_buffer(dev, paddr)))
+ swiotlb_sync_single_for_device(dev, paddr, size, dir);
- if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
-diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
-index 101abeb0a57d..a9907ac262fc 100644
---- a/kernel/dma/swiotlb.c
-+++ b/kernel/dma/swiotlb.c
-@@ -179,6 +179,10 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
- mem->end = mem->start + bytes;
- mem->index = 0;
- mem->late_alloc = late_alloc;
-+
-+ if (swiotlb_force == SWIOTLB_FORCE)
-+ mem->force = true;
-+
- spin_lock_init(&mem->lock);
- for (i = 0; i < mem->nslabs; i++) {
- mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(i);
+ if (!dev_is_dma_coherent(dev))
+@@ -73,7 +73,7 @@ static inline void dma_direct_sync_single_for_cpu(struct device *dev,
+ arch_sync_dma_for_cpu_all();
+ }
+
+- if (unlikely(is_swiotlb_buffer(paddr)))
++ if (unlikely(is_swiotlb_buffer(dev, paddr)))
+ swiotlb_sync_single_for_cpu(dev, paddr, size, dir);
+
+ if (dir == DMA_FROM_DEVICE)
+@@ -113,7 +113,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+ dma_direct_sync_single_for_cpu(dev, addr, size, dir);
+
+- if (unlikely(is_swiotlb_buffer(phys)))
++ if (unlikely(is_swiotlb_buffer(dev, phys)))
+ swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
+ }
+ #endif /* _KERNEL_DMA_DIRECT_H */
--
-2.32.0.272.g935e593368-goog
+2.31.1.751.gd2f1c929bd-goog