--- v13
+++ v7
@@ -1,184 +1,140 @@
-Update is_swiotlb_buffer to add a struct device argument. This will be
-useful later to allow for different pools.
+Add the initialization function to create restricted DMA pools from
+matching reserved-memory nodes.
Signed-off-by: Claire Chang <tientzu@chromium.org>
-Reviewed-by: Christoph Hellwig <hch@lst.de>
-Tested-by: Stefano Stabellini <sstabellini@kernel.org>
-Tested-by: Will Deacon <will@kernel.org>
---
- drivers/iommu/dma-iommu.c | 12 ++++++------
- drivers/xen/swiotlb-xen.c | 2 +-
- include/linux/swiotlb.h | 7 ++++---
- kernel/dma/direct.c | 6 +++---
- kernel/dma/direct.h | 6 +++---
- 5 files changed, 17 insertions(+), 16 deletions(-)
+ include/linux/device.h | 4 +++
+ include/linux/swiotlb.h | 3 +-
+ kernel/dma/swiotlb.c | 76 +++++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 82 insertions(+), 1 deletion(-)
-diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
-index 3087d9fa6065..10997ef541f8 100644
---- a/drivers/iommu/dma-iommu.c
-+++ b/drivers/iommu/dma-iommu.c
-@@ -507,7 +507,7 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr,
-
- __iommu_dma_unmap(dev, dma_addr, size);
-
-- if (unlikely(is_swiotlb_buffer(phys)))
-+ if (unlikely(is_swiotlb_buffer(dev, phys)))
- swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
- }
-
-@@ -578,7 +578,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
- }
-
- iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask);
-- if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(phys))
-+ if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
- swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs);
- return iova;
- }
-@@ -749,7 +749,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev,
- if (!dev_is_dma_coherent(dev))
- arch_sync_dma_for_cpu(phys, size, dir);
-
-- if (is_swiotlb_buffer(phys))
-+ if (is_swiotlb_buffer(dev, phys))
- swiotlb_sync_single_for_cpu(dev, phys, size, dir);
- }
-
-@@ -762,7 +762,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev,
- return;
-
- phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
-- if (is_swiotlb_buffer(phys))
-+ if (is_swiotlb_buffer(dev, phys))
- swiotlb_sync_single_for_device(dev, phys, size, dir);
-
- if (!dev_is_dma_coherent(dev))
-@@ -783,7 +783,7 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev,
- if (!dev_is_dma_coherent(dev))
- arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
-
-- if (is_swiotlb_buffer(sg_phys(sg)))
-+ if (is_swiotlb_buffer(dev, sg_phys(sg)))
- swiotlb_sync_single_for_cpu(dev, sg_phys(sg),
- sg->length, dir);
- }
-@@ -800,7 +800,7 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,
- return;
-
- for_each_sg(sgl, sg, nelems, i) {
-- if (is_swiotlb_buffer(sg_phys(sg)))
-+ if (is_swiotlb_buffer(dev, sg_phys(sg)))
- swiotlb_sync_single_for_device(dev, sg_phys(sg),
- sg->length, dir);
-
-diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
-index 4c89afc0df62..0c6ed09f8513 100644
---- a/drivers/xen/swiotlb-xen.c
-+++ b/drivers/xen/swiotlb-xen.c
-@@ -100,7 +100,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
- * in our domain. Therefore _only_ check address within our domain.
- */
- if (pfn_valid(PFN_DOWN(paddr)))
-- return is_swiotlb_buffer(paddr);
-+ return is_swiotlb_buffer(dev, paddr);
- return 0;
- }
-
+diff --git a/include/linux/device.h b/include/linux/device.h
+index 38a2071cf776..4987608ea4ff 100644
+--- a/include/linux/device.h
++++ b/include/linux/device.h
+@@ -416,6 +416,7 @@ struct dev_links_info {
+ * @dma_pools: Dma pools (if dma'ble device).
+ * @dma_mem: Internal for coherent mem override.
+ * @cma_area: Contiguous memory area for dma allocations
++ * @dma_io_tlb_mem: Internal for swiotlb io_tlb_mem override.
+ * @archdata: For arch-specific additions.
+ * @of_node: Associated device tree node.
+ * @fwnode: Associated device node supplied by platform firmware.
+@@ -521,6 +522,9 @@ struct device {
+ #ifdef CONFIG_DMA_CMA
+ struct cma *cma_area; /* contiguous memory area for dma
+ allocations */
++#endif
++#ifdef CONFIG_DMA_RESTRICTED_POOL
++ struct io_tlb_mem *dma_io_tlb_mem;
+ #endif
+ /* arch specific additions */
+ struct dev_archdata archdata;
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
-index 216854a5e513..d1f3d95881cd 100644
+index 216854a5e513..03ad6e3b4056 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
-@@ -2,6 +2,7 @@
- #ifndef __LINUX_SWIOTLB_H
- #define __LINUX_SWIOTLB_H
+@@ -72,7 +72,8 @@ extern enum swiotlb_force swiotlb_force;
+ * range check to see if the memory was in fact allocated by this
+ * API.
+ * @nslabs: The number of IO TLB blocks (in groups of 64) between @start and
+- * @end. This is command line adjustable via setup_io_tlb_npages.
++ * @end. For default swiotlb, this is command line adjustable via
++ * setup_io_tlb_npages.
+ * @used: The number of used IO TLB block.
+ * @list: The free list describing the number of free entries available
+ * from each index.
+diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
+index b849b01a446f..1d8eb4de0d01 100644
+--- a/kernel/dma/swiotlb.c
++++ b/kernel/dma/swiotlb.c
+@@ -39,6 +39,13 @@
+ #ifdef CONFIG_DEBUG_FS
+ #include <linux/debugfs.h>
+ #endif
++#ifdef CONFIG_DMA_RESTRICTED_POOL
++#include <linux/io.h>
++#include <linux/of.h>
++#include <linux/of_fdt.h>
++#include <linux/of_reserved_mem.h>
++#include <linux/slab.h>
++#endif
-+#include <linux/device.h>
- #include <linux/dma-direction.h>
- #include <linux/init.h>
- #include <linux/types.h>
-@@ -101,9 +102,9 @@ struct io_tlb_mem {
- };
- extern struct io_tlb_mem *io_tlb_default_mem;
+ #include <asm/io.h>
+ #include <asm/dma.h>
+@@ -690,3 +697,72 @@ static int __init swiotlb_create_default_debugfs(void)
+ late_initcall(swiotlb_create_default_debugfs);
--static inline bool is_swiotlb_buffer(phys_addr_t paddr)
-+static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
- {
-- struct io_tlb_mem *mem = io_tlb_default_mem;
-+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
-
- return mem && paddr >= mem->start && paddr < mem->end;
- }
-@@ -115,7 +116,7 @@ bool is_swiotlb_active(void);
- void __init swiotlb_adjust_size(unsigned long size);
- #else
- #define swiotlb_force SWIOTLB_NO_FORCE
--static inline bool is_swiotlb_buffer(phys_addr_t paddr)
-+static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
- {
- return false;
- }
-diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
-index f737e3347059..84c9feb5474a 100644
---- a/kernel/dma/direct.c
-+++ b/kernel/dma/direct.c
-@@ -343,7 +343,7 @@ void dma_direct_sync_sg_for_device(struct device *dev,
- for_each_sg(sgl, sg, nents, i) {
- phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
-
-- if (unlikely(is_swiotlb_buffer(paddr)))
-+ if (unlikely(is_swiotlb_buffer(dev, paddr)))
- swiotlb_sync_single_for_device(dev, paddr, sg->length,
- dir);
-
-@@ -369,7 +369,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
- if (!dev_is_dma_coherent(dev))
- arch_sync_dma_for_cpu(paddr, sg->length, dir);
-
-- if (unlikely(is_swiotlb_buffer(paddr)))
-+ if (unlikely(is_swiotlb_buffer(dev, paddr)))
- swiotlb_sync_single_for_cpu(dev, paddr, sg->length,
- dir);
-
-@@ -504,7 +504,7 @@ size_t dma_direct_max_mapping_size(struct device *dev)
- bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr)
- {
- return !dev_is_dma_coherent(dev) ||
-- is_swiotlb_buffer(dma_to_phys(dev, dma_addr));
-+ is_swiotlb_buffer(dev, dma_to_phys(dev, dma_addr));
- }
-
- /**
-diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
-index 50afc05b6f1d..13e9e7158d94 100644
---- a/kernel/dma/direct.h
-+++ b/kernel/dma/direct.h
-@@ -56,7 +56,7 @@ static inline void dma_direct_sync_single_for_device(struct device *dev,
- {
- phys_addr_t paddr = dma_to_phys(dev, addr);
-
-- if (unlikely(is_swiotlb_buffer(paddr)))
-+ if (unlikely(is_swiotlb_buffer(dev, paddr)))
- swiotlb_sync_single_for_device(dev, paddr, size, dir);
-
- if (!dev_is_dma_coherent(dev))
-@@ -73,7 +73,7 @@ static inline void dma_direct_sync_single_for_cpu(struct device *dev,
- arch_sync_dma_for_cpu_all();
- }
-
-- if (unlikely(is_swiotlb_buffer(paddr)))
-+ if (unlikely(is_swiotlb_buffer(dev, paddr)))
- swiotlb_sync_single_for_cpu(dev, paddr, size, dir);
-
- if (dir == DMA_FROM_DEVICE)
-@@ -113,7 +113,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
- if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
- dma_direct_sync_single_for_cpu(dev, addr, size, dir);
-
-- if (unlikely(is_swiotlb_buffer(phys)))
-+ if (unlikely(is_swiotlb_buffer(dev, phys)))
- swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
- }
- #endif /* _KERNEL_DMA_DIRECT_H */
+ #endif
++
++#ifdef CONFIG_DMA_RESTRICTED_POOL
++static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
++ struct device *dev)
++{
++ struct io_tlb_mem *mem = rmem->priv;
++ unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
++
++ if (dev->dma_io_tlb_mem)
++ return 0;
++
++ /*
++ * Since multiple devices can share the same pool, the private data,
++ * io_tlb_mem struct, will be initialized by the first device attached
++ * to it.
++ */
++ if (!mem) {
++ mem = kzalloc(struct_size(mem, slots, nslabs), GFP_KERNEL);
++ if (!mem)
++ return -ENOMEM;
++
++ if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
++ kfree(mem);
++ return -EINVAL;
++ }
++
++ swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, false);
++
++ rmem->priv = mem;
++
++ if (IS_ENABLED(CONFIG_DEBUG_FS))
++ swiotlb_create_debugfs(mem, rmem->name);
++ }
++
++ dev->dma_io_tlb_mem = mem;
++
++ return 0;
++}
++
++static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
++ struct device *dev)
++{
++ if (dev)
++ dev->dma_io_tlb_mem = NULL;
++}
++
++static const struct reserved_mem_ops rmem_swiotlb_ops = {
++ .device_init = rmem_swiotlb_device_init,
++ .device_release = rmem_swiotlb_device_release,
++};
++
++static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
++{
++ unsigned long node = rmem->fdt_node;
++
++ if (of_get_flat_dt_prop(node, "reusable", NULL) ||
++ of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
++ of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
++ of_get_flat_dt_prop(node, "no-map", NULL))
++ return -EINVAL;
++
++ rmem->ops = &rmem_swiotlb_ops;
++ pr_info("Reserved memory: created device swiotlb memory pool at %pa, size %ld MiB\n",
++ &rmem->base, (unsigned long)rmem->size / SZ_1M);
++ return 0;
++}
++
++RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
++#endif /* CONFIG_DMA_RESTRICTED_POOL */
--
-2.32.0.288.g62a8d224e6-goog
+2.31.1.751.gd2f1c929bd-goog