[RFC PATCH 0/2] DMA pagecache

@@ -9,4 +9,7 @@ enum dma_data_direction {
 	DMA_NONE = 3,
 };
 
+/* Checks if wanted direction is satisfied by current mapping direction*/
+#define DMA_DIR_COMPAT(current, wanted)	(((current) & ~(wanted)) == 0)
+
 #endif

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _IOMMU_CACHE_H
+#define _IOMMU_CACHE_H
+#ifdef __KERNEL__
+
+#include <linux/llist.h>
+#include <linux/xarray.h>
+#include <linux/atomic.h>
+
+struct dmacache {
+	struct llist_head fifo_add;
+	struct llist_head fifo_del;
+	struct xarray cpupages;
+	struct xarray dmapages;
+	atomic64_t cachesize;
+};
+
+#include <asm/iommu.h>
+
+void iommu_cache_init(struct iommu_table *tbl);
+void iommu_dmacache_add(struct iommu_table *tbl, void *page, unsigned int npages, dma_addr_t addr,
+			enum dma_data_direction direction);
+dma_addr_t iommu_dmacache_use(struct iommu_table *tbl, void *page, unsigned int npages,
+			      enum dma_data_direction direction);
+void iommu_dmacache_free(struct iommu_table *tbl, dma_addr_t dma_handle, unsigned int npages);
+
+#define IOMMU_MAP_LIST_MAX	8192
+#define IOMMU_MAP_LIST_THRES	128
+
+#endif /* __KERNEL__ */
+#endif /* _IOMMU_CACHE_H */

@@ -18,6 +18,7 @@
 #include <asm/types.h>
 #include <asm/pci-bridge.h>
 #include <asm/asm-const.h>
+#include <asm/iommu-cache.h>
 
 #define IOMMU_PAGE_SHIFT_4K      12
 #define IOMMU_PAGE_SIZE_4K       (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K)

@@ -114,6 +115,7 @@ struct iommu_table {
 	int it_nid;
 	unsigned long it_reserved_start; /* Start of not-DMA-able (MMIO) area */
 	unsigned long it_reserved_end;
+	struct dmacache cache;
 };
 
 #define IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry) \

@@ -317,6 +319,8 @@ extern void iommu_release_ownership(struct iommu_table *tbl);
 extern enum dma_data_direction iommu_tce_direction(unsigned long tce);
 extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir);
 
+void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages);
+
 #ifdef CONFIG_PPC_CELL_NATIVE
 extern bool iommu_fixed_is_weak;
 #else

@@ -104,7 +104,7 @@ extra-y				+= vmlinux.lds
 obj-$(CONFIG_RELOCATABLE)	+= reloc_$(BITS).o
 
 obj-$(CONFIG_PPC32)		+= entry_32.o setup_32.o early_32.o
-obj-$(CONFIG_PPC64)		+= dma-iommu.o iommu.o
+obj-$(CONFIG_PPC64)		+= dma-iommu.o iommu.o iommu-cache.o
 obj-$(CONFIG_KGDB)		+= kgdb.o
 obj-$(CONFIG_BOOTX_TEXT)	+= btext.o
 obj-$(CONFIG_SMP)		+= smp.o

@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <asm/iommu-cache.h>
+
+struct dma_mapping {
+	struct llist_node mapping;
+	struct llist_node fifo;
+	unsigned long dmapage;
+	unsigned long cpupage;
+	unsigned long size;
+	refcount_t count;
+	enum dma_data_direction direction;
+
+};
+
+struct cpupage_entry {
+	struct llist_node node;
+	struct dma_mapping *data;
+};
+
+/**
+ * iommu_dmacache_use() - Looks for a DMA mapping in cache
+ * @tbl: Device's iommu_table.
+ * @page: Address for which a DMA mapping is desired.
+ * @npages: Page count needed from that address
+ * @direction: DMA direction needed for the mapping
+ *
+ * Looks into the DMA cache for a page/range that is already mapped with given direction.
+ *
+ * Return: DMA mapping for range/direction present in cache
+ *	   DMA_MAPPING_ERROR if not found.
+ */
+dma_addr_t iommu_dmacache_use(struct iommu_table *tbl, void *page, unsigned int npages,
+			      enum dma_data_direction direction)
+{
+	struct cpupage_entry *e;
+	struct dma_mapping *d;
+	const unsigned long start = (unsigned long)page >> tbl->it_page_shift;
+	const unsigned long end = start + npages;
+
+	e = xa_load(&tbl->cache.cpupages, start);
+	if (!e)
+		return DMA_MAPPING_ERROR;
+
+	llist_for_each_entry(e, &e->node, node) {
+		d = e->data;
+		if (start < d->cpupage || end > (d->cpupage + d->size) ||
+		    !DMA_DIR_COMPAT(d->direction, direction))
+			continue;
+
+		refcount_inc(&d->count);
+		return (d->dmapage + start - d->cpupage) << tbl->it_page_shift;
+	}
+
+	return DMA_MAPPING_ERROR;
+}
+
+/**
+ * iommu_dmacache_entry_remove() - Remove a dma mapping from cpupage & dmapage XArrays
+ * @cache: Device's dmacache.
+ * @d: dma_mapping to be removed
+ */
+static void iommu_dmacache_entry_remove(struct dmacache *cache, struct dma_mapping *d)
+{
+	struct cpupage_entry *e, *tmp;
+	dma_addr_t dp = d->dmapage;
+	dma_addr_t end = dp + d->size;
+	unsigned long cp = d->cpupage;
+
+	for (; dp < end; dp++, cp++) {
+		e = xa_erase(&cache->cpupages, cp);
+		if (e && e->node.next) {
+			tmp = llist_entry(e->node.next, struct cpupage_entry, node);
+			xa_store(&cache->cpupages, cp, tmp, GFP_KERNEL);
+		}
+		xa_erase(&cache->dmapages, dp);
+		kfree(e);
+	}
+}
+
+/**
+ * iommu_dmacache_clean() - Clean count mappings from dmacache fifo
+ * @tbl: Device's iommu_table.
+ * @count: number of entries to be removed.
+ */
+static void iommu_dmacache_clean(struct iommu_table *tbl, const long count)
+{
+	struct dma_mapping *d, *tmp;
+	struct llist_node *n;
+	struct dmacache *cache = &tbl->cache;
+	unsigned long removed = 0;
+
+	n = llist_del_all(&cache->fifo_del);
+
+	if (!n)
+		return;
+
+	llist_for_each_entry_safe(d, tmp, n, fifo) {
+		switch (refcount_read(&d->count)) {
+		case 0:
+			/* Fully remove entry */
+			iommu_dmacache_entry_remove(cache, d);
+			__iommu_free(tbl, d->dmapage << tbl->it_page_shift, d->size);
+			kfree(d);
+			removed++;
+			break;
+		case 1:
+			/* Remove entry but don't undo mapping */
+			iommu_dmacache_entry_remove(cache, d);
+			kfree(d);
+			removed++;
+			break;
+		default:
+			/* In use. Re-add it to list. */
+			n = xchg(&tbl->cache.fifo_add.first, &d->fifo);
+			if (!n)
+				n->next = &d->fifo;
+
+			break;
+		}
+
+		if (removed >= count)
+			break;
+	}
+
+	atomic64_sub(removed, &tbl->cache.cachesize);
+
+	xchg(&tbl->cache.fifo_del.first, &tmp->fifo);
+}
+
+/**
+ * iommu_dmacache_free() - Decrement a mapping usage from dmacache and clean when full
+ * @tbl: Device's iommu_table.
+ * @dma_handle: DMA address from the mapping.
+ * @npages: Page count from that address
+ *
+ * Decrements a refcount for a mapping in this dma_handle + npages, and remove
+ * some unused dma mappings from dmacache fifo.
+ */
+void iommu_dmacache_free(struct iommu_table *tbl, dma_addr_t dma_handle,	unsigned int npages)
+{
+	struct dma_mapping *d;
+	long exceeding;
+
+	d = xa_load(&tbl->cache.dmapages, dma_handle >> tbl->it_page_shift);
+	if (!d) {
+		/* Not in list, just free */
+		__iommu_free(tbl, dma_handle, npages);
+		return;
+	}
+
+	refcount_dec(&d->count);
+
+	exceeding = atomic64_read(&tbl->cache.cachesize) - IOMMU_MAP_LIST_MAX;
+
+	if (exceeding > 0)
+		iommu_dmacache_clean(tbl, exceeding + IOMMU_MAP_LIST_THRES);
+}
+
+/**
+ * iommu_dmacache_add() - Create and add a new dma mapping into cache.
+ * @tbl: Device's iommu_table.
+ * @page: Address for which a DMA mapping was created.
+ * @npages: Page count mapped from that address
+ * @addr: DMA address created for that mapping
+ * @direction: DMA direction for the mapping created
+ *
+ * Create a dma_mapping and add it to dmapages and cpupages XArray, then add it to fifo.
+ * On both cpupages and dmapages, an entry will be created for each page in the mapping.
+ * On cpupages, as there may exist many mappings for a single cpupage, each entry has a llist
+ * that starts at the last mapped entry.
+ *
+ */
+void iommu_dmacache_add(struct iommu_table *tbl, void *page, unsigned int npages, dma_addr_t addr,
+			enum dma_data_direction direction)
+{
+	struct dma_mapping *d, *tmp;
+	struct cpupage_entry *e, *old;
+	struct llist_node *n;
+	unsigned long p = (unsigned long)page;
+	unsigned int i;
+
+	d = kmalloc(sizeof(*d), GFP_KERNEL);
+	if (!d)
+		return;
+
+	d->cpupage = (unsigned long)p >> tbl->it_page_shift;
+	d->dmapage = (unsigned long)addr >> tbl->it_page_shift;
+	d->size = npages;
+	d->direction = direction;
+	d->fifo.next = NULL;
+	refcount_set(&d->count, 1);
+
+	p = d->cpupage;
+	addr = d->dmapage;
+
+	for (i = 0; i < npages ; i++) {
+		/* Only one mapping may exist for a DMA address*/
+		tmp = xa_store(&tbl->cache.dmapages, addr++, d, GFP_KERNEL);
+		if (xa_is_err(tmp))
+			break;
+
+		/* Multiple mappings may exist for a page, get them in a list*/
+		e = kmalloc(sizeof(*e), GFP_KERNEL);
+		if (!d)
+			break;
+
+		e->data = d;
+		old = xa_store(&tbl->cache.cpupages, p++, e, GFP_KERNEL);
+		e->node.next = &old->node;
+
+		if (xa_is_err(old)) {
+			kfree(e);
+			break;
+		}
+	}
+
+	n = xchg(&tbl->cache.fifo_add.first, &d->fifo);
+	if (!n)
+		n->next = &d->fifo;
+
+	atomic64_inc(&tbl->cache.cachesize);
+}
+
+void iommu_cache_init(struct iommu_table *tbl)
+{
+	struct dma_mapping *d;
+
+	init_llist_head(&tbl->cache.fifo_add);
+	init_llist_head(&tbl->cache.fifo_del);
+
+	/* First entry for linking both llist_heads */
+	d = kmalloc(sizeof(*d), GFP_KERNEL);
+	if (!d)
+		panic("%s: Can't allocate %ld bytes\n", __func__, sizeof(*d));
+
+	d->cpupage = -1UL;
+	d->dmapage = -1UL;
+	refcount_set(&d->count, 1);
+	llist_add(&d->fifo, &tbl->cache.fifo_add);
+	llist_add(&d->fifo, &tbl->cache.fifo_del);
+
+	xa_init(&tbl->cache.cpupages);
+	xa_init(&tbl->cache.dmapages);
+
+	atomic64_set(&tbl->cache.cachesize, 0);
+}

@@ -40,8 +40,6 @@
 
 static int novmerge;
 
-static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int);
-
 static int __init setup_iommu(char *str)
 {
 	if (!strcmp(str, "novmerge"))

@@ -308,6 +306,10 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
 	dma_addr_t ret = DMA_MAPPING_ERROR;
 	int build_fail;
 
+	ret = iommu_dmacache_use(tbl, page, npages, direction);
+	if (ret != DMA_MAPPING_ERROR)
+		return ret;
+
 	entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
 
 	if (unlikely(entry == DMA_MAPPING_ERROR))

@@ -338,6 +340,8 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
 	/* Make sure updates are seen by hardware */
 	mb();
 
+	iommu_dmacache_add(tbl, page, npages, ret, direction);
+
 	return ret;
 }
 

@@ -388,8 +392,7 @@ static struct iommu_pool *get_pool(struct iommu_table *tbl,
 	return p;
 }
 
-static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
-			 unsigned int npages)
+void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages)
 {
 	unsigned long entry, free_entry;
 	unsigned long flags;

@@ -413,7 +416,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
 		unsigned int npages)
 {
-	__iommu_free(tbl, dma_addr, npages);
+	iommu_dmacache_free(tbl, dma_addr, npages);
 
 	/* Make sure TLB cache is flushed if the HW needs it. We do
 	 * not do an mb() here on purpose, it is not needed on any of

@@ -719,6 +722,8 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid,
 
 	iommu_table_clear(tbl);
 
+	iommu_cache_init(tbl);
+
 	if (!welcomed) {
 		printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n",
 		       novmerge ? "disabled" : "enabled");