--- v5
+++ v10
@@ -1,263 +1,458 @@
-TCE tables might get too big in case of 4K IOMMU pages and DDW enabled
-on huge guests (hundreds of GB of RAM) so the kernel might be unable to
-allocate contiguous chunk of physical memory to store the TCE table.
-
-To address this, POWER8 CPU (actually, IODA2) supports multi-level TCE tables,
-up to 5 levels which splits the table into a tree of smaller subtables.
-
-This adds multi-level TCE tables support to pnv_pci_ioda2_create_table()
-and pnv_pci_ioda2_free_table() callbacks.
+At the moment writing new TCE value to the IOMMU table fails with EBUSY
+if there is a valid entry already. However PAPR specification allows
+the guest to write new TCE value without clearing it first.
+
+Another problem this patch is addressing is the use of pool locks for
+external IOMMU users such as VFIO. The pool locks are to protect
+DMA page allocator rather than entries and since the host kernel does
+not control what pages are in use, there is no point in pool locks and
+exchange()+put_page(oldtce) is sufficient to avoid possible races.
+
+This adds an exchange() callback to iommu_table_ops which does the same
+thing as set() plus it returns replaced TCE and DMA direction so
+the caller can release the pages afterwards. The exchange() receives
+a physical address unlike set() which receives linear mapping address;
+and returns a physical address as the clear() does.
+
+This implements exchange() for P5IOC2/IODA/IODA2. This adds a requirement
+for a platform to have exchange() implemented in order to support VFIO.
+
+This replaces iommu_tce_build() and iommu_clear_tce() with
+a single iommu_tce_xchg().
+
+This makes sure that TCE permission bits are not set in TCE passed to
+IOMMU API as those are to be calculated by platform code from
+DMA direction.
+
+This moves SetPageDirty() to the IOMMU code to make it work for both
+VFIO ioctl interface in in-kernel TCE acceleration (when it becomes
+available later).
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
+[aw: for the vfio related changes]
+Acked-by: Alex Williamson <alex.williamson@redhat.com>
---
- arch/powerpc/include/asm/iommu.h | 2 +
- arch/powerpc/platforms/powernv/pci-ioda.c | 127 ++++++++++++++++++++++++------
- arch/powerpc/platforms/powernv/pci.c | 19 +++++
- 3 files changed, 122 insertions(+), 26 deletions(-)
+Changes:
+v10:
+* did s/tce/hpa/ in iommu_table_ops::exchange and tce_iommu_unuse_page()
+* removed permission bits check from iommu_tce_put_param_check as
+permission bits are not allowed in the address
+* added BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)) to pnv_tce_xchg()
+
+v9:
+* changed exchange() to work with physical addresses as these addresses
+are never accessed by the code and physical addresses are actual values
+we put into the IOMMU table
+---
+ arch/powerpc/include/asm/iommu.h | 22 +++++++++--
+ arch/powerpc/kernel/iommu.c | 59 +++++++++-------------------
+ arch/powerpc/platforms/powernv/pci-ioda.c | 34 ++++++++++++++++
+ arch/powerpc/platforms/powernv/pci-p5ioc2.c | 3 ++
+ arch/powerpc/platforms/powernv/pci.c | 18 +++++++++
+ arch/powerpc/platforms/powernv/pci.h | 2 +
+ drivers/vfio/vfio_iommu_spapr_tce.c | 60 +++++++++++++++++------------
+ 7 files changed, 130 insertions(+), 68 deletions(-)
diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
-index fd118ea..4007432 100644
+index c5375c5..d4ad118 100644
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
-@@ -88,6 +88,8 @@ struct iommu_pool {
- struct iommu_table {
- unsigned long it_busno; /* Bus number this table belongs to */
- unsigned long it_size; /* Size of iommu table in entries */
-+ unsigned long it_indirect_levels;
-+ unsigned long it_level_size;
- unsigned long it_offset; /* Offset into global table */
- unsigned long it_base; /* mapped address of tce table */
- unsigned long it_index; /* which iommu table this is */
+@@ -45,13 +45,29 @@ extern int iommu_is_off;
+ extern int iommu_force_on;
+
+ struct iommu_table_ops {
++ /*
++ * When called with direction==DMA_NONE, it is equal to clear().
++ * uaddr is a linear map address.
++ */
+ int (*set)(struct iommu_table *tbl,
+ long index, long npages,
+ unsigned long uaddr,
+ enum dma_data_direction direction,
+ struct dma_attrs *attrs);
++#ifdef CONFIG_IOMMU_API
++ /*
++ * Exchanges existing TCE with new TCE plus direction bits;
++ * returns old TCE and DMA direction mask.
++ * @tce is a physical address.
++ */
++ int (*exchange)(struct iommu_table *tbl,
++ long index,
++ unsigned long *hpa,
++ enum dma_data_direction *direction);
++#endif
+ void (*clear)(struct iommu_table *tbl,
+ long index, long npages);
++ /* get() returns a physical address */
+ unsigned long (*get)(struct iommu_table *tbl, long index);
+ void (*flush)(struct iommu_table *tbl);
+ };
+@@ -155,6 +171,8 @@ extern void iommu_register_group(struct iommu_table_group *table_group,
+ extern int iommu_add_device(struct device *dev);
+ extern void iommu_del_device(struct device *dev);
+ extern int __init tce_iommu_bus_notifier_init(void);
++extern long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,
++ unsigned long *hpa, enum dma_data_direction *direction);
+ #else
+ static inline void iommu_register_group(struct iommu_table_group *table_group,
+ int pci_domain_number,
+@@ -227,10 +245,6 @@ extern int iommu_tce_clear_param_check(struct iommu_table *tbl,
+ unsigned long npages);
+ extern int iommu_tce_put_param_check(struct iommu_table *tbl,
+ unsigned long ioba, unsigned long tce);
+-extern int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
+- unsigned long hwaddr, enum dma_data_direction direction);
+-extern unsigned long iommu_clear_tce(struct iommu_table *tbl,
+- unsigned long entry);
+
+ extern void iommu_flush_tce(struct iommu_table *tbl);
+ extern int iommu_take_ownership(struct iommu_table *tbl);
+diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
+index 6275164..1287d49 100644
+--- a/arch/powerpc/kernel/iommu.c
++++ b/arch/powerpc/kernel/iommu.c
+@@ -962,10 +962,7 @@ EXPORT_SYMBOL_GPL(iommu_tce_clear_param_check);
+ int iommu_tce_put_param_check(struct iommu_table *tbl,
+ unsigned long ioba, unsigned long tce)
+ {
+- if (!(tce & (TCE_PCI_WRITE | TCE_PCI_READ)))
+- return -EINVAL;
+-
+- if (tce & ~(IOMMU_PAGE_MASK(tbl) | TCE_PCI_WRITE | TCE_PCI_READ))
++ if (tce & ~IOMMU_PAGE_MASK(tbl))
+ return -EINVAL;
+
+ if (ioba & ~IOMMU_PAGE_MASK(tbl))
+@@ -982,44 +979,16 @@ int iommu_tce_put_param_check(struct iommu_table *tbl,
+ }
+ EXPORT_SYMBOL_GPL(iommu_tce_put_param_check);
+
+-unsigned long iommu_clear_tce(struct iommu_table *tbl, unsigned long entry)
++long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry,
++ unsigned long *hpa, enum dma_data_direction *direction)
+ {
+- unsigned long oldtce;
+- struct iommu_pool *pool = get_pool(tbl, entry);
++ long ret;
+
+- spin_lock(&(pool->lock));
++ ret = tbl->it_ops->exchange(tbl, entry, hpa, direction);
+
+- oldtce = tbl->it_ops->get(tbl, entry);
+- if (oldtce & (TCE_PCI_WRITE | TCE_PCI_READ))
+- tbl->it_ops->clear(tbl, entry, 1);
+- else
+- oldtce = 0;
+-
+- spin_unlock(&(pool->lock));
+-
+- return oldtce;
+-}
+-EXPORT_SYMBOL_GPL(iommu_clear_tce);
+-
+-/*
+- * hwaddr is a kernel virtual address here (0xc... bazillion),
+- * tce_build converts it to a physical address.
+- */
+-int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
+- unsigned long hwaddr, enum dma_data_direction direction)
+-{
+- int ret = -EBUSY;
+- unsigned long oldtce;
+- struct iommu_pool *pool = get_pool(tbl, entry);
+-
+- spin_lock(&(pool->lock));
+-
+- oldtce = tbl->it_ops->get(tbl, entry);
+- /* Add new entry if it is not busy */
+- if (!(oldtce & (TCE_PCI_WRITE | TCE_PCI_READ)))
+- ret = tbl->it_ops->set(tbl, entry, 1, hwaddr, direction, NULL);
+-
+- spin_unlock(&(pool->lock));
++ if (!ret && ((*direction == DMA_FROM_DEVICE) ||
++ (*direction == DMA_BIDIRECTIONAL)))
++ SetPageDirty(pfn_to_page(*hpa >> PAGE_SHIFT));
+
+ /* if (unlikely(ret))
+ pr_err("iommu_tce: %s failed on hwaddr=%lx ioba=%lx kva=%lx ret=%d\n",
+@@ -1028,13 +997,23 @@ int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
+
+ return ret;
+ }
+-EXPORT_SYMBOL_GPL(iommu_tce_build);
++EXPORT_SYMBOL_GPL(iommu_tce_xchg);
+
+ int iommu_take_ownership(struct iommu_table *tbl)
+ {
+ unsigned long flags, i, sz = (tbl->it_size + 7) >> 3;
+ int ret = 0;
+
++ /*
++ * VFIO does not control TCE entries allocation and the guest
++ * can write new TCEs on top of existing ones so iommu_tce_build()
++ * must be able to release old pages. This functionality
++ * requires exchange() callback defined so if it is not
++ * implemented, we disallow taking ownership over the table.
++ */
++ if (!tbl->it_ops->exchange)
++ return -EINVAL;
++
+ spin_lock_irqsave(&tbl->large_pool.lock, flags);
+ for (i = 0; i < tbl->nr_pools; i++)
+ spin_lock(&tbl->pools[i].lock);
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
-index 126d803..19b5f36 100644
+index 8e4987d..e3c784d 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
-@@ -47,6 +47,8 @@
- #include "powernv.h"
- #include "pci.h"
-
-+#define POWERNV_IOMMU_DEFAULT_LEVELS 1
-+
- extern void ioda_eeh_tvt_print(struct pnv_phb *phb);
-
- static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
-@@ -1333,16 +1335,79 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
- __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
- }
-
-+static void pnv_free_tce_table(unsigned long addr, unsigned size,
-+ unsigned level)
+@@ -1737,6 +1737,20 @@ static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
+ return ret;
+ }
+
++#ifdef CONFIG_IOMMU_API
++static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
++ unsigned long *hpa, enum dma_data_direction *direction)
+{
-+ addr &= ~(TCE_PCI_READ | TCE_PCI_WRITE);
-+
-+ if (level) {
-+ long i;
-+ u64 *tmp = (u64 *) addr;
-+
-+ for (i = 0; i < size; ++i) {
-+ unsigned long hpa = be64_to_cpu(tmp[i]);
-+
-+ if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE)))
-+ continue;
-+
-+ pnv_free_tce_table((unsigned long) __va(hpa),
-+ size, level - 1);
-+ }
-+ }
-+
-+ free_pages(addr, get_order(size << 3));
++ long ret = pnv_tce_xchg(tbl, index, hpa, direction);
++
++ if (!ret && (tbl->it_type &
++ (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
++ pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);
++
++ return ret;
+}
-+
-+static __be64 *pnv_alloc_tce_table(int nid,
-+ unsigned shift, unsigned levels, unsigned long *left)
++#endif
++
+ static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
+ long npages)
+ {
+@@ -1748,6 +1762,9 @@ static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
+
+ static struct iommu_table_ops pnv_ioda1_iommu_ops = {
+ .set = pnv_ioda1_tce_build,
++#ifdef CONFIG_IOMMU_API
++ .exchange = pnv_ioda1_tce_xchg,
++#endif
+ .clear = pnv_ioda1_tce_free,
+ .get = pnv_tce_get,
+ };
+@@ -1822,6 +1839,20 @@ static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
+ return ret;
+ }
+
++#ifdef CONFIG_IOMMU_API
++static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
++ unsigned long *hpa, enum dma_data_direction *direction)
+{
-+ struct page *tce_mem = NULL;
-+ __be64 *addr, *tmp;
-+ unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT;
-+ unsigned long chunk = 1UL << shift, i;
-+
-+ tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
-+ if (!tce_mem) {
-+ pr_err("Failed to allocate a TCE memory\n");
-+ return NULL;
-+ }
-+
-+ if (!*left)
-+ return NULL;
-+
-+ addr = page_address(tce_mem);
-+ memset(addr, 0, chunk);
-+
-+ --levels;
-+ if (!levels) {
-+ /* This is last level, actual TCEs */
-+ *left -= min(*left, chunk);
-+ return addr;
-+ }
-+
-+ for (i = 0; i < (chunk >> 3); ++i) {
-+ /* We allocated required TCEs, mark the rest "page fault" */
-+ if (!*left) {
-+ addr[i] = cpu_to_be64(0);
-+ continue;
-+ }
-+
-+ tmp = pnv_alloc_tce_table(nid, shift, levels, left);
-+ addr[i] = cpu_to_be64(__pa(tmp) |
-+ TCE_PCI_READ | TCE_PCI_WRITE);
-+ }
-+
-+ return addr;
++ long ret = pnv_tce_xchg(tbl, index, hpa, direction);
++
++ if (!ret && (tbl->it_type &
++ (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE)))
++ pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
++
++ return ret;
+}
-+
- static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
-- __u32 page_shift, __u64 window_size,
-+ __u32 page_shift, __u64 window_size, __u32 levels,
- struct iommu_table *tbl)
- {
- int nid = pe->phb->hose->node;
-- struct page *tce_mem = NULL;
- void *addr;
-- unsigned long tce_table_size;
-- int64_t rc;
-- unsigned order;
-+ unsigned long tce_table_size, left;
-+ unsigned shift;
-
- if ((page_shift != 12) && (page_shift != 16) && (page_shift != 24))
- return -EINVAL;
-@@ -1350,20 +1415,27 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
- if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
- return -EINVAL;
-
-+ if (!levels || (levels > 5))
-+ return -EINVAL;
-+
- tce_table_size = (window_size >> page_shift) * 8;
- tce_table_size = max(0x1000UL, tce_table_size);
-
- /* Allocate TCE table */
-- order = get_order(tce_table_size);
-+#define ROUND_UP(x, n) (((x) + (n) - 1u) & ~((n) - 1u))
-+ shift = ROUND_UP(ilog2(window_size) - page_shift, levels) / levels;
-+ shift += 3;
-+ shift = max_t(unsigned, shift, IOMMU_PAGE_SHIFT_4K);
-+ pr_info("Creating TCE table %08llx, %d levels, TCE table size = %lx\n",
-+ window_size, levels, 1UL << shift);
-
-- tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
-- if (!tce_mem) {
-- pr_err("Failed to allocate a TCE memory, order=%d\n", order);
-- rc = -ENOMEM;
-- goto fail;
-- }
-- addr = page_address(tce_mem);
-- memset(addr, 0, tce_table_size);
-+ tbl->it_level_size = 1ULL << (shift - 3);
-+ left = tce_table_size;
-+ addr = pnv_alloc_tce_table(nid, shift, levels, &left);
-+ if (!addr)
-+ return -ENOMEM;
-+
-+ tbl->it_indirect_levels = levels - 1;
-
- /* Setup linux iommu table */
- pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0,
-@@ -1373,20 +1445,18 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
- iommu_init_table(tbl, nid);
-
- return 0;
--fail:
-- if (tce_mem)
-- __free_pages(tce_mem, get_order(tce_table_size));
--
-- return rc;
- }
-
- static void pnv_pci_free_table(struct iommu_table *tbl)
- {
-+ const unsigned size = tbl->it_indirect_levels ?
-+ tbl->it_level_size : tbl->it_size;
-+
- if (!tbl->it_size)
- return;
-
-- free_pages(tbl->it_base, get_order(tbl->it_size << 3));
-- memset(tbl, 0, sizeof(struct iommu_table));
-+ pnv_free_tce_table(tbl->it_base, size, tbl->it_indirect_levels);
-+ iommu_reset_table(tbl, "ioda2");
- }
-
- static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
-@@ -1395,12 +1465,15 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
- struct pnv_phb *phb = pe->phb;
- const __be64 *swinvp;
- int64_t rc;
-+ const unsigned size = tbl->it_indirect_levels ?
-+ tbl->it_level_size : tbl->it_size;
- const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
- const __u64 win_size = tbl->it_size << tbl->it_page_shift;
-
-- pe_info(pe, "Setting up window at %llx..%llx pagesize=0x%x tablesize=0x%lx\n",
-+ pe_info(pe, "Setting up window at %llx..%llx pagesize=0x%x tablesize=0x%lx levels=%d levelsize=%x\n",
- start_addr, start_addr + win_size - 1,
-- 1UL << tbl->it_page_shift, tbl->it_size << 3);
-+ 1UL << tbl->it_page_shift, tbl->it_size,
-+ tbl->it_indirect_levels + 1, tbl->it_level_size);
-
- pe->table_group.tables[0] = *tbl;
- tbl = &pe->table_group.tables[0];
-@@ -1411,8 +1484,9 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
- * shifted by 1 bit for 32-bits DMA space.
- */
- rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
-- pe->pe_number << 1, 1, __pa(tbl->it_base),
-- tbl->it_size << 3, 1ULL << tbl->it_page_shift);
-+ pe->pe_number << 1, tbl->it_indirect_levels + 1,
-+ __pa(tbl->it_base),
-+ size << 3, 1ULL << tbl->it_page_shift);
- if (rc) {
- pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
- goto fail;
-@@ -1526,7 +1600,8 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
- end);
-
- rc = pnv_pci_ioda2_create_table(pe, IOMMU_PAGE_SHIFT_4K,
-- phb->ioda.m32_pci_base, tbl);
-+ phb->ioda.m32_pci_base,
-+ POWERNV_IOMMU_DEFAULT_LEVELS, tbl);
- if (rc) {
- pe_err(pe, "Failed to create 32-bit TCE table, err %ld", rc);
- return;
++#endif
++
+ static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
+ long npages)
+ {
+@@ -1833,6 +1864,9 @@ static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
+
+ static struct iommu_table_ops pnv_ioda2_iommu_ops = {
+ .set = pnv_ioda2_tce_build,
++#ifdef CONFIG_IOMMU_API
++ .exchange = pnv_ioda2_tce_xchg,
++#endif
+ .clear = pnv_ioda2_tce_free,
+ .get = pnv_tce_get,
+ };
+diff --git a/arch/powerpc/platforms/powernv/pci-p5ioc2.c b/arch/powerpc/platforms/powernv/pci-p5ioc2.c
+index b524b17..94c880c 100644
+--- a/arch/powerpc/platforms/powernv/pci-p5ioc2.c
++++ b/arch/powerpc/platforms/powernv/pci-p5ioc2.c
+@@ -85,6 +85,9 @@ static void pnv_pci_init_p5ioc2_msis(struct pnv_phb *phb) { }
+
+ static struct iommu_table_ops pnv_p5ioc2_iommu_ops = {
+ .set = pnv_tce_build,
++#ifdef CONFIG_IOMMU_API
++ .exchange = pnv_tce_xchg,
++#endif
+ .clear = pnv_tce_free,
+ .get = pnv_tce_get,
+ };
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
-index c5e1f05..9b4a0cf 100644
+index cc82f05..fd14e2c 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
-@@ -592,6 +592,25 @@ struct pci_ops pnv_pci_ops = {
- static __be64 *pnv_tce(struct iommu_table *tbl, long index)
- {
- __be64 *tmp = ((__be64 *)tbl->it_base);
-+ int level = tbl->it_indirect_levels;
-+ const long shift = ilog2(tbl->it_level_size);
-+ unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
-+
-+ if (index >= tbl->it_size)
-+ return NULL;
-+
-+ while (level) {
-+ int n = (index & mask) >> (level * shift);
-+ unsigned long tce = be64_to_cpu(tmp[n]);
-+
-+ if (!(tce & (TCE_PCI_READ | TCE_PCI_WRITE)))
-+ return NULL;
-+
-+ tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
-+ index &= ~mask;
-+ mask >>= shift;
-+ --level;
-+ }
-
- return tmp + index;
- }
+@@ -598,6 +598,24 @@ int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
+ return 0;
+ }
+
++#ifdef CONFIG_IOMMU_API
++int pnv_tce_xchg(struct iommu_table *tbl, long index,
++ unsigned long *hpa, enum dma_data_direction *direction)
++{
++ u64 proto_tce = iommu_direction_to_tce_perm(*direction);
++ unsigned long newtce = *hpa | proto_tce, oldtce;
++ unsigned long idx = index - tbl->it_offset;
++
++ BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
++
++ oldtce = xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce));
++ *hpa = be64_to_cpu(oldtce) & ~(TCE_PCI_READ | TCE_PCI_WRITE);
++ *direction = iommu_tce_direction(oldtce);
++
++ return 0;
++}
++#endif
++
+ void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
+ {
+ long i;
+diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
+index ea97de5..3a72e45 100644
+--- a/arch/powerpc/platforms/powernv/pci.h
++++ b/arch/powerpc/platforms/powernv/pci.h
+@@ -206,6 +206,8 @@ extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
+ unsigned long uaddr, enum dma_data_direction direction,
+ struct dma_attrs *attrs);
+ extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages);
++extern int pnv_tce_xchg(struct iommu_table *tbl, long index,
++ unsigned long *hpa, enum dma_data_direction *direction);
+ extern unsigned long pnv_tce_get(struct iommu_table *tbl, long index);
+
+ void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
+diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
+index 2ead291..0724ec8 100644
+--- a/drivers/vfio/vfio_iommu_spapr_tce.c
++++ b/drivers/vfio/vfio_iommu_spapr_tce.c
+@@ -236,18 +236,11 @@ static void tce_iommu_release(void *iommu_data)
+ }
+
+ static void tce_iommu_unuse_page(struct tce_container *container,
+- unsigned long oldtce)
++ unsigned long hpa)
+ {
+ struct page *page;
+
+- if (!(oldtce & (TCE_PCI_READ | TCE_PCI_WRITE)))
+- return;
+-
+- page = pfn_to_page(oldtce >> PAGE_SHIFT);
+-
+- if (oldtce & TCE_PCI_WRITE)
+- SetPageDirty(page);
+-
++ page = pfn_to_page(hpa >> PAGE_SHIFT);
+ put_page(page);
+ }
+
+@@ -255,14 +248,21 @@ static int tce_iommu_clear(struct tce_container *container,
+ struct iommu_table *tbl,
+ unsigned long entry, unsigned long pages)
+ {
+- unsigned long oldtce;
++ unsigned long oldhpa;
++ long ret;
++ enum dma_data_direction direction;
+
+ for ( ; pages; --pages, ++entry) {
+- oldtce = iommu_clear_tce(tbl, entry);
+- if (!oldtce)
++ direction = DMA_NONE;
++ oldhpa = 0;
++ ret = iommu_tce_xchg(tbl, entry, &oldhpa, &direction);
++ if (ret)
+ continue;
+
+- tce_iommu_unuse_page(container, oldtce);
++ if (direction == DMA_NONE)
++ continue;
++
++ tce_iommu_unuse_page(container, oldhpa);
+ }
+
+ return 0;
+@@ -284,12 +284,13 @@ static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
+
+ static long tce_iommu_build(struct tce_container *container,
+ struct iommu_table *tbl,
+- unsigned long entry, unsigned long tce, unsigned long pages)
++ unsigned long entry, unsigned long tce, unsigned long pages,
++ enum dma_data_direction direction)
+ {
+ long i, ret = 0;
+ struct page *page;
+ unsigned long hpa;
+- enum dma_data_direction direction = iommu_tce_direction(tce);
++ enum dma_data_direction dirtmp;
+
+ for (i = 0; i < pages; ++i) {
+ unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
+@@ -305,8 +306,8 @@ static long tce_iommu_build(struct tce_container *container,
+ }
+
+ hpa |= offset;
+- ret = iommu_tce_build(tbl, entry + i, (unsigned long) __va(hpa),
+- direction);
++ dirtmp = direction;
++ ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp);
+ if (ret) {
+ tce_iommu_unuse_page(container, hpa);
+ pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
+@@ -314,6 +315,10 @@ static long tce_iommu_build(struct tce_container *container,
+ tce, ret);
+ break;
+ }
++
++ if (dirtmp != DMA_NONE)
++ tce_iommu_unuse_page(container, hpa);
++
+ tce += IOMMU_PAGE_SIZE(tbl);
+ }
+
+@@ -378,8 +383,8 @@ static long tce_iommu_ioctl(void *iommu_data,
+ case VFIO_IOMMU_MAP_DMA: {
+ struct vfio_iommu_type1_dma_map param;
+ struct iommu_table *tbl = NULL;
+- unsigned long tce;
+ long num;
++ enum dma_data_direction direction;
+
+ if (!container->enabled)
+ return -EPERM;
+@@ -405,19 +410,26 @@ static long tce_iommu_ioctl(void *iommu_data,
+ return -EINVAL;
+
+ /* iova is checked by the IOMMU API */
+- tce = param.vaddr;
+ if (param.flags & VFIO_DMA_MAP_FLAG_READ)
+- tce |= TCE_PCI_READ;
+- if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
+- tce |= TCE_PCI_WRITE;
++ if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
++ direction = DMA_BIDIRECTIONAL;
++ else
++ direction = DMA_TO_DEVICE;
++ else
++ if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
++ direction = DMA_FROM_DEVICE;
++ else
++ return -EINVAL;
+
+- ret = iommu_tce_put_param_check(tbl, param.iova, tce);
++ ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
+ if (ret)
+ return ret;
+
+ ret = tce_iommu_build(container, tbl,
+ param.iova >> tbl->it_page_shift,
+- tce, param.size >> tbl->it_page_shift);
++ param.vaddr,
++ param.size >> tbl->it_page_shift,
++ direction);
+
+ iommu_flush_tce(tbl);
+
--
-2.0.0
+2.4.0.rc3.8.gfb3e7d5