--- v10
+++ v7
@@ -1,118 +1,501 @@
-This adds a way for the IOMMU user to know how much a new table will
-use so it can be accounted in the locked_vm limit before allocation
-happens.
+At the moment only one group per container is supported.
+POWER8 CPUs have more flexible design and allows naving 2 TCE tables per
+IOMMU group so we can relax this limitation and support multiple groups
+per container.
-This stores the allocated table size in pnv_pci_ioda2_get_table_size()
-so the locked_vm counter can be updated correctly when a table is
-being disposed.
-
-This defines an iommu_table_group_ops callback to let VFIO know
-how much memory will be locked if a table is created.
+This adds TCE table descriptors to a container and uses iommu_table_group_ops
+to create/set DMA windows on IOMMU groups so the same TCE tables will be
+shared between several IOMMU groups.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
---
Changes:
-v10:
-* s/ROUND_UP/_ALIGN_UP/
-* fixed rounding up for @entries_shift (used to use ROUND_UP)
+v7:
+* updated doc
+---
+ Documentation/vfio.txt | 8 +-
+ drivers/vfio/vfio_iommu_spapr_tce.c | 289 ++++++++++++++++++++++++++----------
+ 2 files changed, 214 insertions(+), 83 deletions(-)
-v9:
-* reimplemented the whole patch
----
- arch/powerpc/include/asm/iommu.h | 5 +++++
- arch/powerpc/platforms/powernv/pci-ioda.c | 35 +++++++++++++++++++++++++++++++
- 2 files changed, 40 insertions(+)
-
-diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
-index 2c41115..c8bad21 100644
---- a/arch/powerpc/include/asm/iommu.h
-+++ b/arch/powerpc/include/asm/iommu.h
-@@ -99,6 +99,7 @@ struct iommu_table {
- unsigned long it_size; /* Size of iommu table in entries */
- unsigned long it_indirect_levels;
- unsigned long it_level_size;
-+ unsigned long it_allocated_size;
- unsigned long it_offset; /* Offset into global table */
- unsigned long it_base; /* mapped address of tce table */
- unsigned long it_index; /* which iommu table this is */
-@@ -151,6 +152,10 @@ extern struct iommu_table *iommu_init_table(struct iommu_table * tbl,
- struct iommu_table_group;
-
- struct iommu_table_group_ops {
-+ unsigned long (*get_table_size)(
+diff --git a/Documentation/vfio.txt b/Documentation/vfio.txt
+index 94328c8..7dcf2b5 100644
+--- a/Documentation/vfio.txt
++++ b/Documentation/vfio.txt
+@@ -289,10 +289,12 @@ PPC64 sPAPR implementation note
+
+ This implementation has some specifics:
+
+-1) Only one IOMMU group per container is supported as an IOMMU group
+-represents the minimal entity which isolation can be guaranteed for and
+-groups are allocated statically, one per a Partitionable Endpoint (PE)
++1) On older systems (POWER7 with P5IOC2/IODA1) only one IOMMU group per
++container is supported as an IOMMU table is allocated at the boot time,
++one table per a IOMMU group which is a Partitionable Endpoint (PE)
+ (PE is often a PCI domain but not always).
++Newer systems (POWER8 with IODA2) have improved hardware design which allows
++to remove this limitation and have multiple IOMMU groups per a VFIO container.
+
+ 2) The hardware supports so called DMA windows - the PCI address range
+ within which DMA transfer is allowed, any attempt to access address space
+diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
+index 5049b4f..8cbd239 100644
+--- a/drivers/vfio/vfio_iommu_spapr_tce.c
++++ b/drivers/vfio/vfio_iommu_spapr_tce.c
+@@ -89,10 +89,16 @@ static void decrement_locked_vm(long npages)
+ */
+ struct tce_container {
+ struct mutex lock;
+- struct iommu_group *grp;
+ bool enabled;
+ unsigned long locked_pages;
+ bool v2;
++ struct iommu_table tables[IOMMU_TABLE_GROUP_MAX_TABLES];
++ struct list_head group_list;
++};
++
++struct tce_iommu_group {
++ struct list_head next;
++ struct iommu_group *grp;
+ };
+
+ static long tce_unregister_pages(struct tce_container *container,
+@@ -154,20 +160,20 @@ static bool tce_page_is_contained(struct page *page, unsigned page_shift)
+ return (PAGE_SHIFT + compound_order(compound_head(page))) >= page_shift;
+ }
+
++static inline bool tce_groups_attached(struct tce_container *container)
++{
++ return !list_empty(&container->group_list);
++}
++
+ static struct iommu_table *spapr_tce_find_table(
+ struct tce_container *container,
+ phys_addr_t ioba)
+ {
+ long i;
+ struct iommu_table *ret = NULL;
+- struct iommu_table_group *table_group;
+-
+- table_group = iommu_group_get_iommudata(container->grp);
+- if (!table_group)
+- return NULL;
+
+ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
+- struct iommu_table *tbl = &table_group->tables[i];
++ struct iommu_table *tbl = &container->tables[i];
+ unsigned long entry = ioba >> tbl->it_page_shift;
+ unsigned long start = tbl->it_offset;
+ unsigned long end = start + tbl->it_size;
+@@ -185,11 +191,8 @@ static int tce_iommu_enable(struct tce_container *container)
+ {
+ int ret = 0;
+ unsigned long locked;
+- struct iommu_table *tbl;
+ struct iommu_table_group *table_group;
+-
+- if (!container->grp)
+- return -ENXIO;
++ struct tce_iommu_group *tcegrp;
+
+ if (!current->mm)
+ return -ESRCH; /* process exited */
+@@ -222,12 +225,24 @@ static int tce_iommu_enable(struct tce_container *container)
+ * as this information is only available from KVM and VFIO is
+ * KVM agnostic.
+ */
+- table_group = iommu_group_get_iommudata(container->grp);
++ if (!tce_groups_attached(container))
++ return -ENODEV;
++
++ tcegrp = list_first_entry(&container->group_list,
++ struct tce_iommu_group, next);
++ table_group = iommu_group_get_iommudata(tcegrp->grp);
+ if (!table_group)
+ return -ENODEV;
+
+- tbl = &table_group->tables[0];
+- locked = (tbl->it_size << tbl->it_page_shift) >> PAGE_SHIFT;
++ /*
++ * We do not allow enabling a group if no DMA-able memory was
++ * registered as there is no way to know how much we should
++ * increment the locked_vm counter.
++ */
++ if (!table_group->tce32_size)
++ return -EPERM;
++
++ locked = table_group->tce32_size >> PAGE_SHIFT;
+ ret = try_increment_locked_vm(locked);
+ if (ret)
+ return ret;
+@@ -266,6 +281,8 @@ static void *tce_iommu_open(unsigned long arg)
+ return ERR_PTR(-ENOMEM);
+
+ mutex_init(&container->lock);
++ INIT_LIST_HEAD_RCU(&container->group_list);
++
+ container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
+
+ return container;
+@@ -274,15 +291,35 @@ static void *tce_iommu_open(unsigned long arg)
+ static int tce_iommu_clear(struct tce_container *container,
+ struct iommu_table *tbl,
+ unsigned long entry, unsigned long pages);
++static void tce_iommu_free_table(struct iommu_table *tbl);
+
+ static void tce_iommu_release(void *iommu_data)
+ {
+ struct tce_container *container = iommu_data;
++ struct iommu_table_group *table_group;
++ int i;
++ struct tce_iommu_group *tcegrp;
+
+- WARN_ON(container->grp);
++ while (tce_groups_attached(container)) {
++ tcegrp = list_first_entry(&container->group_list,
++ struct tce_iommu_group, next);
++ table_group = iommu_group_get_iommudata(tcegrp->grp);
++ tce_iommu_detach_group(iommu_data, tcegrp->grp);
++ }
+
+- if (container->grp)
+- tce_iommu_detach_group(iommu_data, container->grp);
++ /* Free tables */
++ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
++ struct iommu_table *tbl = &container->tables[i];
++
++ if (!tbl->it_size)
++ continue;
++
++ tce_iommu_clear(container, tbl,
++ tbl->it_offset, tbl->it_size);
++
++ if (tbl->it_ops && tbl->it_ops->free)
++ tce_iommu_free_table(tbl);
++ }
+
+ tce_iommu_disable(container);
+
+@@ -498,6 +535,44 @@ static long tce_iommu_build_v2(struct tce_container *container,
+ return ret;
+ }
+
++static long tce_iommu_create_table(struct iommu_table_group *table_group,
++ int num,
+ __u32 page_shift,
+ __u64 window_size,
-+ __u32 levels);
- long (*create_table)(struct iommu_table_group *table_group,
- int num,
- __u32 page_shift,
-diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
-index 14d4f34..48aee99 100644
---- a/arch/powerpc/platforms/powernv/pci-ioda.c
-+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
-@@ -41,6 +41,7 @@
- #include <asm/debug.h>
- #include <asm/firmware.h>
- #include <asm/pnv-pci.h>
-+#include <asm/mmzone.h>
-
- #include <misc/cxl.h>
-
-@@ -2052,6 +2053,38 @@ static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
- }
-
- #ifdef CONFIG_IOMMU_API
-+static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
-+ __u64 window_size, __u32 levels)
++ __u32 levels,
++ struct iommu_table *tbl)
+{
-+ unsigned long bytes = 0;
-+ const unsigned window_shift = ilog2(window_size);
-+ unsigned entries_shift = window_shift - page_shift;
-+ unsigned table_shift = entries_shift + 3;
-+ unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
-+ unsigned long direct_table_size;
-+
-+ if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
-+ (window_size > memory_hotplug_max()) ||
-+ !is_power_of_2(window_size))
-+ return 0;
-+
-+ /* Calculate a direct table size from window_size and levels */
-+ entries_shift = (entries_shift + levels - 1) / levels;
-+ table_shift = entries_shift + 3;
-+ table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
-+ direct_table_size = 1UL << table_shift;
-+
-+ for ( ; levels; --levels) {
-+ bytes += _ALIGN_UP(tce_table_size, direct_table_size);
-+
-+ tce_table_size /= direct_table_size;
-+ tce_table_size <<= 3;
-+ tce_table_size = _ALIGN_UP(tce_table_size, direct_table_size);
++ long ret;
++ unsigned long table_size;
++
++ table_size = table_group->ops->get_table_size(page_shift, window_size,
++ levels) >> PAGE_SHIFT;
++
++ ret = try_increment_locked_vm(table_size);
++ if (ret)
++ return ret;
++
++ ret = table_group->ops->create_table(table_group, num,
++ page_shift, window_size, levels, tbl);
++
++ if (ret)
++ decrement_locked_vm(table_size);
++
++ return ret;
++}
++
++static void tce_iommu_free_table(struct iommu_table *tbl)
++{
++ unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
++
++ if (!tbl->it_allocated_size)
++ return;
++
++ tbl->it_ops->free(tbl);
++ decrement_locked_vm(pages);
++ memset(tbl, 0, sizeof(*tbl));
++}
++
+ static long tce_iommu_ioctl(void *iommu_data,
+ unsigned int cmd, unsigned long arg)
+ {
+@@ -521,16 +596,17 @@ static long tce_iommu_ioctl(void *iommu_data,
+
+ case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
+ struct vfio_iommu_spapr_tce_info info;
+- struct iommu_table *tbl;
++ struct tce_iommu_group *tcegrp;
+ struct iommu_table_group *table_group;
+
+- if (WARN_ON(!container->grp))
++ if (!tce_groups_attached(container))
+ return -ENXIO;
+
+- table_group = iommu_group_get_iommudata(container->grp);
++ tcegrp = list_first_entry(&container->group_list,
++ struct tce_iommu_group, next);
++ table_group = iommu_group_get_iommudata(tcegrp->grp);
+
+- tbl = &table_group->tables[0];
+- if (WARN_ON_ONCE(!tbl))
++ if (!table_group)
+ return -ENXIO;
+
+ minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
+@@ -542,9 +618,9 @@ static long tce_iommu_ioctl(void *iommu_data,
+ if (info.argsz < minsz)
+ return -EINVAL;
+
+- info.dma32_window_start = tbl->it_offset << tbl->it_page_shift;
+- info.dma32_window_size = tbl->it_size << tbl->it_page_shift;
+ info.flags = 0;
++ info.dma32_window_start = table_group->tce32_start;
++ info.dma32_window_size = table_group->tce32_size;
+
+ if (copy_to_user((void __user *)arg, &info, minsz))
+ return -EFAULT;
+@@ -721,12 +797,20 @@ static long tce_iommu_ioctl(void *iommu_data,
+ tce_iommu_disable(container);
+ mutex_unlock(&container->lock);
+ return 0;
+- case VFIO_EEH_PE_OP:
+- if (!container->grp)
+- return -ENODEV;
+
+- return vfio_spapr_iommu_eeh_ioctl(container->grp,
+- cmd, arg);
++ case VFIO_EEH_PE_OP: {
++ struct tce_iommu_group *tcegrp;
++
++ ret = 0;
++ list_for_each_entry(tcegrp, &container->group_list, next) {
++ ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp,
++ cmd, arg);
++ if (ret)
++ return ret;
++ }
++ return ret;
+ }
+
-+ return bytes;
-+}
-+
- static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
- __u32 page_shift, __u64 window_size, __u32 levels,
- struct iommu_table *tbl);
-@@ -2131,6 +2164,7 @@ static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
- }
-
- static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
-+ .get_table_size = pnv_pci_ioda2_get_table_size,
- .create_table = pnv_pci_ioda2_create_table,
- .set_window = pnv_pci_ioda2_set_window,
- .unset_window = pnv_pci_ioda2_unset_window,
-@@ -2222,6 +2256,7 @@ static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset,
- page_shift);
- tbl->it_level_size = 1ULL << (level_shift - 3);
- tbl->it_indirect_levels = levels - 1;
-+ tbl->it_allocated_size = tce_table_allocated;
-
- pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n",
- window_size, tce_table_size, bus_offset);
+ }
+
+ return -ENOTTY;
+@@ -735,63 +819,111 @@ static long tce_iommu_ioctl(void *iommu_data,
+ static int tce_iommu_attach_group(void *iommu_data,
+ struct iommu_group *iommu_group)
+ {
+- int ret;
++ int ret, i;
+ struct tce_container *container = iommu_data;
+ struct iommu_table_group *table_group;
++ struct tce_iommu_group *tcegrp = NULL;
++ bool first_group = !tce_groups_attached(container);
+
+ mutex_lock(&container->lock);
+
+ /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
+ iommu_group_id(iommu_group), iommu_group); */
+- if (container->grp) {
+- pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n",
+- iommu_group_id(container->grp),
+- iommu_group_id(iommu_group));
+- ret = -EBUSY;
+- goto unlock_exit;
+- }
+-
+- if (container->enabled) {
+- pr_err("tce_vfio: attaching group #%u to enabled container\n",
+- iommu_group_id(iommu_group));
+- ret = -EBUSY;
+- goto unlock_exit;
+- }
+-
+ table_group = iommu_group_get_iommudata(iommu_group);
+- if (!table_group) {
+- ret = -ENXIO;
++
++ if (!first_group && (!table_group->ops ||
++ !table_group->ops->set_ownership)) {
++ ret = -EBUSY;
++ goto unlock_exit;
++ }
++
++ /* Check if new group has the same iommu_ops (i.e. compatible) */
++ list_for_each_entry(tcegrp, &container->group_list, next) {
++ struct iommu_table_group *table_group_tmp;
++
++ if (tcegrp->grp == iommu_group) {
++ pr_warn("tce_vfio: Group %d is already attached\n",
++ iommu_group_id(iommu_group));
++ ret = -EBUSY;
++ goto unlock_exit;
++ }
++ table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
++ if (table_group_tmp->ops != table_group->ops) {
++ pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
++ iommu_group_id(iommu_group),
++ iommu_group_id(tcegrp->grp));
++ ret = -EPERM;
++ goto unlock_exit;
++ }
++ }
++
++ tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
++ if (!tcegrp) {
++ ret = -ENOMEM;
+ goto unlock_exit;
+ }
+
+ if (!table_group->ops || !table_group->ops->set_ownership) {
+ ret = iommu_take_ownership(table_group);
++ if (!ret)
++ container->tables[0] = table_group->tables[0];
+ } else if (!table_group->ops->create_table ||
+ !table_group->ops->set_window) {
+ WARN_ON_ONCE(1);
+ ret = -EFAULT;
+ } else {
++ table_group->ops->set_ownership(table_group, true);
+ /*
+- * Disable iommu bypass, otherwise the user can DMA to all of
+- * our physical memory via the bypass window instead of just
+- * the pages that has been explicitly mapped into the iommu
++ * If it the first group attached, check if there is any window
++ * created and create one if none.
+ */
+- struct iommu_table tbltmp = { 0 }, *tbl = &tbltmp;
+-
+- table_group->ops->set_ownership(table_group, true);
+- ret = table_group->ops->create_table(table_group, 0,
+- IOMMU_PAGE_SHIFT_4K,
+- table_group->tce32_size, 1, tbl);
+- if (!ret)
+- ret = table_group->ops->set_window(table_group, 0, tbl);
++ if (first_group) {
++ bool found = false;
++
++ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
++ if (!container->tables[i].it_size)
++ continue;
++
++ found = true;
++ break;
++ }
++ if (!found) {
++ struct iommu_table *tbl = &container->tables[0];
++
++ ret = tce_iommu_create_table(
++ table_group, 0,
++ IOMMU_PAGE_SHIFT_4K,
++ table_group->tce32_size, 1,
++ tbl);
++ if (ret)
++ goto unlock_exit;
++ }
++ }
++
++ /* Set all windows to the new group */
++ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
++ struct iommu_table *tbl = &container->tables[i];
++
++ if (!tbl->it_size)
++ continue;
++
++ /* Set the default window to a new group */
++ ret = table_group->ops->set_window(table_group, i, tbl);
++ if (ret)
++ break;
++ }
+ }
+
+ if (ret)
+ goto unlock_exit;
+
+- container->grp = iommu_group;
++ tcegrp->grp = iommu_group;
++ list_add(&tcegrp->next, &container->group_list);
+
+ unlock_exit:
++ if (ret && tcegrp)
++ kfree(tcegrp);
++
+ mutex_unlock(&container->lock);
+
+ return ret;
+@@ -802,25 +934,27 @@ static void tce_iommu_detach_group(void *iommu_data,
+ {
+ struct tce_container *container = iommu_data;
+ struct iommu_table_group *table_group;
++ struct tce_iommu_group *tcegrp, *tcetmp;
+ long i;
++ bool found = false;
+
+ mutex_lock(&container->lock);
+- if (iommu_group != container->grp) {
+- pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n",
+- iommu_group_id(iommu_group),
+- iommu_group_id(container->grp));
++
++ list_for_each_entry_safe(tcegrp, tcetmp, &container->group_list, next) {
++ if (tcegrp->grp != iommu_group)
++ continue;
++ found = true;
++ break;
++ }
++
++ if (!found) {
++ pr_warn("tce_vfio: detaching unattached group #%u\n",
++ iommu_group_id(iommu_group));
+ goto unlock_exit;
+ }
+
+- if (container->enabled) {
+- pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n",
+- iommu_group_id(container->grp));
+- tce_iommu_disable(container);
+- }
+-
+- /* pr_debug("tce_vfio: detaching group #%u from iommu %p\n",
+- iommu_group_id(iommu_group), iommu_group); */
+- container->grp = NULL;
++ list_del(&tcegrp->next);
++ kfree(tcegrp);
+
+ table_group = iommu_group_get_iommudata(iommu_group);
+ BUG_ON(!table_group);
+@@ -828,7 +962,7 @@ static void tce_iommu_detach_group(void *iommu_data,
+ /* Kernel owns the device now, we can restore bypass */
+ if (!table_group->ops || !table_group->ops->set_ownership) {
+ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
+- struct iommu_table *tbl = &table_group->tables[i];
++ struct iommu_table *tbl = &container->tables[i];
+
+ if (!tbl->it_size)
+ continue;
+@@ -837,20 +971,15 @@ static void tce_iommu_detach_group(void *iommu_data,
+ goto unlock_exit;
+ tce_iommu_clear(container, tbl,
+ tbl->it_offset, tbl->it_size);
++
++ memset(tbl, 0, sizeof(*tbl));
+ }
+ iommu_release_ownership(table_group);
+ } else if (!table_group->ops->unset_window) {
+ WARN_ON_ONCE(1);
+ } else {
+ for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
+- struct iommu_table *tbl = &table_group->tables[i];
+-
+ table_group->ops->unset_window(table_group, i);
+- tce_iommu_clear(container, tbl,
+- tbl->it_offset, tbl->it_size);
+-
+- if (tbl->it_ops->free)
+- tbl->it_ops->free(tbl);
+ }
+
+ table_group->ops->set_ownership(table_group, false);
--
-2.4.0.rc3.8.gfb3e7d5
+2.0.0