--- v7
+++ v4
@@ -1,25 +1,18 @@
-TCE tables might get too big in case of 4K IOMMU pages and DDW enabled
-on huge guests (hundreds of GB of RAM) so the kernel might be unable to
-allocate contiguous chunk of physical memory to store the TCE table.
-
-To address this, POWER8 CPU (actually, IODA2) supports multi-level TCE tables,
-up to 5 levels which splits the table into a tree of smaller subtables.
-
This adds multi-level TCE tables support to pnv_pci_ioda2_create_table()
and pnv_pci_ioda2_free_table() callbacks.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
---
- arch/powerpc/include/asm/iommu.h | 2 +
- arch/powerpc/platforms/powernv/pci-ioda.c | 128 ++++++++++++++++++++++++------
+ arch/powerpc/include/asm/iommu.h | 4 +
+ arch/powerpc/platforms/powernv/pci-ioda.c | 125 +++++++++++++++++++++++-------
arch/powerpc/platforms/powernv/pci.c | 19 +++++
- 3 files changed, 123 insertions(+), 26 deletions(-)
+ 3 files changed, 122 insertions(+), 26 deletions(-)
diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
-index 8ed4648..1e0d907 100644
+index cc26eca..283f70f 100644
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
-@@ -90,6 +90,8 @@ struct iommu_pool {
+@@ -85,6 +85,8 @@ struct iommu_pool {
struct iommu_table {
unsigned long it_busno; /* Bus number this table belongs to */
unsigned long it_size; /* Size of iommu table in entries */
@@ -28,22 +21,20 @@
unsigned long it_offset; /* Offset into global table */
unsigned long it_base; /* mapped address of tce table */
unsigned long it_index; /* which iommu table this is */
+@@ -133,6 +135,8 @@ extern struct iommu_table *iommu_init_table(struct iommu_table * tbl,
+
+ #define POWERPC_IOMMU_MAX_TABLES 1
+
++#define POWERPC_IOMMU_DEFAULT_LEVELS 1
++
+ struct powerpc_iommu;
+
+ struct powerpc_iommu_ops {
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
-index 64b7cfe..74e119c 100644
+index 1f725d4..f542819 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
-@@ -47,6 +47,10 @@
- #include "powernv.h"
- #include "pci.h"
-
-+#define POWERNV_IOMMU_DEFAULT_LEVELS 1
-+#define POWERNV_IOMMU_MAX_LEVELS 5
-+#define ROUND_UP(x, n) (((x) + (n) - 1u) & ~((n) - 1u))
-+
- static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
- const char *fmt, ...)
- {
-@@ -1339,16 +1343,82 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
+@@ -1295,16 +1295,79 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
__free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
}
@@ -113,8 +104,8 @@
+}
+
static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
-- __u32 page_shift, __u64 window_size,
-+ __u32 page_shift, __u64 window_size, __u32 levels,
+- __u32 page_shift, __u32 window_shift,
++ __u32 page_shift, __u32 window_shift, __u32 levels,
struct iommu_table *tbl)
{
int nid = pe->phb->hose->node;
@@ -125,22 +116,27 @@
- unsigned order;
+ unsigned long tce_table_size, left;
+ unsigned shift;
-+
-+ if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS))
+
+ if ((page_shift != 12) && (page_shift != 16) && (page_shift != 24))
+ return -EINVAL;
+@@ -1312,20 +1375,27 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
+ if ((1ULL << window_shift) > memory_hotplug_max())
+ return -EINVAL;
+
++ if (!levels || (levels > 5))
+ return -EINVAL;
-
- if ((window_size > memory_hotplug_max()) || !is_power_of_2(window_size))
- return -EINVAL;
-@@ -1357,16 +1427,19 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
++
+ tce_table_size = (1ULL << (window_shift - page_shift)) * 8;
tce_table_size = max(0x1000UL, tce_table_size);
/* Allocate TCE table */
- order = get_order(tce_table_size);
-+ shift = ROUND_UP(ilog2(window_size) - page_shift, levels) / levels;
++#define ROUND_UP(x, n) (((x) + (n) - 1u) & ~((n) - 1u))
++ shift = ROUND_UP(window_shift - page_shift, levels) / levels;
+ shift += 3;
+ shift = max_t(unsigned, shift, IOMMU_PAGE_SHIFT_4K);
+ pr_info("Creating TCE table %08llx, %d levels, TCE table size = %lx\n",
-+ window_size, levels, 1UL << shift);
++ 1ULL << window_shift, levels, 1UL << shift);
- tce_mem = alloc_pages_node(nid, GFP_KERNEL, order);
- if (!tce_mem) {
@@ -160,8 +156,8 @@
/* Setup linux iommu table */
pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0,
-@@ -1375,20 +1448,18 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
- tbl->it_ops = &pnv_ioda2_iommu_ops;
+@@ -1335,20 +1405,18 @@ static long pnv_pci_ioda2_create_table(struct pnv_ioda_pe *pe,
+ iommu_init_table(tbl, nid);
return 0;
-fail:
@@ -171,7 +167,7 @@
- return rc;
}
- static void pnv_pci_free_table(struct iommu_table *tbl)
+ static void pnv_pci_ioda2_free_table(struct iommu_table *tbl)
{
+ const unsigned size = tbl->it_indirect_levels ?
+ tbl->it_level_size : tbl->it_size;
@@ -186,7 +182,7 @@
}
static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
-@@ -1397,12 +1468,15 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
+@@ -1357,12 +1425,15 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
struct pnv_phb *phb = pe->phb;
const __be64 *swinvp;
int64_t rc;
@@ -202,9 +198,9 @@
+ 1UL << tbl->it_page_shift, tbl->it_size,
+ tbl->it_indirect_levels + 1, tbl->it_level_size);
- pe->table_group.tables[0] = *tbl;
- tbl = &pe->table_group.tables[0];
-@@ -1413,8 +1487,9 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
+ pe->iommu.tables[0] = *tbl;
+ tbl = &pe->iommu.tables[0];
+@@ -1373,8 +1444,9 @@ static long pnv_pci_ioda2_set_window(struct pnv_ioda_pe *pe,
* shifted by 1 bit for 32-bits DMA space.
*/
rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
@@ -216,21 +212,21 @@
if (rc) {
pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
goto fail;
-@@ -1530,7 +1605,8 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
+@@ -1488,7 +1560,8 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
end);
rc = pnv_pci_ioda2_create_table(pe, IOMMU_PAGE_SHIFT_4K,
-- phb->ioda.m32_pci_base, tbl);
-+ phb->ioda.m32_pci_base,
-+ POWERNV_IOMMU_DEFAULT_LEVELS, tbl);
+- ilog2(phb->ioda.m32_pci_base), tbl);
++ ilog2(phb->ioda.m32_pci_base),
++ POWERPC_IOMMU_DEFAULT_LEVELS, tbl);
if (rc) {
pe_err(pe, "Failed to create 32-bit TCE table, err %ld", rc);
return;
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
-index a9797dd..e734e37 100644
+index cf8206b..e98495a 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
-@@ -592,6 +592,25 @@ struct pci_ops pnv_pci_ops = {
+@@ -605,6 +605,25 @@ static unsigned long pnv_dmadir_to_flags(enum dma_data_direction direction)
static __be64 *pnv_tce(struct iommu_table *tbl, long index)
{
__be64 *tmp = ((__be64 *)tbl->it_base);