[PATCH v7 9/9] iommu/exynos: add bus notifier for registering System MMU

Regards,
Prathyush

On Fri, Jul 5, 2013 at 5:59 PM, Cho KyongHo [off-list ref] wrote:
When a device driver is registered, all constructs to handle System MMU
is prepared by bus notifier call.

Signed-off-by: Cho KyongHo <redacted>
---
 drivers/iommu/exynos-iommu.c |  778 ++++++++++++++++++++++++++++++-----------
 1 files changed, 569 insertions(+), 209 deletions(-)

diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index b151e51..51d43bb 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c

@@ -27,6 +27,9 @@
 #include <linux/memblock.h>
 #include <linux/export.h>
 #include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/pm_domain.h>
+#include <linux/notifier.h>

 #include <asm/cacheflush.h>
 #include <asm/pgtable.h>

@@ -80,6 +83,14 @@
 #define CTRL_BLOCK     0x7
 #define CTRL_DISABLE   0x0

+#define CFG_LRU                0x1
+#define CFG_QOS(n)     ((n & 0xF) << 7)
+#define CFG_MASK       0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
+#define CFG_ACGEN      (1 << 24) /* System MMU 3.3 only */
+#define CFG_SYSSEL     (1 << 22) /* System MMU 3.2 only */
+#define CFG_FLPDCACHE  (1 << 20) /* System MMU 3.2+ only */
+#define CFG_SHAREABLE  (1 << 12) /* System MMU 3.x only */
+
 #define REG_MMU_CTRL           0x000
 #define REG_MMU_CFG            0x004
 #define REG_MMU_STATUS         0x008

@@ -96,6 +107,9 @@

 #define REG_MMU_VERSION                0x034

+#define MMU_MAJ_VER(reg)       (reg >> 28)
+#define MMU_MIN_VER(reg)       ((reg >> 21) & 0x7F)
+
 #define REG_PB0_SADDR          0x04C
 #define REG_PB0_EADDR          0x050
 #define REG_PB1_SADDR          0x054

@@ -126,16 +140,6 @@ enum exynos_sysmmu_inttype {
        SYSMMU_FAULTS_NUM
 };

-/*
- * @itype: type of fault.
- * @pgtable_base: the physical address of page table base. This is 0 if @itype
- *                is SYSMMU_BUSERROR.
- * @fault_addr: the device (virtual) address that the System MMU tried to
- *             translated. This is 0 if @itype is SYSMMU_BUSERROR.
- */
-typedef int (*sysmmu_fault_handler_t)(enum exynos_sysmmu_inttype itype,
-                       unsigned long pgtable_base, unsigned long fault_addr);
-
 static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
        REG_PAGE_FAULT_ADDR,
        REG_AR_FAULT_ADDR,

@@ -159,6 +163,14 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
        "UNKNOWN FAULT"
 };

+struct exynos_iommu_client {
+       struct list_head node;  /* entry of exynos_iommu_domain.clients */
+       struct device *dev;
+       spinlock_t lock;
+       int num_sysmmu;
+       struct device *sysmmu[0];
+};
+
 struct exynos_iommu_domain {
        struct list_head clients; /* list of sysmmu_drvdata.node */
        unsigned long *pgtable; /* lv1 page table, 16KB */

@@ -170,13 +182,13 @@ struct exynos_iommu_domain {
 struct sysmmu_drvdata {
        struct list_head node; /* entry of exynos_iommu_domain.clients */
        struct device *sysmmu;  /* System MMU's device descriptor */
-       struct device *dev;     /* Owner of system MMU */
+       struct device *master;  /* Owner of system MMU */
        int nsfrs;
        struct clk *clk;
        int activations;
        spinlock_t lock;
        struct iommu_domain *domain;
-       sysmmu_fault_handler_t fault_handler;
+       bool runtime_active;
        unsigned long pgtable;
        void __iomem *sfrbases[0];
 };

@@ -200,6 +212,20 @@ static bool is_sysmmu_active(struct sysmmu_drvdata *data)
        return data->activations > 0;
 }

+static unsigned int __sysmmu_version(struct sysmmu_drvdata *data,
+                                       int idx, unsigned int *minor)
+{
+       unsigned int major;
+
+       major = readl(data->sfrbases[idx] + REG_MMU_VERSION);
+
+       if (minor)
+               *minor = MMU_MIN_VER(major);
+       major = MMU_MAJ_VER(major);
+
+       return major;
+}
+
 static void sysmmu_unblock(void __iomem *sfrbase)
 {
        __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);

@@ -235,7 +261,6 @@ static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
 static void __sysmmu_set_ptbase(void __iomem *sfrbase,
                                       unsigned long pgd)
 {
-       __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
        __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);

        __sysmmu_tlb_invalidate(sfrbase);

@@ -292,34 +317,17 @@ finish:
        spin_unlock_irqrestore(&data->lock, flags);
 }

-static void __set_fault_handler(struct sysmmu_drvdata *data,
-                                       sysmmu_fault_handler_t handler)
-{
-       unsigned long flags;
-
-       spin_lock_irqsave(&data->lock, flags);
-       data->fault_handler = handler;
-       spin_unlock_irqrestore(&data->lock, flags);
-}
-
-void exynos_sysmmu_set_fault_handler(struct device *dev,
-                                       sysmmu_fault_handler_t handler)
-{
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-
-       __set_fault_handler(data, handler);
-}
-
-static int default_fault_handler(enum exynos_sysmmu_inttype itype,
-                    unsigned long pgtable_base, unsigned long fault_addr)
+static int show_fault_information(const char *name,
+               enum exynos_sysmmu_inttype itype,
+               unsigned long pgtable_base, unsigned long fault_addr)
 {
        unsigned long *ent;

        if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
                itype = SYSMMU_FAULT_UNKNOWN;

-       pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n",
-                       sysmmu_fault_name[itype], fault_addr, pgtable_base);
+       pr_err("%s occurred at 0x%lx by %s(Page table base: 0x%lx)\n",
+               sysmmu_fault_name[itype], fault_addr, name, pgtable_base);

        ent = section_entry(__va(pgtable_base), fault_addr);
        pr_err("\tLv1 entry: 0x%lx\n", *ent);

@@ -340,25 +348,34 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
 {
        /* SYSMMU is in blocked when interrupt occurred. */
        struct sysmmu_drvdata *data = dev_id;
-       struct resource *irqres;
-       struct platform_device *pdev;
+       struct exynos_iommu_client *client = NULL;
+       struct resource *memres;
        enum exynos_sysmmu_inttype itype;
        unsigned long addr = -1;
-
        int i, ret = -ENOSYS;

-       spin_lock(&data->lock);
+       if (data->master)
+               client = data->master->archdata.iommu;

        WARN_ON(!is_sysmmu_active(data));

-       pdev = to_platform_device(data->sysmmu);
-       for (i = 0; i < (pdev->num_resources / 2); i++) {
-               irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
-               if (irqres && ((int)irqres->start == irq))
+       for (i = 0; i < data->nsfrs; i++) {
+               struct resource *irqres;
+               irqres = platform_get_resource(to_platform_device(data->sysmmu),
+                                               IORESOURCE_IRQ, i);
+               if (irqres && ((int)irqres->start == irq)) {
+                       memres = platform_get_resource(
+                                       to_platform_device(data->sysmmu),
+                                       IORESOURCE_MEM, i);
                        break;
+               }
        }

-       if (i == pdev->num_resources) {
+       if (client)
+               spin_lock(&client->lock);
+       spin_lock(&data->lock);
+
+       if (i == data->nsfrs) {
                itype = SYSMMU_FAULT_UNKNOWN;
        } else {
                itype = (enum exynos_sysmmu_inttype)

@@ -371,197 +388,263 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
        }

        if (data->domain)
-               ret = report_iommu_fault(data->domain, data->dev,
+               ret = report_iommu_fault(data->domain, data->master,
                                addr, itype);

-       if ((ret == -ENOSYS) && data->fault_handler) {
-               unsigned long base = data->pgtable;
-               if (itype != SYSMMU_FAULT_UNKNOWN)
-                       base = __raw_readl(
-                                       data->sfrbases[i] + REG_PT_BASE_ADDR);
-               ret = data->fault_handler(itype, base, addr);
-       }
-
        if (!ret && (itype != SYSMMU_FAULT_UNKNOWN))
                __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
-       else
-               dev_dbg(data->sysmmu, "%s is not handled.\n",
-                               sysmmu_fault_name[itype]);
+       else {
+               unsigned long ba = data->pgtable;
+               if (itype != SYSMMU_FAULT_UNKNOWN)
+                       ba = __raw_readl(data->sfrbases[i] + REG_PT_BASE_ADDR);
+               show_fault_information(dev_name(data->sysmmu),
+                                       itype, ba, addr);
+       }

        if (itype != SYSMMU_FAULT_UNKNOWN)
                sysmmu_unblock(data->sfrbases[i]);

        spin_unlock(&data->lock);
+       if (client)
+               spin_unlock(&client->lock);

        return IRQ_HANDLED;
 }

-static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data)
+static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
 {
+               int i;
+
+               for (i = 0; i < data->nsfrs; i++) {
+                       __raw_writel(CTRL_DISABLE,
+                                    data->sfrbases[i] + REG_MMU_CTRL);
+                       __raw_writel(0, data->sfrbases[i] + REG_MMU_CFG);
+               }
+
+               clk_disable(data->clk);
+}
+
+static bool __sysmmu_disable(struct sysmmu_drvdata *data)
+{
+       bool disabled;
        unsigned long flags;
-       bool disabled = false;
-       int i;

        spin_lock_irqsave(&data->lock, flags);

-       if (!set_sysmmu_inactive(data))
-               goto finish;
+       disabled = set_sysmmu_inactive(data);

-       for (i = 0; i < data->nsfrs; i++)
-               __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL);
+       if (disabled) {
+               data->pgtable = 0;
+               data->domain = NULL;

-       if (data->clk)
-               clk_disable(data->clk);
+               if (data->runtime_active)
+                       __sysmmu_disable_nocount(data);

-       disabled = true;
-       data->pgtable = 0;
-       data->domain = NULL;
-finish:
-       spin_unlock_irqrestore(&data->lock, flags);
-
-       if (disabled)
                dev_dbg(data->sysmmu, "Disabled\n");
-       else
-               dev_dbg(data->sysmmu, "%d times left to be disabled\n",
+       } else  {
+               dev_dbg(data->sysmmu, "%d times left to disable\n",
                                        data->activations);
+       }
+
+       spin_unlock_irqrestore(&data->lock, flags);

        return disabled;
 }

-/* __exynos_sysmmu_enable: Enables System MMU
- *
- * returns -error if an error occurred and System MMU is not enabled,
- * 0 if the System MMU has been just enabled and 1 if System MMU was already
- * enabled before.
- */
-static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data,
+
+static void __sysmmu_init_config(struct sysmmu_drvdata *data, int idx)
+{
+       unsigned long cfg = CFG_LRU;
+       int maj, min = 0;
+
+       maj = __sysmmu_version(data, idx, &min);
+       if (maj == 3) {
+               cfg |= CFG_SHAREABLE;
+               if (min > 1) {
+                       cfg |= CFG_FLPDCACHE;
+                       cfg |= (min == 2) ? CFG_SYSSEL : CFG_ACGEN;
+               }
+       }
+
+       cfg |= __raw_readl(data->sfrbases[idx] + REG_MMU_CFG) & ~CFG_MASK;
+       __raw_writel(cfg, data->sfrbases[idx] + REG_MMU_CFG);
+}
+
+static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
+{
+       int i;
+
+       clk_enable(data->clk);
+
+       for (i = 0; i < data->nsfrs; i++) {
+               BUG_ON(__raw_readl(data->sfrbases[i] + REG_MMU_CTRL)
+                                                               & CTRL_ENABLE);

We are assuming that the REG_MMU_CTRL will be 0 in the beginning.
In the newer versions of smmus (e.g. FIMD's sysmmu in Exynos5420), the reset value of REG_MMU_CTRL is 1.
So this BUG will be triggered when sysmmu gets enabled for the first time.

One way to fix this is to pass an additional parameter to this function 'init':
The BUG_ON should not be checked when sysmmu is enabled for first time i.e. from __sysmmu_enable (init = true)
And the BUG_ON can be checked when this function is called from else where (init = false)

 
+
+               __sysmmu_init_config(data, i);
+
+               __sysmmu_set_ptbase(data->sfrbases[i], data->pgtable);
+
+               __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL);
+       }
+}
+
+static int __sysmmu_enable(struct sysmmu_drvdata *data,
                        unsigned long pgtable, struct iommu_domain *domain)
 {
-       int i, ret = 0;
+       int ret = 0;
        unsigned long flags;

        spin_lock_irqsave(&data->lock, flags);
+       if (set_sysmmu_active(data)) {
+               data->pgtable = pgtable;
+               data->domain = domain;

-       if (!set_sysmmu_active(data)) {
-               if (WARN_ON(pgtable != data->pgtable)) {
-                       ret = -EBUSY;
-                       set_sysmmu_inactive(data);
-               } else {
-                       ret = 1;
-               }
+               if (data->runtime_active)
+                       __sysmmu_enable_nocount(data);

-               dev_dbg(data->sysmmu, "Already enabled\n");
-               goto finish;
+               dev_dbg(data->sysmmu, "Enabled\n");
+       } else {
+               ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
+
+               dev_dbg(data->sysmmu, "already enabled\n");
        }

-       if (data->clk)
-               clk_enable(data->clk);
+       if (WARN_ON(ret < 0))
+               set_sysmmu_inactive(data); /* decrement count */

-       data->pgtable = pgtable;
+       spin_unlock_irqrestore(&data->lock, flags);

-       for (i = 0; i < data->nsfrs; i++) {
-               __sysmmu_set_ptbase(data->sfrbases[i], pgtable);
+       return ret;
+}

-               if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) {
-                       /* System MMU version is 3.x */
-                       __raw_writel((1 << 12) | (2 << 28),
-                                       data->sfrbases[i] + REG_MMU_CFG);
-                       __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0);
-                       __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1);
-               }
+/* __exynos_sysmmu_enable: Enables System MMU
+ *
+ * returns -error if an error occurred and System MMU is not enabled,
+ * 0 if the System MMU has been just enabled and 1 if System MMU was already
+ * enabled before.
+ */
+static int __exynos_sysmmu_enable(struct device *dev, unsigned long pgtable,
+                                 struct iommu_domain *domain)
+{
+       int ret = 0;
+       unsigned long flags;
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int i;

-               __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL);
-       }
+       if (WARN_ON(!client))
+               return -ENODEV;

-       data->domain = domain;
+       spin_lock_irqsave(&client->lock, flags);

-       dev_dbg(data->sysmmu, "Enabled\n");
-finish:
-       spin_unlock_irqrestore(&data->lock, flags);
+       for (i = 0; i < client->num_sysmmu; i++) {
+               struct sysmmu_drvdata *data =
+                               dev_get_drvdata(client->sysmmu[i]);
+               ret = __sysmmu_enable(data, pgtable, domain);
+               if (ret < 0) {
+                       int j;
+                       for (j = 0; j < i; j++)
+                               __sysmmu_disable(data);
+                       break;
+               } else {
+                       data->master = dev;
+               }
+       }
+
+       spin_unlock_irqrestore(&client->lock, flags);

        return ret;
 }

 int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable)
 {
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
        int ret;

        BUG_ON(!memblock_is_memory(pgtable));

-       ret = pm_runtime_get_sync(data->sysmmu);
-       if (ret < 0) {
-               dev_dbg(data->sysmmu, "Failed to enable\n");
-               return ret;
-       }
-
-       ret = __exynos_sysmmu_enable(data, pgtable, NULL);
-       if (WARN_ON(ret < 0)) {
-               pm_runtime_put(data->sysmmu);
-               dev_err(data->sysmmu,
-                       "Already enabled with page table %#lx\n",
-                       data->pgtable);
-       } else {
-               data->dev = dev;
-       }
+       ret = __exynos_sysmmu_enable(dev, pgtable, NULL);

        return ret;
 }

 static bool exynos_sysmmu_disable(struct device *dev)
 {
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-       bool disabled;
+       unsigned long flags;
+       bool disabled = true;
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int i;
+
+       if (WARN_ON(!client))
+               return true;
+
+       spin_lock_irqsave(&client->lock, flags);
+
+       /* Every call to __sysmmu_disable() must return same result */
+       for (i = 0; i < client->num_sysmmu; i++) {
+               struct sysmmu_drvdata *data =
+                               dev_get_drvdata(client->sysmmu[i]);
+               disabled = __sysmmu_disable(data);
+               if (disabled)
+                       data->master = NULL;
+       }

-       disabled = __exynos_sysmmu_disable(data);
-       pm_runtime_put(data->sysmmu);
+       spin_unlock_irqrestore(&client->lock, flags);

        return disabled;
 }

 static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova)
 {
-       unsigned long flags;
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int i;

-       spin_lock_irqsave(&data->lock, flags);
+       for (i = 0; i < client->num_sysmmu; i++) {
+               unsigned long flags;
+               struct sysmmu_drvdata *data;

-       if (is_sysmmu_active(data)) {
-               int i;
-               for (i = 0; i < data->nsfrs; i++) {
-                       if (sysmmu_block(data->sfrbases[i])) {
+               data = dev_get_drvdata(client->sysmmu[i]);
+
+               spin_lock_irqsave(&data->lock, flags);
+               if (is_sysmmu_active(data) && data->runtime_active) {
+                       int i;
+                       for (i = 0; i < data->nsfrs; i++)
                                __sysmmu_tlb_invalidate_entry(
                                                data->sfrbases[i], iova);
-                               sysmmu_unblock(data->sfrbases[i]);
-                       }
+               } else {
+                       dev_dbg(dev,
+                               "disabled. Skipping TLB invalidation @ %#lx\n",
+                               iova);
                }
-       } else {
-               dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n");
+               spin_unlock_irqrestore(&data->lock, flags);
        }
-
-       spin_unlock_irqrestore(&data->lock, flags);
 }

 void exynos_sysmmu_tlb_invalidate(struct device *dev)
 {
-       unsigned long flags;
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-
-       spin_lock_irqsave(&data->lock, flags);
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int i;

-       if (is_sysmmu_active(data)) {
-               int i;
-               for (i = 0; i < data->nsfrs; i++) {
-                       if (sysmmu_block(data->sfrbases[i])) {
-                               __sysmmu_tlb_invalidate(data->sfrbases[i]);
-                               sysmmu_unblock(data->sfrbases[i]);
+       for (i = 0; i < client->num_sysmmu; i++) {
+               unsigned long flags;
+               struct sysmmu_drvdata *data;
+
+               data = dev_get_drvdata(client->sysmmu[i]);
+
+               spin_lock_irqsave(&data->lock, flags);
+               if (is_sysmmu_active(data) &&
+                               data->runtime_active) {
+                       int i;
+                       for (i = 0; i < data->nsfrs; i++) {
+                               if (sysmmu_block(data->sfrbases[i])) {
+                                       __sysmmu_tlb_invalidate(
+                                                       data->sfrbases[i]);
+                                       sysmmu_unblock(data->sfrbases[i]);
+                               }
                        }
+               } else {
+                       dev_dbg(dev, "disabled. Skipping TLB invalidation\n");
                }
-       } else {
-               dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n");
+               spin_unlock_irqrestore(&data->lock, flags);
        }
-
-       spin_unlock_irqrestore(&data->lock, flags);
 }

 static int __init exynos_sysmmu_probe(struct platform_device *pdev)

@@ -622,8 +705,6 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)

        pm_runtime_enable(dev);

-       __set_fault_handler(data, &default_fault_handler);
-
        data->sysmmu = dev;
        data->clk = devm_clk_get(dev, "sysmmu");
        if (IS_ERR(data->clk)) {

@@ -647,6 +728,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
                }
        }

+       data->runtime_active = !pm_runtime_enabled(dev);

We are enabling pm_runtime in the probe itself. Is this check still required?
We can directly set data->runtime_active to false since sysmmus get probed first.

+
        spin_lock_init(&data->lock);
        INIT_LIST_HEAD(&data->node);

@@ -656,6 +739,34 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
        return ret;
 }

+#ifdef CONFIG_PM_SLEEP
+static int sysmmu_suspend(struct device *dev)
+{
+       struct sysmmu_drvdata *data = dev_get_drvdata(dev);
+       unsigned long flags;
+       spin_lock_irqsave(&data->lock, flags);
+       if (is_sysmmu_active(data) &&
+               (!pm_runtime_enabled(dev) || data->runtime_active))
+               __sysmmu_disable_nocount(data);
+       spin_unlock_irqrestore(&data->lock, flags);
+       return 0;
+}
+
+static int sysmmu_resume(struct device *dev)
+{
+       struct sysmmu_drvdata *data = dev_get_drvdata(dev);
+       unsigned long flags;
+       spin_lock_irqsave(&data->lock, flags);
+       if (is_sysmmu_active(data) &&
+               (!pm_runtime_enabled(dev) || data->runtime_active))
+               __sysmmu_enable_nocount(data);
+       spin_unlock_irqrestore(&data->lock, flags);
+       return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(sysmmu_pm_ops, sysmmu_suspend, sysmmu_resume);
+
 #ifdef CONFIG_OF
 static struct of_device_id sysmmu_of_match[] __initconst = {
        { .compatible   = "samsung,exynos4210-sysmmu", },

@@ -668,6 +779,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = {
        .driver = {
                .owner          = THIS_MODULE,
                .name           = "exynos-sysmmu",
+               .pm             = &sysmmu_pm_ops,
                .of_match_table = of_match_ptr(sysmmu_of_match),
        }
 };

@@ -720,7 +832,7 @@ err_pgtable:
 static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
 {
        struct exynos_iommu_domain *priv = domain->priv;
-       struct sysmmu_drvdata *data;
+       struct exynos_iommu_client *client;
        unsigned long flags;
        int i;

@@ -728,11 +840,14 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)

        spin_lock_irqsave(&priv->lock, flags);

-       list_for_each_entry(data, &priv->clients, node) {
-               while (!exynos_sysmmu_disable(data->dev))
+       list_for_each_entry(client, &priv->clients, node) {
+               while (!exynos_sysmmu_disable(client->dev))
                        ; /* until System MMU is actually disabled */
        }

+       while (!list_empty(&priv->clients))
+               list_del_init(priv->clients.next);
+
        spin_unlock_irqrestore(&priv->lock, flags);

        for (i = 0; i < NUM_LV1ENTRIES; i++)

@@ -749,41 +864,26 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
 static int exynos_iommu_attach_device(struct iommu_domain *domain,
                                   struct device *dev)
 {
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+       struct exynos_iommu_client *client = dev->archdata.iommu;
        struct exynos_iommu_domain *priv = domain->priv;
        unsigned long flags;
        int ret;

-       ret = pm_runtime_get_sync(data->sysmmu);
-       if (ret < 0)
-               return ret;
-
-       ret = 0;
-
        spin_lock_irqsave(&priv->lock, flags);

-       ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain);
-
-       if (ret == 0) {
-               /* 'data->node' must not be appeared in priv->clients */
-               BUG_ON(!list_empty(&data->node));
-               data->dev = dev;
-               list_add_tail(&data->node, &priv->clients);
-       }
+       ret = __exynos_sysmmu_enable(dev, __pa(priv->pgtable), domain);
+       if (ret == 0)
+               list_add_tail(&client->node, &priv->clients);

        spin_unlock_irqrestore(&priv->lock, flags);

-       if (ret < 0) {
+       if (ret < 0)
                dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n",
                                __func__, __pa(priv->pgtable));
-               pm_runtime_put(data->sysmmu);
-       } else if (ret > 0) {
-               dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
-                                       __func__, __pa(priv->pgtable));
-       } else {
-               dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n",
-                                       __func__, __pa(priv->pgtable));
-       }
+       else
+               dev_dbg(dev, "%s: Attached IOMMU with pgtable 0x%lx%s\n",
+                                       __func__, __pa(priv->pgtable),
+                                       (ret == 0) ? "" : ", again");

        return ret;
 }

@@ -791,39 +891,27 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain,
 static void exynos_iommu_detach_device(struct iommu_domain *domain,
                                    struct device *dev)
 {
-       struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+       struct exynos_iommu_client *client = NULL;
        struct exynos_iommu_domain *priv = domain->priv;
-       struct list_head *pos;
        unsigned long flags;
-       bool found = false;

        spin_lock_irqsave(&priv->lock, flags);

-       list_for_each(pos, &priv->clients) {
-               if (list_entry(pos, struct sysmmu_drvdata, node) == data) {
-                       found = true;
+       list_for_each_entry(client, &priv->clients, node) {
+               if (client == dev->archdata.iommu) {
+                       if (exynos_sysmmu_disable(dev))
+                               list_del_init(&client->node);
                        break;
                }
        }

-       if (!found)
-               goto finish;
+       spin_unlock_irqrestore(&priv->lock, flags);

-       if (__exynos_sysmmu_disable(data)) {
+       if (client == dev->archdata.iommu)
                dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n",
                                        __func__, __pa(priv->pgtable));
-               list_del_init(&data->node);
-
-       } else {
-               dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed",
-                                       __func__, __pa(priv->pgtable));
-       }
-
-finish:
-       spin_unlock_irqrestore(&priv->lock, flags);
-
-       if (found)
-               pm_runtime_put(data->sysmmu);
+       else
+               dev_dbg(dev, "%s: No IOMMU is attached\n", __func__);
 }

 static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova,

@@ -940,7 +1028,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
                                               unsigned long iova, size_t size)
 {
        struct exynos_iommu_domain *priv = domain->priv;
-       struct sysmmu_drvdata *data;
+       struct exynos_iommu_client *client;
        unsigned long flags;
        unsigned long *ent;
        size_t err_page;

@@ -997,8 +1085,8 @@ done:
        spin_unlock_irqrestore(&priv->pgtablelock, flags);

        spin_lock_irqsave(&priv->lock, flags);
-       list_for_each_entry(data, &priv->clients, node)
-               sysmmu_tlb_invalidate_entry(data->dev, iova);
+       list_for_each_entry(client, &priv->clients, node)
+               sysmmu_tlb_invalidate_entry(client->dev, iova);
        spin_unlock_irqrestore(&priv->lock, flags);

        return size;

@@ -1082,3 +1170,275 @@ static int __init exynos_iommu_init(void)
        return ret;
 }
 subsys_initcall(exynos_iommu_init);
+
+#ifdef CONFIG_PM_SLEEP
+static int sysmmu_pm_genpd_suspend(struct device *dev)
+{
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int ret = 0;
+       int i;
+
+       for (i = 0; i < client->num_sysmmu; i++) {
+               ret = pm_generic_suspend(client->sysmmu[i]);
+               if (ret)
+                       break;
+       }
+
+       if (!ret)
+               ret = pm_generic_suspend(dev);
+
+       if (ret) {
+               int j;
+
+               for (j = 0; j < i; j++)
+                       pm_generic_resume(client->sysmmu[j]);
+       }
+
+       return ret;
+}
+
+static int sysmmu_pm_genpd_resume(struct device *dev)
+{
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int ret = 0;
+       int i;
+
+       for (i = 0; i < client->num_sysmmu; i++) {
+               ret = pm_generic_resume(client->sysmmu[i]);
+               if (ret)
+                       break;
+       }
+
+       if (!ret)
+               ret = pm_generic_resume(dev);
+
+       if (ret) {
+               int j;
+
+               for (j = 0; j < i; j++)
+                       pm_generic_suspend(client->sysmmu[j]);
+       }
+
+       return ret;
+}
+#endif
+
+#ifdef CONFIG_PM_RUNTIME
+static void sysmmu_restore_state(struct device *sysmmu)
+{
+       struct sysmmu_drvdata *data = dev_get_drvdata(sysmmu);
+       unsigned long flags;
+
+       spin_lock_irqsave(&data->lock, flags);
+       data->runtime_active = true;
+       if (is_sysmmu_active(data))
+               __sysmmu_enable_nocount(data);
+       spin_unlock_irqrestore(&data->lock, flags);
+}
+
+static void sysmmu_save_state(struct device *sysmmu)
+{
+       struct sysmmu_drvdata *data = dev_get_drvdata(sysmmu);
+       unsigned long flags;
+
+       spin_lock_irqsave(&data->lock, flags);
+       if (is_sysmmu_active(data))
+               __sysmmu_disable_nocount(data);
+       data->runtime_active = false;
+       spin_unlock_irqrestore(&data->lock, flags);
+}
+
+static int sysmmu_pm_genpd_save_state(struct device *dev)
+{
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int (*cb)(struct device *__dev);
+       int i;
+
+       if (dev->type && dev->type->pm)
+               cb = dev->type->pm->runtime_suspend;
+       else if (dev->class && dev->class->pm)
+               cb = dev->class->pm->runtime_suspend;
+       else if (dev->bus && dev->bus->pm)
+               cb = dev->bus->pm->runtime_suspend;
+       else
+               cb = NULL;
+
+       if (!cb && dev->driver && dev->driver->pm)
+               cb = dev->driver->pm->runtime_suspend;
+
+       if (cb) {
+               int ret;
+
+               ret = cb(dev);
+               if (ret)
+                       return ret;
+       }
+
+       for (i = 0; i < client->num_sysmmu; i++)
+               sysmmu_save_state(client->sysmmu[i]);
+
+       return 0;
+}
+
+static int sysmmu_pm_genpd_restore_state(struct device *dev)
+{
+       struct exynos_iommu_client *client = dev->archdata.iommu;
+       int (*cb)(struct device *__dev);
+       int i;
+
+       if (dev->type && dev->type->pm)
+               cb = dev->type->pm->runtime_resume;
+       else if (dev->class && dev->class->pm)
+               cb = dev->class->pm->runtime_resume;
+       else if (dev->bus && dev->bus->pm)
+               cb = dev->bus->pm->runtime_resume;
+       else
+               cb = NULL;
+
+       if (!cb && dev->driver && dev->driver->pm)
+               cb = dev->driver->pm->runtime_resume;
+
+       for (i = 0; i < client->num_sysmmu; i++)
+               sysmmu_restore_state(client->sysmmu[i]);
+
+       if (cb) {
+               int ret;
+               ret = cb(dev);
+               if (ret) {
+                       for (i = 0; i < client->num_sysmmu; i++)
+                               sysmmu_save_state(client->sysmmu[i]);
+                       return ret;
+               }
+       }
+
+       return 0;
+}
+#endif
+
+#ifdef CONFIG_PM_GENERIC_DOMAINS
+struct gpd_dev_ops sysmmu_devpm_ops = {
+#ifdef CONFIG_PM_RUNTIME
+       .save_state = &sysmmu_pm_genpd_save_state,
+       .restore_state = &sysmmu_pm_genpd_restore_state,
+#endif
+#ifdef CONFIG_PM_SLEEP
+       .suspend = &sysmmu_pm_genpd_suspend,
+       .resume = &sysmmu_pm_genpd_resume,
+#endif
+};
+#endif /* CONFIG_PM_GENERIC_DOMAINS */
+
+static int sysmmu_hook_driver_register(struct notifier_block *nb,
+                                       unsigned long val,
+                                       void *p)
+{
+       struct device *dev = p;
+
+       switch (val) {
+       case BUS_NOTIFY_BIND_DRIVER:
+       {
+               int i = 0;
+               int size = 0;
+               const __be32 *phandle;
+               struct exynos_iommu_client *client;
+
+               phandle = of_get_property(dev->of_node, "iommu", &size);
+               if (!phandle)
+                       break;
+
+               size = size / sizeof(*phandle); /* number of elements */
+
+               client = devm_kzalloc(dev, sizeof(*client) * size, GFP_KERNEL);
+               if (!client) {
+                       dev_err(dev, "No Memory for exynos_iommu_client\n");
+                       return -ENOMEM;
+               }
+
+               client->num_sysmmu = size;
+               client->dev = dev;
+               INIT_LIST_HEAD(&client->node);
+               spin_lock_init(&client->lock);
+
+               for (i = 0; i < size; i++) {
+                       struct device_node *np;
+                       struct platform_device *sysmmu;
+
+                       /* this always success: see above of_find_property() */
+                       np = of_parse_phandle(dev->of_node, "iommu", i);
+
+                       sysmmu = of_find_device_by_node(np);
+                       if (!sysmmu) {
+                               dev_err(dev,
+                                       "sysmmu node '%s' is not found\n",
+                                       np->name);
+                               break;
+                       }
+
+                       client->sysmmu[i] = &sysmmu->dev;
+               }
+
+               if (i < size) {
+                       while (--i >= 0)
+                               of_node_put(client->sysmmu[i]->of_node);
+                       devm_kfree(dev, client);
+                       return -ENODEV;
+               }
+
+               i = pm_genpd_add_callbacks(dev, &sysmmu_devpm_ops, NULL);
+               if (i && (i != -ENOSYS)) {
+                       dev_err(dev,
+                               "Failed to register 'dev_pm_ops' for iommu\n");
+                       devm_kfree(dev, client);
+                       return i;
+               }
+
+               dev->archdata.iommu = client;
+               break;
+       }
+       case BUS_NOTIFY_BOUND_DRIVER:
+       {
+               struct exynos_iommu_client *client = dev->archdata.iommu;
+               if (dev->archdata.iommu &&
+                               (!pm_runtime_enabled(dev) ||
+                                        IS_ERR(dev_to_genpd(dev)))) {
+                       int i;
+                       for (i = 0; i < client->num_sysmmu; i++) {
+                               struct sysmmu_drvdata *data;
+                               pm_runtime_disable(client->sysmmu[i]);
+                               data = dev_get_drvdata(client->sysmmu[i]);
+                               if (!data)
+                                       continue;
+                               data->runtime_active =
+                                       !pm_runtime_enabled(data->sysmmu);
+                               if (data->runtime_active &&
+                                               is_sysmmu_active(data))
+                                       __sysmmu_enable_nocount(data);
+                       }
+               }
+               break;
+       }
+       case BUS_NOTIFY_UNBOUND_DRIVER:
+       {
+               if (dev->archdata.iommu) {
+                       __pm_genpd_remove_callbacks(dev, false);
+
+                       devm_kfree(dev, dev->archdata.iommu);
+
+                       dev->archdata.iommu = NULL;
+               }
+               break;
+       }
+       } /* switch (val) */
+
+       return 0;
+}
+
+static struct notifier_block sysmmu_notifier = {
+       .notifier_call = &sysmmu_hook_driver_register,
+};
+
+static int __init exynos_iommu_prepare(void)
+{
+       return bus_register_notifier(&platform_bus_type, &sysmmu_notifier);
+}
+arch_initcall(exynos_iommu_prepare);
--

1.7.2.5