Thread (28 messages) 28 messages, 6 authors, 2015-09-29
STALE3945d

[PATCH v2 5/9] x86, pmem: push fallback handling to arch code

From: Dan Williams <hidden>
Date: 2015-08-26 01:33:39
Also in: lkml, nvdimm
Subsystem: acpi, generic include/asm header files, libnvdimm pmem: persistent memory driver, libnvdimm: non-volatile memory device subsystem, the rest, x86 architecture (32-bit and 64-bit) · Maintainers: "Rafael J. Wysocki", Arnd Bergmann, Dan Williams, Vishal Verma, Dave Jiang, Alison Schofield, Linus Torvalds, Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen

The decision of when to fallback to the default pmem apis is currently
done at too high of a level.  In particular the test for
arch_has_pmem_api() in memcpy_to_pmem() really wants to decide whether
the arch_memcpy_to_pmem() implementation is placing data in a location
that a subsequent wmb_pmem() can flush.

For x86 this equates to an arch_memcpy_to_pmem() implementation that
guarantees that write data is at most sitting in the local cpu write
buffer.  The current usage of __copy_from_user_inatomic_nocache()
guarantees this property on all 64-bit x86 implementations (at least
according to the Intel SDM that says Pentium M implementations may leave
dirty-data in the cache after a non-temporal store).  In the 32-bit case
waiting until memcpy_to_pmem() time to perform a fallback is too late.
Instead 32-bit x86 is converted to use write-through mappings for pmem.

arch_has_pmem_api() is updated to only indicate whether the arch
provides the proper helpers.  Code that cares whether wmb_pmem()
actually flushes writes to pmem must now call arch_has_wmb_pmem()
directly.

Cc: Thomas Gleixner <redacted>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Toshi Kani <redacted>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Ross Zwisler <redacted>
Signed-off-by: Dan Williams <redacted>
---
 arch/x86/include/asm/io.h   |    2 -
 arch/x86/include/asm/pmem.h |   41 ++++++++++++++++++++++--
 drivers/acpi/nfit.c         |    2 +
 drivers/nvdimm/pmem.c       |    2 +
 include/asm-generic/pmem.h  |   72 ++++++++++++++++++++++++++++++++++++++++++
 include/linux/pmem.h        |   73 +++++++------------------------------------
 6 files changed, 123 insertions(+), 69 deletions(-)
 create mode 100644 include/asm-generic/pmem.h
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index d241fbd5c87b..83ec9b1d77cc 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -248,8 +248,6 @@ static inline void flush_write_buffers(void)
 #endif
 }
 
-#define ARCH_MEMREMAP_PMEM MEMREMAP_WB
-
 #endif /* __KERNEL__ */
 
 extern void native_io_delay(void);
diff --git a/arch/x86/include/asm/pmem.h b/arch/x86/include/asm/pmem.h
index a3a0df6545ee..6eb3c1da5d57 100644
--- a/arch/x86/include/asm/pmem.h
+++ b/arch/x86/include/asm/pmem.h
@@ -16,9 +16,12 @@
 #include <linux/uaccess.h>
 #include <asm/cacheflush.h>
 #include <asm/cpufeature.h>
+#include <asm-generic/pmem.h>
 #include <asm/special_insns.h>
 
 #ifdef CONFIG_ARCH_HAS_PMEM_API
+#ifdef CONFIG_X86_64
+#define ARCH_MEMREMAP_PMEM MEMREMAP_WB
 /**
  * arch_memcpy_to_pmem - copy data to persistent memory
  * @dst: destination buffer for the copy
@@ -141,18 +144,48 @@ static inline void arch_clear_pmem(void __pmem *addr, size_t size)
 	__arch_wb_cache_pmem(vaddr, size);
 }
 
-static inline bool arch_has_wmb_pmem(void)
+static inline bool __arch_has_wmb_pmem(void)
 {
-#ifdef CONFIG_X86_64
 	/*
 	 * We require that wmb() be an 'sfence', that is only guaranteed on
 	 * 64-bit builds
 	 */
 	return static_cpu_has(X86_FEATURE_PCOMMIT);
+}
 #else
+/*
+ * Some 32-bit implementations may leave dirty-data in cache after a
+ * series of non-temporal stores, so set pmem ranges to write-through
+ * caching.
+ */
+#define ARCH_MEMREMAP_PMEM MEMREMAP_WT
+
+static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
+		size_t n)
+{
+	default_memcpy_pmem(dst, src, n);
+}
+
+static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes,
+		struct iov_iter *i)
+{
+	return default_copy_from_iter_pmem(addr, bytes, i);
+}
+
+static inline void arch_clear_pmem(void __pmem *addr, size_t size)
+{
+	default_clear_pmem(addr, size);
+}
+
+static inline void arch_wmb_pmem(void)
+{
+	wmb();
+}
+
+static inline bool __arch_has_wmb_pmem(void)
+{
 	return false;
-#endif
 }
+#endif /* CONFIG_X86_64 */
 #endif /* CONFIG_ARCH_HAS_PMEM_API */
-
 #endif /* __ASM_X86_PMEM_H__ */
diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c
index 7c2638f914a9..c3fe20635562 100644
--- a/drivers/acpi/nfit.c
+++ b/drivers/acpi/nfit.c
@@ -1364,7 +1364,7 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
 			return -ENOMEM;
 	}
 
-	if (!arch_has_pmem_api() && !nfit_blk->nvdimm_flush)
+	if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush)
 		dev_warn(dev, "unable to guarantee persistence of writes\n");
 
 	if (mmio->line_size == 0)
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 3b5b9cb758b6..20bf122328da 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -125,7 +125,7 @@ static struct pmem_device *pmem_alloc(struct device *dev,
 
 	pmem->phys_addr = res->start;
 	pmem->size = resource_size(res);
-	if (!arch_has_pmem_api())
+	if (!arch_has_wmb_pmem())
 		dev_warn(dev, "unable to guarantee persistence of writes\n");
 
 	if (!devm_request_mem_region(dev, pmem->phys_addr, pmem->size,
diff --git a/include/asm-generic/pmem.h b/include/asm-generic/pmem.h
new file mode 100644
index 000000000000..95d1a6ac0df7
--- /dev/null
+++ b/include/asm-generic/pmem.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#ifndef __ASM_GENERIC_PMEM_H__
+#define __ASM_GENERIC_PMEM_H__
+/*
+ * These defaults seek to offer decent performance and minimize the
+ * window between i/o completion and writes being durable on media.
+ * However, it is undefined / architecture specific whether
+ * default_memremap_pmem + default_memcpy_to_pmem is sufficient for
+ * making data durable relative to i/o completion.
+ */
+static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src,
+		size_t size)
+{
+	memcpy((void __force *) dst, src, size);
+}
+
+static inline size_t default_copy_from_iter_pmem(void __pmem *addr,
+		size_t bytes, struct iov_iter *i)
+{
+	return copy_from_iter_nocache((void __force *)addr, bytes, i);
+}
+
+static inline void default_clear_pmem(void __pmem *addr, size_t size)
+{
+	if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0)
+		clear_page((void __force *)addr);
+	else
+		memset((void __force *)addr, 0, size);
+}
+
+#ifndef CONFIG_ARCH_HAS_PMEM_API
+/*
+ * These are simply here to enable compilation, all call sites gate
+ * calling these symbols with arch_has_pmem_api() and redirect to the
+ * implementation in asm/pmem.h.
+ */
+
+static inline bool __arch_has_wmb_pmem(void)
+{
+	return false;
+}
+
+static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
+		size_t n)
+{
+	BUG();
+}
+
+static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes,
+		struct iov_iter *i)
+{
+	BUG();
+	return 0;
+}
+
+static inline void arch_clear_pmem(void __pmem *addr, size_t size)
+{
+	BUG();
+}
+#endif /* CONFIG_ARCH_HAS_PMEM_API */
+#endif /* __ASM_GENERIC_PMEM_H__ */
diff --git a/include/linux/pmem.h b/include/linux/pmem.h
index a9d84bf335ee..f7f5a713a860 100644
--- a/include/linux/pmem.h
+++ b/include/linux/pmem.h
@@ -15,37 +15,9 @@
 
 #include <linux/io.h>
 #include <linux/uio.h>
-
+#include <asm-generic/pmem.h>
 #ifdef CONFIG_ARCH_HAS_PMEM_API
 #include <asm/pmem.h>
-#else
-static inline void arch_wmb_pmem(void)
-{
-	BUG();
-}
-
-static inline bool arch_has_wmb_pmem(void)
-{
-	return false;
-}
-
-static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
-		size_t n)
-{
-	BUG();
-}
-
-static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes,
-		struct iov_iter *i)
-{
-	BUG();
-	return 0;
-}
-
-static inline void arch_clear_pmem(void __pmem *addr, size_t size)
-{
-	BUG();
-}
 #endif
 
 /*
@@ -53,7 +25,6 @@ static inline void arch_clear_pmem(void __pmem *addr, size_t size)
  * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(),
  * arch_copy_from_iter_pmem(), arch_clear_pmem() and arch_has_wmb_pmem().
  */
-
 static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size)
 {
 	memcpy(dst, (void __force const *) src, size);
@@ -64,8 +35,13 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr)
 	devm_memunmap(dev, (void __force *) addr);
 }
 
+static inline bool arch_has_pmem_api(void)
+{
+	return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API);
+}
+
 /**
- * arch_has_pmem_api - true if wmb_pmem() ensures durability
+ * arch_has_wmb_pmem - true if wmb_pmem() ensures durability
  *
  * For a given cpu implementation within an architecture it is possible
  * that wmb_pmem() resolves to a nop.  In the case this returns
@@ -73,36 +49,9 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr)
  * fall back to a different data consistency model, or otherwise notify
  * the user.
  */
-static inline bool arch_has_pmem_api(void)
-{
-	return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && arch_has_wmb_pmem();
-}
-
-/*
- * These defaults seek to offer decent performance and minimize the
- * window between i/o completion and writes being durable on media.
- * However, it is undefined / architecture specific whether
- * default_memremap_pmem + default_memcpy_to_pmem is sufficient for
- * making data durable relative to i/o completion.
- */
-static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src,
-		size_t size)
-{
-	memcpy((void __force *) dst, src, size);
-}
-
-static inline size_t default_copy_from_iter_pmem(void __pmem *addr,
-		size_t bytes, struct iov_iter *i)
-{
-	return copy_from_iter_nocache((void __force *)addr, bytes, i);
-}
-
-static inline void default_clear_pmem(void __pmem *addr, size_t size)
+static inline bool arch_has_wmb_pmem(void)
 {
-	if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0)
-		clear_page((void __force *)addr);
-	else
-		memset((void __force *)addr, 0, size);
+	return arch_has_pmem_api() && __arch_has_wmb_pmem();
 }
 
 /**
@@ -158,8 +107,10 @@ static inline void memcpy_to_pmem(void __pmem *dst, const void *src, size_t n)
  */
 static inline void wmb_pmem(void)
 {
-	if (arch_has_pmem_api())
+	if (arch_has_wmb_pmem())
 		arch_wmb_pmem();
+	else
+		wmb();
 }
 
 /**

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help