The decision of when to fallback to the default pmem apis is currently done at too high of a level. In particular the test for arch_has_pmem_api() in memcpy_to_pmem() really wants to decide whether the arch_memcpy_to_pmem() implementation is placing data in a location that a subsequent wmb_pmem() can flush. For x86 this equates to an arch_memcpy_to_pmem() implementation that guarantees that write data is at most sitting in the local cpu write buffer. The current usage of __copy_from_user_inatomic_nocache() guarantees this property on all 64-bit x86 implementations (at least according to the Intel SDM that says Pentium M implementations may leave dirty-data in the cache after a non-temporal store). In the 32-bit case waiting until memcpy_to_pmem() time to perform a fallback is too late. Instead 32-bit x86 is converted to use write-through mappings for pmem. arch_has_pmem_api() is updated to only indicate whether the arch provides the proper helpers. Code that cares whether wmb_pmem() actually flushes writes to pmem must now call arch_has_wmb_pmem() directly. Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx> Cc: Ingo Molnar <mingo@xxxxxxxxxx> Cc: "H. Peter Anvin" <hpa@xxxxxxxxx> Cc: Toshi Kani <toshi.kani@xxxxxx> Cc: Christoph Hellwig <hch@xxxxxx> Cc: Ross Zwisler <ross.zwisler@xxxxxxxxxxxxxxx> Signed-off-by: Dan Williams <dan.j.williams@xxxxxxxxx> --- arch/x86/include/asm/io.h | 2 - arch/x86/include/asm/pmem.h | 41 ++++++++++++++++++++++-- drivers/acpi/nfit.c | 2 + drivers/nvdimm/pmem.c | 2 + include/asm-generic/pmem.h | 72 ++++++++++++++++++++++++++++++++++++++++++ include/linux/pmem.h | 73 +++++++------------------------------------ 6 files changed, 123 insertions(+), 69 deletions(-) create mode 100644 include/asm-generic/pmem.h diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index d241fbd5c87b..83ec9b1d77cc 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -248,8 +248,6 @@ static inline void flush_write_buffers(void) #endif } -#define ARCH_MEMREMAP_PMEM MEMREMAP_WB - #endif /* __KERNEL__ */ extern void native_io_delay(void); diff --git a/arch/x86/include/asm/pmem.h b/arch/x86/include/asm/pmem.h index a3a0df6545ee..6eb3c1da5d57 100644 --- a/arch/x86/include/asm/pmem.h +++ b/arch/x86/include/asm/pmem.h @@ -16,9 +16,12 @@ #include <linux/uaccess.h> #include <asm/cacheflush.h> #include <asm/cpufeature.h> +#include <asm-generic/pmem.h> #include <asm/special_insns.h> #ifdef CONFIG_ARCH_HAS_PMEM_API +#ifdef CONFIG_X86_64 +#define ARCH_MEMREMAP_PMEM MEMREMAP_WB /** * arch_memcpy_to_pmem - copy data to persistent memory * @dst: destination buffer for the copy @@ -141,18 +144,48 @@ static inline void arch_clear_pmem(void __pmem *addr, size_t size) __arch_wb_cache_pmem(vaddr, size); } -static inline bool arch_has_wmb_pmem(void) +static inline bool __arch_has_wmb_pmem(void) { -#ifdef CONFIG_X86_64 /* * We require that wmb() be an 'sfence', that is only guaranteed on * 64-bit builds */ return static_cpu_has(X86_FEATURE_PCOMMIT); +} #else +/* + * Some 32-bit implementations may leave dirty-data in cache after a + * series of non-temporal stores, so set pmem ranges to write-through + * caching. + */ +#define ARCH_MEMREMAP_PMEM MEMREMAP_WT + +static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, + size_t n) +{ + default_memcpy_pmem(dst, src, n); +} + +static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, + struct iov_iter *i) +{ + return default_copy_from_iter_pmem(addr, bytes, i); +} + +static inline void arch_clear_pmem(void __pmem *addr, size_t size) +{ + default_clear_pmem(addr, size); +} + +static inline void arch_wmb_pmem(void) +{ + wmb(); +} + +static inline bool __arch_has_wmb_pmem(void) +{ return false; -#endif } +#endif /* CONFIG_X86_64 */ #endif /* CONFIG_ARCH_HAS_PMEM_API */ - #endif /* __ASM_X86_PMEM_H__ */ diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index 7c2638f914a9..c3fe20635562 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -1364,7 +1364,7 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, return -ENOMEM; } - if (!arch_has_pmem_api() && !nfit_blk->nvdimm_flush) + if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush) dev_warn(dev, "unable to guarantee persistence of writes\n"); if (mmio->line_size == 0) diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 3b5b9cb758b6..20bf122328da 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -125,7 +125,7 @@ static struct pmem_device *pmem_alloc(struct device *dev, pmem->phys_addr = res->start; pmem->size = resource_size(res); - if (!arch_has_pmem_api()) + if (!arch_has_wmb_pmem()) dev_warn(dev, "unable to guarantee persistence of writes\n"); if (!devm_request_mem_region(dev, pmem->phys_addr, pmem->size, diff --git a/include/asm-generic/pmem.h b/include/asm-generic/pmem.h new file mode 100644 index 000000000000..95d1a6ac0df7 --- /dev/null +++ b/include/asm-generic/pmem.h @@ -0,0 +1,72 @@ +/* + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#ifndef __ASM_GENERIC_PMEM_H__ +#define __ASM_GENERIC_PMEM_H__ +/* + * These defaults seek to offer decent performance and minimize the + * window between i/o completion and writes being durable on media. + * However, it is undefined / architecture specific whether + * default_memremap_pmem + default_memcpy_to_pmem is sufficient for + * making data durable relative to i/o completion. + */ +static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, + size_t size) +{ + memcpy((void __force *) dst, src, size); +} + +static inline size_t default_copy_from_iter_pmem(void __pmem *addr, + size_t bytes, struct iov_iter *i) +{ + return copy_from_iter_nocache((void __force *)addr, bytes, i); +} + +static inline void default_clear_pmem(void __pmem *addr, size_t size) +{ + if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0) + clear_page((void __force *)addr); + else + memset((void __force *)addr, 0, size); +} + +#ifndef CONFIG_ARCH_HAS_PMEM_API +/* + * These are simply here to enable compilation, all call sites gate + * calling these symbols with arch_has_pmem_api() and redirect to the + * implementation in asm/pmem.h. + */ + +static inline bool __arch_has_wmb_pmem(void) +{ + return false; +} + +static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, + size_t n) +{ + BUG(); +} + +static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, + struct iov_iter *i) +{ + BUG(); + return 0; +} + +static inline void arch_clear_pmem(void __pmem *addr, size_t size) +{ + BUG(); +} +#endif /* CONFIG_ARCH_HAS_PMEM_API */ +#endif /* __ASM_GENERIC_PMEM_H__ */ diff --git a/include/linux/pmem.h b/include/linux/pmem.h index a9d84bf335ee..f7f5a713a860 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -15,37 +15,9 @@ #include <linux/io.h> #include <linux/uio.h> - +#include <asm-generic/pmem.h> #ifdef CONFIG_ARCH_HAS_PMEM_API #include <asm/pmem.h> -#else -static inline void arch_wmb_pmem(void) -{ - BUG(); -} - -static inline bool arch_has_wmb_pmem(void) -{ - return false; -} - -static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t n) -{ - BUG(); -} - -static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, - struct iov_iter *i) -{ - BUG(); - return 0; -} - -static inline void arch_clear_pmem(void __pmem *addr, size_t size) -{ - BUG(); -} #endif /* @@ -53,7 +25,6 @@ static inline void arch_clear_pmem(void __pmem *addr, size_t size) * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), * arch_copy_from_iter_pmem(), arch_clear_pmem() and arch_has_wmb_pmem(). */ - static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size) { memcpy(dst, (void __force const *) src, size); @@ -64,8 +35,13 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr) devm_memunmap(dev, (void __force *) addr); } +static inline bool arch_has_pmem_api(void) +{ + return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API); +} + /** - * arch_has_pmem_api - true if wmb_pmem() ensures durability + * arch_has_wmb_pmem - true if wmb_pmem() ensures durability * * For a given cpu implementation within an architecture it is possible * that wmb_pmem() resolves to a nop. In the case this returns @@ -73,36 +49,9 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr) * fall back to a different data consistency model, or otherwise notify * the user. */ -static inline bool arch_has_pmem_api(void) -{ - return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && arch_has_wmb_pmem(); -} - -/* - * These defaults seek to offer decent performance and minimize the - * window between i/o completion and writes being durable on media. - * However, it is undefined / architecture specific whether - * default_memremap_pmem + default_memcpy_to_pmem is sufficient for - * making data durable relative to i/o completion. - */ -static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t size) -{ - memcpy((void __force *) dst, src, size); -} - -static inline size_t default_copy_from_iter_pmem(void __pmem *addr, - size_t bytes, struct iov_iter *i) -{ - return copy_from_iter_nocache((void __force *)addr, bytes, i); -} - -static inline void default_clear_pmem(void __pmem *addr, size_t size) +static inline bool arch_has_wmb_pmem(void) { - if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0) - clear_page((void __force *)addr); - else - memset((void __force *)addr, 0, size); + return arch_has_pmem_api() && __arch_has_wmb_pmem(); } /** @@ -158,8 +107,10 @@ static inline void memcpy_to_pmem(void __pmem *dst, const void *src, size_t n) */ static inline void wmb_pmem(void) { - if (arch_has_pmem_api()) + if (arch_has_wmb_pmem()) arch_wmb_pmem(); + else + wmb(); } /** -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx"> email@xxxxxxxxx </a>