Expose incoherent clearing primitives (clear_pages_movnt(),
clear_pages_clzero()) as alternatives via clear_pages_incoherent().
Fallback to clear_pages() if, X86_FEATURE_MOVNT_SLOW is set and
the CPU does not have X86_FEATURE_CLZERO.
Both these primitives use weakly-ordered stores. To ensure that
callers don't mix accesses to different types of address_spaces,
annotate clear_user_pages_incoherent(), and clear_pages_incoherent()
as taking __incoherent pointers as arguments.
Also add clear_page_make_coherent() which provides the necessary
store fence to make access to these __incoherent regions safe.
Signed-off-by: Ankur Arora <ankur.a.arora@xxxxxxxxxx>
---
arch/x86/include/asm/page.h | 13 +++++++++++++
arch/x86/include/asm/page_64.h | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h
index 045eaab08f43..8fc6cc6759b9 100644
--- a/arch/x86/include/asm/page.h
+++ b/arch/x86/include/asm/page.h
@@ -40,6 +40,19 @@ static inline void clear_user_page(void *page, unsigned long vaddr,
clear_page(page);
}
+#ifdef __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT /* x86_64 */
+/*
+ * clear_pages_incoherent: valid on only __incoherent memory regions.
+ */
+static inline void clear_user_pages_incoherent(__incoherent void *page,
+ unsigned long vaddr,
+ struct page *pg,
+ unsigned int npages)
+{
+ clear_pages_incoherent(page, npages);
+}
+#endif /* __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT */
+
static inline void copy_user_page(void *to, void *from, unsigned long vaddr,
struct page *topage)
{
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index e8d4698fda65..78417f63f522 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -69,6 +69,40 @@ static inline void clear_pages(void *page, unsigned int npages)
: "cc", "memory", "rax", "rcx");
}
+#define __HAVE_ARCH_CLEAR_USER_PAGES_INCOHERENT
+/*
+ * clear_pages_incoherent: only allowed on __incoherent memory regions.
+ */
+static inline void clear_pages_incoherent(__incoherent void *page,
+ unsigned int npages)
+{
+ alternative_call_2(clear_pages_movnt,
+ clear_pages, X86_FEATURE_MOVNT_SLOW,
+ clear_pages_clzero, X86_FEATURE_CLZERO,
+ "=D" (page), "S" ((unsigned long) npages),
+ "0" (page)
+ : "cc", "memory", "rax", "rcx");
+}
+
+/*
+ * clear_page_make_coherent: execute the necessary store fence
+ * after which __incoherent regions can be safely accessed.
+ */
+static inline void clear_page_make_coherent(void)
+{
+ /*
+ * Keep the sfence for oldinstr and clzero separate to guard against
+ * the possibility that a CPU has both X86_FEATURE_MOVNT_SLOW and
+ * X86_FEATURE_CLZERO.
+ *
+ * The alternatives need to be in the same order as the ones
+ * in clear_pages_incoherent().
+ */
+ alternative_2("sfence",
+ "", X86_FEATURE_MOVNT_SLOW,
+ "sfence", X86_FEATURE_CLZERO);
+}
+
void copy_page(void *to, void *from);
#ifdef CONFIG_X86_5LEVEL
--
2.31.1
