With the introduction of movdir64b instruction, there is now an instruction
that can write 64 bytes of data atomicaly.
Quoting from Intel SDM:
"There is no atomicity guarantee provided for the 64-byte load operation
from source address, and processor implementations may use multiple
load operations to read the 64-bytes. The 64-byte direct-store issued
by MOVDIR64B guarantees 64-byte write-completion atomicity. This means
that the data arrives at the destination in a single undivided 64-byte
write transaction."
We have identified at least 3 different use cases for this instruction in
the format of func(dst, src, count):
1) Clear poison / Initialize MKTME memory
Destination is normal memory.
Source in normal memory. Does not increment. (Copy same line to all
targets)
Count (to clear/init multiple lines)
2) Submit command(s) to new devices
Destination is a special MMIO region for a device. Does not increment.
Source is normal memory. Increments.
Count usually is 1, but can be multiple.
3) Copy to iomem in big chunks
Destination is iomem and increments
Source in normal memory and increments
Count is number of chunks to copy
This commit adds support for case #2 to support device that will accept
commands via this instruction.
Signed-off-by: Dave Jiang <dave.jiang@xxxxxxxxx>
---
arch/x86/include/asm/io.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
include/linux/io.h | 11 +++++++++++
2 files changed, 55 insertions(+)
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 6bed97ff6db2..3126f6e1d5b8 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -403,5 +403,49 @@ extern bool arch_memremap_can_ram_remap(resource_size_t offset,
extern bool phys_mem_access_encrypted(unsigned long phys_addr,
unsigned long size);
+#include <linux/cpufeature.h>
+static inline bool cpu_has_write512(void)
+{
+ return cpu_feature_enabled(X86_FEATURE_MOVDIR64B);
+}
+
+#define cpu_has_write512 cpu_has_write512
+
+static inline void __iowrite512(void __iomem *__dst, const void *src)
+{
+ volatile struct { char _[64]; } *dst = __dst;
+
+ /* movdir64b [rdx], rax */
+ asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+ : "=m" (dst)
+ : "d" (src), "a" (dst));
+}
+
+/**
+ * iosubmit_cmds512 - copy data to single MMIO location, in 512-bit units
+ * @dst: destination, in MMIO space (must be 512-bit aligned)
+ * @src: source
+ * @count: number of 512 bits quantities to submit
+ *
+ * Submit data from kernel space to MMIO space, in units of 512 bits at a
+ * time. Order of access is not guaranteed, nor is a memory barrier
+ * performed afterwards.
+ */
+static inline void iosubmit_cmds512(void __iomem *dst, const void *src,
+ size_t count)
+{
+ const u8 *from = src;
+ const u8 *end = from + count * 64;
+
+ if (!cpu_has_write512())
+ return;
+
+ while (from < end) {
+ __iowrite512(dst, from);
+ from += 64;
+ }
+}
+
+#define iosubmit_cmds512 iosubmit_cmds512
#endif /* _ASM_X86_IO_H */
diff --git a/include/linux/io.h b/include/linux/io.h
index accac822336a..ca14af3e84de 100644
--- a/include/linux/io.h
+++ b/include/linux/io.h
@@ -20,6 +20,17 @@ __visible void __iowrite32_copy(void __iomem *to, const void *from, size_t count
void __ioread32_copy(void *to, const void __iomem *from, size_t count);
void __iowrite64_copy(void __iomem *to, const void *from, size_t count);
+#ifndef cpu_has_write512
+#define cpu_has_write512() (0)
+#endif
+
+#ifndef iosubmit_cmds512
+static inline void iosubmit_cmds512(void __iomem *to, const void *from,
+ size_t count)
+{
+}
+#endif
+
#ifdef CONFIG_MMU
int ioremap_page_range(unsigned long addr, unsigned long end,
phys_addr_t phys_addr, pgprot_t prot);
