This patch implements dynamic memory consistency switching on RISC-V
using the Ssdtso ISA extension.
Ssdtso is a RISC-V ISA extension, which allows to switch the memory
consistency model of less privileged modes from RVWMO to TSO at runtime.
The active model is controlled by a DTSO bit in the {m,h,s}envcfg CSRs
(per-hart state).
TSO is a stronger memory ordering than RVWMO, which means that
executing software that was written for RVWMO can also run under TSO
without causing memory consistency issues.
Since RVWMO is the default model, switching to TSO is safe.
The patch introduces Ssdtso basic support:
* enable dynamic memory consistency switching if Ssdtso support is
enabled in the kernel config
* define the relevant envcfg bits
* add the relevant code to store/restore the DTSO state
* register the the extension in hwcap/cpufeatures
* extend task_struct to keep the state across context switches
* add a Kconfig symbol to disable Ssdtso support
Signed-off-by: Christoph Müllner <christoph.muellner@xxxxxxxx>
---
arch/riscv/Kconfig | 11 ++++
arch/riscv/include/asm/csr.h | 1 +
arch/riscv/include/asm/dtso.h | 97 ++++++++++++++++++++++++++++++
arch/riscv/include/asm/hwcap.h | 1 +
arch/riscv/include/asm/switch_to.h | 3 +
arch/riscv/kernel/asm-offsets.c | 3 +
arch/riscv/kernel/cpufeature.c | 1 +
7 files changed, 117 insertions(+)
create mode 100644 arch/riscv/include/asm/dtso.h
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index bffbd869a068..1b26797e7cdd 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -23,6 +23,7 @@ config RISCV
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DEBUG_VM_PGTABLE
select ARCH_HAS_DEBUG_WX
+ select ARCH_HAS_DYNAMIC_MEMORY_CONSISTENCY_MODEL if RISCV_ISA_SSDTSO
select ARCH_HAS_FORTIFY_SOURCE
select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_HAS_GIGANTIC_PAGE
@@ -480,6 +481,16 @@ config RISCV_ISA_C
If you don't know what to do here, say Y.
+config RISCV_ISA_SSDTSO
+ bool "Ssdtso extension support for dynamic TSO memory ordering"
+ default y
+ help
+ Adds support to dynamically detect the presence of the Ssdtso
+ ISA-extension and allows user-space processes to activate/deactivate
+ the TSO memory ordering model at run-time.
+
+ If you don't know what to do here, say Y.
+
config RISCV_ISA_SVNAPOT
bool "Svnapot extension support for supervisor mode NAPOT pages"
depends on 64BIT && MMU
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 510014051f5d..83e5737d720d 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -194,6 +194,7 @@
/* xENVCFG flags */
#define ENVCFG_STCE (_AC(1, ULL) << 63)
#define ENVCFG_PBMTE (_AC(1, ULL) << 62)
+#define ENVCFG_DTSO (_AC(1, UL) << 8)
#define ENVCFG_CBZE (_AC(1, UL) << 7)
#define ENVCFG_CBCFE (_AC(1, UL) << 6)
#define ENVCFG_CBIE_SHIFT 4
diff --git a/arch/riscv/include/asm/dtso.h b/arch/riscv/include/asm/dtso.h
new file mode 100644
index 000000000000..25f9bb30884e
--- /dev/null
+++ b/arch/riscv/include/asm/dtso.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2023 Christoph Muellner <christoph.muellner@xxxxxxxx>
+ */
+
+#ifndef __ASM_RISCV_DTSO_H
+#define __ASM_RISCV_DTSO_H
+
+#define RISCV_MEMORY_CONSISTENCY_MODEL_WMO 0
+#define RISCV_MEMORY_CONSISTENCY_MODEL_TSO 1
+
+#ifdef CONFIG_RISCV_ISA_SSDTSO
+
+#include <linux/sched/task_stack.h>
+#include <asm/cpufeature.h>
+#include <asm/csr.h>
+
+static __always_inline bool has_dtso(void)
+{
+ return riscv_has_extension_unlikely(RISCV_ISA_EXT_SSDTSO);
+}
+
+static __always_inline bool has_ztso(void)
+{
+ return riscv_has_extension_unlikely(RISCV_ISA_EXT_ZTSO);
+}
+
+static inline bool dtso_is_enabled(void)
+{
+ if (has_dtso())
+ return csr_read(CSR_SENVCFG) & ENVCFG_DTSO;
+ return 0;
+}
+
+static inline void dtso_disable(void)
+{
+ if (has_dtso() && !has_ztso())
+ csr_clear(CSR_SENVCFG, ENVCFG_DTSO);
+}
+
+static inline void dtso_enable(void)
+{
+ if (has_dtso() && !has_ztso())
+ csr_set(CSR_SENVCFG, ENVCFG_DTSO);
+}
+
+static inline unsigned long get_memory_consistency_model(
+ struct task_struct *task)
+{
+ return task->memory_consistency_model;
+}
+
+static inline void set_memory_consitency_model(struct task_struct *task,
+ unsigned long model)
+{
+ task->memory_consistency_model = model;
+}
+
+static inline void dtso_restore(struct task_struct *task)
+{
+ unsigned long cur_model = get_memory_consistency_model(task);
+
+ if (cur_model == RISCV_MEMORY_CONSISTENCY_MODEL_TSO)
+ dtso_enable();
+ else
+ dtso_disable();
+}
+
+static inline void __switch_to_dtso(struct task_struct *prev,
+ struct task_struct *next)
+{
+ struct pt_regs *regs;
+
+ regs = task_pt_regs(prev);
+
+ /*
+ * We don't need to save the DTSO bit, because we don't expect it to
+ * change. So any mechanism that changes the DTSO bit, needs to take
+ * care to write to task->memory_consistency_model (and reschedule
+ * all threads of the process).
+ */
+
+ dtso_restore(next);
+}
+
+#else /* ! CONFIG_RISCV_ISA_SSDTSO */
+
+static __always_inline bool has_dtso(void) { return false; }
+static __always_inline bool dtso_is_enabled(void) { return false; }
+#define dtso_disable() do { } while (0)
+#define dtso_enable() do { } while (0)
+#define dtso_restore(task) do { } while (0)
+#define __switch_to_dtso(prev, next) do { } while (0)
+
+#endif /* CONFIG_RISCV_ISA_SSDTSO */
+
+#endif /* ! __ASM_RISCV_DTSO_H */
diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index 5340f818746b..88740f419d13 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -80,6 +80,7 @@
#define RISCV_ISA_EXT_ZFA 71
#define RISCV_ISA_EXT_ZTSO 72
#define RISCV_ISA_EXT_ZACAS 73
+#define RISCV_ISA_EXT_SSDTSO 74
#define RISCV_ISA_EXT_MAX 128
#define RISCV_ISA_EXT_INVALID U32_MAX
diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h
index 7efdb0584d47..bedf7fe12c1d 100644
--- a/arch/riscv/include/asm/switch_to.h
+++ b/arch/riscv/include/asm/switch_to.h
@@ -9,6 +9,7 @@
#include <linux/jump_label.h>
#include <linux/sched/task_stack.h>
#include <asm/vector.h>
+#include <asm/dtso.h>
#include <asm/cpufeature.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
@@ -80,6 +81,8 @@ do { \
__switch_to_fpu(__prev, __next); \
if (has_vector()) \
__switch_to_vector(__prev, __next); \
+ if (has_dtso()) \
+ __switch_to_dtso(__prev, __next); \
((last) = __switch_to(__prev, __next)); \
} while (0)
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index a03129f40c46..b5dc39788c41 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -80,6 +80,9 @@ void asm_offsets(void)
#ifdef CONFIG_STACKPROTECTOR
OFFSET(TSK_STACK_CANARY, task_struct, stack_canary);
#endif
+#ifdef CONFIG_DYNAMIC_MEMORY_CONSISTENCY_MODEL
+ OFFSET(TASK_MEM_CONSISTENCY_MODEL, task_struct, memory_consistency_model);
+#endif
DEFINE(PT_SIZE, sizeof(struct pt_regs));
OFFSET(PT_EPC, pt_regs, epc);
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 89920f84d0a3..b63d6b699238 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -303,6 +303,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
__RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
__RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
__RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
+ __RISCV_ISA_EXT_DATA(ssdtso, RISCV_ISA_EXT_SSDTSO),
__RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
__RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
__RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
--
2.43.0
