We can use the PR_{S,G}ET_MEMORY_CONSISTENCY_MODEL prctl calls to change
the memory consistency model at run-time if we have Ssdtso.
This patch registers RISCV_WMO and RISCV_TSO as valid arguments
for these prctl calls and implements the glue code to switch
between these.
Signed-off-by: Christoph Müllner <christoph.muellner@xxxxxxxx>
---
.../mm/dynamic-memory-consistency-model.rst | 12 +++-
arch/riscv/include/asm/processor.h | 7 ++
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/dtso.c | 67 +++++++++++++++++++
include/uapi/linux/prctl.h | 2 +
5 files changed, 88 insertions(+), 1 deletion(-)
create mode 100644 arch/riscv/kernel/dtso.c
diff --git a/Documentation/mm/dynamic-memory-consistency-model.rst b/Documentation/mm/dynamic-memory-consistency-model.rst
index 1fce855a1fad..c8188c174e27 100644
--- a/Documentation/mm/dynamic-memory-consistency-model.rst
+++ b/Documentation/mm/dynamic-memory-consistency-model.rst
@@ -73,4 +73,14 @@ Supported memory consistency models
This section defines the memory consistency models which are supported
by the prctl interface.
-<none>
+RISC-V
+------
+
+RISC-V uses RVWMO (RISC-V weak memory ordering) as default memory consistency
+model. TSO (total store ordering) is another specified model and provides
+additional ordering guarantees. Switching user-mode processes from RVWMO to TSO
+is possible when the Ssdtso extension is available.
+
+* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO`: RISC-V weak memory ordering (default).
+
+* :c:macro:`PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO`: RISC-V total store ordering.
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index a8509cc31ab2..05e05fddc94d 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -184,6 +184,13 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
#define GET_UNALIGN_CTL(tsk, addr) get_unalign_ctl((tsk), (addr))
#define SET_UNALIGN_CTL(tsk, val) set_unalign_ctl((tsk), (val))
+#ifdef CONFIG_RISCV_ISA_SSDTSO
+extern int dtso_set_memory_consistency_model(unsigned long arg);
+extern int dtso_get_memory_consistency_model(void);
+#define SET_MEMORY_CONSISTENCY_MODEL(arg) dtso_set_memory_consistency_model(arg)
+#define GET_MEMORY_CONSISTENCY_MODEL() dtso_get_memory_consistency_model()
+#endif /* CONIG_RISCV_ISA_SSDTSO */
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_RISCV_PROCESSOR_H */
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f71910718053..85f7291da498 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o
obj-$(CONFIG_FPU) += fpu.o
obj-$(CONFIG_RISCV_ISA_V) += vector.o
obj-$(CONFIG_RISCV_ISA_V) += kernel_mode_vector.o
+obj-$(CONFIG_RISCV_ISA_SSDTSO) += dtso.o
obj-$(CONFIG_SMP) += smpboot.o
obj-$(CONFIG_SMP) += smp.o
obj-$(CONFIG_SMP) += cpu_ops.o
diff --git a/arch/riscv/kernel/dtso.c b/arch/riscv/kernel/dtso.c
new file mode 100644
index 000000000000..591d5f9de0f5
--- /dev/null
+++ b/arch/riscv/kernel/dtso.c
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2024 Christoph Muellner <christoph.muellner@xxxxxxxx>
+ */
+
+#include <linux/cpu.h>
+#include <linux/smp.h>
+#include <linux/prctl.h>
+
+#include <asm/cpu.h>
+#include <asm/dtso.h>
+
+#include <trace/events/ipi.h>
+
+int dtso_set_memory_consistency_model(unsigned long arg)
+{
+ int cpu;
+ unsigned long cur_model = get_memory_consistency_model(current);
+ unsigned long new_model;
+
+ switch (arg) {
+ case PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO:
+ new_model = RISCV_MEMORY_CONSISTENCY_MODEL_WMO;
+ break;
+ case PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO:
+ new_model = RISCV_MEMORY_CONSISTENCY_MODEL_TSO;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* No change requested. */
+ if (cur_model == new_model)
+ return 0;
+
+ /* Enabling TSO only works if DTSO is available. */
+ if (new_model == PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO && !has_dtso())
+ return -EINVAL;
+
+ /* Switching TSO->WMO is not allowed. */
+ if (new_model == RISCV_MEMORY_CONSISTENCY_MODEL_WMO)
+ return -EINVAL;
+
+ /* Set the new model in the task struct. */
+ set_memory_consitency_model(current, new_model);
+
+ /*
+ * We need to reschedule all threads of the current process.
+ * Let's do this by rescheduling all CPUs.
+ * This is stricter than necessary, but since this call is
+ * not expected to happen frequently the impact is low.
+ */
+ for_each_cpu(cpu, cpu_online_mask)
+ smp_send_reschedule(cpu);
+
+ return 0;
+}
+
+int dtso_get_memory_consistency_model(void)
+{
+ unsigned long cur_model = get_memory_consistency_model(current);
+
+ if (cur_model == RISCV_MEMORY_CONSISTENCY_MODEL_TSO)
+ return PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO;
+
+ return PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO;
+}
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 579662731eaa..20264bdc3092 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -308,5 +308,7 @@ struct prctl_mm_map {
#define PR_SET_MEMORY_CONSISTENCY_MODEL 71
#define PR_GET_MEMORY_CONSISTENCY_MODEL 72
+# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_WMO 1
+# define PR_MEMORY_CONSISTENCY_MODEL_RISCV_TSO 2
#endif /* _LINUX_PRCTL_H */
--
2.43.0
