Introduce __vdso_flush_icache function for performing cache
flush in userspace whenever possible.
It will use ibar in userspace if CPU comes with ICACHET support,
and fallback to syscall if not. It also made infra ready for
possible future userspace cache flush instruction.
Signed-off-by: Jiaxun Yang <jiaxun.yang@xxxxxxxxxxx>
---
arch/loongarch/include/asm/vdso/vdso.h | 10 ++++++
arch/loongarch/include/asm/vdso/vsyscall.h | 1 +
arch/loongarch/kernel/vdso.c | 2 ++
arch/loongarch/mm/cache.c | 3 ++
arch/loongarch/vdso/Makefile | 2 +-
arch/loongarch/vdso/flush_icache.c | 50 ++++++++++++++++++++++++++++++
arch/loongarch/vdso/vdso.lds.S | 5 +++
7 files changed, 72 insertions(+), 1 deletion(-)
diff --git a/arch/loongarch/include/asm/vdso/vdso.h b/arch/loongarch/include/asm/vdso/vdso.h
index 1c183a9b2115a29a997ec8db0e788d87fb191dce..215b8c85fb2347a2ba9b53167a4343086efb9af3 100644
--- a/arch/loongarch/include/asm/vdso/vdso.h
+++ b/arch/loongarch/include/asm/vdso/vdso.h
@@ -13,6 +13,15 @@
#include <asm/page.h>
#include <asm/vdso.h>
+enum vdso_icacle_flush_mode {
+ VDSO_ICACLE_FLUSH_IBAR,
+ VDSO_ICACLE_FLUSH_FALLBACK,
+};
+
+struct vdso_icache_flush_data {
+ enum vdso_icacle_flush_mode mode;
+};
+
struct vdso_pcpu_data {
u32 node;
} ____cacheline_aligned_in_smp;
@@ -20,6 +29,7 @@ struct vdso_pcpu_data {
struct loongarch_vdso_data {
struct vdso_pcpu_data pdata[NR_CPUS];
struct vdso_rng_data rng_data;
+ struct vdso_icache_flush_data icache_flush_data;
};
/*
diff --git a/arch/loongarch/include/asm/vdso/vsyscall.h b/arch/loongarch/include/asm/vdso/vsyscall.h
index 8987e951d0a93c34ca75de676fb9c191ff4ef3c2..a02663ddf7cdccbe4b0e4c4d87b65874ab14070a 100644
--- a/arch/loongarch/include/asm/vdso/vsyscall.h
+++ b/arch/loongarch/include/asm/vdso/vsyscall.h
@@ -8,6 +8,7 @@
extern struct vdso_data *vdso_data;
extern struct vdso_rng_data *vdso_rng_data;
+extern struct vdso_icache_flush_data *vdso_icache_flush_data;
static __always_inline
struct vdso_data *__loongarch_get_k_vdso_data(void)
diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c
index 05e5fbac102a902016e633db75d9aff7ed550c50..4085452b3e1081e115d346f5870d54dad5c6ef54 100644
--- a/arch/loongarch/kernel/vdso.c
+++ b/arch/loongarch/kernel/vdso.c
@@ -36,6 +36,8 @@ static union {
struct vdso_data *vdso_data = generic_vdso_data.data;
struct vdso_pcpu_data *vdso_pdata = loongarch_vdso_data.vdata.pdata;
struct vdso_rng_data *vdso_rng_data = &loongarch_vdso_data.vdata.rng_data;
+struct vdso_icache_flush_data *vdso_icache_flush_data =
+ &loongarch_vdso_data.vdata.icache_flush_data;
static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma)
{
diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c
index 6be04d36ca0769658a2b52d25af50dd6ad7e07e0..a424a9b24827e5eb83a8d4db7da76b42ba3f4d8b 100644
--- a/arch/loongarch/mm/cache.c
+++ b/arch/loongarch/mm/cache.c
@@ -24,6 +24,7 @@
#include <asm/numa.h>
#include <asm/processor.h>
#include <asm/setup.h>
+#include <asm/vdso/vsyscall.h>
void cache_error_setup(void)
{
@@ -156,6 +157,8 @@ void cpu_cache_init(void)
current_cpu_data.cache_leaves_present = leaf;
current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
+
+ vdso_icache_flush_data->mode = VDSO_ICACLE_FLUSH_IBAR;
}
static const pgprot_t protection_map[16] = {
diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile
index fdde1bcd4e2663bd400dcc6becc4261b7d5dce3a..8407d85548a4d4ff03571f35317ba2a0881f684c 100644
--- a/arch/loongarch/vdso/Makefile
+++ b/arch/loongarch/vdso/Makefile
@@ -4,7 +4,7 @@
# Include the generic Makefile to check the built vdso.
include $(srctree)/lib/vdso/Makefile
-obj-vdso-y := elf.o vgetcpu.o vgettimeofday.o vgetrandom.o \
+obj-vdso-y := elf.o flush_icache.o vgetcpu.o vgettimeofday.o vgetrandom.o \
vgetrandom-chacha.o sigreturn.o
# Common compiler flags between ABIs.
diff --git a/arch/loongarch/vdso/flush_icache.c b/arch/loongarch/vdso/flush_icache.c
new file mode 100644
index 0000000000000000000000000000000000000000..e1f95572175a0bf4c136afd1107ea2f8d8933b21
--- /dev/null
+++ b/arch/loongarch/vdso/flush_icache.c
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Fast user context implementation of getcpu()
+ */
+
+#include <asm/vdso.h>
+#include <asm/unistd.h>
+
+static int flush_icache_ibar(void)
+{
+ __asm__ __volatile__ ("\tibar 0\n"::);
+
+ return 0;
+}
+
+static int flush_icache_fallback(uintptr_t start, uintptr_t end,
+ uintptr_t flags)
+{
+ register long _num __asm__ ("a7") = __NR_loongarch_flush_icache;
+ register long _arg1 __asm__ ("a0") = (long)(start);
+ register long _arg2 __asm__ ("a1") = (long)(end);
+ register long _arg3 __asm__ ("a2") = (long)(flags);
+
+ __asm__ volatile (
+ "syscall 0\n"
+ : "+r"(_arg1)
+ : "r"(_arg2), "r"(_arg3),
+ "r"(_num)
+ : "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5",
+ "$t6", "$t7", "$t8"
+ );
+
+ return _arg1;
+}
+
+extern int __vdso_flush_icache(uintptr_t start, uintptr_t end,
+ uintptr_t flags);
+int __vdso_flush_icache(uintptr_t start, uintptr_t end, uintptr_t flags)
+{
+
+ switch (_loongarch_data.icache_flush_data.mode) {
+ case VDSO_ICACLE_FLUSH_IBAR:
+ return flush_icache_ibar();
+ case VDSO_ICACLE_FLUSH_FALLBACK:
+ default:
+ return flush_icache_fallback(start, end, flags);
+ }
+
+ return -EINVAL;
+}
diff --git a/arch/loongarch/vdso/vdso.lds.S b/arch/loongarch/vdso/vdso.lds.S
index 160cfaef2de45b1243502c7356f8a913658548fe..a1023d10fbb7b45c2a70ead0304c2753c05f9654 100644
--- a/arch/loongarch/vdso/vdso.lds.S
+++ b/arch/loongarch/vdso/vdso.lds.S
@@ -72,6 +72,11 @@ VERSION
__vdso_rt_sigreturn;
local: *;
};
+ LINUX_6.14 {
+ global:
+ __vdso_flush_icache;
+ local: *;
+ };
}
/*
--
2.43.0
