Currently, when the range to flush covers more than one page (a 4K page or
a hugepage), __flush_tlb_range() flushes the whole tlb. Flushing the whole
tlb comes with a greater cost than flushing a single entry so we should
flush single entries up to a certain threshold so that:
threshold * cost of flushing a single entry < cost of flushing the whole
tlb.
This threshold is microarchitecture dependent and can/should be
overwritten by vendors.
Co-developed-by: Mayuresh Chitale <mchitale@xxxxxxxxxxxxxxxx>
Signed-off-by: Mayuresh Chitale <mchitale@xxxxxxxxxxxxxxxx>
Signed-off-by: Alexandre Ghiti <alexghiti@xxxxxxxxxxxx>
---
arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++++++++++++++++--
1 file changed, 39 insertions(+), 2 deletions(-)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 3e4acef1f6bc..de61ecaa218a 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -24,13 +24,48 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
: "memory");
}
+/*
+ * Flush entire TLB if number of entries to be flushed is greater
+ * than the threshold below. Platforms may override the threshold
+ * value based on marchid, mvendorid, and mimpid.
+ */
+unsigned long tlb_flush_all_threshold __read_mostly = 64;
+
+static void local_flush_tlb_range_threshold_asid(unsigned long start,
+ unsigned long size,
+ unsigned long stride,
+ unsigned long asid)
+{
+ u16 nr_ptes_in_range = DIV_ROUND_UP(size, stride);
+ int i;
+
+ if (nr_ptes_in_range > tlb_flush_all_threshold) {
+ if (asid != -1)
+ local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_all();
+ return;
+ }
+
+ for (i = 0; i < nr_ptes_in_range; ++i) {
+ if (asid != -1)
+ local_flush_tlb_page_asid(start, asid);
+ else
+ local_flush_tlb_page(start);
+ start += stride;
+ }
+}
+
static inline void local_flush_tlb_range(unsigned long start,
unsigned long size, unsigned long stride)
{
if (size <= stride)
local_flush_tlb_page(start);
- else
+ else if (size == (unsigned long)-1)
local_flush_tlb_all();
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride, -1);
+
}
static inline void local_flush_tlb_range_asid(unsigned long start,
@@ -38,8 +73,10 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
{
if (size <= stride)
local_flush_tlb_page_asid(start, asid);
- else
+ else if (size == (unsigned long)-1)
local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride, asid);
}
static void __ipi_flush_tlb_all(void *info)
--
2.39.2
