On Thu, Jul 27, 2023 at 08:55:52PM +0200, Alexandre Ghiti wrote:
> Currently, when the range to flush covers more than one page (a 4K page or
> a hugepage), __flush_tlb_range() flushes the whole tlb. Flushing the whole
> tlb comes with a greater cost than flushing a single entry so we should
> flush single entries up to a certain threshold so that:
> threshold * cost of flushing a single entry < cost of flushing the whole
> tlb.
>
> This threshold is microarchitecture dependent and can/should be
> overwritten by vendors.
>
> Co-developed-by: Mayuresh Chitale <mchitale@xxxxxxxxxxxxxxxx>
> Signed-off-by: Mayuresh Chitale <mchitale@xxxxxxxxxxxxxxxx>
> Signed-off-by: Alexandre Ghiti <alexghiti@xxxxxxxxxxxx>
> ---
> arch/riscv/mm/tlbflush.c | 41 ++++++++++++++++++++++++++++++++++++++--
> 1 file changed, 39 insertions(+), 2 deletions(-)
>
> diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
> index 3e4acef1f6bc..8017d2130e27 100644
> --- a/arch/riscv/mm/tlbflush.c
> +++ b/arch/riscv/mm/tlbflush.c
> @@ -24,13 +24,48 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
> : "memory");
> }
>
> +/*
> + * Flush entire TLB if number of entries to be flushed is greater
> + * than the threshold below. Platforms may override the threshold
> + * value based on marchid, mvendorid, and mimpid.
> + */
> +static unsigned long tlb_flush_all_threshold __read_mostly = 64;
> +
> +static void local_flush_tlb_range_threshold_asid(unsigned long start,
> + unsigned long size,
> + unsigned long stride,
> + unsigned long asid)
> +{
> + u16 nr_ptes_in_range = DIV_ROUND_UP(size, stride);
> + int i;
> +
> + if (nr_ptes_in_range > tlb_flush_all_threshold) {
> + if (asid != -1)
> + local_flush_tlb_all_asid(asid);
> + else
> + local_flush_tlb_all();
> + return;
> + }
> +
> + for (i = 0; i < nr_ptes_in_range; ++i) {
> + if (asid != -1)
> + local_flush_tlb_page_asid(start, asid);
> + else
> + local_flush_tlb_page(start);
> + start += stride;
> + }
> +}
> +
> static inline void local_flush_tlb_range(unsigned long start,
> unsigned long size, unsigned long stride)
> {
> if (size <= stride)
> local_flush_tlb_page(start);
> - else
> + else if (size == (unsigned long)-1)
The more we scatter this -1 around, especially now that we also need to
cast it, the more I think we should introduce a #define for it.
> local_flush_tlb_all();
> + else
> + local_flush_tlb_range_threshold_asid(start, size, stride, -1);
> +
> }
>
> static inline void local_flush_tlb_range_asid(unsigned long start,
> @@ -38,8 +73,10 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
> {
> if (size <= stride)
> local_flush_tlb_page_asid(start, asid);
> - else
> + else if (size == (unsigned long)-1)
> local_flush_tlb_all_asid(asid);
> + else
> + local_flush_tlb_range_threshold_asid(start, size, stride, asid);
> }
>
> static void __ipi_flush_tlb_all(void *info)
> --
> 2.39.2
>
Otherwise,
Reviewed-by: Andrew Jones <ajones@xxxxxxxxxxxxxxxx>
Thanks,
drew
