On Sat, Mar 06, 2021 at 01:15AM +0100, Andrey Konovalov wrote:
> This change adds an argument to mte_set_mem_tag_range() that allows
> to enable memory initialization when settinh the allocation tags.
> The implementation uses stzg instruction instead of stg when this
> argument indicates to initialize memory.
>
> Combining setting allocation tags with memory initialization will
> improve HW_TAGS KASAN performance when init_on_alloc/free is enabled.
>
> This change doesn't integrate memory initialization with KASAN,
> this is done is subsequent patches in this series.
>
> Signed-off-by: Andrey Konovalov <andreyknvl@xxxxxxxxxx>
Acked-by: Marco Elver <elver@xxxxxxxxxx>
> ---
> arch/arm64/include/asm/memory.h | 4 ++--
> arch/arm64/include/asm/mte-kasan.h | 20 ++++++++++++++------
> mm/kasan/kasan.h | 9 +++++----
> 3 files changed, 21 insertions(+), 12 deletions(-)
>
> diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
> index c759faf7a1ff..f1ba48b4347d 100644
> --- a/arch/arm64/include/asm/memory.h
> +++ b/arch/arm64/include/asm/memory.h
> @@ -248,8 +248,8 @@ static inline const void *__tag_set(const void *addr, u8 tag)
> #define arch_init_tags(max_tag) mte_init_tags(max_tag)
> #define arch_get_random_tag() mte_get_random_tag()
> #define arch_get_mem_tag(addr) mte_get_mem_tag(addr)
> -#define arch_set_mem_tag_range(addr, size, tag) \
> - mte_set_mem_tag_range((addr), (size), (tag))
> +#define arch_set_mem_tag_range(addr, size, tag, init) \
> + mte_set_mem_tag_range((addr), (size), (tag), (init))
> #endif /* CONFIG_KASAN_HW_TAGS */
>
> /*
> diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
> index 7ab500e2ad17..35fe549f7ea4 100644
> --- a/arch/arm64/include/asm/mte-kasan.h
> +++ b/arch/arm64/include/asm/mte-kasan.h
> @@ -53,7 +53,8 @@ static inline u8 mte_get_random_tag(void)
> * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
> * size must be non-zero and MTE_GRANULE_SIZE aligned.
> */
> -static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
> +static inline void mte_set_mem_tag_range(void *addr, size_t size,
> + u8 tag, bool init)
> {
> u64 curr, end;
>
> @@ -68,10 +69,16 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
> * 'asm volatile' is required to prevent the compiler to move
> * the statement outside of the loop.
> */
> - asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
> - :
> - : "r" (curr)
> - : "memory");
> + if (init)
> + asm volatile(__MTE_PREAMBLE "stzg %0, [%0]"
> + :
> + : "r" (curr)
> + : "memory");
> + else
> + asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
> + :
> + : "r" (curr)
> + : "memory");
>
> curr += MTE_GRANULE_SIZE;
> } while (curr != end);
> @@ -100,7 +107,8 @@ static inline u8 mte_get_random_tag(void)
> return 0xFF;
> }
>
> -static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag)
> +static inline void mte_set_mem_tag_range(void *addr, size_t size,
> + u8 tag, bool init)
> {
> }
>
> diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h
> index 8c55634d6edd..7fbb32234414 100644
> --- a/mm/kasan/kasan.h
> +++ b/mm/kasan/kasan.h
> @@ -291,7 +291,7 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
> #define arch_get_mem_tag(addr) (0xFF)
> #endif
> #ifndef arch_set_mem_tag_range
> -#define arch_set_mem_tag_range(addr, size, tag) ((void *)(addr))
> +#define arch_set_mem_tag_range(addr, size, tag, init) ((void *)(addr))
> #endif
>
> #define hw_enable_tagging() arch_enable_tagging()
> @@ -299,7 +299,8 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag)
> #define hw_set_tagging_report_once(state) arch_set_tagging_report_once(state)
> #define hw_get_random_tag() arch_get_random_tag()
> #define hw_get_mem_tag(addr) arch_get_mem_tag(addr)
> -#define hw_set_mem_tag_range(addr, size, tag) arch_set_mem_tag_range((addr), (size), (tag))
> +#define hw_set_mem_tag_range(addr, size, tag, init) \
> + arch_set_mem_tag_range((addr), (size), (tag), (init))
>
> #else /* CONFIG_KASAN_HW_TAGS */
>
> @@ -343,7 +344,7 @@ static inline void kasan_poison(const void *addr, size_t size, u8 value)
> if (WARN_ON(size & KASAN_GRANULE_MASK))
> return;
>
> - hw_set_mem_tag_range((void *)addr, size, value);
> + hw_set_mem_tag_range((void *)addr, size, value, false);
> }
>
> static inline void kasan_unpoison(const void *addr, size_t size)
> @@ -360,7 +361,7 @@ static inline void kasan_unpoison(const void *addr, size_t size)
> return;
> size = round_up(size, KASAN_GRANULE_SIZE);
>
> - hw_set_mem_tag_range((void *)addr, size, tag);
> + hw_set_mem_tag_range((void *)addr, size, tag, false);
> }
>
> static inline bool kasan_byte_accessible(const void *addr)
> --
> 2.30.1.766.gb4fecdf3b7-goog
>
