From: Alexander Potapenko <glider@xxxxxxxxxx> Subject: kfence, kasan: make KFENCE compatible with KASAN Make KFENCE compatible with KASAN. Currently this helps test KFENCE itself, where KASAN can catch potential corruptions to KFENCE state, or other corruptions that may be a result of freepointer corruptions in the main allocators. [akpm@xxxxxxxxxxxxxxxxxxxx: merge fixup] [andreyknvl@xxxxxxxxxx: untag addresses for KFENCE] Link: https://lkml.kernel.org/r/9dc196006921b191d25d10f6e611316db7da2efc.1611946152.git.andreyknvl@xxxxxxxxxx Link: https://lkml.kernel.org/r/20201103175841.3495947-7-elver@xxxxxxxxxx Signed-off-by: Marco Elver <elver@xxxxxxxxxx> Signed-off-by: Alexander Potapenko <glider@xxxxxxxxxx> Signed-off-by: Andrey Konovalov <andreyknvl@xxxxxxxxxx> Reviewed-by: Dmitry Vyukov <dvyukov@xxxxxxxxxx> Reviewed-by: Jann Horn <jannh@xxxxxxxxxx> Co-developed-by: Marco Elver <elver@xxxxxxxxxx> Cc: Andrey Konovalov <andreyknvl@xxxxxxxxxx> Cc: Andrey Ryabinin <aryabinin@xxxxxxxxxxxxx> Cc: Andy Lutomirski <luto@xxxxxxxxxx> Cc: Borislav Petkov <bp@xxxxxxxxx> Cc: Catalin Marinas <catalin.marinas@xxxxxxx> Cc: Christopher Lameter <cl@xxxxxxxxx> Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx> Cc: David Rientjes <rientjes@xxxxxxxxxx> Cc: Eric Dumazet <edumazet@xxxxxxxxxx> Cc: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> Cc: Hillf Danton <hdanton@xxxxxxxx> Cc: "H. Peter Anvin" <hpa@xxxxxxxxx> Cc: Ingo Molnar <mingo@xxxxxxxxxx> Cc: Joern Engel <joern@xxxxxxxxxxxxxxx> Cc: Jonathan Corbet <corbet@xxxxxxx> Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx> Cc: Kees Cook <keescook@xxxxxxxxxxxx> Cc: Mark Rutland <mark.rutland@xxxxxxx> Cc: Paul E. McKenney <paulmck@xxxxxxxxxx> Cc: Pekka Enberg <penberg@xxxxxxxxxx> Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx> Cc: SeongJae Park <sjpark@xxxxxxxxx> Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx> Cc: Vlastimil Babka <vbabka@xxxxxxx> Cc: Will Deacon <will@xxxxxxxxxx> Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx> --- lib/Kconfig.kfence | 2 +- mm/kasan/common.c | 6 ++++++ mm/kasan/generic.c | 3 ++- mm/kasan/kasan.h | 21 ++++++++++++++++++--- mm/kasan/shadow.c | 13 +++++++++++++ 5 files changed, 40 insertions(+), 5 deletions(-) --- a/lib/Kconfig.kfence~kfence-kasan-make-kfence-compatible-with-kasan +++ a/lib/Kconfig.kfence @@ -5,7 +5,7 @@ config HAVE_ARCH_KFENCE menuconfig KFENCE bool "KFENCE: low-overhead sampling-based memory safety error detector" - depends on HAVE_ARCH_KFENCE && !KASAN && (SLAB || SLUB) + depends on HAVE_ARCH_KFENCE && (SLAB || SLUB) select STACKTRACE help KFENCE is a low-overhead sampling-based detector of heap out-of-bounds --- a/mm/kasan/common.c~kfence-kasan-make-kfence-compatible-with-kasan +++ a/mm/kasan/common.c @@ -335,6 +335,9 @@ static bool ____kasan_slab_free(struct k tagged_object = object; object = kasan_reset_tag(object); + if (is_kfence_address(object)) + return false; + if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) != object)) { kasan_report_invalid_free(tagged_object, ip); @@ -413,6 +416,9 @@ static void *____kasan_kmalloc(struct km if (unlikely(object == NULL)) return NULL; + if (is_kfence_address(kasan_reset_tag(object))) + return (void *)object; + redzone_start = round_up((unsigned long)(object + size), KASAN_GRANULE_SIZE); redzone_end = round_up((unsigned long)object + cache->object_size, --- a/mm/kasan/generic.c~kfence-kasan-make-kfence-compatible-with-kasan +++ a/mm/kasan/generic.c @@ -14,6 +14,7 @@ #include <linux/init.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/kfence.h> #include <linux/kmemleak.h> #include <linux/linkage.h> #include <linux/memblock.h> @@ -331,7 +332,7 @@ void kasan_record_aux_stack(void *addr) struct kasan_alloc_meta *alloc_meta; void *object; - if (!(page && PageSlab(page))) + if (is_kfence_address(addr) || !(page && PageSlab(page))) return; cache = page->slab_cache; --- a/mm/kasan/kasan.h~kfence-kasan-make-kfence-compatible-with-kasan +++ a/mm/kasan/kasan.h @@ -3,6 +3,7 @@ #define __MM_KASAN_KASAN_H #include <linux/kasan.h> +#include <linux/kfence.h> #include <linux/stackdepot.h> #ifdef CONFIG_KASAN_HW_TAGS @@ -331,14 +332,28 @@ static inline u8 kasan_random_tag(void) static inline void kasan_poison(const void *address, size_t size, u8 value) { - hw_set_mem_tag_range(kasan_reset_tag(address), + address = kasan_reset_tag(address); + + /* Skip KFENCE memory if called explicitly outside of sl*b. */ + if (is_kfence_address(address)) + return; + + hw_set_mem_tag_range((void *)address, round_up(size, KASAN_GRANULE_SIZE), value); } static inline void kasan_unpoison(const void *address, size_t size) { - hw_set_mem_tag_range(kasan_reset_tag(address), - round_up(size, KASAN_GRANULE_SIZE), get_tag(address)); + u8 tag = get_tag(address); + + address = kasan_reset_tag(address); + + /* Skip KFENCE memory if called explicitly outside of sl*b. */ + if (is_kfence_address(address)) + return; + + hw_set_mem_tag_range((void *)address, + round_up(size, KASAN_GRANULE_SIZE), tag); } static inline bool kasan_byte_accessible(const void *addr) --- a/mm/kasan/shadow.c~kfence-kasan-make-kfence-compatible-with-kasan +++ a/mm/kasan/shadow.c @@ -13,6 +13,7 @@ #include <linux/init.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/kfence.h> #include <linux/kmemleak.h> #include <linux/memory.h> #include <linux/mm.h> @@ -84,6 +85,10 @@ void kasan_poison(const void *address, s address = kasan_reset_tag(address); size = round_up(size, KASAN_GRANULE_SIZE); + /* Skip KFENCE memory if called explicitly outside of sl*b. */ + if (is_kfence_address(address)) + return; + shadow_start = kasan_mem_to_shadow(address); shadow_end = kasan_mem_to_shadow(address + size); @@ -102,6 +107,14 @@ void kasan_unpoison(const void *address, */ address = kasan_reset_tag(address); + /* + * Skip KFENCE memory if called explicitly outside of sl*b. Also note + * that calls to ksize(), where size is not a multiple of machine-word + * size, would otherwise poison the invalid portion of the word. + */ + if (is_kfence_address(address)) + return; + kasan_poison(address, size, tag); if (size & KASAN_GRANULE_MASK) { _
