Kfence only needs its pool to be mapped as page granularity, if it is
inited early. Previous judgement was a bit over protected. From [1], Mark
suggested to "just map the KFENCE region a page granularity". So I
decouple it from judgement and do page granularity mapping for kfence
pool only. Need to be noticed that late init of kfence pool still requires
page granularity mapping.
Page granularity mapping in theory cost more(2M per 1GB) memory on arm64
platform. Like what I've tested on QEMU(emulated 1GB RAM) with
gki_defconfig, also turning off rodata protection:
Before:
[root@liebao ]# cat /proc/meminfo
MemTotal: 999484 kB
After:
[root@liebao ]# cat /proc/meminfo
MemTotal: 1001480 kB
To implement this, also relocate the kfence pool allocation before the
linear mapping setting up, arm64_kfence_alloc_pool is to allocate phys
addr, __kfence_pool is to be set after linear mapping set up.
LINK: [1] https://lore.kernel.org/linux-arm-kernel/Y+IsdrvDNILA59UN@FVFF77S0Q05N/
Suggested-by: Mark Rutland <mark.rutland@xxxxxxx>
Signed-off-by: Zhenhua Huang <quic_zhenhuah@xxxxxxxxxxx>
---
arch/arm64/include/asm/kfence.h | 16 +++++++++++
arch/arm64/mm/mmu.c | 59 +++++++++++++++++++++++++++++++++++++++++
arch/arm64/mm/pageattr.c | 9 +++++--
include/linux/kfence.h | 1 +
mm/kfence/core.c | 4 +++
5 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/kfence.h b/arch/arm64/include/asm/kfence.h
index aa855c6..8143c91 100644
--- a/arch/arm64/include/asm/kfence.h
+++ b/arch/arm64/include/asm/kfence.h
@@ -10,6 +10,22 @@
#include <asm/set_memory.h>
+extern phys_addr_t early_kfence_pool;
+
+#ifdef CONFIG_KFENCE
+
+extern char *__kfence_pool;
+static inline void kfence_set_pool(phys_addr_t addr)
+{
+ __kfence_pool = phys_to_virt(addr);
+}
+
+#else
+
+static inline void kfence_set_pool(phys_addr_t addr) { }
+
+#endif
+
static inline bool arch_kfence_init_pool(void) { return true; }
static inline bool kfence_protect_page(unsigned long addr, bool protect)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 6f9d889..61944c70 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -24,6 +24,7 @@
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/set_memory.h>
+#include <linux/kfence.h>
#include <asm/barrier.h>
#include <asm/cputype.h>
@@ -38,6 +39,7 @@
#include <asm/ptdump.h>
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>
+#include <asm/kfence.h>
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
@@ -525,6 +527,48 @@ static int __init enable_crash_mem_map(char *arg)
}
early_param("crashkernel", enable_crash_mem_map);
+#ifdef CONFIG_KFENCE
+
+static bool kfence_early_init __initdata = !!CONFIG_KFENCE_SAMPLE_INTERVAL;
+/*
+ * early_param can be parsed before linear mapping
+ * set up
+ */
+static int __init parse_kfence_early_init(char *p)
+{
+ int val;
+
+ if (get_option(&p, &val))
+ kfence_early_init = !!val;
+ return 0;
+}
+early_param("kfence.sample_interval", parse_kfence_early_init);
+
+static phys_addr_t arm64_kfence_alloc_pool(void)
+{
+ phys_addr_t kfence_pool;
+
+ if (!kfence_early_init)
+ return 0;
+
+ kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE);
+ if (!kfence_pool)
+ pr_err("failed to allocate kfence pool\n");
+
+ return kfence_pool;
+}
+
+#else
+
+static phys_addr_t arm64_kfence_alloc_pool(void)
+{
+ return 0;
+}
+
+#endif
+
+phys_addr_t early_kfence_pool;
+
static void __init map_mem(pgd_t *pgdp)
{
static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN);
@@ -543,6 +587,10 @@ static void __init map_mem(pgd_t *pgdp)
*/
BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
+ early_kfence_pool = arm64_kfence_alloc_pool();
+ if (early_kfence_pool)
+ memblock_mark_nomap(early_kfence_pool, KFENCE_POOL_SIZE);
+
if (can_set_direct_map())
flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
@@ -608,6 +656,17 @@ static void __init map_mem(pgd_t *pgdp)
}
}
#endif
+
+ /* Kfence pool needs page-level mapping */
+ if (early_kfence_pool) {
+ __map_memblock(pgdp, early_kfence_pool,
+ early_kfence_pool + KFENCE_POOL_SIZE,
+ pgprot_tagged(PAGE_KERNEL),
+ NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
+ memblock_clear_nomap(early_kfence_pool, KFENCE_POOL_SIZE);
+ /* kfence_pool really mapped now */
+ kfence_set_pool(early_kfence_pool);
+ }
