From: "Russell King (Oracle)" <rmk+kernel@xxxxxxxxxxxxxxx>
Kernel text replication needs to maintain separate per-node page
tables for the kernel text. In order to do this without affecting
other kernel memory mappings, placing the kernel such that it does
not share a L0 page table entry with any other mapping is desirable.
Prior to this commit, the layout without KASLR was:
+----------+
| vmalloc |
+----------+
| Kernel |
+----------+ MODULES_END, VMALLOC_START, KIMAGE_VADDR =
| Modules | MODULES_VADDR + MODULES_VSIZE
+----------+ MODULES_VADDR = _PAGE_END(VA_BITS_MIN)
| VA space |
+----------+ 0
This becomes:
+----------+
| vmalloc |
+----------+ VMALLOC_START = MODULES_END + PGDIR_SIZE
| Kernel |
+----------+ MODULES_END, KIMAGE_VADDR = _PAGE_END(VA_BITS_MIN) +
| Modules | max(PGDIR_SIZE, MODULES_VSIZE)
+----------+ MODULES_VADDR = MODULES_END - MODULES_VSIZE
| VA space |
+----------+ 0
This assumes MODULES_VSIZE (128M) <= PGDIR_SIZE.
One side effect of this change is that KIMAGE_VADDR's definition now
includes PGDIR_SIZE (to leave room for the modules) but this is not
defined when asm/memory.h is included. This means KIMAGE_VADDR can
not be used in inline functions within this file, so we convert
kaslr_offset() and kaslr_enabled() to be macros instead.
Signed-off-by: Russell King (Oracle) <rmk+kernel@xxxxxxxxxxxxxxx>
---
arch/arm64/include/asm/memory.h | 28 +++++++++++++++++++++-------
arch/arm64/include/asm/pgtable.h | 2 +-
arch/arm64/kernel/kaslr.c | 1 +
3 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index d82305ab420f..c73820fb36a3 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -43,9 +43,26 @@
#define VA_BITS (CONFIG_ARM64_VA_BITS)
#define _PAGE_OFFSET(va) (-(UL(1) << (va)))
#define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS))
-#define KIMAGE_VADDR (MODULES_END)
-#define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
-#define MODULES_VADDR (_PAGE_END(VA_BITS_MIN))
+
+/*
+ * Setting KIMAGE_VADDR has got a lot harder, ideally we'd like to use
+ * min(PGDIR_SIZE, MODULES_VSIZE) but this can't work because this is used
+ * both in assembly as C, where it causes problems. min_t() solves the
+ * C problems but can't be used in assembly.
+ * CONFIG_ARM64_4K_PAGES, PGDIR_SIZE is 2M, 1G, 512G
+ * CONFIG_ARM64_16K_PAGES, PGDIR_SIZE is 32M, 64G or 128T
+ * CONFIG_ARM64_64K_PAGES, PGDIR_SIZE is 512M or 4T
+ */
+#if (CONFIG_ARM64_4K_PAGES && CONFIG_PGTABLE_LEVELS < 4) || \
+ (CONFIG_ARM64_16K_PAGES && CONFIG_PGTABLE_LEVELS < 3) || \
+ (CONFIG_ARM64_64K_PAGES && CONFIG_PGTABLE_LEVELS < 2)
+#define KIMAGE_OFFSET MODULES_VSIZE
+#else
+#define KIMAGE_OFFSET PGDIR_SIZE
+#endif
+#define KIMAGE_VADDR (_PAGE_END(VA_BITS_MIN) + KIMAGE_OFFSET)
+#define MODULES_END (KIMAGE_VADDR)
+#define MODULES_VADDR (MODULES_END - MODULES_VSIZE)
#define MODULES_VSIZE (SZ_2G)
#define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT)))
#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE)
@@ -223,10 +240,7 @@ extern s64 memstart_addr;
/* the offset between the kernel virtual and physical mappings */
extern u64 kimage_voffset;
-static inline unsigned long kaslr_offset(void)
-{
- return (u64)&_text - KIMAGE_VADDR;
-}
+#define kaslr_offset() ((unsigned long)((u64)&_text - KIMAGE_VADDR))
#ifdef CONFIG_RANDOMIZE_BASE
void kaslr_init(void);
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 79ce70fbb751..97d2127d64eb 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -21,7 +21,7 @@
* VMALLOC_END: extends to the available space below vmemmap, PCI I/O space
* and fixed mappings
*/
-#define VMALLOC_START (MODULES_END)
+#define VMALLOC_START (MODULES_END + PGDIR_SIZE)
#define VMALLOC_END (VMEMMAP_START - SZ_256M)
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c
index 12c7f3c8ba76..1af065280d86 100644
--- a/arch/arm64/kernel/kaslr.c
+++ b/arch/arm64/kernel/kaslr.c
@@ -9,6 +9,7 @@
#include <asm/cpufeature.h>
#include <asm/memory.h>
+#include <asm/pgtable.h>
u16 __initdata memstart_offset_seed;
--
2.20.1
