Kernel text replication needs to maintain separate per-node page
tables for the kernel text. In order to do this without affecting
other kernel memory mappings, placing the kernel such that it does
not share a L0 page table entry with any other mapping is desirable.
Prior to this commit, the layout without KASLR was:
+----------+
| vmalloc |
+----------+
| Kernel |
+----------+ MODULES_END, VMALLOC_START, KIMAGE_VADDR =
| Modules | MODULES_VADDR + MODULES_VSIZE
+----------+ MODULES_VADDR = _PAGE_END(VA_BITS_MIN)
| VA space |
+----------+ 0
This becomes:
+----------+
| vmalloc |
+----------+ VMALLOC_START = MODULES_END + PGDIR_SIZE
| Kernel |
+----------+ MODULES_END, KIMAGE_VADDR = _PAGE_END(VA_BITS_MIN) + PGDIR_SIZE
| Modules |
+----------+ MODULES_VADDR = MODULES_END - MODULES_VSIZE
| VA space |
+----------+ 0
This assumes MODULES_VSIZE (128M) <= PGDIR_SIZE.
One side effect of this change is that KIMAGE_VADDR's definition now
includes PGDIR_SIZE (to leave room for the modules) but this is not
defined when asm/memory.h is included. This means KIMAGE_VADDR can
not be used in inline functions within this file, so we convert
kaslr_offset() and kaslr_enabled() to be macros instead.
Signed-off-by: Russell King (Oracle) <rmk+kernel@xxxxxxxxxxxxxxx>
---
arch/arm64/include/asm/memory.h | 26 ++++++++++----------------
arch/arm64/include/asm/pgtable.h | 2 +-
arch/arm64/mm/mmu.c | 3 ++-
3 files changed, 13 insertions(+), 18 deletions(-)
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index c735afdf639b..089f556b7387 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -43,9 +43,9 @@
#define VA_BITS (CONFIG_ARM64_VA_BITS)
#define _PAGE_OFFSET(va) (-(UL(1) << (va)))
#define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS))
-#define KIMAGE_VADDR (MODULES_END)
-#define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
-#define MODULES_VADDR (_PAGE_END(VA_BITS_MIN))
+#define KIMAGE_VADDR (_PAGE_END(VA_BITS_MIN) + PGDIR_SIZE)
+#define MODULES_END (KIMAGE_VADDR)
+#define MODULES_VADDR (MODULES_END - MODULES_VSIZE)
#define MODULES_VSIZE (SZ_128M)
#define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT)))
#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE)
@@ -199,20 +199,14 @@ extern u64 kimage_vaddr;
/* the offset between the kernel virtual and physical mappings */
extern u64 kimage_voffset;
-static inline unsigned long kaslr_offset(void)
-{
- return kimage_vaddr - KIMAGE_VADDR;
-}
+#define kaslr_offset() ((unsigned long)(kimage_vaddr - KIMAGE_VADDR))
-static inline bool kaslr_enabled(void)
-{
- /*
- * The KASLR offset modulo MIN_KIMG_ALIGN is taken from the physical
- * placement of the image rather than from the seed, so a displacement
- * of less than MIN_KIMG_ALIGN means that no seed was provided.
- */
- return kaslr_offset() >= MIN_KIMG_ALIGN;
-}
+/*
+ * The KASLR offset modulo MIN_KIMG_ALIGN is taken from the physical
+ * placement of the image rather than from the seed, so a displacement
+ * of less than MIN_KIMG_ALIGN means that no seed was provided.
+ */
+#define kaslr_enabled() (kaslr_offset() >= MIN_KIMG_ALIGN)
/*
* Allow all memory at the discovery stage. We will clip it later.
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 0bd18de9fd97..cb526e69299d 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -21,7 +21,7 @@
* VMALLOC_END: extends to the available space below vmemmap, PCI I/O space
* and fixed mappings
*/
-#define VMALLOC_START (MODULES_END)
+#define VMALLOC_START (MODULES_END + PGDIR_SIZE)
#define VMALLOC_END (VMEMMAP_START - SZ_256M)
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 4829abe017e9..baf74d0c43c9 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -478,7 +478,8 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{
- if ((virt >= PAGE_END) && (virt < VMALLOC_START)) {
+ if ((virt >= PAGE_END) && (virt < VMALLOC_START) &&
+ !is_kernel(virt)) {
pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
&phys, virt);
return;
--
2.30.2
