Encrypt memory areas in place when possible (e.g. zero page, etc.) so
that special handling isn't needed afterwards.
Signed-off-by: Tom Lendacky <thomas.lendacky@xxxxxxx>
---
arch/x86/kernel/head64.c | 93 ++++++++++++++++++++++++++++++++++++++++++++--
arch/x86/kernel/setup.c | 8 ++++
2 files changed, 96 insertions(+), 5 deletions(-)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 88c7bae..358d7bc 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -47,12 +47,12 @@ static void __init reset_early_page_tables(void)
}
/* Create a new PMD entry */
-int __init early_make_pgtable(unsigned long address)
+static int __init __early_make_pgtable(unsigned long address, pmdval_t pmd)
{
unsigned long physaddr = address - __PAGE_OFFSET;
pgdval_t pgd, *pgd_p;
pudval_t pud, *pud_p;
- pmdval_t pmd, *pmd_p;
+ pmdval_t *pmd_p;
/* Invalid address or early pgt is done ? */
if (physaddr >= MAXMEM || read_cr3() != __sme_pa_nodebug(early_level4_pgt))
@@ -94,12 +94,95 @@ again:
memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
}
- pmd = (physaddr & PMD_MASK) + early_pmd_flags;
pmd_p[pmd_index(address)] = pmd;
return 0;
}
+int __init early_make_pgtable(unsigned long address)
+{
+ unsigned long physaddr = address - __PAGE_OFFSET;
+ pmdval_t pmd;
+
+ pmd = (physaddr & PMD_MASK) + early_pmd_flags;
+
+ return __early_make_pgtable(address, pmd);
+}
+
+static void __init create_unencrypted_mapping(void *address, unsigned long size)
+{
+ unsigned long physaddr = (unsigned long)address - __PAGE_OFFSET;
+ pmdval_t pmd_flags, pmd;
+
+ if (!sme_me_mask)
+ return;
+
+ /* Clear the encryption mask from the early_pmd_flags */
+ pmd_flags = early_pmd_flags & ~sme_me_mask;
+
+ do {
+ pmd = (physaddr & PMD_MASK) + pmd_flags;
+ __early_make_pgtable((unsigned long)address, pmd);
+
+ address += PMD_SIZE;
+ physaddr += PMD_SIZE;
+ size = (size < PMD_SIZE) ? 0 : size - PMD_SIZE;
+ } while (size);
+}
+
+static void __init __clear_mapping(unsigned long address)
+{
+ unsigned long physaddr = address - __PAGE_OFFSET;
+ pgdval_t pgd, *pgd_p;
+ pudval_t pud, *pud_p;
+ pmdval_t *pmd_p;
+
+ /* Invalid address or early pgt is done ? */
+ if (physaddr >= MAXMEM ||
+ read_cr3() != __sme_pa_nodebug(early_level4_pgt))
+ return;
+
+ pgd_p = &early_level4_pgt[pgd_index(address)].pgd;
+ pgd = *pgd_p;
+
+ if (!pgd)
+ return;
+
+ /*
+ * The use of __START_KERNEL_map rather than __PAGE_OFFSET here matches
+ * __early_make_pgtable where the entry was created.
+ */
+ pud_p = (pudval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+ pud_p += pud_index(address);
+ pud = *pud_p;
+
+ if (!pud)
+ return;
+
+ pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
+ pmd_p[pmd_index(address)] = 0;
+}
+
+static void __init clear_mapping(void *address, unsigned long size)
+{
+ if (!sme_me_mask)
+ return;
+
+ do {
+ __clear_mapping((unsigned long)address);
+
+ address += PMD_SIZE;
+ size = (size < PMD_SIZE) ? 0 : size - PMD_SIZE;
+ } while (size);
+}
+
+static void __init sme_memcpy(void *dst, void *src, unsigned long size)
+{
+ create_unencrypted_mapping(src, size);
+ memcpy(dst, src, size);
+ clear_mapping(src, size);
+}
+
/* Don't add a printk in there. printk relies on the PDA which is not initialized
yet. */
static void __init clear_bss(void)
@@ -122,12 +205,12 @@ static void __init copy_bootdata(char *real_mode_data)
char * command_line;
unsigned long cmd_line_ptr;
- memcpy(&boot_params, real_mode_data, sizeof boot_params);
+ sme_memcpy(&boot_params, real_mode_data, sizeof boot_params);
sanitize_boot_params(&boot_params);
cmd_line_ptr = get_cmd_line_ptr();
if (cmd_line_ptr) {
command_line = __va(cmd_line_ptr);
- memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
+ sme_memcpy(boot_command_line, command_line, COMMAND_LINE_SIZE);
}
}
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 1489da8..1fdaa11 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -114,6 +114,7 @@
#include <asm/microcode.h>
#include <asm/mmu_context.h>
#include <asm/kaslr.h>
+#include <asm/mem_encrypt.h>
/*
* max_low_pfn_mapped: highest direct mapped pfn under 4GB
@@ -376,6 +377,13 @@ static void __init reserve_initrd(void)
!ramdisk_image || !ramdisk_size)
return; /* No initrd provided by bootloader */
+ /*
+ * This memory is marked encrypted by the kernel but the ramdisk
+ * was loaded in the clear by the bootloader, so make sure that
+ * the ramdisk image is encrypted.
+ */
+ sme_early_mem_enc(ramdisk_image, ramdisk_end - ramdisk_image);
+
initrd_start = 0;
mapped_size = memblock_mem_size(max_pfn_mapped);
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
