Hello Sean,
On 1/24/2025 6:39 PM, Sean Christopherson wrote:
> On Fri, Jan 24, 2025, Ashish Kalra wrote:
>> With discussions with the AMD IOMMU team, here is the AMD IOMMU
>> initialization flow:
>
> ..
>
>> IOMMU SNP check
>> Core IOMMU subsystem init is done during iommu_subsys_init() via
>> subsys_initcall. This function does change the DMA mode depending on
>> kernel config. Hence, SNP check should be done after subsys_initcall.
>> That's why its done currently during IOMMU PCI init (IOMMU_PCI_INIT stage).
>> And for that reason snp_rmptable_init() is currently invoked via
>> device_initcall().
>>
>> The summary is that we cannot move snp_rmptable_init() to subsys_initcall as
>> core IOMMU subsystem gets initialized via subsys_initcall.
>
> Just explicitly invoke RMP initialization during IOMMU SNP setup. Pretending
> there's no connection when snp_rmptable_init() checks amd_iommu_snp_en and has
> a comment saying it needs to come after IOMMU SNP setup is ridiculous.
>
Thanks for the suggestion and the patch, i have tested it works for all cases
and scenarios. I will post the next version of the patch-set based on this
patch.
Ashish
> Compile tested only.
>
> ---
> From: Sean Christopherson <seanjc@xxxxxxxxxx>
> Date: Fri, 24 Jan 2025 16:25:58 -0800
> Subject: [PATCH] x86/sev: iommu/amd: Explicitly init SNP's RMP table during
> IOMMU SNP setup
>
> Explicitly initialize the RMP table during IOMMU SNP setup, as there is a
> hard dependency on the IOMMU being configured first, and dancing around
> the dependency with initcall shenanigans and a comment is all kinds of
> stupid.
>
> The RMP is blatantly not a device; initializing it via a device_initcall()
> is confusing and "works" only because of dumb luck: due to kernel build
> order, when the the PSP driver is built-in, its effective device_initcall()
> just so happens to be invoked after snp_rmptable_init().
>
> That all falls apart if the order is changed in any way. E.g. if KVM
> is built-in and attempts to access the RMP during its device_initcall(),
> chaos ensues.
>
> Signed-off-by: Sean Christopherson <seanjc@xxxxxxxxxx>
> ---
> arch/x86/include/asm/sev.h | 1 +
> arch/x86/virt/svm/sev.c | 25 ++++++++-----------------
> drivers/iommu/amd/init.c | 7 ++++++-
> 3 files changed, 15 insertions(+), 18 deletions(-)
>
> diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
> index 91f08af31078..30da0fc15923 100644
> --- a/arch/x86/include/asm/sev.h
> +++ b/arch/x86/include/asm/sev.h
> @@ -503,6 +503,7 @@ static inline void snp_kexec_begin(void) { }
>
> #ifdef CONFIG_KVM_AMD_SEV
> bool snp_probe_rmptable_info(void);
> +int __init snp_rmptable_init(void);
> int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level);
> void snp_dump_hva_rmpentry(unsigned long address);
> int psmash(u64 pfn);
> diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
> index 9a6a943d8e41..d932aa21340b 100644
> --- a/arch/x86/virt/svm/sev.c
> +++ b/arch/x86/virt/svm/sev.c
> @@ -189,19 +189,19 @@ void __init snp_fixup_e820_tables(void)
> * described in the SNP_INIT_EX firmware command description in the SNP
> * firmware ABI spec.
> */
> -static int __init snp_rmptable_init(void)
> +int __init snp_rmptable_init(void)
> {
> u64 max_rmp_pfn, calc_rmp_sz, rmptable_size, rmp_end, val;
> void *rmptable_start;
>
> - if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
> - return 0;
> + if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP)))
> + return -ENOSYS;
>
> - if (!amd_iommu_snp_en)
> - goto nosnp;
> + if (WARN_ON_ONCE(!amd_iommu_snp_en))
> + return -ENOSYS;
>
> if (!probed_rmp_size)
> - goto nosnp;
> + return -ENOSYS;
>
> rmp_end = probed_rmp_base + probed_rmp_size - 1;
>
> @@ -218,13 +218,13 @@ static int __init snp_rmptable_init(void)
> if (calc_rmp_sz > probed_rmp_size) {
> pr_err("Memory reserved for the RMP table does not cover full system RAM (expected 0x%llx got 0x%llx)\n",
> calc_rmp_sz, probed_rmp_size);
> - goto nosnp;
> + return -ENOSYS;
> }
>
> rmptable_start = memremap(probed_rmp_base, probed_rmp_size, MEMREMAP_WB);
> if (!rmptable_start) {
> pr_err("Failed to map RMP table\n");
> - goto nosnp;
> + return -ENOMEM;
> }
>
> /*
> @@ -261,17 +261,8 @@ static int __init snp_rmptable_init(void)
> crash_kexec_post_notifiers = true;
>
> return 0;
> -
> -nosnp:
> - cc_platform_clear(CC_ATTR_HOST_SEV_SNP);
> - return -ENOSYS;
> }
>
> -/*
> - * This must be called after the IOMMU has been initialized.
> - */
> -device_initcall(snp_rmptable_init);
> -
> static struct rmpentry *get_rmpentry(u64 pfn)
> {
> if (WARN_ON_ONCE(pfn > rmptable_max_pfn))
> diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
> index 0e0a531042ac..d00530156a72 100644
> --- a/drivers/iommu/amd/init.c
> +++ b/drivers/iommu/amd/init.c
> @@ -3171,7 +3171,7 @@ static bool __init detect_ivrs(void)
> return true;
> }
>
> -static void iommu_snp_enable(void)
> +static __init void iommu_snp_enable(void)
> {
> #ifdef CONFIG_KVM_AMD_SEV
> if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
> @@ -3196,6 +3196,11 @@ static void iommu_snp_enable(void)
> goto disable_snp;
> }
>
> + if (snp_rmptable_init()) {
> + pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n");
> + goto disable_snp;
> + }
> +
> pr_info("IOMMU SNP support enabled.\n");
> return;
>
>
> base-commit: ac80076177131f6e3291737c851a6fe32cc03fd3
