On Tue, Jun 27, 2023 at 02:12:41AM +1200, Kai Huang wrote:
> Start to transit out the "multi-steps" to construct a list of "TD Memory
> Regions" (TDMRs) to cover all TDX-usable memory regions.
>
> The kernel configures TDX-usable memory regions by passing a list of
> TDMRs "TD Memory Regions" (TDMRs) to the TDX module. Each TDMR contains
> the information of the base/size of a memory region, the base/size of the
> associated Physical Address Metadata Table (PAMT) and a list of reserved
> areas in the region.
>
> Do the first step to fill out a number of TDMRs to cover all TDX memory
> regions. To keep it simple, always try to use one TDMR for each memory
> region. As the first step only set up the base/size for each TDMR.
>
> Each TDMR must be 1G aligned and the size must be in 1G granularity.
> This implies that one TDMR could cover multiple memory regions. If a
> memory region spans the 1GB boundary and the former part is already
> covered by the previous TDMR, just use a new TDMR for the remaining
> part.
>
> TDX only supports a limited number of TDMRs. Disable TDX if all TDMRs
> are consumed but there is more memory region to cover.
>
> There are fancier things that could be done like trying to merge
> adjacent TDMRs. This would allow more pathological memory layouts to be
> supported. But, current systems are not even close to exhausting the
> existing TDMR resources in practice. For now, keep it simple.
>
> Signed-off-by: Kai Huang <kai.huang@xxxxxxxxx>
> Reviewed-by: Kirill A. Shutemov <kirill.shutemov@xxxxxxxxxxxxxxx>
> Reviewed-by: Kuppuswamy Sathyanarayanan <sathyanarayanan.kuppuswamy@xxxxxxxxxxxxxxx>
> ---
Reviewed-by: Yuan Yao <yuan.yao@xxxxxxxxx>
>
> v11 -> v12:
> - Improved comments around looping over TDX memblock to create TDMRs.
> (Dave).
> - Added code to pr_warn() when consumed TDMRs reaching maximum TDMRs
> (Dave).
> - BIT_ULL(30) -> SZ_1G (Kirill)
> - Removed unused TDMR_PFN_ALIGNMENT (Sathy)
> - Added tags from Kirill/Sathy
>
> v10 -> v11:
> - No update
>
> v9 -> v10:
> - No change.
>
> v8 -> v9:
>
> - Added the last paragraph in the changelog (Dave).
> - Removed unnecessary type cast in tdmr_entry() (Dave).
>
> ---
> arch/x86/virt/vmx/tdx/tdx.c | 103 +++++++++++++++++++++++++++++++++++-
> arch/x86/virt/vmx/tdx/tdx.h | 3 ++
> 2 files changed, 105 insertions(+), 1 deletion(-)
>
> diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
> index e28615b60f9b..2ffc1517a93b 100644
> --- a/arch/x86/virt/vmx/tdx/tdx.c
> +++ b/arch/x86/virt/vmx/tdx/tdx.c
> @@ -341,6 +341,102 @@ static void free_tdmr_list(struct tdmr_info_list *tdmr_list)
> tdmr_list->max_tdmrs * tdmr_list->tdmr_sz);
> }
>
> +/* Get the TDMR from the list at the given index. */
> +static struct tdmr_info *tdmr_entry(struct tdmr_info_list *tdmr_list,
> + int idx)
> +{
> + int tdmr_info_offset = tdmr_list->tdmr_sz * idx;
> +
> + return (void *)tdmr_list->tdmrs + tdmr_info_offset;
> +}
> +
> +#define TDMR_ALIGNMENT SZ_1G
> +#define TDMR_ALIGN_DOWN(_addr) ALIGN_DOWN((_addr), TDMR_ALIGNMENT)
> +#define TDMR_ALIGN_UP(_addr) ALIGN((_addr), TDMR_ALIGNMENT)
> +
> +static inline u64 tdmr_end(struct tdmr_info *tdmr)
> +{
> + return tdmr->base + tdmr->size;
> +}
> +
> +/*
> + * Take the memory referenced in @tmb_list and populate the
> + * preallocated @tdmr_list, following all the special alignment
> + * and size rules for TDMR.
> + */
> +static int fill_out_tdmrs(struct list_head *tmb_list,
> + struct tdmr_info_list *tdmr_list)
> +{
> + struct tdx_memblock *tmb;
> + int tdmr_idx = 0;
> +
> + /*
> + * Loop over TDX memory regions and fill out TDMRs to cover them.
> + * To keep it simple, always try to use one TDMR to cover one
> + * memory region.
> + *
> + * In practice TDX supports at least 64 TDMRs. A 2-socket system
> + * typically only consumes less than 10 of those. This code is
> + * dumb and simple and may use more TMDRs than is strictly
> + * required.
> + */
> + list_for_each_entry(tmb, tmb_list, list) {
> + struct tdmr_info *tdmr = tdmr_entry(tdmr_list, tdmr_idx);
> + u64 start, end;
> +
> + start = TDMR_ALIGN_DOWN(PFN_PHYS(tmb->start_pfn));
> + end = TDMR_ALIGN_UP(PFN_PHYS(tmb->end_pfn));
> +
> + /*
> + * A valid size indicates the current TDMR has already
> + * been filled out to cover the previous memory region(s).
> + */
> + if (tdmr->size) {
> + /*
> + * Loop to the next if the current memory region
> + * has already been fully covered.
> + */
> + if (end <= tdmr_end(tdmr))
> + continue;
> +
> + /* Otherwise, skip the already covered part. */
> + if (start < tdmr_end(tdmr))
> + start = tdmr_end(tdmr);
> +
> + /*
> + * Create a new TDMR to cover the current memory
> + * region, or the remaining part of it.
> + */
> + tdmr_idx++;
> + if (tdmr_idx >= tdmr_list->max_tdmrs) {
> + pr_warn("initialization failed: TDMRs exhausted.\n");
> + return -ENOSPC;
> + }
> +
> + tdmr = tdmr_entry(tdmr_list, tdmr_idx);
> + }
> +
> + tdmr->base = start;
> + tdmr->size = end - start;
> + }
> +
> + /* @tdmr_idx is always the index of the last valid TDMR. */
> + tdmr_list->nr_consumed_tdmrs = tdmr_idx + 1;
> +
> + /*
> + * Warn early that kernel is about to run out of TDMRs.
> + *
> + * This is an indication that TDMR allocation has to be
> + * reworked to be smarter to not run into an issue.
> + */
> + if (tdmr_list->max_tdmrs - tdmr_list->nr_consumed_tdmrs < TDMR_NR_WARN)
> + pr_warn("consumed TDMRs reaching limit: %d used out of %d\n",
> + tdmr_list->nr_consumed_tdmrs,
> + tdmr_list->max_tdmrs);
> +
> + return 0;
> +}
> +
> /*
> * Construct a list of TDMRs on the preallocated space in @tdmr_list
> * to cover all TDX memory regions in @tmb_list based on the TDX module
> @@ -350,10 +446,15 @@ static int construct_tdmrs(struct list_head *tmb_list,
> struct tdmr_info_list *tdmr_list,
> struct tdsysinfo_struct *sysinfo)
> {
> + int ret;
> +
> + ret = fill_out_tdmrs(tmb_list, tdmr_list);
> + if (ret)
> + return ret;
> +
> /*
> * TODO:
> *
> - * - Fill out TDMRs to cover all TDX memory regions.
> * - Allocate and set up PAMTs for each TDMR.
> * - Designate reserved areas for each TDMR.
> *
> diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
> index 193764afc602..3086f7ad0522 100644
> --- a/arch/x86/virt/vmx/tdx/tdx.h
> +++ b/arch/x86/virt/vmx/tdx/tdx.h
> @@ -123,6 +123,9 @@ struct tdx_memblock {
> unsigned long end_pfn;
> };
>
> +/* Warn if kernel has less than TDMR_NR_WARN TDMRs after allocation */
> +#define TDMR_NR_WARN 4
> +
> struct tdmr_info_list {
> void *tdmrs; /* Flexible array to hold 'tdmr_info's */
> int nr_consumed_tdmrs; /* How many 'tdmr_info's are in use */
> --
> 2.40.1
>
