Re: [PATCH] arm64: Add support for PTE contiguous bit.

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 15 September 2015 at 19:01, David Woods <dwoods@xxxxxxxxxx> wrote:
> The arm64 MMU supports a Contiguous bit which is a hint that the TTE
> is one of a set of contiguous entries which can be cached in a single
> TLB entry.  Supporting this bit adds new intermediate huge page sizes.
>
> The set of huge page sizes available depends on the base page size.
> Without using contiguous pages the huge page sizes are as follows.
>
>  4KB:   2MB  1GB
> 64KB: 512MB  4TB

We just have 512MB for a 64KB granule.
As per [1] D4.2.6 - "The VMSAv8-64 translation table format" page D4-1668.

>
> With 4KB pages, the contiguous bit groups together sets of 16 pages
> and with 64KB pages it groups sets of 32 pages.  This enables two new
> huge page sizes in each case, so that the full set of available sizes
> is as follows.
>
>  4KB:  64KB   2MB  32MB  1GB
> 64KB:   2MB 512MB  16GB  4TB
>
> If the base page size is set to 64KB then 2MB pages are enabled by
> default.  It is possible in the future to make 2MB the default huge
> page size for both 4KB and 64KB pages.
>

Hi David,
Thanks for posting this, and apologies in advance for talking about
the ARM ARM[1]...

D4.4.2 "Other fields in the VMSAv8-64 translation table format
descriptors" (page D4-1715)
Only gives examples of the contiguous bit being used for level 3
descriptors (i.e. PTEs) when running with a 4KB and 64KB granule.

With a 16KB granule we *can* have a contiguous bit being used by level
2 descriptors (i.e. PMDs), so the pmd_contig logic could perhaps be
used in combination with Suzuki's 16KB PAGE_SIZE series at:
http://lists.infradead.org/pipermail/linux-arm-kernel/2015-September/370117.html

I will read through the rest of the patch and post more feedback

Cheers,
--
Steve

[1] - http://infocenter.arm.com/help/topic/com.arm.doc.ddi0487a.g/index.html




> Signed-off-by: David Woods <dwoods@xxxxxxxxxx>
> Reviewed-by: Chris Metcalf <cmetcalf@xxxxxxxxxx>
> ---
>  arch/arm64/Kconfig                     |   3 -
>  arch/arm64/include/asm/hugetlb.h       |   4 +
>  arch/arm64/include/asm/pgtable-hwdef.h |  15 +++
>  arch/arm64/include/asm/pgtable.h       |  30 +++++-
>  arch/arm64/mm/hugetlbpage.c            | 165 ++++++++++++++++++++++++++++++++-
>  5 files changed, 210 insertions(+), 7 deletions(-)
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 7d95663..8310e38 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -447,9 +447,6 @@ config HW_PERF_EVENTS
>  config SYS_SUPPORTS_HUGETLBFS
>         def_bool y
>
> -config ARCH_WANT_GENERAL_HUGETLB
> -       def_bool y
> -
>  config ARCH_WANT_HUGE_PMD_SHARE
>         def_bool y if !ARM64_64K_PAGES
>
> diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
> index bb4052e..e5af553 100644
> --- a/arch/arm64/include/asm/hugetlb.h
> +++ b/arch/arm64/include/asm/hugetlb.h
> @@ -97,4 +97,8 @@ static inline void arch_clear_hugepage_flags(struct page *page)
>         clear_bit(PG_dcache_clean, &page->flags);
>  }
>
> +extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
> +                               struct page *page, int writable);
> +#define arch_make_huge_pte arch_make_huge_pte
> +
>  #endif /* __ASM_HUGETLB_H */
> diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
> index 24154b0..da73243 100644
> --- a/arch/arm64/include/asm/pgtable-hwdef.h
> +++ b/arch/arm64/include/asm/pgtable-hwdef.h
> @@ -55,6 +55,19 @@
>  #define SECTION_MASK           (~(SECTION_SIZE-1))
>
>  /*
> + * Contiguous large page definitions.
> + */
> +#ifdef CONFIG_ARM64_64K_PAGES
> +#define        CONTIG_SHIFT            5
> +#define CONTIG_PAGES           32
> +#else
> +#define        CONTIG_SHIFT            4
> +#define CONTIG_PAGES           16
> +#endif
> +#define        CONTIG_PTE_SIZE         (CONTIG_PAGES * PAGE_SIZE)
> +#define        CONTIG_PTE_MASK         (~(CONTIG_PTE_SIZE - 1))
> +
> +/*
>   * Hardware page table definitions.
>   *
>   * Level 1 descriptor (PUD).
> @@ -83,6 +96,7 @@
>  #define PMD_SECT_S             (_AT(pmdval_t, 3) << 8)
>  #define PMD_SECT_AF            (_AT(pmdval_t, 1) << 10)
>  #define PMD_SECT_NG            (_AT(pmdval_t, 1) << 11)
> +#define PMD_SECT_CONTIG                (_AT(pmdval_t, 1) << 52)
>  #define PMD_SECT_PXN           (_AT(pmdval_t, 1) << 53)
>  #define PMD_SECT_UXN           (_AT(pmdval_t, 1) << 54)
>
> @@ -105,6 +119,7 @@
>  #define PTE_AF                 (_AT(pteval_t, 1) << 10)        /* Access Flag */
>  #define PTE_NG                 (_AT(pteval_t, 1) << 11)        /* nG */
>  #define PTE_DBM                        (_AT(pteval_t, 1) << 51)        /* Dirty Bit Management */
> +#define PTE_CONTIG             (_AT(pteval_t, 1) << 52)        /* Contiguous */
>  #define PTE_PXN                        (_AT(pteval_t, 1) << 53)        /* Privileged XN */
>  #define PTE_UXN                        (_AT(pteval_t, 1) << 54)        /* User XN */
>
> diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
> index 6900b2d9..df5ec64 100644
> --- a/arch/arm64/include/asm/pgtable.h
> +++ b/arch/arm64/include/asm/pgtable.h
> @@ -144,6 +144,7 @@ extern struct page *empty_zero_page;
>  #define pte_special(pte)       (!!(pte_val(pte) & PTE_SPECIAL))
>  #define pte_write(pte)         (!!(pte_val(pte) & PTE_WRITE))
>  #define pte_exec(pte)          (!(pte_val(pte) & PTE_UXN))
> +#define pte_contig(pte)                (!!(pte_val(pte) & PTE_CONTIG))
>
>  #ifdef CONFIG_ARM64_HW_AFDBM
>  #define pte_hw_dirty(pte)      (!(pte_val(pte) & PTE_RDONLY))
> @@ -206,6 +207,9 @@ static inline pte_t pte_mkspecial(pte_t pte)
>         return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
>  }
>
> +extern pte_t pte_mkcontig(pte_t pte);
> +extern pmd_t pmd_mkcontig(pmd_t pmd);
> +
>  static inline void set_pte(pte_t *ptep, pte_t pte)
>  {
>         *ptep = pte;
> @@ -275,7 +279,7 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
>  /*
>   * Hugetlb definitions.
>   */
> -#define HUGE_MAX_HSTATE                2
> +#define HUGE_MAX_HSTATE                ((2 * CONFIG_PGTABLE_LEVELS) - 1)
>  #define HPAGE_SHIFT            PMD_SHIFT
>  #define HPAGE_SIZE             (_AC(1, UL) << HPAGE_SHIFT)
>  #define HPAGE_MASK             (~(HPAGE_SIZE - 1))
> @@ -372,7 +376,8 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
>  #define pmd_none(pmd)          (!pmd_val(pmd))
>  #define pmd_present(pmd)       (pmd_val(pmd))
>
> -#define pmd_bad(pmd)           (!(pmd_val(pmd) & 2))
> +#define pmd_bad(pmd)           (!(pmd_val(pmd) & \
> +                                  (PMD_TABLE_BIT | PMD_SECT_CONTIG)))
>
>  #define pmd_table(pmd)         ((pmd_val(pmd) & PMD_TYPE_MASK) == \
>                                  PMD_TYPE_TABLE)
> @@ -500,7 +505,8 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr)
>  static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
>  {
>         const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
> -                             PTE_PROT_NONE | PTE_WRITE | PTE_TYPE_MASK;
> +                             PTE_PROT_NONE | PTE_WRITE | PTE_TYPE_MASK |
> +                             PTE_CONTIG;
>         /* preserve the hardware dirty information */
>         if (pte_hw_dirty(pte))
>                 newprot |= PTE_DIRTY;
> @@ -513,6 +519,24 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
>         return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
>  }
>
> +static inline pte_t pte_modify_pfn(pte_t pte, unsigned long newpfn)
> +{
> +       const pteval_t mask = PHYS_MASK & PAGE_MASK;
> +
> +       pte_val(pte) = pfn_pte(newpfn, (pte_val(pte) & ~mask));
> +       return pte;
> +}
> +
> +#if CONFIG_PGTABLE_LEVELS > 2
> +static inline pmd_t pmd_modify_pfn(pmd_t pmd, unsigned long newpfn)
> +{
> +       const pmdval_t mask = PHYS_MASK & PAGE_MASK;
> +
> +       pmd = pfn_pmd(newpfn, (pmd_val(pmd) & ~mask));
> +       return pmd;
> +}
> +#endif
> +
>  #ifdef CONFIG_ARM64_HW_AFDBM
>  /*
>   * Atomic pte/pmd modifications.
> diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
> index 383b03f..f5bbbbc 100644
> --- a/arch/arm64/mm/hugetlbpage.c
> +++ b/arch/arm64/mm/hugetlbpage.c
> @@ -41,6 +41,155 @@ int pud_huge(pud_t pud)
>  #endif
>  }
>
> +pte_t *huge_pte_alloc(struct mm_struct *mm,
> +                       unsigned long addr, unsigned long sz)
> +{
> +       pgd_t *pgd;
> +       pud_t *pud;
> +       pte_t *pte = NULL;
> +       int i;
> +
> +       pgd = pgd_offset(mm, addr);
> +       pud = pud_alloc(mm, pgd, addr);
> +       if (pud) {
> +               if (sz == PUD_SIZE) {
> +                       pte = (pte_t *)pud;
> +               } else if (sz == PMD_SIZE) {
> +#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
> +                       if (pud_none(*pud))
> +                               pte = huge_pmd_share(mm, addr, pud);
> +                       else
> +#endif
> +                               pte = (pte_t *)pmd_alloc(mm, pud, addr);
> +               } else if (sz == (PAGE_SIZE * CONTIG_PAGES)) {
> +                       pmd_t *pmd = pmd_alloc(mm, pud, addr);
> +
> +                       WARN_ON(addr & (sz - 1));
> +                       pte = pte_alloc_map(mm, NULL, pmd, addr);
> +                       if (pte_present(*pte)) {
> +                               unsigned long pfn;
> +                               *pte = pte_mkcontig(*pte);
> +                               pfn = pte_pfn(*pte);
> +                               for (i = 0; i < CONTIG_PAGES; i++) {
> +                                       set_pte(&pte[i],
> +                                               pte_modify_pfn(*pte, pfn + i));
> +                               }
> +                       }
> +#if CONFIG_PGTABLE_LEVELS > 2
> +               } else if (sz == (PMD_SIZE * CONTIG_PAGES)) {
> +                       pmd_t *pmd;
> +
> +                       pmd = pmd_alloc(mm, pud, addr);
> +                       WARN_ON(addr & (sz - 1));
> +                       if (pmd && pmd_present(*pmd)) {
> +                               unsigned long pfn;
> +                               pmd_t pmdval;
> +
> +                               pmdval = *pmd = pmd_mkcontig(*pmd);
> +                               pfn = pmd_pfn(*pmd);
> +                               for (i = 0; i < CONTIG_PAGES; i++) {
> +                                       unsigned long newpfn = pfn +
> +                                               (i << (PMD_SHIFT - PAGE_SHIFT));
> +                                       if (!pmd_present(pmd[i]))
> +                                               atomic_long_inc(&mm->nr_ptes);
> +                                       set_pmd(&pmd[i],
> +                                               pmd_modify_pfn(pmdval, newpfn));
> +                               }
> +                       }
> +                       return pmd;
> +#endif
> +               }
> +       }
> +
> +       return pte;
> +}
> +
> +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
> +{
> +       pgd_t *pgd;
> +       pud_t *pud;
> +       pmd_t *pmd = NULL;
> +       pte_t *pte = NULL;
> +
> +       pgd = pgd_offset(mm, addr);
> +       if (pgd_present(*pgd)) {
> +               pud = pud_offset(pgd, addr);
> +               if (pud_present(*pud)) {
> +                       if (pud_huge(*pud))
> +                               return (pte_t *)pud;
> +                       pmd = pmd_offset(pud, addr);
> +                       if (pmd_present(*pmd)) {
> +                               if (pmd_huge(*pmd))
> +                                       return (pte_t *)pmd;
> +                               pte = pte_offset_kernel(pmd, addr);
> +                               if (pte_present(*pte) && pte_contig(*pte)) {
> +                                       pte = pte_offset_kernel(
> +                                               pmd, (addr & CONTIG_PTE_MASK));
> +                                       return pte;
> +                               }
> +                       }
> +               }
> +       }
> +       return (pte_t *) NULL;
> +}
> +
> +pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
> +                        struct page *page, int writable)
> +{
> +       size_t pagesize = huge_page_size(hstate_vma(vma));
> +       pte_t nent = {0};
> +
> +       if (pagesize == PUD_SIZE || pagesize == PMD_SIZE)
> +               nent = entry;
> +       else if (pagesize == (PAGE_SIZE * CONTIG_PAGES))
> +               nent = pte_mkcontig(entry);
> +#if CONFIG_PGTABLE_LEVELS > 2
> +       else if (pagesize == (PMD_SIZE * CONTIG_PAGES) ||
> +                pagesize == (PUD_SIZE * CONTIG_PAGES))
> +               nent = pmd_mkcontig(entry);
> +#endif
> +       else {
> +               pr_warn("%s: unrecognized huge page size 0x%lx\n",
> +                      __func__, pagesize);
> +       }
> +       return nent;
> +}
> +
> +pte_t pte_mkcontig(pte_t pte)
> +{
> +       pte = set_pte_bit(pte, __pgprot(PTE_CONTIG));
> +       pte = set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
> +       return pte;
> +}
> +
> +pmd_t pmd_mkcontig(pmd_t pmd)
> +{
> +       pmd = __pmd(pmd_val(pmd) | PMD_SECT_CONTIG);
> +       return pmd;
> +}
> +
> +struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
> +               pmd_t *pmd, int write)
> +{
> +       struct page *page;
> +
> +       page = pte_page(*(pte_t *)pmd);
> +       if (page)
> +               page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
> +       return page;
> +}
> +
> +struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
> +               pud_t *pud, int write)
> +{
> +       struct page *page;
> +
> +       page = pte_page(*(pte_t *)pud);
> +       if (page)
> +               page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
> +       return page;
> +}
> +
>  static __init int setup_hugepagesz(char *opt)
>  {
>         unsigned long ps = memparse(opt, &opt);
> @@ -48,10 +197,24 @@ static __init int setup_hugepagesz(char *opt)
>                 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
>         } else if (ps == PUD_SIZE) {
>                 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
> +       } else if (ps == (PAGE_SIZE * CONTIG_PAGES)) {
> +               hugetlb_add_hstate(CONTIG_SHIFT);
> +       } else if (ps == (PMD_SIZE * CONTIG_PAGES)) {
> +               hugetlb_add_hstate((PMD_SHIFT + CONTIG_SHIFT) - PAGE_SHIFT);
>         } else {
> -               pr_err("hugepagesz: Unsupported page size %lu M\n", ps >> 20);
> +               pr_err("hugepagesz: Unsupported page size %lu K\n", ps >> 10);
>                 return 0;
>         }
>         return 1;
>  }
>  __setup("hugepagesz=", setup_hugepagesz);
> +
> +#ifdef CONFIG_ARM64_64K_PAGES
> +static __init int add_default_hugepagesz(void)
> +{
> +       if (size_to_hstate(CONTIG_PAGES * PAGE_SIZE) == NULL)
> +               hugetlb_add_hstate(CONTIG_SHIFT);
> +       return 0;
> +}
> +arch_initcall(add_default_hugepagesz);
> +#endif
> --
> 2.1.2
>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>



[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [ECOS]     [Asterisk Internet PBX]     [Linux API]