On Wed, Jan 29, 2020 at 11:39:45AM +0100, Peter Zijlstra wrote:
In addition to the PGD/PMD table size (128*4) add a PTE table size
(64*4) to the table allocator. This completely removes the pte-table
overhead compared to the old code, even for dense tables.
Notes:
- the allocator gained __flush_page_to_ram(), since the old
page-based allocator had that.
- the allocator gained a list_empty() check to deal with there not
being any pages at all.
- the free mask is extended to cover more than the 8 bits required
for the (512 byte) PGD/PMD tables.
- NR_PAGETABLE accounting is restored.
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
arch/m68k/include/asm/motorola_pgalloc.h | 24 +++++-----
arch/m68k/mm/init.c | 6 +-
arch/m68k/mm/memory.c | 70 ++++++++++++++++++++-----------
3 files changed, 61 insertions(+), 39 deletions(-)
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -5,61 +5,61 @@
#include <asm/tlb.h>
#include <asm/tlbflush.h>
-extern pmd_t *get_pointer_table(void);
-extern int free_pointer_table(pmd_t *);
+extern void *get_pointer_table(int type);
Could be prettier/obfuscated with an enum type?
--- a/arch/m68k/mm/memory.c
+++ b/arch/m68k/mm/memory.c
@@ -27,24 +27,34 @@
arch/sparc/mm/srmmu.c ... */
typedef struct list_head ptable_desc;
-static LIST_HEAD(ptable_list);
+
+static struct list_head ptable_list[2] = {
+ LIST_HEAD_INIT(ptable_list[0]),
+ LIST_HEAD_INIT(ptable_list[1]),
+};
#define PD_PTABLE(page) ((ptable_desc *)&(virt_to_page(page)->lru))
#define PD_PAGE(ptable) (list_entry(ptable, struct page, lru))
-#define PD_MARKBITS(dp) (*(unsigned char *)&PD_PAGE(dp)->index)
+#define PD_MARKBITS(dp) (*(unsigned int *)&PD_PAGE(dp)->index)
+
+static const int ptable_shift[2] = {
+ 7+2, /* PGD, PMD */
+ 6+2, /* PTE */
+};
-#define PTABLE_SIZE (PTRS_PER_PMD * sizeof(pmd_t))
+#define ptable_size(type) (1U << ptable_shift[type])
+#define ptable_mask(type) ((1U << (PAGE_SIZE / ptable_size(type))) - 1)
-void __init init_pointer_table(unsigned long ptable)
+void __init init_pointer_table(unsigned long ptable, int type)
{
ptable_desc *dp;
unsigned long page = ptable & PAGE_MASK;
- unsigned char mask = 1 << ((ptable - page)/PTABLE_SIZE);
+ unsigned int mask = 1U << ((ptable - page)/ptable_size(type));
dp = PD_PTABLE(page);
if (!(PD_MARKBITS(dp) & mask)) {
- PD_MARKBITS(dp) = 0xff;
- list_add(dp, &ptable_list);
+ PD_MARKBITS(dp) = ptable_mask(type);
+ list_add(dp, &ptable_list[type]);
}
PD_MARKBITS(dp) &= ~mask;
@@ -57,12 +67,10 @@ void __init init_pointer_table(unsigned
return;
}
-pmd_t *get_pointer_table (void)
+void *get_pointer_table (int type)
{
- ptable_desc *dp = ptable_list.next;
- unsigned char mask = PD_MARKBITS (dp);
- unsigned char tmp;
- unsigned int off;
+ ptable_desc *dp = ptable_list[type].next;
+ unsigned int mask, tmp, off;
nit, but if you do:
unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp);
then you can leave the existing mask logic as-is.
Will
