The patch titled
Subject: mm: change the interface for __tlb_remove_page()
has been added to the -mm tree. Its filename is
mm-change-the-interface-for-__tlb_remove_page.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-change-the-interface-for-__tlb_remove_page.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-change-the-interface-for-__tlb_remove_page.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/SubmitChecklist when testing your code ***
The -mm tree is included into linux-next and is updated
there every 3-4 working days
------------------------------------------------------
From: "Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxxxxxxx>
Subject: mm: change the interface for __tlb_remove_page()
This updates the generic and arch specific implementation to return true
if we need to do a tlb flush. That means if a __tlb_remove_page indicate
a flush is needed, the page we try to remove need to be tracked and added
again after the flush. We need to track it because we have already update
the pte to none and we can't just loop back.
This change is done to enable us to do a tlb_flush when we try to flush a
range that consists of different page sizes. For architectures like
ppc64, we can do a range based tlb flush and we need to track page size
for that. When we try to remove a huge page, we will force a tlb flush
and starts a new mmu gather.
Link: http://lkml.kernel.org/r/1464860389-29019-2-git-send-email-aneesh.kumar@xxxxxxxxxxxxxxxxxx
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxxxxxxx>
Cc: Benjamin Herrenschmidt <benh@xxxxxxxxxxxxxxxxxxx>
Cc: Michael Ellerman <mpe@xxxxxxxxxxxxxx>
Cc: Hugh Dickins <hughd@xxxxxxxxxx>
Cc: "Kirill A. Shutemov" <kirill.shutemov@xxxxxxxxxxxxxxx>
Cc: Andrea Arcangeli <aarcange@xxxxxxxxxx>
Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Vlastimil Babka <vbabka@xxxxxxx>
Cc: Minchan Kim <minchan.kim@xxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
arch/arm/include/asm/tlb.h | 11 +++++++----
arch/ia64/include/asm/tlb.h | 13 ++++++++-----
arch/s390/include/asm/tlb.h | 4 ++--
arch/sh/include/asm/tlb.h | 2 +-
arch/um/include/asm/tlb.h | 2 +-
include/asm-generic/tlb.h | 18 ++++++++++++++++--
mm/memory.c | 20 ++++++++++++++------
7 files changed, 49 insertions(+), 21 deletions(-)
diff -puN arch/arm/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page arch/arm/include/asm/tlb.h
--- a/arch/arm/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/arch/arm/include/asm/tlb.h
@@ -209,17 +209,20 @@ tlb_end_vma(struct mmu_gather *tlb, stru
tlb_flush(tlb);
}
-static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
+ if (tlb->nr == tlb->max)
+ return true;
tlb->pages[tlb->nr++] = page;
- VM_BUG_ON(tlb->nr > tlb->max);
- return tlb->max - tlb->nr;
+ return false;
}
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
- if (!__tlb_remove_page(tlb, page))
+ if (__tlb_remove_page(tlb, page)) {
tlb_flush_mmu(tlb);
+ __tlb_remove_page(tlb, page);
+ }
}
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
diff -puN arch/ia64/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page arch/ia64/include/asm/tlb.h
--- a/arch/ia64/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/arch/ia64/include/asm/tlb.h
@@ -205,17 +205,18 @@ tlb_finish_mmu(struct mmu_gather *tlb, u
* must be delayed until after the TLB has been flushed (see comments at the beginning of
* this file).
*/
-static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
+ if (tlb->nr == tlb->max)
+ return true;
+
tlb->need_flush = 1;
if (!tlb->nr && tlb->pages == tlb->local)
__tlb_alloc_page(tlb);
tlb->pages[tlb->nr++] = page;
- VM_BUG_ON(tlb->nr > tlb->max);
-
- return tlb->max - tlb->nr;
+ return false;
}
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
@@ -235,8 +236,10 @@ static inline void tlb_flush_mmu(struct
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
- if (!__tlb_remove_page(tlb, page))
+ if (__tlb_remove_page(tlb, page)) {
tlb_flush_mmu(tlb);
+ __tlb_remove_page(tlb, page);
+ }
}
/*
diff -puN arch/s390/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page arch/s390/include/asm/tlb.h
--- a/arch/s390/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/arch/s390/include/asm/tlb.h
@@ -87,10 +87,10 @@ static inline void tlb_finish_mmu(struct
* tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page
* has already been freed, so just do free_page_and_swap_cache.
*/
-static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
free_page_and_swap_cache(page);
- return 1; /* avoid calling tlb_flush_mmu */
+ return false; /* avoid calling tlb_flush_mmu */
}
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
diff -puN arch/sh/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page arch/sh/include/asm/tlb.h
--- a/arch/sh/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/arch/sh/include/asm/tlb.h
@@ -101,7 +101,7 @@ static inline void tlb_flush_mmu(struct
static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
free_page_and_swap_cache(page);
- return 1; /* avoid calling tlb_flush_mmu */
+ return false; /* avoid calling tlb_flush_mmu */
}
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
diff -puN arch/um/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page arch/um/include/asm/tlb.h
--- a/arch/um/include/asm/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/arch/um/include/asm/tlb.h
@@ -102,7 +102,7 @@ static inline int __tlb_remove_page(stru
{
tlb->need_flush = 1;
free_page_and_swap_cache(page);
- return 1; /* avoid calling tlb_flush_mmu */
+ return false; /* avoid calling tlb_flush_mmu */
}
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
diff -puN include/asm-generic/tlb.h~mm-change-the-interface-for-__tlb_remove_page include/asm-generic/tlb.h
--- a/include/asm-generic/tlb.h~mm-change-the-interface-for-__tlb_remove_page
+++ a/include/asm-generic/tlb.h
@@ -107,6 +107,11 @@ struct mmu_gather {
struct mmu_gather_batch local;
struct page *__pages[MMU_GATHER_BUNDLE];
unsigned int batch_count;
+ /*
+ * __tlb_adjust_range will track the new addr here,
+ * that that we can adjust the range after the flush
+ */
+ unsigned long addr;
};
#define HAVE_GENERIC_MMU_GATHER
@@ -115,7 +120,7 @@ void tlb_gather_mmu(struct mmu_gather *t
void tlb_flush_mmu(struct mmu_gather *tlb);
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
unsigned long end);
-int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
+bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
/* tlb_remove_page
* Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
@@ -123,8 +128,11 @@ int __tlb_remove_page(struct mmu_gather
*/
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
- if (!__tlb_remove_page(tlb, page))
+ if (__tlb_remove_page(tlb, page)) {
tlb_flush_mmu(tlb);
+ __tlb_adjust_range(tlb, tlb->addr);
+ __tlb_remove_page(tlb, page);
+ }
}
static inline void __tlb_adjust_range(struct mmu_gather *tlb,
@@ -132,6 +140,12 @@ static inline void __tlb_adjust_range(st
{
tlb->start = min(tlb->start, address);
tlb->end = max(tlb->end, address + PAGE_SIZE);
+ /*
+ * Track the last address with which we adjusted the range. This
+ * will be used later to adjust again after a mmu_flush due to
+ * failed __tlb_remove_page
+ */
+ tlb->addr = address;
}
static inline void __tlb_reset_range(struct mmu_gather *tlb)
diff -puN mm/memory.c~mm-change-the-interface-for-__tlb_remove_page mm/memory.c
--- a/mm/memory.c~mm-change-the-interface-for-__tlb_remove_page
+++ a/mm/memory.c
@@ -292,23 +292,24 @@ void tlb_finish_mmu(struct mmu_gather *t
* handling the additional races in SMP caused by other CPUs caching valid
* mappings in their TLBs. Returns the number of free page slots left.
* When out of page slots we must call tlb_flush_mmu().
+ *returns true if the caller should flush.
*/
-int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
struct mmu_gather_batch *batch;
VM_BUG_ON(!tlb->end);
batch = tlb->active;
- batch->pages[batch->nr++] = page;
if (batch->nr == batch->max) {
if (!tlb_next_batch(tlb))
- return 0;
+ return true;
batch = tlb->active;
}
VM_BUG_ON_PAGE(batch->nr > batch->max, page);
- return batch->max - batch->nr;
+ batch->pages[batch->nr++] = page;
+ return false;
}
#endif /* HAVE_GENERIC_MMU_GATHER */
@@ -1109,6 +1110,7 @@ static unsigned long zap_pte_range(struc
pte_t *start_pte;
pte_t *pte;
swp_entry_t entry;
+ struct page *pending_page = NULL;
again:
init_rss_vec(rss);
@@ -1160,8 +1162,9 @@ again:
page_remove_rmap(page, false);
if (unlikely(page_mapcount(page) < 0))
print_bad_pte(vma, addr, ptent, page);
- if (unlikely(!__tlb_remove_page(tlb, page))) {
+ if (unlikely(__tlb_remove_page(tlb, page))) {
force_flush = 1;
+ pending_page = page;
addr += PAGE_SIZE;
break;
}
@@ -1202,7 +1205,12 @@ again:
if (force_flush) {
force_flush = 0;
tlb_flush_mmu_free(tlb);
-
+ if (pending_page) {
+ /* remove the page with new size */
+ __tlb_adjust_range(tlb, tlb->addr);
+ __tlb_remove_page(tlb, pending_page);
+ pending_page = NULL;
+ }
if (addr != end)
goto again;
}
_
Patches currently in -mm which might be from aneesh.kumar@xxxxxxxxxxxxxxxxxx are
mm-debug-add-vm_warn-which-maps-to-warn.patch
powerpc-mm-check-for-irq-disabled-only-if-debug_vm-is-enabled.patch
mm-hugetlb-simplify-hugetlb-unmap.patch
mm-change-the-interface-for-__tlb_remove_page.patch
mm-mmu_gather-track-page-size-with-mmu-gather-and-force-flush-if-page-size-change.patch
powerpc-mm-radix-implement-tlb-mmu-gather-flush-efficiently.patch
a.patch
--
To unsubscribe from this list: send the line "unsubscribe mm-commits" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
