Signed-off-by: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
---
include/asm-generic/tlb.h | 85 ++++++++++++++++++++++++++--------------------
mm/memory.c | 6 ---
2 files changed, 50 insertions(+), 41 deletions(-)
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -21,6 +21,40 @@
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
+/*
+ * If we can't allocate a page to make a big batch of page pointers
+ * to work on, then just handle a few from the on-stack structure.
+ */
+#define MMU_GATHER_BUNDLE 8
+
+struct mmu_gather_batch {
+ struct mmu_gather_batch *next;
+ unsigned int nr;
+ unsigned int max;
+ struct page *pages[0];
+};
+
+#define MAX_GATHER_BATCH \
+ ((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
+
+/* struct mmu_gather is an opaque type used by the mm code for passing around
+ * any data needed by arch specific code for tlb_remove_page.
+ */
+struct mmu_gather {
+ struct mm_struct *mm;
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+ struct mmu_table_batch *batch;
+#endif
+ unsigned int need_flush : 1, /* Did free PTEs */
+ fast_mode : 1; /* No batching */
+
+ unsigned int fullmm;
+
+ struct mmu_gather_batch *active;
+ struct mmu_gather_batch local;
+ struct page *__pages[MMU_GATHER_BUNDLE];
+};
+
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
/*
* Semi RCU freeing of the page directories.
@@ -59,51 +93,30 @@ struct mmu_table_batch {
#define MAX_TABLE_BATCH \
((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
+static inline void tlb_table_init(struct mmu_gather *tlb)
+{
+ tlb->batch = NULL;
+}
+
extern void tlb_table_flush(struct mmu_gather *tlb);
extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
-#else
+#else /* CONFIG_HAVE_RCU_TABLE_FREE */
-static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
+static inline void tlb_table_init(struct mmu_gather *tlb)
{
- tlb_remove_page(tlb, table);
}
-#endif
-
-/*
- * If we can't allocate a page to make a big batch of page pointers
- * to work on, then just handle a few from the on-stack structure.
- */
-#define MMU_GATHER_BUNDLE 8
-
-struct mmu_gather_batch {
- struct mmu_gather_batch *next;
- unsigned int nr;
- unsigned int max;
- struct page *pages[0];
-};
-
-#define MAX_GATHER_BATCH \
- ((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
-
-/* struct mmu_gather is an opaque type used by the mm code for passing around
- * any data needed by arch specific code for tlb_remove_page.
- */
-struct mmu_gather {
- struct mm_struct *mm;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
- struct mmu_table_batch *batch;
-#endif
- unsigned int need_flush : 1, /* Did free PTEs */
- fast_mode : 1; /* No batching */
+static inline void tlb_table_flush(struct mmu_gather *tlb)
+{
+}
- unsigned int fullmm;
+static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
+{
+ tlb_remove_page(tlb, table);
+}
- struct mmu_gather_batch *active;
- struct mmu_gather_batch local;
- struct page *__pages[MMU_GATHER_BUNDLE];
-};
+#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
#define HAVE_GENERIC_MMU_GATHER
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -214,9 +214,7 @@ void tlb_gather_mmu(struct mmu_gather *t
tlb->local.max = ARRAY_SIZE(tlb->__pages);
tlb->active = &tlb->local;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
- tlb->batch = NULL;
-#endif
+ tlb_table_init(tlb);
}
void tlb_flush_mmu(struct mmu_gather *tlb)
@@ -227,9 +225,7 @@ void tlb_flush_mmu(struct mmu_gather *tl
return;
tlb->need_flush = 0;
tlb_flush(tlb);
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
tlb_table_flush(tlb);
-#endif
if (tlb_fast_mode(tlb))
return;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>