[PATCH v3 09/11] mm/slub: Fold slab_update_freelist()

[Peter Zijlstra <peterz@xxxxxxxxxxxxx>]

The two functions slab_update_freelist() and __slab_update_freelist()
are nearly identical, fold and add a boolean argument and rely on
constant propagation.

Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
---
 mm/slub.c |   80 +++++++++++++++++++++-----------------------------------------
 1 file changed, 28 insertions(+), 52 deletions(-)

--- a/mm/slub.c
+++ b/mm/slub.c
@@ -559,53 +559,29 @@ __update_freelist_slow(struct slab *slab
  * allocation/ free operation in hardirq context. Therefore nothing can
  * interrupt the operation.
  */
-static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
-		void *freelist_old, unsigned long counters_old,
-		void *freelist_new, unsigned long counters_new,
-		const char *n)
+static __always_inline
+bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
+			  void *freelist_old, unsigned long counters_old,
+			  void *freelist_new, unsigned long counters_new,
+			  bool irq_save, const char *n)
 {
 	bool ret;
 
-	if (USE_LOCKLESS_FAST_PATH())
+	if (!irq_save && USE_LOCKLESS_FAST_PATH())
 		lockdep_assert_irqs_disabled();
 
 	if (s->flags & __CMPXCHG_DOUBLE) {
 		ret = __update_freelist_fast(slab, freelist_old, counters_old,
 				            freelist_new, counters_new);
 	} else {
-		ret = __update_freelist_slow(slab, freelist_old, counters_old,
-				            freelist_new, counters_new);
-	}
-	if (likely(ret))
-		return true;
-
-	cpu_relax();
-	stat(s, CMPXCHG_DOUBLE_FAIL);
-
-#ifdef SLUB_DEBUG_CMPXCHG
-	pr_info("%s %s: cmpxchg double redo ", n, s->name);
-#endif
-
-	return false;
-}
-
-static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
-		void *freelist_old, unsigned long counters_old,
-		void *freelist_new, unsigned long counters_new,
-		const char *n)
-{
-	bool ret;
-
-	if (s->flags & __CMPXCHG_DOUBLE) {
-		ret = __update_freelist_fast(slab, freelist_old, counters_old,
-				            freelist_new, counters_new);
-	} else {
 		unsigned long flags;
 
-		local_irq_save(flags);
+		if (irq_save)
+			local_irq_save(flags);
 		ret = __update_freelist_slow(slab, freelist_old, counters_old,
 				            freelist_new, counters_new);
-		local_irq_restore(flags);
+		if (irq_save)
+			local_irq_restore(flags);
 	}
 	if (likely(ret))
 		return true;
@@ -2250,10 +2226,10 @@ static inline void *acquire_slab(struct
 	VM_BUG_ON(new.frozen);
 	new.frozen = 1;
 
-	if (!__slab_update_freelist(s, slab,
-			freelist, counters,
-			new.freelist, new.counters,
-			"acquire_slab"))
+	if (!slab_update_freelist(s, slab,
+				  freelist, counters,
+				  new.freelist, new.counters,
+				  false, "acquire_slab"))
 		return NULL;
 
 	remove_partial(n, slab);
@@ -2577,9 +2553,9 @@ static void deactivate_slab(struct kmem_
 
 
 	if (!slab_update_freelist(s, slab,
-				old.freelist, old.counters,
-				new.freelist, new.counters,
-				"unfreezing slab")) {
+				  old.freelist, old.counters,
+				  new.freelist, new.counters,
+				  true, "unfreezing slab")) {
 		if (mode == M_PARTIAL)
 			spin_unlock_irqrestore(&n->list_lock, flags);
 		goto redo;
@@ -2633,10 +2609,10 @@ static void __unfreeze_partials(struct k
 
 			new.frozen = 0;
 
-		} while (!__slab_update_freelist(s, slab,
-				old.freelist, old.counters,
-				new.freelist, new.counters,
-				"unfreezing slab"));
+		} while (!slab_update_freelist(s, slab,
+					       old.freelist, old.counters,
+					       new.freelist, new.counters,
+					       false, "unfreezing slab"));
 
 		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
 			slab->next = slab_to_discard;
@@ -3072,10 +3048,10 @@ static inline void *get_freelist(struct
 		new.inuse = slab->objects;
 		new.frozen = freelist != NULL;
 
-	} while (!__slab_update_freelist(s, slab,
-		freelist, counters,
-		NULL, new.counters,
-		"get_freelist"));
+	} while (!slab_update_freelist(s, slab,
+				       freelist, counters,
+				       NULL, new.counters,
+				       false, "get_freelist"));
 
 	return freelist;
 }
@@ -3666,9 +3642,9 @@ static void __slab_free(struct kmem_cach
 		}
 
 	} while (!slab_update_freelist(s, slab,
-		prior, counters,
-		head, new.counters,
-		"__slab_free"));
+				       prior, counters,
+				       head, new.counters,
+				       true, "__slab_free"));
 
 	if (likely(!n)) {