SLUB fastpaths use __always_inline to avoid function calls. With
CONFIG_SLUB_TINY we would rather save the memory. Add a
__fastpath_inline macro that's __always_inline normally but empty with
CONFIG_SLUB_TINY.
bloat-o-meter results on x86_64 mm/slub.o:
add/remove: 3/1 grow/shrink: 1/8 up/down: 865/-1784 (-919)
Function old new delta
kmem_cache_free 20 281 +261
slab_alloc_node.isra - 245 +245
slab_free.constprop.isra - 231 +231
__kmem_cache_alloc_lru.isra - 128 +128
__kmem_cache_release 88 83 -5
__kmem_cache_create 1446 1436 -10
__kmem_cache_free 271 142 -129
kmem_cache_alloc_node 330 127 -203
kmem_cache_free_bulk.part 826 613 -213
__kmem_cache_alloc_node 230 10 -220
kmem_cache_alloc_lru 325 12 -313
kmem_cache_alloc 325 10 -315
kmem_cache_free.part 376 - -376
Total: Before=26103, After=25184, chg -3.52%
Signed-off-by: Vlastimil Babka <vbabka@xxxxxxx>
---
mm/slub.c | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index 7f1cd702c3b4..d54466e76503 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -187,6 +187,12 @@ do { \
#define USE_LOCKLESS_FAST_PATH() (false)
#endif
+#ifndef CONFIG_SLUB_TINY
+#define __fastpath_inline __always_inline
+#else
+#define __fastpath_inline
+#endif
+
#ifdef CONFIG_SLUB_DEBUG
#ifdef CONFIG_SLUB_DEBUG_ON
DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
@@ -3386,7 +3392,7 @@ static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
*
* Otherwise we can simply pick the next object from the lockless free list.
*/
-static __always_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
+static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
{
void *object;
@@ -3412,13 +3418,13 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s, struct list_l
return object;
}
-static __always_inline void *slab_alloc(struct kmem_cache *s, struct list_lru *lru,
+static __fastpath_inline void *slab_alloc(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags, unsigned long addr, size_t orig_size)
{
return slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, addr, orig_size);
}
-static __always_inline
+static __fastpath_inline
void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags)
{
@@ -3733,7 +3739,7 @@ static void do_slab_free(struct kmem_cache *s,
}
#endif /* CONFIG_SLUB_TINY */
-static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
+static __fastpath_inline void slab_free(struct kmem_cache *s, struct slab *slab,
void *head, void *tail, void **p, int cnt,
unsigned long addr)
{
--
2.38.1
