On Thu, Mar 24, 2022 at 07:08:27PM +0100, Vlastimil Babka wrote: > On 3/8/22 12:41, Hyeonggon Yoo wrote: > > There is not much benefit for serving large objects in kmalloc(). > > Let's pass large requests to page allocator like SLUB for better > > maintenance of common code. > > > > [ vbabka@xxxxxxx: Enable and disable irq around free_large_kmalloc(). > > Do not lose NUMA locality in __do_kmalloc_node(). > > Use folio_slab(folio)->slab_cache instead of virt_to_cache(). > > Remove large sizes in __kmalloc_index(). ] A bit late to reply but better late than never... > > Thanks for the mention but that's generally only done like this if I took > your patch and made those changes myself. But I just suggested them. Small > suggested changes like this are usually just mentioned in e.g. v1->v2 > changelogs. Ah, okay. I didn't know about the convention. thanks for letting me know! > > Signed-off-by: Hyeonggon Yoo <42.hyeyoo@xxxxxxxxx> > > --- > > include/linux/slab.h | 23 +++++----------------- > > mm/slab.c | 45 ++++++++++++++++++++++++++++++-------------- > > mm/slab.h | 3 +++ > > mm/slab_common.c | 25 +++++++++++++++++------- > > mm/slub.c | 19 ------------------- > > 5 files changed, 57 insertions(+), 58 deletions(-) > > > > diff --git a/include/linux/slab.h b/include/linux/slab.h > > index dfcc8301d969..9ced225a3ea3 100644 > > --- a/include/linux/slab.h > > +++ b/include/linux/slab.h > > @@ -226,27 +226,17 @@ void kmem_dump_obj(void *object); > > > > #ifdef CONFIG_SLAB > > /* > > - * The largest kmalloc size supported by the SLAB allocators is > > - * 32 megabyte (2^25) or the maximum allocatable page order if that is > > - * less than 32 MB. > > - * > > - * WARNING: Its not easy to increase this value since the allocators have > > - * to do various tricks to work around compiler limitations in order to > > - * ensure proper constant folding. > > + * SLAB and SLUB directly allocates requests fitting in to an order-1 page > > + * (PAGE_SIZE*2). Larger requests are passed to the page allocator. > > */ > > -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ > > - (MAX_ORDER + PAGE_SHIFT - 1) : 25) > > -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH > > +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) > > +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) > > #ifndef KMALLOC_SHIFT_LOW > > #define KMALLOC_SHIFT_LOW 5 > > #endif > > #endif > > > > #ifdef CONFIG_SLUB > > -/* > > - * SLUB directly allocates requests fitting in to an order-1 page > > - * (PAGE_SIZE*2). Larger requests are passed to the page allocator. > > - */ > > #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) > > #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) > > #ifndef KMALLOC_SHIFT_LOW > > @@ -398,10 +388,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size, > > if (size <= 512 * 1024) return 19; > > if (size <= 1024 * 1024) return 20; > > if (size <= 2 * 1024 * 1024) return 21; > > - if (size <= 4 * 1024 * 1024) return 22; > > - if (size <= 8 * 1024 * 1024) return 23; > > - if (size <= 16 * 1024 * 1024) return 24; > > - if (size <= 32 * 1024 * 1024) return 25; > > > > if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) > > BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); > > @@ -411,6 +397,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size, > > /* Will never be reached. Needed because the compiler may complain */ > > return -1; > > } > > +static_assert(PAGE_SHIFT <= 20); > > #define kmalloc_index(s) __kmalloc_index(s, true) > > #endif /* !CONFIG_SLOB */ > > > > diff --git a/mm/slab.c b/mm/slab.c > > index 6ebf509bf2de..f0041f0125ba 100644 > > --- a/mm/slab.c > > +++ b/mm/slab.c > > @@ -3568,7 +3568,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) > > void *ret; > > > > if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) > > - return NULL; > > + return kmalloc_large_node(size, flags, node); > > Similar issue with caller not traced. > Actually I moved tracepoint into kmalloc_large_node(), but the problem I think was I write patches hard to review. in v2 I split some patches to be more reviewable. thanks!! > > cachep = kmalloc_slab(size, flags); > > if (unlikely(ZERO_OR_NULL_PTR(cachep))) > > return cachep; > > @@ -3642,15 +3642,25 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) > > { > > struct kmem_cache *s; > > size_t i; > > + struct folio *folio; > > > > local_irq_disable(); > > for (i = 0; i < size; i++) { > > void *objp = p[i]; > > > > - if (!orig_s) /* called via kfree_bulk */ > > - s = virt_to_cache(objp); > > - else > > + if (!orig_s) { > > + folio = virt_to_folio(objp); > > + /* called via kfree_bulk */ > > + if (!folio_test_slab(folio)) { > > + local_irq_enable(); > > + free_large_kmalloc(folio, objp); > > + local_irq_disable(); > > + continue; > > + } > > + s = folio_slab(folio)->slab_cache; > > + } else > > s = cache_from_obj(orig_s, objp); > > + > > if (!s) > > continue; > > > > @@ -3679,20 +3689,25 @@ void kfree(const void *objp) > > { > > struct kmem_cache *c; > > unsigned long flags; > > + struct folio *folio; > > + void *x = (void *) objp; > > I think you don't need to add 'x', just do the cast while calling > free_large_kmalloc(), same as done for __cache_free(). > in fact also SLUB's kfree defines x. But your suggestion sounds better. Anyway did it in v2. thanks! > > > > trace_kfree(_RET_IP_, objp); > > > > if (unlikely(ZERO_OR_NULL_PTR(objp))) > > return; > > - local_irq_save(flags); > > - kfree_debugcheck(objp); > > - c = virt_to_cache(objp); > > - if (!c) { > > - local_irq_restore(flags); > > + > > + folio = virt_to_folio(objp); > > + if (!folio_test_slab(folio)) { > > + free_large_kmalloc(folio, x); > > return; > > } > > - debug_check_no_locks_freed(objp, c->object_size); > > > > + c = folio_slab(folio)->slab_cache; > > + > > + local_irq_save(flags); > > + kfree_debugcheck(objp); > > + debug_check_no_locks_freed(objp, c->object_size); > > debug_check_no_obj_freed(objp, c->object_size); > > __cache_free(c, (void *)objp, _RET_IP_); > > local_irq_restore(flags); -- Thanks, Hyeonggon