slub - Use local_t protection Use local_enter/local_exit for protection in the fast path. Depends on the cmpxchg_local slub patch. Changelog: Add new primitives to switch from local critical section to interrupt disabled section. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@xxxxxxxxxx> CC: Christoph Lameter <clameter@xxxxxxx> --- mm/slub.c | 55 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 23 deletions(-) Index: linux-2.6-lttng/mm/slub.c =================================================================== --- linux-2.6-lttng.orig/mm/slub.c 2007-09-05 09:01:00.000000000 -0400 +++ linux-2.6-lttng/mm/slub.c 2007-09-05 09:05:34.000000000 -0400 @@ -1065,9 +1065,6 @@ static struct page *new_slab(struct kmem BUG_ON(flags & GFP_SLAB_BUG_MASK); - if (flags & __GFP_WAIT) - local_irq_enable(); - page = allocate_slab(s, flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); if (!page) @@ -1100,8 +1097,6 @@ static struct page *new_slab(struct kmem page->freelist = start; page->inuse = 0; out: - if (flags & __GFP_WAIT) - local_irq_disable(); return page; } @@ -1455,8 +1450,7 @@ static void *__slab_alloc(struct kmem_ca struct page *new; unsigned long flags; - local_irq_save(flags); - put_cpu_no_resched(); + local_nest_irq_save(flags); if (!c->page) /* Slab was flushed */ goto new_slab; @@ -1479,8 +1473,7 @@ load_freelist: c->freelist = object[c->offset]; out: slab_unlock(c->page); - local_irq_restore(flags); - preempt_check_resched(); + local_nest_irq_restore(flags); if (unlikely((gfpflags & __GFP_ZERO))) memset(object, 0, c->objsize); return object; @@ -1494,8 +1487,16 @@ new_slab: c->page = new; goto load_freelist; } - + if (gfpflags & __GFP_WAIT) { + local_nest_irq_enable(); + local_exit(); + } new = new_slab(s, gfpflags, node); + if (gfpflags & __GFP_WAIT) { + local_enter(); + local_nest_irq_disable(); + } + if (new) { c = get_cpu_slab(s, smp_processor_id()); if (c->page) { @@ -1523,8 +1524,7 @@ new_slab: c->page = new; goto load_freelist; } - local_irq_restore(flags); - preempt_check_resched(); + local_nest_irq_restore(flags); return NULL; debug: object = c->page->freelist; @@ -1552,8 +1552,11 @@ static void __always_inline *slab_alloc( { void **object; struct kmem_cache_cpu *c; + unsigned long flags; + void *ret; - c = get_cpu_slab(s, get_cpu()); + local_enter_save(flags); + c = get_cpu_slab(s, smp_processor_id()); redo: object = c->freelist; if (unlikely(!object)) @@ -1566,14 +1569,15 @@ redo: object[c->offset]) != object)) goto redo; - put_cpu(); + local_exit_restore(flags); if (unlikely((gfpflags & __GFP_ZERO))) memset(object, 0, c->objsize); return object; slow: - return __slab_alloc(s, gfpflags, node, addr, c); - + ret = __slab_alloc(s, gfpflags, node, addr, c); + local_exit_restore(flags); + return ret; } void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) @@ -1605,8 +1609,7 @@ static void __slab_free(struct kmem_cach void **object = (void *)x; unsigned long flags; - put_cpu(); - local_irq_save(flags); + local_nest_irq_save(flags); slab_lock(page); if (unlikely(SlabDebug(page))) @@ -1632,7 +1635,7 @@ checks_ok: out_unlock: slab_unlock(page); - local_irq_restore(flags); + local_nest_irq_restore(flags); return; slab_empty: @@ -1643,7 +1646,7 @@ slab_empty: remove_partial(s, page); slab_unlock(page); - local_irq_restore(flags); + local_nest_irq_restore(flags); discard_slab(s, page); return; @@ -1670,10 +1673,12 @@ static void __always_inline slab_free(st void **object = (void *)x; void **freelist; struct kmem_cache_cpu *c; + unsigned long flags; debug_check_no_locks_freed(object, s->objsize); - c = get_cpu_slab(s, get_cpu()); + local_enter_save(flags); + c = get_cpu_slab(s, smp_processor_id()); if (unlikely(c->node < 0)) goto slow; redo: @@ -1687,10 +1692,11 @@ redo: != freelist)) goto redo; - put_cpu(); + local_exit_restore(flags); return; slow: __slab_free(s, page, x, addr, c->offset); + local_exit_restore(flags); } void kmem_cache_free(struct kmem_cache *s, void *x) @@ -2026,8 +2032,11 @@ static struct kmem_cache_node *early_kme BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node)); + if (gfpflags & __GFP_WAIT) + local_irq_enable(); page = new_slab(kmalloc_caches, gfpflags, node); - + if (gfpflags & __GFP_WAIT) + local_irq_disable(); BUG_ON(!page); if (page_to_nid(page) != node) { printk(KERN_ERR "SLUB: Unable to allocate memory from " -- Mathieu Desnoyers Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68 - To unsubscribe from this list: send the line "unsubscribe linux-ia64" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
![]() |