We want to add some extra cache invalidation logic to vmalloc() area list unlocks - for that first abstract away the vmap_area_lock operations into vmap_lock()/vmap_unlock(). Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx> Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx> Cc: Rik van Riel <riel@xxxxxxxxxx> Cc: linux-mm@xxxxxxxxx Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx> --- mm/vmalloc.c | 55 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 2faaa2976447..605138083880 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -277,6 +277,17 @@ EXPORT_SYMBOL(vmalloc_to_pfn); #define VM_VM_AREA 0x04 static DEFINE_SPINLOCK(vmap_area_lock); + +static inline void vmap_lock(void) +{ + spin_lock(&vmap_area_lock); +} + +static inline void vmap_unlock(void) +{ + spin_unlock(&vmap_area_lock); +} + /* Export for kexec only */ LIST_HEAD(vmap_area_list); static struct rb_root vmap_area_root = RB_ROOT; @@ -373,7 +384,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); retry: - spin_lock(&vmap_area_lock); + vmap_lock(); /* * Invalidate cache if we have more permissive parameters. * cached_hole_size notes the largest hole noticed _below_ @@ -452,7 +463,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, va->flags = 0; __insert_vmap_area(va); free_vmap_cache = &va->rb_node; - spin_unlock(&vmap_area_lock); + vmap_unlock(); BUG_ON(va->va_start & (align-1)); BUG_ON(va->va_start < vstart); @@ -461,7 +472,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, return va; overflow: - spin_unlock(&vmap_area_lock); + vmap_unlock(); if (!purged) { purge_vmap_area_lazy(); purged = 1; @@ -514,9 +525,9 @@ static void __free_vmap_area(struct vmap_area *va) */ static void free_vmap_area(struct vmap_area *va) { - spin_lock(&vmap_area_lock); + vmap_lock(); __free_vmap_area(va); - spin_unlock(&vmap_area_lock); + vmap_unlock(); } /* @@ -642,10 +653,10 @@ static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, flush_tlb_kernel_range(*start, *end); if (nr) { - spin_lock(&vmap_area_lock); + vmap_lock(); list_for_each_entry_safe(va, n_va, &valist, purge_list) __free_vmap_area(va); - spin_unlock(&vmap_area_lock); + vmap_unlock(); } spin_unlock(&purge_lock); } @@ -707,9 +718,9 @@ static struct vmap_area *find_vmap_area(unsigned long addr) { struct vmap_area *va; - spin_lock(&vmap_area_lock); + vmap_lock(); va = __find_vmap_area(addr); - spin_unlock(&vmap_area_lock); + vmap_unlock(); return va; } @@ -1304,14 +1315,14 @@ EXPORT_SYMBOL_GPL(map_vm_area); static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, unsigned long flags, const void *caller) { - spin_lock(&vmap_area_lock); + vmap_lock(); vm->flags = flags; vm->addr = (void *)va->va_start; vm->size = va->va_end - va->va_start; vm->caller = caller; va->vm = vm; va->flags |= VM_VM_AREA; - spin_unlock(&vmap_area_lock); + vmap_unlock(); } static void clear_vm_uninitialized_flag(struct vm_struct *vm) @@ -1433,10 +1444,10 @@ struct vm_struct *remove_vm_area(const void *addr) if (va && va->flags & VM_VM_AREA) { struct vm_struct *vm = va->vm; - spin_lock(&vmap_area_lock); + vmap_lock(); va->vm = NULL; va->flags &= ~VM_VM_AREA; - spin_unlock(&vmap_area_lock); + vmap_unlock(); vmap_debug_free_range(va->va_start, va->va_end); kasan_free_shadow(vm); @@ -2008,7 +2019,7 @@ long vread(char *buf, char *addr, unsigned long count) if ((unsigned long) addr + count < count) count = -(unsigned long) addr; - spin_lock(&vmap_area_lock); + vmap_lock(); list_for_each_entry(va, &vmap_area_list, list) { if (!count) break; @@ -2040,7 +2051,7 @@ long vread(char *buf, char *addr, unsigned long count) count -= n; } finished: - spin_unlock(&vmap_area_lock); + vmap_unlock(); if (buf == buf_start) return 0; @@ -2090,7 +2101,7 @@ long vwrite(char *buf, char *addr, unsigned long count) count = -(unsigned long) addr; buflen = count; - spin_lock(&vmap_area_lock); + vmap_lock(); list_for_each_entry(va, &vmap_area_list, list) { if (!count) break; @@ -2121,7 +2132,7 @@ long vwrite(char *buf, char *addr, unsigned long count) count -= n; } finished: - spin_unlock(&vmap_area_lock); + vmap_unlock(); if (!copied) return 0; return buflen; @@ -2435,7 +2446,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, goto err_free; } retry: - spin_lock(&vmap_area_lock); + vmap_lock(); /* start scanning - we scan from the top, begin with the last area */ area = term_area = last_area; @@ -2457,7 +2468,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, * comparing. */ if (base + last_end < vmalloc_start + last_end) { - spin_unlock(&vmap_area_lock); + vmap_unlock(); if (!purged) { purge_vmap_area_lazy(); purged = true; @@ -2512,7 +2523,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, vmap_area_pcpu_hole = base + offsets[last_area]; - spin_unlock(&vmap_area_lock); + vmap_unlock(); /* insert all vm's */ for (area = 0; area < nr_vms; area++) @@ -2557,7 +2568,7 @@ static void *s_start(struct seq_file *m, loff_t *pos) loff_t n = *pos; struct vmap_area *va; - spin_lock(&vmap_area_lock); + vmap_lock(); va = list_entry((&vmap_area_list)->next, typeof(*va), list); while (n > 0 && &va->list != &vmap_area_list) { n--; @@ -2585,7 +2596,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos) static void s_stop(struct seq_file *m, void *p) __releases(&vmap_area_lock) { - spin_unlock(&vmap_area_lock); + vmap_unlock(); } static void show_numa_info(struct seq_file *m, struct vm_struct *v) -- 2.1.4 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx"> email@xxxxxxxxx </a>