Members of struct vmap_area are not being used at the same time.
For example,
(1)members @flags, @purge_list, and @vm are unused when vmap area is
in the *FREE* tree;
(2)members @subtree_max_size and @purge_list are unused when vmap area is
in the *BUSY* tree and not in the purge list;
(3)members @subtree_max_size and @vm are unused when vmap area is
in the *BUSY* tree and in the purge list.
Since members @subtree_max_size, @purge_list and @vm are not used
at the same time, so they can be placed in a union to reduce the
size of struct vmap_area.
Besides, rename @flags to @_vm_valid to indicate if @vm is valid.
The reason why @_vm_valid can be placed in a union with @va_start
is that if @vm is valid, then @va_start can be known by @vm.
Signed-off-by: Pengfei Li <lpf.vector@xxxxxxxxx>
---
include/linux/vmalloc.h | 28 ++++++++++----
mm/vmalloc.c | 85 +++++++++++++++++++++++++++++++----------
2 files changed, 85 insertions(+), 28 deletions(-)
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 51e131245379..7b99de5ccbec 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -48,18 +48,30 @@ struct vm_struct {
};
struct vmap_area {
- unsigned long va_start;
+ union {
+ unsigned long va_start;
+ /*
+ * Determine whether vm is valid according to
+ * the value of _vm_valid
+ */
+ unsigned long _vm_valid;
+ };
+
unsigned long va_end;
- /*
- * Largest available free size in subtree.
- */
- unsigned long subtree_max_size;
- unsigned long flags;
+ union {
+ /* Only used when vmap area in *FREE* vmap_area tree */
+ unsigned long subtree_max_size;
+
+ /* Only used when vmap area in vmap_purge_list */
+ struct llist_node purge_list;
+
+ /* Only used when va_vm_is_valid() return true */
+ struct vm_struct *vm;
+ };
+
struct rb_node rb_node; /* address sorted rbtree */
struct list_head list; /* address sorted list */
- struct llist_node purge_list; /* "lazy purge" list */
- struct vm_struct *vm;
};
/*
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 4148d6fdfb6d..89b93ee0ec04 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -329,8 +329,28 @@ EXPORT_SYMBOL(vmalloc_to_pfn);
#define DEBUG_AUGMENT_PROPAGATE_CHECK 0
#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
-#define VM_LAZY_FREE 0x02
-#define VM_VM_AREA 0x04
+#define VA_VM_VALID 0x01UL
+
+static __always_inline void
+__va_set_vm(struct vmap_area *va, struct vm_struct *vm)
+{
+ /* Overwrite va->va_start by va->_vm_valid */
+ va->_vm_valid = VA_VM_VALID;
+ va->vm = vm;
+}
+
+static __always_inline void
+__va_unset_vm(struct vmap_area *va)
+{
+ /* Restore va->va_start and overwrite va->_vm_valid */
+ va->va_start = (unsigned long)va->vm->addr;
+}
+
+static __always_inline bool
+va_vm_is_valid(struct vmap_area *va)
+{
+ return (va->_vm_valid == VA_VM_VALID);
+}
static DEFINE_SPINLOCK(vmap_area_lock);
/* Export for kexec only */
@@ -399,15 +419,26 @@ static void purge_vmap_area_lazy(void);
static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
static unsigned long lazy_max_pages(void);
+/*
+ * Search the *BUSY* vmap_area tree. If va_vm_is_valid() return true,
+ * then va->va_start has been overwritten by va->_vm_valid,
+ * otherwise va->va_start remains the original value.
+ */
static struct vmap_area *__search_va_from_busy_tree(unsigned long addr)
{
struct rb_node *n = vmap_area_root.rb_node;
while (n) {
struct vmap_area *va;
+ unsigned long start;
va = rb_entry(n, struct vmap_area, rb_node);
- if (addr < va->va_start)
+ if (va_vm_is_valid(va))
+ start = (unsigned long)va->vm->addr;
+ else
+ start = va->va_start;
+
+ if (addr < start)
n = n->rb_left;
else if (addr >= va->va_end)
n = n->rb_right;
@@ -429,8 +460,13 @@ find_va_links(struct vmap_area *va,
{
struct vmap_area *tmp_va;
struct rb_node **link;
+ unsigned long start;
+ bool is_busy_va_tree = false;
if (root) {
+ if (root == &vmap_area_root)
+ is_busy_va_tree = true;
+
link = &root->rb_node;
if (unlikely(!*link)) {
*parent = NULL;
@@ -447,6 +483,10 @@ find_va_links(struct vmap_area *va,
*/
do {
tmp_va = rb_entry(*link, struct vmap_area, rb_node);
+ if (is_busy_va_tree && va_vm_is_valid(tmp_va))
+ start = (unsigned long)tmp_va->vm->addr;
+ else
+ start = tmp_va->va_start;
/*
* During the traversal we also do some sanity check.
@@ -454,9 +494,9 @@ find_va_links(struct vmap_area *va,
* or full overlaps.
*/
if (va->va_start < tmp_va->va_end &&
- va->va_end <= tmp_va->va_start)
+ va->va_end <= start)
link = &(*link)->rb_left;
- else if (va->va_end > tmp_va->va_start &&
+ else if (va->va_end > start &&
va->va_start >= tmp_va->va_end)
link = &(*link)->rb_right;
else
@@ -1079,8 +1119,8 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
va->va_start = addr;
va->va_end = addr + size;
- va->flags = 0;
insert_va_to_busy_tree(va);
+ va->vm = NULL;
spin_unlock(&vmap_area_lock);
@@ -1872,11 +1912,11 @@ void __init vmalloc_init(void)
if (WARN_ON_ONCE(!va))
continue;
- va->flags = VM_VM_AREA;
va->va_start = (unsigned long)tmp->addr;
va->va_end = va->va_start + tmp->size;
- va->vm = tmp;
insert_va_to_busy_tree(va);
+
+ __va_set_vm(va, tmp);
}
/*
@@ -1969,8 +2009,9 @@ static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
vm->addr = (void *)va->va_start;
vm->size = va->va_end - va->va_start;
vm->caller = caller;
- va->vm = vm;
- va->flags |= VM_VM_AREA;
+
+ __va_set_vm(va, vm);
+
spin_unlock(&vmap_area_lock);
}
@@ -2075,7 +2116,7 @@ struct vm_struct *find_vm_area(const void *addr)
struct vmap_area *va;
va = find_vmap_area((unsigned long)addr);
- if (va && va->flags & VM_VM_AREA)
+ if (va && va_vm_is_valid(va))
return va->vm;
return NULL;
@@ -2098,13 +2139,15 @@ struct vm_struct *remove_vm_area(const void *addr)
might_sleep();
va = find_vmap_area((unsigned long)addr);
- if (va && va->flags & VM_VM_AREA) {
+ if (va && va_vm_is_valid(va)) {
struct vm_struct *vm = va->vm;
spin_lock(&vmap_area_lock);
- va->vm = NULL;
- va->flags &= ~VM_VM_AREA;
- va->flags |= VM_LAZY_FREE;
+ /*
+ * Call __va_unset_vm() to restore the value of va->va_start
+ * before calling free_unmap_vmap_area() to add it to purge list
+ */
+ __va_unset_vm(va);
spin_unlock(&vmap_area_lock);
kasan_free_shadow(vm);
@@ -2813,7 +2856,7 @@ long vread(char *buf, char *addr, unsigned long count)
if (!count)
break;
- if (!(va->flags & VM_VM_AREA))
+ if (!va_vm_is_valid(va))
continue;
vm = va->vm;
@@ -2893,7 +2936,7 @@ long vwrite(char *buf, char *addr, unsigned long count)
if (!count)
break;
- if (!(va->flags & VM_VM_AREA))
+ if (!va_vm_is_valid(va))
continue;
vm = va->vm;
@@ -3407,14 +3450,16 @@ static int s_show(struct seq_file *m, void *p)
va = list_entry(p, struct vmap_area, list);
/*
- * s_show can encounter race with remove_vm_area, !VM_VM_AREA on
+ * s_show can encounter race with remove_vm_area, !va_vm_is_valid() on
* behalf of vmap area is being tear down or vm_map_ram allocation.
+ * And if va->vm != NULL then vmap area is being tear down,
+ * otherwise vmap area is allocated by vm_map_ram().
*/
- if (!(va->flags & VM_VM_AREA)) {
+ if (!va_vm_is_valid(va)) {
seq_printf(m, "0x%pK-0x%pK %7ld %s\n",
(void *)va->va_start, (void *)va->va_end,
va->va_end - va->va_start,
- va->flags & VM_LAZY_FREE ? "unpurged vm_area" : "vm_map_ram");
+ va->vm ? "unpurged vm_area" : "vm_map_ram");
return 0;
}
--
2.21.0
