On Wed, Nov 16, 2022 at 05:06:16PM -0800, Song Liu wrote:
> +static void move_vmap_to_free_text_tree(void *addr)
> +{
> + struct vmap_area *va;
> +
> + /* remove from vmap_area_root */
> + spin_lock(&vmap_area_lock);
> + va = __find_vmap_area((unsigned long)addr, &vmap_area_root);
> + if (WARN_ON_ONCE(!va)) {
> + spin_unlock(&vmap_area_lock);
> + return;
> + }
> + unlink_va(va, &vmap_area_root);
> + spin_unlock(&vmap_area_lock);
> +
> + /* make the memory RO+X */
> + memset(addr, 0, va->va_end - va->va_start);
> + set_memory_ro(va->va_start, (va->va_end - va->va_start) >> PAGE_SHIFT);
> + set_memory_x(va->va_start, (va->va_end - va->va_start) >> PAGE_SHIFT);
> +
> + /* add to all_text_vm */
> + va->vm->next = all_text_vm;
> + all_text_vm = va->vm;
> +
> + /* add to free_text_area_root */
> + spin_lock(&free_text_area_lock);
> + merge_or_add_vmap_area_augment(va, &free_text_area_root, &free_text_area_list);
> + spin_unlock(&free_text_area_lock);
> +}
<-- snip -->
> +void *execmem_alloc(unsigned long size, unsigned long align)
> +{
> + struct vmap_area *va, *tmp;
> + unsigned long addr;
> + enum fit_type type;
> + int ret;
> +
> + va = kmem_cache_alloc_node(vmap_area_cachep, GFP_KERNEL, NUMA_NO_NODE);
> + if (unlikely(!va))
> + return NULL;
> +
> +again:
> + preload_this_cpu_lock(&free_text_area_lock, GFP_KERNEL, NUMA_NO_NODE);
> + tmp = find_vmap_lowest_match(&free_text_area_root, size, align, 1, false);
> +
> + if (!tmp) {
> + unsigned long alloc_size;
> + void *ptr;
> +
> + spin_unlock(&free_text_area_lock);
> +
> + /*
> + * Not enough continuous space in free_text_area_root, try
> + * allocate more memory. The memory is first added to
> + * vmap_area_root, and then moved to free_text_area_root.
> + */
> + alloc_size = roundup(size, PMD_SIZE * num_online_nodes());
> + ptr = __vmalloc_node_range(alloc_size, PMD_SIZE, EXEC_MEM_START,
> + EXEC_MEM_END, GFP_KERNEL, PAGE_KERNEL,
> + VM_ALLOW_HUGE_VMAP | VM_NO_GUARD,
> + NUMA_NO_NODE, __builtin_return_address(0));
> + if (unlikely(!ptr))
> + goto err_out;
> +
> + move_vmap_to_free_text_tree(ptr);
It's not perfectly clear to me how we know for sure nothing can take
this underneath our noses.
Luis
