On Wed, Oct 23, 2024 at 1:08 PM Suren Baghdasaryan <surenb@xxxxxxxxxx> wrote:
>
> When a module gets unloaded there is a possibility that some of the
> allocations it made are still used and therefore the allocation tags
> corresponding to these allocations are still referenced. As such, the
> memory for these tags can't be freed. This is currently handled as an
> abnormal situation and module's data section is not being unloaded.
> To handle this situation without keeping module's data in memory,
> allow codetags with longer lifespan than the module to be loaded into
> their own separate memory. The in-use memory areas and gaps after
> module unloading in this separate memory are tracked using maple trees.
> Allocation tags arrange their separate memory so that it is virtually
> contiguous and that will allow simple allocation tag indexing later on
> in this patchset. The size of this virtually contiguous memory is set
> to store up to 100000 allocation tags.
>
> Signed-off-by: Suren Baghdasaryan <surenb@xxxxxxxxxx>
Reviewed-by: Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
> ---
> include/asm-generic/codetag.lds.h | 19 +++
> include/linux/alloc_tag.h | 13 +-
> include/linux/codetag.h | 37 ++++-
> kernel/module/main.c | 80 ++++++----
> lib/alloc_tag.c | 249 +++++++++++++++++++++++++++---
> lib/codetag.c | 100 +++++++++++-
> scripts/module.lds.S | 5 +-
> 7 files changed, 441 insertions(+), 62 deletions(-)
>
> diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h
> index 64f536b80380..372c320c5043 100644
> --- a/include/asm-generic/codetag.lds.h
> +++ b/include/asm-generic/codetag.lds.h
> @@ -11,4 +11,23 @@
> #define CODETAG_SECTIONS() \
> SECTION_WITH_BOUNDARIES(alloc_tags)
>
> +/*
> + * Module codetags which aren't used after module unload, therefore have the
> + * same lifespan as the module and can be safely unloaded with the module.
> + */
> +#define MOD_CODETAG_SECTIONS()
> +
> +#define MOD_SEPARATE_CODETAG_SECTION(_name) \
> + .codetag.##_name : { \
> + SECTION_WITH_BOUNDARIES(_name) \
> + }
> +
> +/*
> + * For codetags which might be used after module unload, therefore might stay
> + * longer in memory. Each such codetag type has its own section so that we can
> + * unload them individually once unused.
> + */
> +#define MOD_SEPARATE_CODETAG_SECTIONS() \
> + MOD_SEPARATE_CODETAG_SECTION(alloc_tags)
> +
> #endif /* __ASM_GENERIC_CODETAG_LDS_H */
> diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
> index 1f0a9ff23a2c..7431757999c5 100644
> --- a/include/linux/alloc_tag.h
> +++ b/include/linux/alloc_tag.h
> @@ -30,6 +30,13 @@ struct alloc_tag {
> struct alloc_tag_counters __percpu *counters;
> } __aligned(8);
>
> +struct alloc_tag_module_section {
> + unsigned long start_addr;
> + unsigned long end_addr;
> + /* used size */
> + unsigned long size;
> +};
> +
> #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
>
> #define CODETAG_EMPTY ((void *)1)
> @@ -54,6 +61,8 @@ static inline void set_codetag_empty(union codetag_ref *ref) {}
>
> #ifdef CONFIG_MEM_ALLOC_PROFILING
>
> +#define ALLOC_TAG_SECTION_NAME "alloc_tags"
> +
> struct codetag_bytes {
> struct codetag *ct;
> s64 bytes;
> @@ -76,7 +85,7 @@ DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
>
> #define DEFINE_ALLOC_TAG(_alloc_tag) \
> static struct alloc_tag _alloc_tag __used __aligned(8) \
> - __section("alloc_tags") = { \
> + __section(ALLOC_TAG_SECTION_NAME) = { \
> .ct = CODE_TAG_INIT, \
> .counters = &_shared_alloc_tag };
>
> @@ -85,7 +94,7 @@ DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
> #define DEFINE_ALLOC_TAG(_alloc_tag) \
> static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \
> static struct alloc_tag _alloc_tag __used __aligned(8) \
> - __section("alloc_tags") = { \
> + __section(ALLOC_TAG_SECTION_NAME) = { \
> .ct = CODE_TAG_INIT, \
> .counters = &_alloc_tag_cntr };
>
> diff --git a/include/linux/codetag.h b/include/linux/codetag.h
> index c2a579ccd455..d10bd9810d32 100644
> --- a/include/linux/codetag.h
> +++ b/include/linux/codetag.h
> @@ -35,8 +35,15 @@ struct codetag_type_desc {
> size_t tag_size;
> void (*module_load)(struct codetag_type *cttype,
> struct codetag_module *cmod);
> - bool (*module_unload)(struct codetag_type *cttype,
> + void (*module_unload)(struct codetag_type *cttype,
> struct codetag_module *cmod);
> +#ifdef CONFIG_MODULES
> + void (*module_replaced)(struct module *mod, struct module *new_mod);
> + bool (*needs_section_mem)(struct module *mod, unsigned long size);
> + void *(*alloc_section_mem)(struct module *mod, unsigned long size,
> + unsigned int prepend, unsigned long align);
> + void (*free_section_mem)(struct module *mod, bool used);
> +#endif
> };
>
> struct codetag_iterator {
> @@ -71,11 +78,31 @@ struct codetag_type *
> codetag_register_type(const struct codetag_type_desc *desc);
>
> #if defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES)
> +
> +bool codetag_needs_module_section(struct module *mod, const char *name,
> + unsigned long size);
> +void *codetag_alloc_module_section(struct module *mod, const char *name,
> + unsigned long size, unsigned int prepend,
> + unsigned long align);
> +void codetag_free_module_sections(struct module *mod);
> +void codetag_module_replaced(struct module *mod, struct module *new_mod);
> void codetag_load_module(struct module *mod);
> -bool codetag_unload_module(struct module *mod);
> -#else
> +void codetag_unload_module(struct module *mod);
> +
> +#else /* defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) */
> +
> +static inline bool
> +codetag_needs_module_section(struct module *mod, const char *name,
> + unsigned long size) { return false; }
> +static inline void *
> +codetag_alloc_module_section(struct module *mod, const char *name,
> + unsigned long size, unsigned int prepend,
> + unsigned long align) { return NULL; }
> +static inline void codetag_free_module_sections(struct module *mod) {}
> +static inline void codetag_module_replaced(struct module *mod, struct module *new_mod) {}
> static inline void codetag_load_module(struct module *mod) {}
> -static inline bool codetag_unload_module(struct module *mod) { return true; }
> -#endif
> +static inline void codetag_unload_module(struct module *mod) {}
> +
> +#endif /* defined(CONFIG_CODE_TAGGING) && defined(CONFIG_MODULES) */
>
> #endif /* _LINUX_CODETAG_H */
> diff --git a/kernel/module/main.c b/kernel/module/main.c
> index ef54733bd7d2..1787686e5cae 100644
> --- a/kernel/module/main.c
> +++ b/kernel/module/main.c
> @@ -1254,22 +1254,17 @@ static int module_memory_alloc(struct module *mod, enum mod_mem_type type)
> return 0;
> }
>
> -static void module_memory_free(struct module *mod, enum mod_mem_type type,
> - bool unload_codetags)
> +static void module_memory_free(struct module *mod, enum mod_mem_type type)
> {
> struct module_memory *mem = &mod->mem[type];
> - void *ptr = mem->base;
>
> if (mem->is_rox)
> vfree(mem->rw_copy);
>
> - if (!unload_codetags && mod_mem_type_is_core_data(type))
> - return;
> -
> - execmem_free(ptr);
> + execmem_free(mem->base);
> }
>
> -static void free_mod_mem(struct module *mod, bool unload_codetags)
> +static void free_mod_mem(struct module *mod)
> {
> for_each_mod_mem_type(type) {
> struct module_memory *mod_mem = &mod->mem[type];
> @@ -1280,25 +1275,20 @@ static void free_mod_mem(struct module *mod, bool unload_codetags)
> /* Free lock-classes; relies on the preceding sync_rcu(). */
> lockdep_free_key_range(mod_mem->base, mod_mem->size);
> if (mod_mem->size)
> - module_memory_free(mod, type, unload_codetags);
> + module_memory_free(mod, type);
> }
>
> /* MOD_DATA hosts mod, so free it at last */
> lockdep_free_key_range(mod->mem[MOD_DATA].base, mod->mem[MOD_DATA].size);
> - module_memory_free(mod, MOD_DATA, unload_codetags);
> + module_memory_free(mod, MOD_DATA);
> }
>
> /* Free a module, remove from lists, etc. */
> static void free_module(struct module *mod)
> {
> - bool unload_codetags;
> -
> trace_module_free(mod);
>
> - unload_codetags = codetag_unload_module(mod);
> - if (!unload_codetags)
> - pr_warn("%s: memory allocation(s) from the module still alive, cannot unload cleanly\n",
> - mod->name);
> + codetag_unload_module(mod);
>
> mod_sysfs_teardown(mod);
>
> @@ -1341,7 +1331,7 @@ static void free_module(struct module *mod)
> kfree(mod->args);
> percpu_modfree(mod);
>
> - free_mod_mem(mod, unload_codetags);
> + free_mod_mem(mod);
> }
>
> void *__symbol_get(const char *symbol)
> @@ -1606,6 +1596,20 @@ static void __layout_sections(struct module *mod, struct load_info *info, bool i
> if (WARN_ON_ONCE(type == MOD_INVALID))
> continue;
>
> + /*
> + * Do not allocate codetag memory as we load it into
> + * preallocated contiguous memory.
> + */
> + if (codetag_needs_module_section(mod, sname, s->sh_size)) {
> + /*
> + * s->sh_entsize won't be used but populate the
> + * type field to avoid confusion.
> + */
> + s->sh_entsize = ((unsigned long)(type) & SH_ENTSIZE_TYPE_MASK)
> + << SH_ENTSIZE_TYPE_SHIFT;
> + continue;
> + }
> +
> s->sh_entsize = module_get_offset_and_type(mod, type, s, i);
> pr_debug("\t%s\n", sname);
> }
> @@ -2280,6 +2284,7 @@ static int move_module(struct module *mod, struct load_info *info)
> int i;
> enum mod_mem_type t = 0;
> int ret = -ENOMEM;
> + bool codetag_section_found = false;
>
> for_each_mod_mem_type(type) {
> if (!mod->mem[type].size) {
> @@ -2291,7 +2296,7 @@ static int move_module(struct module *mod, struct load_info *info)
> ret = module_memory_alloc(mod, type);
> if (ret) {
> t = type;
> - goto out_enomem;
> + goto out_err;
> }
> }
>
> @@ -2300,15 +2305,33 @@ static int move_module(struct module *mod, struct load_info *info)
> for (i = 0; i < info->hdr->e_shnum; i++) {
> void *dest;
> Elf_Shdr *shdr = &info->sechdrs[i];
> - enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
> - unsigned long offset = shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK;
> + const char *sname;
> unsigned long addr;
>
> if (!(shdr->sh_flags & SHF_ALLOC))
> continue;
>
> - addr = (unsigned long)mod->mem[type].base + offset;
> - dest = mod->mem[type].rw_copy + offset;
> + sname = info->secstrings + shdr->sh_name;
> + /*
> + * Load codetag sections separately as they might still be used
> + * after module unload.
> + */
> + if (codetag_needs_module_section(mod, sname, shdr->sh_size)) {
> + dest = codetag_alloc_module_section(mod, sname, shdr->sh_size,
> + arch_mod_section_prepend(mod, i), shdr->sh_addralign);
> + if (IS_ERR(dest)) {
> + ret = PTR_ERR(dest);
> + goto out_err;
> + }
> + addr = (unsigned long)dest;
> + codetag_section_found = true;
> + } else {
> + enum mod_mem_type type = shdr->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
> + unsigned long offset = shdr->sh_entsize & SH_ENTSIZE_OFFSET_MASK;
> +
> + addr = (unsigned long)mod->mem[type].base + offset;
> + dest = mod->mem[type].rw_copy + offset;
> + }
>
> if (shdr->sh_type != SHT_NOBITS) {
> /*
> @@ -2320,7 +2343,7 @@ static int move_module(struct module *mod, struct load_info *info)
> if (i == info->index.mod &&
> (WARN_ON_ONCE(shdr->sh_size != sizeof(struct module)))) {
> ret = -ENOEXEC;
> - goto out_enomem;
> + goto out_err;
> }
> memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
> }
> @@ -2336,9 +2359,12 @@ static int move_module(struct module *mod, struct load_info *info)
> }
>
> return 0;
> -out_enomem:
> +out_err:
> for (t--; t >= 0; t--)
> - module_memory_free(mod, t, true);
> + module_memory_free(mod, t);
> + if (codetag_section_found)
> + codetag_free_module_sections(mod);
> +
> return ret;
> }
>
> @@ -2459,6 +2485,8 @@ static struct module *layout_and_allocate(struct load_info *info, int flags)
> /* Module has been copied to its final place now: return it. */
> mod = (void *)info->sechdrs[info->index.mod].sh_addr;
> kmemleak_load_module(mod, info);
> + codetag_module_replaced(info->mod, mod);
> +
> return mod;
> }
>
> @@ -2468,7 +2496,7 @@ static void module_deallocate(struct module *mod, struct load_info *info)
> percpu_modfree(mod);
> module_arch_freeing_init(mod);
>
> - free_mod_mem(mod, true);
> + free_mod_mem(mod);
> }
>
> int __weak module_finalize(const Elf_Ehdr *hdr,
> diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
> index 435aa837e550..d9f51169ffeb 100644
> --- a/lib/alloc_tag.c
> +++ b/lib/alloc_tag.c
> @@ -1,5 +1,6 @@
> // SPDX-License-Identifier: GPL-2.0-only
> #include <linux/alloc_tag.h>
> +#include <linux/execmem.h>
> #include <linux/fs.h>
> #include <linux/gfp.h>
> #include <linux/module.h>
> @@ -9,6 +10,7 @@
> #include <linux/seq_file.h>
>
> #define ALLOCINFO_FILE_NAME "allocinfo"
> +#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
>
> #ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
> static bool mem_profiling_support __meminitdata = true;
> @@ -174,31 +176,226 @@ static void __init procfs_init(void)
> }
> }
>
> -static bool alloc_tag_module_unload(struct codetag_type *cttype,
> - struct codetag_module *cmod)
> +#ifdef CONFIG_MODULES
> +
> +static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
> +/* A dummy object used to indicate an unloaded module */
> +static struct module unloaded_mod;
> +/* A dummy object used to indicate a module prepended area */
> +static struct module prepend_mod;
> +
> +static struct alloc_tag_module_section module_tags;
> +
> +static bool needs_section_mem(struct module *mod, unsigned long size)
> {
> - struct codetag_iterator iter = codetag_get_ct_iter(cttype);
> - struct alloc_tag_counters counter;
> - bool module_unused = true;
> - struct alloc_tag *tag;
> - struct codetag *ct;
> + return size >= sizeof(struct alloc_tag);
> +}
> +
> +static struct alloc_tag *find_used_tag(struct alloc_tag *from, struct alloc_tag *to)
> +{
> + while (from <= to) {
> + struct alloc_tag_counters counter;
>
> - for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) {
> - if (iter.cmod != cmod)
> + counter = alloc_tag_read(from);
> + if (counter.bytes)
> + return from;
> + from++;
> + }
> +
> + return NULL;
> +}
> +
> +/* Called with mod_area_mt locked */
> +static void clean_unused_module_areas_locked(void)
> +{
> + MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
> + struct module *val;
> +
> + mas_for_each(&mas, val, module_tags.size) {
> + if (val != &unloaded_mod)
> continue;
>
> - tag = ct_to_alloc_tag(ct);
> - counter = alloc_tag_read(tag);
> + /* Release area if all tags are unused */
> + if (!find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
> + (struct alloc_tag *)(module_tags.start_addr + mas.last)))
> + mas_erase(&mas);
> + }
> +}
> +
> +/* Called with mod_area_mt locked */
> +static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
> + unsigned long size, unsigned int prepend, unsigned long align)
> +{
> + bool cleanup_done = false;
> +
> +repeat:
> + /* Try finding exact size and hope the start is aligned */
> + if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
> + if (IS_ALIGNED(mas->index + prepend, align))
> + return true;
> +
> + /* Try finding larger area to align later */
> + mas_reset(mas);
> + if (!mas_empty_area(mas, 0, section_size - 1,
> + size + prepend + align - 1))
> + return true;
> + }
>
> - if (WARN(counter.bytes,
> - "%s:%u module %s func:%s has %llu allocated at module unload",
> - ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes))
> - module_unused = false;
> + /* No free area, try cleanup stale data and repeat the search once */
> + if (!cleanup_done) {
> + clean_unused_module_areas_locked();
> + cleanup_done = true;
> + mas_reset(mas);
> + goto repeat;
> }
>
> - return module_unused;
> + return false;
> +}
> +
> +static void *reserve_module_tags(struct module *mod, unsigned long size,
> + unsigned int prepend, unsigned long align)
> +{
> + unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
> + MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
> + unsigned long offset;
> + void *ret = NULL;
> +
> + /* If no tags return NULL */
> + if (size < sizeof(struct alloc_tag))
> + return NULL;
> +
> + /*
> + * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
> + * align 0 or 1 means no alignment, to simplify set to 1.
> + */
> + if (!align)
> + align = 1;
> +
> + mas_lock(&mas);
> + if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
> + ret = ERR_PTR(-ENOMEM);
> + goto unlock;
> + }
> +
> + /* Mark found area as reserved */
> + offset = mas.index;
> + offset += prepend;
> + offset = ALIGN(offset, align);
> + if (offset != mas.index) {
> + unsigned long pad_start = mas.index;
> +
> + mas.last = offset - 1;
> + mas_store(&mas, &prepend_mod);
> + if (mas_is_err(&mas)) {
> + ret = ERR_PTR(xa_err(mas.node));
> + goto unlock;
> + }
> + mas.index = offset;
> + mas.last = offset + size - 1;
> + mas_store(&mas, mod);
> + if (mas_is_err(&mas)) {
> + mas.index = pad_start;
> + mas_erase(&mas);
> + ret = ERR_PTR(xa_err(mas.node));
> + }
> + } else {
> + mas.last = offset + size - 1;
> + mas_store(&mas, mod);
> + if (mas_is_err(&mas))
> + ret = ERR_PTR(xa_err(mas.node));
> + }
> +unlock:
> + mas_unlock(&mas);
> +
> + if (IS_ERR(ret))
> + return ret;
> +
> + if (module_tags.size < offset + size)
> + module_tags.size = offset + size;
> +
> + return (struct alloc_tag *)(module_tags.start_addr + offset);
> }
>
> +static void release_module_tags(struct module *mod, bool used)
> +{
> + MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
> + struct alloc_tag *tag;
> + struct module *val;
> +
> + mas_lock(&mas);
> + mas_for_each_rev(&mas, val, 0)
> + if (val == mod)
> + break;
> +
> + if (!val) /* module not found */
> + goto out;
> +
> + if (!used)
> + goto release_area;
> +
> + /* Find out if the area is used */
> + tag = find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
> + (struct alloc_tag *)(module_tags.start_addr + mas.last));
> + if (tag) {
> + struct alloc_tag_counters counter = alloc_tag_read(tag);
> +
> + pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
> + tag->ct.filename, tag->ct.lineno, tag->ct.modname,
> + tag->ct.function, counter.bytes);
> + } else {
> + used = false;
> + }
> +release_area:
> + mas_store(&mas, used ? &unloaded_mod : NULL);
> + val = mas_prev_range(&mas, 0);
> + if (val == &prepend_mod)
> + mas_store(&mas, NULL);
> +out:
> + mas_unlock(&mas);
> +}
> +
> +static void replace_module(struct module *mod, struct module *new_mod)
> +{
> + MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
> + struct module *val;
> +
> + mas_lock(&mas);
> + mas_for_each(&mas, val, module_tags.size) {
> + if (val != mod)
> + continue;
> +
> + mas_store_gfp(&mas, new_mod, GFP_KERNEL);
> + break;
> + }
> + mas_unlock(&mas);
> +}
> +
> +static int __init alloc_mod_tags_mem(void)
> +{
> + /* Allocate space to copy allocation tags */
> + module_tags.start_addr = (unsigned long)execmem_alloc(EXECMEM_MODULE_DATA,
> + MODULE_ALLOC_TAG_VMAP_SIZE);
> + if (!module_tags.start_addr)
> + return -ENOMEM;
> +
> + module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
> +
> + return 0;
> +}
> +
> +static void __init free_mod_tags_mem(void)
> +{
> + execmem_free((void *)module_tags.start_addr);
> + module_tags.start_addr = 0;
> +}
> +
> +#else /* CONFIG_MODULES */
> +
> +static inline int alloc_mod_tags_mem(void) { return 0; }
> +static inline void free_mod_tags_mem(void) {}
> +
> +#endif /* CONFIG_MODULES */
> +
> static int __init setup_early_mem_profiling(char *str)
> {
> bool enable;
> @@ -274,14 +471,26 @@ static inline void sysctl_init(void) {}
> static int __init alloc_tag_init(void)
> {
> const struct codetag_type_desc desc = {
> - .section = "alloc_tags",
> - .tag_size = sizeof(struct alloc_tag),
> - .module_unload = alloc_tag_module_unload,
> + .section = ALLOC_TAG_SECTION_NAME,
> + .tag_size = sizeof(struct alloc_tag),
> +#ifdef CONFIG_MODULES
> + .needs_section_mem = needs_section_mem,
> + .alloc_section_mem = reserve_module_tags,
> + .free_section_mem = release_module_tags,
> + .module_replaced = replace_module,
> +#endif
> };
> + int res;
> +
> + res = alloc_mod_tags_mem();
> + if (res)
> + return res;
>
> alloc_tag_cttype = codetag_register_type(&desc);
> - if (IS_ERR(alloc_tag_cttype))
> + if (IS_ERR(alloc_tag_cttype)) {
> + free_mod_tags_mem();
> return PTR_ERR(alloc_tag_cttype);
> + }
>
> sysctl_init();
> procfs_init();
> diff --git a/lib/codetag.c b/lib/codetag.c
> index d1fbbb7c2ec3..654496952f86 100644
> --- a/lib/codetag.c
> +++ b/lib/codetag.c
> @@ -207,6 +207,94 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod)
> }
>
> #ifdef CONFIG_MODULES
> +#define CODETAG_SECTION_PREFIX ".codetag."
> +
> +/* Some codetag types need a separate module section */
> +bool codetag_needs_module_section(struct module *mod, const char *name,
> + unsigned long size)
> +{
> + const char *type_name;
> + struct codetag_type *cttype;
> + bool ret = false;
> +
> + if (strncmp(name, CODETAG_SECTION_PREFIX, strlen(CODETAG_SECTION_PREFIX)))
> + return false;
> +
> + type_name = name + strlen(CODETAG_SECTION_PREFIX);
> + mutex_lock(&codetag_lock);
> + list_for_each_entry(cttype, &codetag_types, link) {
> + if (strcmp(type_name, cttype->desc.section) == 0) {
> + if (!cttype->desc.needs_section_mem)
> + break;
> +
> + down_write(&cttype->mod_lock);
> + ret = cttype->desc.needs_section_mem(mod, size);
> + up_write(&cttype->mod_lock);
> + break;
> + }
> + }
> + mutex_unlock(&codetag_lock);
> +
> + return ret;
> +}
> +
> +void *codetag_alloc_module_section(struct module *mod, const char *name,
> + unsigned long size, unsigned int prepend,
> + unsigned long align)
> +{
> + const char *type_name = name + strlen(CODETAG_SECTION_PREFIX);
> + struct codetag_type *cttype;
> + void *ret = NULL;
> +
> + mutex_lock(&codetag_lock);
> + list_for_each_entry(cttype, &codetag_types, link) {
> + if (strcmp(type_name, cttype->desc.section) == 0) {
> + if (WARN_ON(!cttype->desc.alloc_section_mem))
> + break;
> +
> + down_write(&cttype->mod_lock);
> + ret = cttype->desc.alloc_section_mem(mod, size, prepend, align);
> + up_write(&cttype->mod_lock);
> + break;
> + }
> + }
> + mutex_unlock(&codetag_lock);
> +
> + return ret;
> +}
> +
> +void codetag_free_module_sections(struct module *mod)
> +{
> + struct codetag_type *cttype;
> +
> + mutex_lock(&codetag_lock);
> + list_for_each_entry(cttype, &codetag_types, link) {
> + if (!cttype->desc.free_section_mem)
> + continue;
> +
> + down_write(&cttype->mod_lock);
> + cttype->desc.free_section_mem(mod, false);
> + up_write(&cttype->mod_lock);
> + }
> + mutex_unlock(&codetag_lock);
> +}
> +
> +void codetag_module_replaced(struct module *mod, struct module *new_mod)
> +{
> + struct codetag_type *cttype;
> +
> + mutex_lock(&codetag_lock);
> + list_for_each_entry(cttype, &codetag_types, link) {
> + if (!cttype->desc.module_replaced)
> + continue;
> +
> + down_write(&cttype->mod_lock);
> + cttype->desc.module_replaced(mod, new_mod);
> + up_write(&cttype->mod_lock);
> + }
> + mutex_unlock(&codetag_lock);
> +}
> +
> void codetag_load_module(struct module *mod)
> {
> struct codetag_type *cttype;
> @@ -220,13 +308,12 @@ void codetag_load_module(struct module *mod)
> mutex_unlock(&codetag_lock);
> }
>
> -bool codetag_unload_module(struct module *mod)
> +void codetag_unload_module(struct module *mod)
> {
> struct codetag_type *cttype;
> - bool unload_ok = true;
>
> if (!mod)
> - return true;
> + return;
>
> /* await any module's kfree_rcu() operations to complete */
> kvfree_rcu_barrier();
> @@ -246,18 +333,17 @@ bool codetag_unload_module(struct module *mod)
> }
> if (found) {
> if (cttype->desc.module_unload)
> - if (!cttype->desc.module_unload(cttype, cmod))
> - unload_ok = false;
> + cttype->desc.module_unload(cttype, cmod);
>
> cttype->count -= range_size(cttype, &cmod->range);
> idr_remove(&cttype->mod_idr, mod_id);
> kfree(cmod);
> }
> up_write(&cttype->mod_lock);
> + if (found && cttype->desc.free_section_mem)
> + cttype->desc.free_section_mem(mod, true);
> }
> mutex_unlock(&codetag_lock);
> -
> - return unload_ok;
> }
> #endif /* CONFIG_MODULES */
>
> diff --git a/scripts/module.lds.S b/scripts/module.lds.S
> index 3f43edef813c..711c6e029936 100644
> --- a/scripts/module.lds.S
> +++ b/scripts/module.lds.S
> @@ -50,7 +50,7 @@ SECTIONS {
> .data : {
> *(.data .data.[0-9a-zA-Z_]*)
> *(.data..L*)
> - CODETAG_SECTIONS()
> + MOD_CODETAG_SECTIONS()
> }
>
> .rodata : {
> @@ -59,9 +59,10 @@ SECTIONS {
> }
> #else
> .data : {
> - CODETAG_SECTIONS()
> + MOD_CODETAG_SECTIONS()
> }
> #endif
> + MOD_SEPARATE_CODETAG_SECTIONS()
> }
>
> /* bring in arch-specific sections */
> --
> 2.47.0.105.g07ac214952-goog
>
