On Mon, Aug 06, 2018 at 04:52:24PM +1000, Rashmica Gupta wrote:
> When hot-removing memory release_mem_region_adjustable() splits
> iomem resources if they are not the exact size of the memory being
> hot-deleted. Adding this memory back to the kernel adds a new
> resource.
>
> Eg a node has memory 0x0 - 0xfffffffff. Offlining and hot-removing
> 1GB from 0xf40000000 results in the single resource 0x0-0xfffffffff being
> split into two resources: 0x0-0xf3fffffff and 0xf80000000-0xfffffffff.
>
> When we hot-add the memory back we now have three resources:
> 0x0-0xf3fffffff, 0xf40000000-0xf7fffffff, and 0xf80000000-0xfffffffff.
>
> Now if we try to remove a section of memory that overlaps these resources,
> like 2GB from 0xf40000000, release_mem_region_adjustable() fails as it
> expects the chunk of memory to be within the boundaries of a single
> resource.
>
> This patch adds a function request_resource_and_merge(). This is called
> instead of request_resource_conflict() when registering a resource in
> add_memory(). It calls request_resource_conflict() and if hot-removing is
> enabled (if it isn't we won't get resource fragmentation) we attempt to
> merge contiguous resources on the node.
>
> Signed-off-by: Rashmica Gupta <rashmica.g@xxxxxxxxx>
> ---
> v1->v2: Only attempt to merge resources if hot-remove is enabled.
>
> include/linux/ioport.h | 2 +
> include/linux/memory_hotplug.h | 2 +-
> kernel/resource.c | 116 +++++++++++++++++++++++++++++++++++++++++
> mm/memory_hotplug.c | 22 ++++----
> 4 files changed, 130 insertions(+), 12 deletions(-)
>
> diff --git a/include/linux/ioport.h b/include/linux/ioport.h
> index da0ebaec25f0..f5b93a711e86 100644
> --- a/include/linux/ioport.h
> +++ b/include/linux/ioport.h
> @@ -189,6 +189,8 @@ extern int allocate_resource(struct resource *root, struct resource *new,
> resource_size_t,
> resource_size_t),
> void *alignf_data);
> +extern struct resource *request_resource_and_merge(struct resource *parent,
> + struct resource *new, int nid);
> struct resource *lookup_resource(struct resource *root, resource_size_t start);
> int adjust_resource(struct resource *res, resource_size_t start,
> resource_size_t size);
> diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
> index 4e9828cda7a2..9c00f97c8cc6 100644
> --- a/include/linux/memory_hotplug.h
> +++ b/include/linux/memory_hotplug.h
> @@ -322,7 +322,7 @@ static inline void remove_memory(int nid, u64 start, u64 size) {}
> extern int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
> void *arg, int (*func)(struct memory_block *, void *));
> extern int add_memory(int nid, u64 start, u64 size);
> -extern int add_memory_resource(int nid, struct resource *resource, bool online);
> +extern int add_memory_resource(int nid, u64 start, u64 size, bool online);
The add_memory_resource() function is also used by Xen balloon. It should
be updated as well.
> extern int arch_add_memory(int nid, u64 start, u64 size,
> struct vmem_altmap *altmap, bool want_memblock);
> extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
> diff --git a/kernel/resource.c b/kernel/resource.c
> index 30e1bc68503b..5967061f9cea 100644
> --- a/kernel/resource.c
> +++ b/kernel/resource.c
> @@ -1621,3 +1621,119 @@ static int __init strict_iomem(char *str)
> }
>
> __setup("iomem=", strict_iomem);
> +
> +#ifdef CONFIG_MEMORY_HOTPLUG
> +#ifdef CONFIG_MEMORY_HOTREMOVE
> +/*
> + * Attempt to merge resource and it's sibling
> + */
> +static int merge_resources(struct resource *res)
> +{
> + struct resource *next;
> + struct resource *tmp;
> + uint64_t size;
> + int ret = -EINVAL;
> +
> + next = res->sibling;
> +
> + /*
> + * Not sure how to handle two different children. So only attempt
> + * to merge two resources if neither have children, only one has a
> + * child or if both have the same child.
> + */
> + if ((res->child && next->child) && (res->child != next->child))
> + return ret;
> +
> + if (res->end + 1 != next->start)
> + return ret;
> +
> + if (res->flags != next->flags)
> + return ret;
> +
> + /* Update sibling and child of resource */
> + res->sibling = next->sibling;
> + tmp = res->child;
> + if (!res->child)
> + res->child = next->child;
> +
> + size = next->end - res->start + 1;
> + ret = __adjust_resource(res, res->start, size);
> + if (ret) {
> + /* Failed so restore resource to original state */
> + res->sibling = next;
> + res->child = tmp;
> + return ret;
> + }
> +
> + free_resource(next);
> +
> + return ret;
> +}
> +
> +/*
> + * Attempt to merge resources on the node
> + */
> +static void merge_node_resources(int nid, struct resource *parent)
> +{
> + struct resource *res;
> + uint64_t start_addr;
> + uint64_t end_addr;
> + int ret;
> +
> + start_addr = node_start_pfn(nid) << PAGE_SHIFT;
> + end_addr = node_end_pfn(nid) << PAGE_SHIFT;
> +
> + write_lock(&resource_lock);
> +
> + /* Get the first resource */
> + res = parent->child;
> +
> + while (res) {
> + /* Check that the resource is within the node */
> + if (res->start < start_addr) {
> + res = res->sibling;
> + continue;
> + }
> + /* Exit if resource is past end of node */
> + if (res->sibling->end > end_addr)
> + break;
> +
> + ret = merge_resources(res);
> + if (!ret)
> + continue;
> + res = res->sibling;
> + }
> + write_unlock(&resource_lock);
> +}
> +#endif /* CONFIG_MEMORY_HOTREMOVE */
> +
> +/**
> + * request_resource_and_merge() - request an I/O or memory resource for hot-add
> + * @parent: parent resource descriptor
> + * @new: resource descriptor desired by caller
> + * @nid: node id of the node we want the resource on
> + *
> + * Returns NULL for success and conflict resource on error.
> + * If no conflict resource then attempt to merge resources on the node.
Please put the return value description at the end and start it with
'Return:' rather than 'Returns'
> + *
> + * This is intended to cleanup the fragmentation of resources that occurs when
> + * hot-removing memory (see release_mem_region_adjustable). If hot-removing is
> + * not enabled then there is no point trying to merge resources.
I think it's worth noting that inability to merge the resources is not an
error.
> + */
> +struct resource *request_resource_and_merge(struct resource *parent,
> + struct resource *new, int nid)
> +{
> + struct resource *conflict;
> +
> + conflict = request_resource_conflict(parent, new);
> +
> + if (conflict)
> + return conflict;
> +
> +#ifdef CONFIG_MEMORY_HOTREMOVE
> + merge_node_resources(nid, parent);
> +#endif /* CONFIG_MEMORY_HOTREMOVE */
> +
> + return NULL;
> +}
> +#endif /* CONFIG_MEMORY_HOTPLUG */
> diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
> index 7deb49f69e27..2e342f5ce322 100644
> --- a/mm/memory_hotplug.c
> +++ b/mm/memory_hotplug.c
> @@ -97,7 +97,7 @@ void mem_hotplug_done(void)
> }
>
> /* add this memory to iomem resource */
> -static struct resource *register_memory_resource(u64 start, u64 size)
> +static struct resource *register_memory_resource(int nid, u64 start, u64 size)
> {
> struct resource *res, *conflict;
> res = kzalloc(sizeof(struct resource), GFP_KERNEL);
> @@ -108,7 +108,7 @@ static struct resource *register_memory_resource(u64 start, u64 size)
> res->start = start;
> res->end = start + size - 1;
> res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
> - conflict = request_resource_conflict(&iomem_resource, res);
> + conflict = request_resource_and_merge(&iomem_resource, res, nid);
> if (conflict) {
> if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) {
> pr_debug("Device unaddressable memory block "
> @@ -122,11 +122,15 @@ static struct resource *register_memory_resource(u64 start, u64 size)
> return res;
> }
>
> -static void release_memory_resource(struct resource *res)
> +static void release_memory_resource(struct resource *res, u64 start, u64 size)
> {
> if (!res)
> return;
> +#ifdef CONFIG_MEMORY_HOTREMOVE
> + release_mem_region_adjustable(&iomem_resource, start, size);
> +#else
> release_resource(res);
> +#endif
> kfree(res);
> return;
> }
> @@ -1096,17 +1100,13 @@ static int online_memory_block(struct memory_block *mem, void *arg)
> }
>
> /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
> -int __ref add_memory_resource(int nid, struct resource *res, bool online)
> +int __ref add_memory_resource(int nid, u64 start, u64 size, bool online)
> {
> - u64 start, size;
> pg_data_t *pgdat = NULL;
> bool new_pgdat;
> bool new_node;
> int ret;
>
> - start = res->start;
> - size = resource_size(res);
> -
> ret = check_hotplug_memory_range(start, size);
> if (ret)
> return ret;
> @@ -1195,13 +1195,13 @@ int __ref add_memory(int nid, u64 start, u64 size)
> struct resource *res;
> int ret;
>
> - res = register_memory_resource(start, size);
> + res = register_memory_resource(nid, start, size);
> if (IS_ERR(res))
> return PTR_ERR(res);
>
> - ret = add_memory_resource(nid, res, memhp_auto_online);
> + ret = add_memory_resource(nid, start, size, memhp_auto_online);
> if (ret < 0)
> - release_memory_resource(res);
> + release_memory_resource(res, start, size);
> return ret;
> }
> EXPORT_SYMBOL_GPL(add_memory);
> --
> 2.14.4
>
--
Sincerely yours,
Mike.
