On Tue, Feb 25, 2025 at 03:39:24PM -0400, Jason Gunthorpe wrote:
> These are called in a lot of places and are not trivial. Move them to the
> core module.
>
> Tidy some of the comments and function arguments, fold
> __iommu_alloc_account() into its only caller, change
> __iommu_free_account() into __iommu_free_page() to remove some
> duplication.
>
> Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
Reviewed-by: Mostafa Saleh <smostafa@xxxxxxxxxx>
> ---
> drivers/iommu/Makefile | 1 +
> drivers/iommu/iommu-pages.c | 84 +++++++++++++++++++++++++++++
> drivers/iommu/iommu-pages.h | 103 ++----------------------------------
> 3 files changed, 90 insertions(+), 98 deletions(-)
> create mode 100644 drivers/iommu/iommu-pages.c
>
> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
> index 5e5a83c6c2aae2..fe91d770abe16c 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -1,6 +1,7 @@
> # SPDX-License-Identifier: GPL-2.0
> obj-y += amd/ intel/ arm/ iommufd/ riscv/
> obj-$(CONFIG_IOMMU_API) += iommu.o
> +obj-$(CONFIG_IOMMU_SUPPORT) += iommu-pages.o
> obj-$(CONFIG_IOMMU_API) += iommu-traces.o
> obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
> obj-$(CONFIG_IOMMU_DEBUGFS) += iommu-debugfs.o
> diff --git a/drivers/iommu/iommu-pages.c b/drivers/iommu/iommu-pages.c
> new file mode 100644
> index 00000000000000..31ff83ffaf0106
> --- /dev/null
> +++ b/drivers/iommu/iommu-pages.c
> @@ -0,0 +1,84 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (c) 2024, Google LLC.
> + * Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
> + */
> +#include "iommu-pages.h"
> +#include <linux/gfp.h>
> +#include <linux/mm.h>
> +
> +/**
> + * iommu_alloc_pages_node - Allocate a zeroed page of a given order from
> + * specific NUMA node
> + * @nid: memory NUMA node id
> + * @gfp: buddy allocator flags
> + * @order: page order
> + *
> + * Returns the virtual address of the allocated page. The page must be
> + * freed either by calling iommu_free_pages() or via iommu_put_pages_list().
> + */
> +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)
> +{
> + const unsigned long pgcnt = 1UL << order;
> + struct page *page;
> +
> + page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order);
> + if (unlikely(!page))
> + return NULL;
> +
> + /*
> + * All page allocations that should be reported to as "iommu-pagetables"
> + * to userspace must use one of the functions below. This includes
> + * allocations of page-tables and other per-iommu_domain configuration
> + * structures.
> + *
> + * This is necessary for the proper accounting as IOMMU state can be
> + * rather large, i.e. multiple gigabytes in size.
> + */
> + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);
> + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);
> +
> + return page_address(page);
> +}
> +EXPORT_SYMBOL_GPL(iommu_alloc_pages_node);
> +
> +static void __iommu_free_page(struct page *page)
> +{
> + unsigned int order = folio_order(page_folio(page));
> + const unsigned long pgcnt = 1UL << order;
> +
> + mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);
> + mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);
> + put_page(page);
> +}
> +
> +/**
> + * iommu_free_pages - free pages
> + * @virt: virtual address of the page to be freed.
> + *
> + * The page must have have been allocated by iommu_alloc_pages_node()
> + */
> +void iommu_free_pages(void *virt)
> +{
> + if (!virt)
> + return;
> + __iommu_free_page(virt_to_page(virt));
> +}
> +EXPORT_SYMBOL_GPL(iommu_free_pages);
> +
> +/**
> + * iommu_put_pages_list - free a list of pages.
> + * @head: the head of the lru list to be freed.
> + *
> + * Frees a list of pages allocated by iommu_alloc_pages_node().
> + */
> +void iommu_put_pages_list(struct list_head *head)
> +{
> + while (!list_empty(head)) {
> + struct page *p = list_entry(head->prev, struct page, lru);
> +
> + list_del(&p->lru);
> + __iommu_free_page(p);
> + }
> +}
> +EXPORT_SYMBOL_GPL(iommu_put_pages_list);
> diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h
> index fcd17b94f7b830..e3c35aa14ad716 100644
> --- a/drivers/iommu/iommu-pages.h
> +++ b/drivers/iommu/iommu-pages.h
> @@ -7,67 +7,12 @@
> #ifndef __IOMMU_PAGES_H
> #define __IOMMU_PAGES_H
>
> -#include <linux/vmstat.h>
> -#include <linux/gfp.h>
> -#include <linux/mm.h>
> +#include <linux/types.h>
> +#include <linux/topology.h>
>
> -/*
> - * All page allocations that should be reported to as "iommu-pagetables" to
> - * userspace must use one of the functions below. This includes allocations of
> - * page-tables and other per-iommu_domain configuration structures.
> - *
> - * This is necessary for the proper accounting as IOMMU state can be rather
> - * large, i.e. multiple gigabytes in size.
> - */
> -
> -/**
> - * __iommu_alloc_account - account for newly allocated page.
> - * @page: head struct page of the page.
> - * @order: order of the page
> - */
> -static inline void __iommu_alloc_account(struct page *page, int order)
> -{
> - const long pgcnt = 1l << order;
> -
> - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);
> - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);
> -}
> -
> -/**
> - * __iommu_free_account - account a page that is about to be freed.
> - * @page: head struct page of the page.
> - * @order: order of the page
> - */
> -static inline void __iommu_free_account(struct page *page)
> -{
> - unsigned int order = folio_order(page_folio(page));
> - const long pgcnt = 1l << order;
> -
> - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);
> - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);
> -}
> -
> -/**
> - * iommu_alloc_pages_node - allocate a zeroed page of a given order from
> - * specific NUMA node.
> - * @nid: memory NUMA node id
> - * @gfp: buddy allocator flags
> - * @order: page order
> - *
> - * returns the virtual address of the allocated page
> - */
> -static inline void *iommu_alloc_pages_node(int nid, gfp_t gfp, int order)
> -{
> - struct page *page =
> - alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order);
> -
> - if (unlikely(!page))
> - return NULL;
> -
> - __iommu_alloc_account(page, order);
> -
> - return page_address(page);
> -}
> +void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order);
> +void iommu_free_pages(void *virt);
> +void iommu_put_pages_list(struct list_head *head);
>
> /**
> * iommu_alloc_pages - allocate a zeroed page of a given order
> @@ -104,42 +49,4 @@ static inline void *iommu_alloc_page(gfp_t gfp)
> return iommu_alloc_pages_node(numa_node_id(), gfp, 0);
> }
>
> -/**
> - * iommu_free_pages - free pages
> - * @virt: virtual address of the page to be freed.
> - *
> - * The page must have have been allocated by iommu_alloc_pages_node()
> - */
> -static inline void iommu_free_pages(void *virt)
> -{
> - struct page *page;
> -
> - if (!virt)
> - return;
> -
> - page = virt_to_page(virt);
> - __iommu_free_account(page);
> - put_page(page);
> -}
> -
> -/**
> - * iommu_put_pages_list - free a list of pages.
> - * @page: the head of the lru list to be freed.
> - *
> - * There are no locking requirement for these pages, as they are going to be
> - * put on a free list as soon as refcount reaches 0. Pages are put on this LRU
> - * list once they are removed from the IOMMU page tables. However, they can
> - * still be access through debugfs.
> - */
> -static inline void iommu_put_pages_list(struct list_head *page)
> -{
> - while (!list_empty(page)) {
> - struct page *p = list_entry(page->prev, struct page, lru);
> -
> - list_del(&p->lru);
> - __iommu_free_account(p);
> - put_page(p);
> - }
> -}
> -
> #endif /* __IOMMU_PAGES_H */
> --
> 2.43.0
>
