Currently, udmabuf handles folio by create an unpin list to record
each folio obtained from the list and unpinning them when released. To
maintain this, many struct have been established.
However, maintain this requires a significant amount of memory and
iter the list is a substantial overhead, which is not friendly to the
CPU cache.
When create, we arranged the folio array in the order of pin and set
the offset according to pgcnt. So, if record each pinned folio when
create, then can easy unpin it. Compare to use list to record it,
an array also can do this.
Hence, this patch setup a pinned_folios array(size is the pgcnt) to
instead of udmabuf_folio struct, it record each folio which pinned when
invoke memfd_pin_folios, then unpin folio by iter pinned_folios.
Note that, since a folio may be pinned multiple times, each folio can be
added to pinned_folios multiple times, depend on how many times the
folio has been pinned when create.
Compare to udmabuf_folio(24 byte size), a folio pointer is 8 byte, if no
large folio - each folio is PAGE_SIZE - and need to unpin when release.
So need to record each folio, by this patch, each folio can save 16 byte.
But if large folio used, depend on the large folio's number, the
pinned_folios array may take more memory, but it still can makes unpin
access more cache-friendly.
Signed-off-by: Huan Yang <link@xxxxxxxx>
Acked-by: Vivek Kasireddy <vivek.kasireddy@xxxxxxxxx>
---
drivers/dma-buf/udmabuf.c | 65 +++++++++++++++++----------------------
1 file changed, 29 insertions(+), 36 deletions(-)
diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
index 40ebff2c77f4..24badfefa6b4 100644
--- a/drivers/dma-buf/udmabuf.c
+++ b/drivers/dma-buf/udmabuf.c
@@ -27,15 +27,21 @@ MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is
struct udmabuf {
pgoff_t pagecount;
struct folio **folios;
+
+ /**
+ * Unlike folios, pinned_folios is only used for unpin.
+ * So, nr_pinned is not the same to pagecount, the pinned_folios
+ * only set each folio which already pinned when udmabuf_create.
+ * Note that, since a folio may be pinned multiple times, each folio
+ * can be added to pinned_folios multiple times, depending on how many
+ * times the folio has been pinned when create.
+ */
+ pgoff_t nr_pinned;
+ struct folio **pinned_folios;
+
struct sg_table *sg;
struct miscdevice *device;
pgoff_t *offsets;
- struct list_head unpin_list;
-};
-
-struct udmabuf_folio {
- struct folio *folio;
- struct list_head list;
};
static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
@@ -196,38 +202,18 @@ static void unmap_udmabuf(struct dma_buf_attachment *at,
return put_sg_table(at->dev, sg, direction);
}
-static void unpin_all_folios(struct list_head *unpin_list)
+static void unpin_all_folios(struct udmabuf *ubuf)
{
- struct udmabuf_folio *ubuf_folio;
+ pgoff_t i;
- while (!list_empty(unpin_list)) {
- ubuf_folio = list_first_entry(unpin_list,
- struct udmabuf_folio, list);
- unpin_folio(ubuf_folio->folio);
+ for (i = 0; i < ubuf->nr_pinned; ++i)
+ unpin_folio(ubuf->pinned_folios[i]);
- list_del(&ubuf_folio->list);
- kfree(ubuf_folio);
- }
-}
-
-static int add_to_unpin_list(struct list_head *unpin_list,
- struct folio *folio)
-{
- struct udmabuf_folio *ubuf_folio;
-
- ubuf_folio = kzalloc(sizeof(*ubuf_folio), GFP_KERNEL);
- if (!ubuf_folio)
- return -ENOMEM;
-
- ubuf_folio->folio = folio;
- list_add_tail(&ubuf_folio->list, unpin_list);
- return 0;
+ kvfree(ubuf->pinned_folios);
}
static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt)
{
- INIT_LIST_HEAD(&ubuf->unpin_list);
-
ubuf->folios = kvmalloc_array(pgcnt, sizeof(*ubuf->folios), GFP_KERNEL);
if (!ubuf->folios)
return -ENOMEM;
@@ -236,12 +222,18 @@ static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt)
if (!ubuf->offsets)
return -ENOMEM;
+ ubuf->pinned_folios = kvmalloc_array(pgcnt,
+ sizeof(*ubuf->pinned_folios),
+ GFP_KERNEL);
+ if (!ubuf->pinned_folios)
+ return -ENOMEM;
+
return 0;
}
static __always_inline void deinit_udmabuf(struct udmabuf *ubuf)
{
- unpin_all_folios(&ubuf->unpin_list);
+ unpin_all_folios(ubuf);
kvfree(ubuf->offsets);
kvfree(ubuf->folios);
}
@@ -348,9 +340,11 @@ static int export_udmabuf(struct udmabuf *ubuf,
static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd,
loff_t start, loff_t size)
{
- pgoff_t pgoff, pgcnt, upgcnt = ubuf->pagecount;
+ pgoff_t nr_pinned = ubuf->nr_pinned;
+ pgoff_t upgcnt = ubuf->pagecount;
struct folio **folios = NULL;
u32 cur_folio, cur_pgcnt;
+ pgoff_t pgoff, pgcnt;
long nr_folios;
long ret = 0;
loff_t end;
@@ -372,9 +366,7 @@ static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd,
pgoff_t subpgoff = pgoff;
size_t fsize = folio_size(folios[cur_folio]);
- ret = add_to_unpin_list(&ubuf->unpin_list, folios[cur_folio]);
- if (ret < 0)
- goto end;
+ ubuf->pinned_folios[nr_pinned++] = folios[cur_folio];
for (; subpgoff < fsize; subpgoff += PAGE_SIZE) {
ubuf->folios[upgcnt] = folios[cur_folio];
@@ -395,6 +387,7 @@ static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd,
}
end:
ubuf->pagecount = upgcnt;
+ ubuf->nr_pinned = nr_pinned;
kvfree(folios);
return ret;
}
--
2.45.2
