For now, zswap_store_pages() will sequentially compress these pages with
zswap_compress().
These changes are follow-up to code review comments received for [1], and
are intended to set up zswap_store() for batching with Intel IAA.
[1]: https://patchwork.kernel.org/project/linux-mm/patch/20241123070127.332773-11-kanchana.p.sridhar@xxxxxxxxx/
Signed-off-by: Kanchana P Sridhar <kanchana.p.sridhar@xxxxxxxxx>
---
include/linux/zswap.h | 1 +
mm/zswap.c | 154 ++++++++++++++++++++++++------------------
2 files changed, 88 insertions(+), 67 deletions(-)
diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index d961ead91bf1..05a81e750744 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -7,6 +7,7 @@
struct lruvec;
+#define SWAP_CRYPTO_BATCH_SIZE 8UL
extern atomic_long_t zswap_stored_pages;
#ifdef CONFIG_ZSWAP
diff --git a/mm/zswap.c b/mm/zswap.c
index f6316b66fb23..b09d1023e775 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1409,78 +1409,96 @@ static void shrink_worker(struct work_struct *w)
* main API
**********************************/
-static ssize_t zswap_store_page(struct page *page,
- struct obj_cgroup *objcg,
- struct zswap_pool *pool)
+/*
+ * Store multiple pages in @folio, starting from the page at index @si up to
+ * and including the page at index @ei.
+ */
+static ssize_t zswap_store_pages(struct folio *folio,
+ long si,
+ long ei,
+ struct obj_cgroup *objcg,
+ struct zswap_pool *pool)
{
- swp_entry_t page_swpentry = page_swap_entry(page);
+ struct page *page;
+ swp_entry_t page_swpentry;
struct zswap_entry *entry, *old;
+ size_t compressed_bytes = 0;
+ u8 nr_pages = ei - si + 1;
+ u8 i;
+
+ for (i = 0; i < nr_pages; ++i) {
+ page = folio_page(folio, si + i);
+ page_swpentry = page_swap_entry(page);
+
+ /* allocate entry */
+ entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
+ if (!entry) {
+ zswap_reject_kmemcache_fail++;
+ return -EINVAL;
+ }
- /* allocate entry */
- entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
- if (!entry) {
- zswap_reject_kmemcache_fail++;
- return -EINVAL;
- }
-
- if (!zswap_compress(page, entry, pool))
- goto compress_failed;
+ if (!zswap_compress(page, entry, pool))
+ goto compress_failed;
- old = xa_store(swap_zswap_tree(page_swpentry),
- swp_offset(page_swpentry),
- entry, GFP_KERNEL);
- if (xa_is_err(old)) {
- int err = xa_err(old);
+ old = xa_store(swap_zswap_tree(page_swpentry),
+ swp_offset(page_swpentry),
+ entry, GFP_KERNEL);
+ if (xa_is_err(old)) {
+ int err = xa_err(old);
- WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
- zswap_reject_alloc_fail++;
- goto store_failed;
- }
+ WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
+ zswap_reject_alloc_fail++;
+ goto store_failed;
+ }
- /*
- * We may have had an existing entry that became stale when
- * the folio was redirtied and now the new version is being
- * swapped out. Get rid of the old.
- */
- if (old)
- zswap_entry_free(old);
+ /*
+ * We may have had an existing entry that became stale when
+ * the folio was redirtied and now the new version is being
+ * swapped out. Get rid of the old.
+ */
+ if (old)
+ zswap_entry_free(old);
- /*
- * The entry is successfully compressed and stored in the tree, there is
- * no further possibility of failure. Grab refs to the pool and objcg.
- * These refs will be dropped by zswap_entry_free() when the entry is
- * removed from the tree.
- */
- zswap_pool_get(pool);
- if (objcg)
- obj_cgroup_get(objcg);
+ /*
+ * The entry is successfully compressed and stored in the tree, there is
+ * no further possibility of failure. Grab refs to the pool and objcg.
+ * These refs will be dropped by zswap_entry_free() when the entry is
+ * removed from the tree.
+ */
+ zswap_pool_get(pool);
+ if (objcg)
+ obj_cgroup_get(objcg);
- /*
- * We finish initializing the entry while it's already in xarray.
- * This is safe because:
- *
- * 1. Concurrent stores and invalidations are excluded by folio lock.
- *
- * 2. Writeback is excluded by the entry not being on the LRU yet.
- * The publishing order matters to prevent writeback from seeing
- * an incoherent entry.
- */
- entry->pool = pool;
- entry->swpentry = page_swpentry;
- entry->objcg = objcg;
- entry->referenced = true;
- if (entry->length) {
- INIT_LIST_HEAD(&entry->lru);
- zswap_lru_add(&zswap_list_lru, entry);
- }
+ /*
+ * We finish initializing the entry while it's already in xarray.
+ * This is safe because:
+ *
+ * 1. Concurrent stores and invalidations are excluded by folio lock.
+ *
+ * 2. Writeback is excluded by the entry not being on the LRU yet.
+ * The publishing order matters to prevent writeback from seeing
+ * an incoherent entry.
+ */
+ entry->pool = pool;
+ entry->swpentry = page_swpentry;
+ entry->objcg = objcg;
+ entry->referenced = true;
+ if (entry->length) {
+ INIT_LIST_HEAD(&entry->lru);
+ zswap_lru_add(&zswap_list_lru, entry);
+ }
- return entry->length;
+ compressed_bytes += entry->length;
+ continue;
store_failed:
- zpool_free(pool->zpool, entry->handle);
+ zpool_free(pool->zpool, entry->handle);
compress_failed:
- zswap_entry_cache_free(entry);
- return -EINVAL;
+ zswap_entry_cache_free(entry);
+ return -EINVAL;
+ }
+
+ return compressed_bytes;
}
bool zswap_store(struct folio *folio)
@@ -1492,7 +1510,7 @@ bool zswap_store(struct folio *folio)
struct zswap_pool *pool;
size_t compressed_bytes = 0;
bool ret = false;
- long index;
+ long si, ei, incr = SWAP_CRYPTO_BATCH_SIZE;
VM_WARN_ON_ONCE(!folio_test_locked(folio));
VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
@@ -1526,11 +1544,13 @@ bool zswap_store(struct folio *folio)
mem_cgroup_put(memcg);
}
- for (index = 0; index < nr_pages; ++index) {
- struct page *page = folio_page(folio, index);
+ /* Store the folio in batches of SWAP_CRYPTO_BATCH_SIZE pages. */
+ for (si = 0, ei = min(si + incr - 1, nr_pages - 1);
+ ((si < nr_pages) && (ei < nr_pages));
+ si = ei + 1, ei = min(si + incr - 1, nr_pages - 1)) {
ssize_t bytes;
- bytes = zswap_store_page(page, objcg, pool);
+ bytes = zswap_store_pages(folio, si, ei, objcg, pool);
if (bytes < 0)
goto put_pool;
compressed_bytes += bytes;
@@ -1565,9 +1585,9 @@ bool zswap_store(struct folio *folio)
struct zswap_entry *entry;
struct xarray *tree;
- for (index = 0; index < nr_pages; ++index) {
- tree = swap_zswap_tree(swp_entry(type, offset + index));
- entry = xa_erase(tree, offset + index);
+ for (si = 0; si < nr_pages; ++si) {
+ tree = swap_zswap_tree(swp_entry(type, offset + si));
+ entry = xa_erase(tree, offset + si);
if (entry)
zswap_entry_free(entry);
}
