From: Kairui Song <kasong@xxxxxxxxxxx>
Currently, shmem uses cluster readahead for all swap backends. Cluster
readahead is not a good solution for ramdisk based device (ZRAM) at all.
After switching to the new helper, most benchmarks showed a good result:
- Single file sequence read:
perf stat --repeat 20 dd if=/tmpfs/test of=/dev/null bs=1M count=8192
(/tmpfs/test is a zero filled file, using brd as swap, 4G memcg limit)
Before: 22.248 +- 0.549
After: 22.021 +- 0.684 (-1.1%)
- Random read stress test:
fio -name=tmpfs --numjobs=16 --directory=/tmpfs \
--size=256m --ioengine=mmap --rw=randread --random_distribution=random \
--time_based --ramp_time=1m --runtime=5m --group_reporting
(using brd as swap, 2G memcg limit)
Before: 1818MiB/s
After: 1888MiB/s (+3.85%)
- Zipf biased random read stress test:
fio -name=tmpfs --numjobs=16 --directory=/tmpfs \
--size=256m --ioengine=mmap --rw=randread --random_distribution=zipf:1.2 \
--time_based --ramp_time=1m --runtime=5m --group_reporting
(using brd as swap, 2G memcg limit)
Before: 31.1GiB/s
After: 32.3GiB/s (+3.86%)
So cluster readahead doesn't help much even for single sequence read,
and for random stress test, the performance is better without it.
Considering both memory and swap device will get more fragmented
slowly, and commonly used ZRAM consumes much more CPU than plain
ramdisk, false readahead could occur more frequently and waste
more CPU. Direct SWAP is cheaper, so use the new helper and skip
read ahead for SWP_SYNCHRONOUS_IO device.
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
mm/shmem.c | 67 +++++++++++++++++++++++++------------------------
mm/swap.h | 9 -------
mm/swap_state.c | 11 ++++++--
3 files changed, 43 insertions(+), 44 deletions(-)
diff --git a/mm/shmem.c b/mm/shmem.c
index 9da9f7a0e620..3c0729fe934d 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1564,20 +1564,6 @@ static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info,
pgoff_t index, unsigned int order, pgoff_t *ilx);
-static struct folio *shmem_swapin_cluster(swp_entry_t swap, gfp_t gfp,
- struct shmem_inode_info *info, pgoff_t index)
-{
- struct mempolicy *mpol;
- pgoff_t ilx;
- struct folio *folio;
-
- mpol = shmem_get_pgoff_policy(info, index, 0, &ilx);
- folio = swap_cluster_readahead(swap, gfp, mpol, ilx);
- mpol_cond_put(mpol);
-
- return folio;
-}
-
/*
* Make sure huge_gfp is always more limited than limit_gfp.
* Some of the flags set permissions, while others set limitations.
@@ -1851,9 +1837,12 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
{
struct address_space *mapping = inode->i_mapping;
struct shmem_inode_info *info = SHMEM_I(inode);
+ enum swap_cache_result cache_result;
struct swap_info_struct *si;
struct folio *folio = NULL;
+ struct mempolicy *mpol;
swp_entry_t swap;
+ pgoff_t ilx;
int error;
VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
@@ -1871,36 +1860,40 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
return -EINVAL;
}
- /* Look it up and read it in.. */
- folio = swap_cache_get_folio(swap, NULL, 0, NULL);
+ mpol = shmem_get_pgoff_policy(info, index, 0, &ilx);
+ folio = swapin_entry_mpol(swap, gfp, mpol, ilx, &cache_result);
+ mpol_cond_put(mpol);
+
if (!folio) {
- /* Or update major stats only when swapin succeeds?? */
+ error = -ENOMEM;
+ goto failed;
+ }
+ if (cache_result != SWAP_CACHE_HIT) {
if (fault_type) {
*fault_type |= VM_FAULT_MAJOR;
count_vm_event(PGMAJFAULT);
count_memcg_event_mm(fault_mm, PGMAJFAULT);
}
- /* Here we actually start the io */
- folio = shmem_swapin_cluster(swap, gfp, info, index);
- if (!folio) {
- error = -ENOMEM;
- goto failed;
- }
}
/* We have to do this with folio locked to prevent races */
folio_lock(folio);
- if (!folio_test_swapcache(folio) ||
- folio->swap.val != swap.val ||
- !shmem_confirm_swap(mapping, index, swap)) {
+ if (cache_result != SWAP_CACHE_BYPASS) {
+ /* With cache bypass, folio is new allocated, sync, and not in cache */
+ if (!folio_test_swapcache(folio) || folio->swap.val != swap.val) {
+ error = -EEXIST;
+ goto unlock;
+ }
+ if (!folio_test_uptodate(folio)) {
+ error = -EIO;
+ goto failed;
+ }
+ folio_wait_writeback(folio);
+ }
+ if (!shmem_confirm_swap(mapping, index, swap)) {
error = -EEXIST;
goto unlock;
}
- if (!folio_test_uptodate(folio)) {
- error = -EIO;
- goto failed;
- }
- folio_wait_writeback(folio);
/*
* Some architectures may have to restore extra metadata to the
@@ -1908,12 +1901,19 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
*/
arch_swap_restore(swap, folio);
- if (shmem_should_replace_folio(folio, gfp)) {
+ /* With cache bypass, folio is new allocated and always respect gfp flags */
+ if (cache_result != SWAP_CACHE_BYPASS && shmem_should_replace_folio(folio, gfp)) {
error = shmem_replace_folio(&folio, gfp, info, index);
if (error)
goto failed;
}
+ /*
+ * The expected value checking below should be enough to ensure
+ * only one up-to-date swapin success. swap_free() is called after
+ * this, so the entry can't be reused. As long as the mapping still
+ * has the old entry value, it's never swapped in or modified.
+ */
error = shmem_add_to_page_cache(folio, mapping, index,
swp_to_radix_entry(swap), gfp);
if (error)
@@ -1924,7 +1924,8 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
if (sgp == SGP_WRITE)
folio_mark_accessed(folio);
- delete_from_swap_cache(folio);
+ if (cache_result != SWAP_CACHE_BYPASS)
+ delete_from_swap_cache(folio);
folio_mark_dirty(folio);
swap_free(swap);
put_swap_device(si);
diff --git a/mm/swap.h b/mm/swap.h
index 8f790a67b948..20f4048c971c 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -57,9 +57,6 @@ void __delete_from_swap_cache(struct folio *folio,
void delete_from_swap_cache(struct folio *folio);
void clear_shadow_from_swap_cache(int type, unsigned long begin,
unsigned long end);
-struct folio *swap_cache_get_folio(swp_entry_t entry,
- struct vm_area_struct *vma, unsigned long addr,
- void **shadowp);
struct folio *filemap_get_incore_folio(struct address_space *mapping,
pgoff_t index);
@@ -123,12 +120,6 @@ static inline int swap_writepage(struct page *p, struct writeback_control *wbc)
return 0;
}
-static inline struct folio *swap_cache_get_folio(swp_entry_t entry,
- struct vm_area_struct *vma, unsigned long addr)
-{
- return NULL;
-}
-
static inline
struct folio *filemap_get_incore_folio(struct address_space *mapping,
pgoff_t index)
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 3edf4b63158d..10eec68475dd 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -318,7 +318,14 @@ void free_pages_and_swap_cache(struct encoded_page **pages, int nr)
static inline bool swap_use_no_readahead(struct swap_info_struct *si, swp_entry_t entry)
{
- return data_race(si->flags & SWP_SYNCHRONOUS_IO) && __swap_count(entry) == 1;
+ int count;
+
+ if (!data_race(si->flags & SWP_SYNCHRONOUS_IO))
+ return false;
+
+ count = __swap_count(entry);
+
+ return (count == 1 || count == SWAP_MAP_SHMEM);
}
static inline bool swap_use_vma_readahead(void)
@@ -334,7 +341,7 @@ static inline bool swap_use_vma_readahead(void)
*
* Caller must lock the swap device or hold a reference to keep it valid.
*/
-struct folio *swap_cache_get_folio(swp_entry_t entry,
+static struct folio *swap_cache_get_folio(swp_entry_t entry,
struct vm_area_struct *vma, unsigned long addr, void **shadowp)
{
struct folio *folio;
--
2.43.0
