On Thu, Sep 5, 2024 at 11:44 AM Barry Song <21cnbao@xxxxxxxxx> wrote: > > On Wed, Sep 4, 2024 at 11:14 PM Usama Arif <usamaarif642@xxxxxxxxx> wrote: > > > > > > > > On 04/09/2024 06:55, Barry Song wrote: > > > On Thu, Jun 13, 2024 at 12:48 AM Usama Arif <usamaarif642@xxxxxxxxx> wrote: > > >> > > >> Approximately 10-20% of pages to be swapped out are zero pages [1]. > > >> Rather than reading/writing these pages to flash resulting > > >> in increased I/O and flash wear, a bitmap can be used to mark these > > >> pages as zero at write time, and the pages can be filled at > > >> read time if the bit corresponding to the page is set. > > >> With this patch, NVMe writes in Meta server fleet decreased > > >> by almost 10% with conventional swap setup (zswap disabled). > > >> > > >> [1] https://lore.kernel.org/all/20171018104832epcms5p1b2232e2236258de3d03d1344dde9fce0@epcms5p1/ > > >> > > >> Signed-off-by: Usama Arif <usamaarif642@xxxxxxxxx> > > >> --- > > >> include/linux/swap.h | 1 + > > >> mm/page_io.c | 114 ++++++++++++++++++++++++++++++++++++++++++- > > >> mm/swapfile.c | 24 ++++++++- > > >> 3 files changed, 136 insertions(+), 3 deletions(-) > > >> > > >> diff --git a/include/linux/swap.h b/include/linux/swap.h > > >> index a11c75e897ec..e88563978441 100644 > > >> --- a/include/linux/swap.h > > >> +++ b/include/linux/swap.h > > >> @@ -299,6 +299,7 @@ struct swap_info_struct { > > >> signed char type; /* strange name for an index */ > > >> unsigned int max; /* extent of the swap_map */ > > >> unsigned char *swap_map; /* vmalloc'ed array of usage counts */ > > >> + unsigned long *zeromap; /* vmalloc'ed bitmap to track zero pages */ > > >> struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ > > >> struct swap_cluster_list free_clusters; /* free clusters list */ > > >> unsigned int lowest_bit; /* index of first free in swap_map */ > > >> diff --git a/mm/page_io.c b/mm/page_io.c > > >> index a360857cf75d..39fc3919ce15 100644 > > >> --- a/mm/page_io.c > > >> +++ b/mm/page_io.c > > >> @@ -172,6 +172,88 @@ int generic_swapfile_activate(struct swap_info_struct *sis, > > >> goto out; > > >> } > > >> > > >> +static bool is_folio_page_zero_filled(struct folio *folio, int i) > > >> +{ > > >> + unsigned long *data; > > >> + unsigned int pos, last_pos = PAGE_SIZE / sizeof(*data) - 1; > > >> + bool ret = false; > > >> + > > >> + data = kmap_local_folio(folio, i * PAGE_SIZE); > > >> + if (data[last_pos]) > > >> + goto out; > > >> + for (pos = 0; pos < PAGE_SIZE / sizeof(*data); pos++) { > > >> + if (data[pos]) > > >> + goto out; > > >> + } > > >> + ret = true; > > >> +out: > > >> + kunmap_local(data); > > >> + return ret; > > >> +} > > >> + > > >> +static bool is_folio_zero_filled(struct folio *folio) > > >> +{ > > >> + unsigned int i; > > >> + > > >> + for (i = 0; i < folio_nr_pages(folio); i++) { > > >> + if (!is_folio_page_zero_filled(folio, i)) > > >> + return false; > > >> + } > > >> + return true; > > >> +} > > >> + > > >> +static void folio_zero_fill(struct folio *folio) > > >> +{ > > >> + unsigned int i; > > >> + > > >> + for (i = 0; i < folio_nr_pages(folio); i++) > > >> + clear_highpage(folio_page(folio, i)); > > >> +} > > >> + > > >> +static void swap_zeromap_folio_set(struct folio *folio) > > >> +{ > > >> + struct swap_info_struct *sis = swp_swap_info(folio->swap); > > >> + swp_entry_t entry; > > >> + unsigned int i; > > >> + > > >> + for (i = 0; i < folio_nr_pages(folio); i++) { > > >> + entry = page_swap_entry(folio_page(folio, i)); > > >> + set_bit(swp_offset(entry), sis->zeromap); > > >> + } > > >> +} > > >> + > > >> +static void swap_zeromap_folio_clear(struct folio *folio) > > >> +{ > > >> + struct swap_info_struct *sis = swp_swap_info(folio->swap); > > >> + swp_entry_t entry; > > >> + unsigned int i; > > >> + > > >> + for (i = 0; i < folio_nr_pages(folio); i++) { > > >> + entry = page_swap_entry(folio_page(folio, i)); > > >> + clear_bit(swp_offset(entry), sis->zeromap); > > >> + } > > >> +} > > >> + > > >> +/* > > >> + * Return the index of the first subpage which is not zero-filled > > >> + * according to swap_info_struct->zeromap. > > >> + * If all pages are zero-filled according to zeromap, it will return > > >> + * folio_nr_pages(folio). > > >> + */ > > >> +static unsigned int swap_zeromap_folio_test(struct folio *folio) > > >> +{ > > >> + struct swap_info_struct *sis = swp_swap_info(folio->swap); > > >> + swp_entry_t entry; > > >> + unsigned int i; > > >> + > > >> + for (i = 0; i < folio_nr_pages(folio); i++) { > > >> + entry = page_swap_entry(folio_page(folio, i)); > > >> + if (!test_bit(swp_offset(entry), sis->zeromap)) > > >> + return i; > > >> + } > > >> + return i; > > >> +} > > >> + > > >> /* > > >> * We may have stale swap cache pages in memory: notice > > >> * them here and get rid of the unnecessary final write. > > >> @@ -195,6 +277,13 @@ int swap_writepage(struct page *page, struct writeback_control *wbc) > > >> folio_unlock(folio); > > >> return ret; > > >> } > > >> + > > >> + if (is_folio_zero_filled(folio)) { > > >> + swap_zeromap_folio_set(folio); > > >> + folio_unlock(folio); > > >> + return 0; > > >> + } > > >> + swap_zeromap_folio_clear(folio); > > >> if (zswap_store(folio)) { > > >> folio_start_writeback(folio); > > >> folio_unlock(folio); > > >> @@ -426,6 +515,26 @@ static void sio_read_complete(struct kiocb *iocb, long ret) > > >> mempool_free(sio, sio_pool); > > >> } > > >> > > >> +static bool swap_read_folio_zeromap(struct folio *folio) > > >> +{ > > >> + unsigned int idx = swap_zeromap_folio_test(folio); > > >> + > > >> + if (idx == 0) > > >> + return false; > > >> + > > >> + /* > > >> + * Swapping in a large folio that is partially in the zeromap is not > > >> + * currently handled. Return true without marking the folio uptodate so > > >> + * that an IO error is emitted (e.g. do_swap_page() will sigbus). > > >> + */ > > >> + if (WARN_ON_ONCE(idx < folio_nr_pages(folio))) > > >> + return true; > > > > > > Hi Usama, Yosry, > > > > > > I feel the warning is wrong as we could have the case where idx==0 > > > is not zeromap but idx=1 is zeromap. idx == 0 doesn't necessarily > > > mean we should return false. > > > > > > What about the below change which both fixes the warning and unblocks > > > large folios swap-in? > > > > > Hi Barry, > > > > I remembered when resending the zeromap series about the comment Yosry had made earlier, but checked that the mTHP swap-in was not in mm-unstable. > > I should have checked the mailing list and commented! > > > > I have not tested the below diff yet (will do in a few hours). But there might be a small issue with it. Have commented inline. > > > > > diff --git a/mm/page_io.c b/mm/page_io.c > > > index 4bc77d1c6bfa..7d7ff7064e2b 100644 > > > --- a/mm/page_io.c > > > +++ b/mm/page_io.c > > > @@ -226,26 +226,6 @@ static void swap_zeromap_folio_clear(struct folio *folio) > > > } > > > } > > > > > > -/* > > > - * Return the index of the first subpage which is not zero-filled > > > - * according to swap_info_struct->zeromap. > > > - * If all pages are zero-filled according to zeromap, it will return > > > - * folio_nr_pages(folio). > > > - */ > > > -static unsigned int swap_zeromap_folio_test(struct folio *folio) > > > -{ > > > - struct swap_info_struct *sis = swp_swap_info(folio->swap); > > > - swp_entry_t entry; > > > - unsigned int i; > > > - > > > - for (i = 0; i < folio_nr_pages(folio); i++) { > > > - entry = page_swap_entry(folio_page(folio, i)); > > > - if (!test_bit(swp_offset(entry), sis->zeromap)) > > > - return i; > > > - } > > > - return i; > > > -} > > > - > > > /* > > > * We may have stale swap cache pages in memory: notice > > > * them here and get rid of the unnecessary final write. > > > @@ -524,9 +504,10 @@ static void sio_read_complete(struct kiocb *iocb, long ret) > > > > > > static bool swap_read_folio_zeromap(struct folio *folio) > > > { > > > - unsigned int idx = swap_zeromap_folio_test(folio); > > > + unsigned int nr_pages = folio_nr_pages(folio); > > > + unsigned int nr = swap_zeromap_entries_count(folio->swap, nr_pages); > > > > > > - if (idx == 0) > > > + if (nr == 0) > > > return false; > > > > > > /* > > > @@ -534,7 +515,7 @@ static bool swap_read_folio_zeromap(struct folio *folio) > > > * currently handled. Return true without marking the folio uptodate so > > > * that an IO error is emitted (e.g. do_swap_page() will sigbus). > > > */ > > > - if (WARN_ON_ONCE(idx < folio_nr_pages(folio))) > > > + if (WARN_ON_ONCE(nr < nr_pages)) > > > return true; > > > > > > folio_zero_range(folio, 0, folio_size(folio)); > > > diff --git a/mm/swap.h b/mm/swap.h > > > index f8711ff82f84..2d59e9d89e95 100644 > > > --- a/mm/swap.h > > > +++ b/mm/swap.h > > > @@ -80,6 +80,32 @@ static inline unsigned int folio_swap_flags(struct folio *folio) > > > { > > > return swp_swap_info(folio->swap)->flags; > > > } > > > + > > > +/* > > > + * Return the number of entries which are zero-filled according to > > > + * swap_info_struct->zeromap. It isn't precise if the return value > > > + * is larger than 0 and smaller than nr to avoid extra iterations, > > > + * In this case, it means entries haven't consistent zeromap. > > > + */ > > > +static inline unsigned int swap_zeromap_entries_count(swp_entry_t entry, int nr) > > > +{ > > > + struct swap_info_struct *sis = swp_swap_info(entry); > > > + unsigned long offset = swp_offset(entry); > > > + unsigned int type = swp_type(entry); > > > + unsigned int n = 0; > > > + > > > + for (int i = 0; i < nr; i++) { > > > + entry = swp_entry(type, offset + i); > > > + if (test_bit(offset + i, sis->zeromap)) { > > > > Should this be if (test_bit(swp_offset(entry), sis->zeromap)) > > > > well. i feel i have a much cheaper way to implement this, which > can entirely iteration even in your original code: > > +/* > + * Return the number of entries which are zero-filled according to > + * swap_info_struct->zeromap. It isn't precise if the return value > + * is 1 for nr > 1. In this case, it means entries have inconsistent > + * zeromap. > + */ > +static inline unsigned int swap_zeromap_entries_count(swp_entry_t > entry, int nr) > +{ > + struct swap_info_struct *sis = swp_swap_info(entry); > + unsigned long start = swp_offset(entry); > + unsigned long end = start + nr; > + unsigned long idx = 0; > + > + idx = find_next_bit(sis->zeromap, end, start); > + if (idx == end) > + return 0; > + if (idx > start) > + return 1; missed a case here: if (nr > 1 && find_next_zero_bit(sis->zeromap, end, start) != end) return 1; > + return nr; > +} > + > > > > > > Also, are you going to use this in alloc_swap_folio? > > You mentioned above that this unblocks large folios swap-in, but I don't see > > it in the diff here. I am guessing there is some change in alloc_swap_info that > > uses swap_zeromap_entries_count? > > > > Thanks > > Usama > > > > > + if (i != n) > > > + return i; > > > + n++; > > > + } > > > + } > > > + > > > + return n; > > > +} > > > + > > > #else /* CONFIG_SWAP */ > > > struct swap_iocb; > > > static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) > > > @@ -171,6 +197,11 @@ static inline unsigned int folio_swap_flags(struct folio *folio) > > > { > > > return 0; > > > } > > > + > > > +static inline unsigned int swap_zeromap_entries_count(swp_entry_t entry, int nr) > > > +{ > > > + return 0; > > > +} > > > #endif /* CONFIG_SWAP */ > > > > > > #endif /* _MM_SWAP_H */ > > > > > Thanks > Barry
