From: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
To implement workingset detection for anonymous page, the page that is
swapped-in by readahead but not yet touched in the swap cache is also
charged to the memcg. However, if these readahead pages are not ours,
these pages are wrongly charged to us and our memcg could be wrongly
pressured.
To solve the problem, this patch implements to avoid readahead if the
previous owner of the swap entry isn't me. With this patch, readahead
only happens for mine and then there is no wrongly memcg pressure issue.
The purpose of the readahead is to improve the performance by using
locality of the swap space. If the entry isn't mine, there is no locality.
Therefore, it's safe to avoid readahead.
Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
---
include/linux/swap.h | 9 ++++++---
mm/shmem.c | 2 +-
mm/swap_state.c | 34 ++++++++++++++++++++++++++--------
mm/workingset.c | 16 ++++++++++++++++
mm/zswap.c | 2 +-
5 files changed, 50 insertions(+), 13 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 97e8a2e..d204cc7 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -307,6 +307,7 @@ struct vma_swap_readahead {
};
/* linux/mm/workingset.c */
+bool shadow_from_memcg(void *shadow, struct mem_cgroup *memcg);
void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);
void workingset_refault(struct page *page, void *shadow);
void workingset_activation(struct page *page);
@@ -410,7 +411,8 @@ extern void show_swap_cache_info(void);
extern int add_to_swap(struct page *page);
extern void *get_shadow_from_swap_cache(swp_entry_t entry);
extern int add_to_swap_cache(struct page *page, swp_entry_t entry,
- struct vm_area_struct *vma, gfp_t gfp, void **shadowp);
+ struct vm_area_struct *vma, gfp_t gfp,
+ void **shadowp, bool readahead);
extern int __add_to_swap_cache(struct page *page, swp_entry_t entry);
extern void __delete_from_swap_cache(struct page *page,
swp_entry_t entry, void *shadow);
@@ -425,7 +427,7 @@ extern struct page *read_swap_cache_async(swp_entry_t, gfp_t,
bool do_poll);
extern struct page *__read_swap_cache_async(swp_entry_t, gfp_t,
struct vm_area_struct *vma, unsigned long addr,
- bool *new_page_allocated);
+ bool *new_page_allocated, bool readahead);
extern struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
struct vm_fault *vmf);
extern struct page *swapin_readahead(swp_entry_t entry, gfp_t flag,
@@ -573,7 +575,8 @@ static inline void *get_shadow_from_swap_cache(swp_entry_t entry)
}
static inline int add_to_swap_cache(struct page *page, swp_entry_t entry,
- struct vm_area_struct *vma, gfp_t gfp, void **shadowp)
+ struct vm_area_struct *vma, gfp_t gfp,
+ void **shadowp, bool readahead)
{
return -1;
}
diff --git a/mm/shmem.c b/mm/shmem.c
index 8e28c1f..483c32d 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1371,7 +1371,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
if (add_to_swap_cache(page, swap, NULL,
__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
- NULL) == 0) {
+ NULL, false) == 0) {
spin_lock_irq(&info->lock);
shmem_recalc_inode(inode);
info->swapped++;
diff --git a/mm/swap_state.c b/mm/swap_state.c
index de994f9..9bf0251f 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -126,7 +126,8 @@ void *get_shadow_from_swap_cache(swp_entry_t entry)
* but sets SwapCache flag and private instead of mapping and index.
*/
int add_to_swap_cache(struct page *page, swp_entry_t entry,
- struct vm_area_struct *vma, gfp_t gfp, void **shadowp)
+ struct vm_area_struct *vma, gfp_t gfp,
+ void **shadowp, bool readahead)
{
struct address_space *address_space = swap_address_space(entry);
pgoff_t idx = swp_offset(entry);
@@ -143,6 +144,23 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry,
VM_BUG_ON_PAGE(PageSwapCache(page), page);
VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
+ if (readahead) {
+ void *shadow = get_shadow_from_swap_cache(entry);
+
+ /*
+ * In readahead case, check the memcgid of the shadow entry
+ * in order to stop to readahead other's page
+ */
+ if (shadow) {
+ memcg = get_mem_cgroup_from_mm(mm);
+ if (memcg && !shadow_from_memcg(shadow, memcg)) {
+ mem_cgroup_put(memcg);
+ return -EINVAL;
+ }
+ mem_cgroup_put(memcg);
+ }
+ }
+
page_ref_add(page, nr);
/* PageSwapCache() prevent the page from being re-charged */
SetPageSwapCache(page);
@@ -253,7 +271,7 @@ int add_to_swap(struct page *page)
* Add it to the swap cache.
*/
err = add_to_swap_cache(page, entry, NULL,
- __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL);
+ __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL, false);
if (err)
/*
* add_to_swap_cache() doesn't return -EEXIST, so we can safely
@@ -402,7 +420,7 @@ struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma,
struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
struct vm_area_struct *vma, unsigned long addr,
- bool *new_page_allocated)
+ bool *new_page_allocated, bool readahead)
{
struct page *found_page = NULL, *new_page = NULL;
struct swap_info_struct *si;
@@ -465,7 +483,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
__SetPageSwapBacked(new_page);
shadow = NULL;
err = add_to_swap_cache(new_page, entry, vma,
- gfp_mask & GFP_KERNEL, &shadow);
+ gfp_mask & GFP_KERNEL, &shadow, readahead);
if (likely(!err)) {
/* Initiate read into locked page */
SetPageWorkingset(new_page);
@@ -481,7 +499,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
* clear SWAP_HAS_CACHE flag.
*/
put_swap_page(new_page, entry);
- } while (err != -ENOMEM);
+ } while (err != -ENOMEM && err != -EINVAL);
if (new_page)
put_page(new_page);
@@ -499,7 +517,7 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
{
bool page_was_allocated;
struct page *retpage = __read_swap_cache_async(entry, gfp_mask,
- vma, addr, &page_was_allocated);
+ vma, addr, &page_was_allocated, false);
if (page_was_allocated)
swap_readpage(retpage, do_poll);
@@ -624,7 +642,7 @@ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
/* Ok, do the async read-ahead now */
page = __read_swap_cache_async(
swp_entry(swp_type(entry), offset),
- gfp_mask, vma, addr, &page_allocated);
+ gfp_mask, vma, addr, &page_allocated, true);
if (!page)
continue;
if (page_allocated) {
@@ -796,7 +814,7 @@ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask,
if (unlikely(non_swap_entry(entry)))
continue;
page = __read_swap_cache_async(entry, gfp_mask, vma,
- vmf->address, &page_allocated);
+ vmf->address, &page_allocated, true);
if (!page)
continue;
if (page_allocated) {
diff --git a/mm/workingset.c b/mm/workingset.c
index 871b867..1fa46cf 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -231,6 +231,22 @@ static void unpack_shadow(void *shadow, int *memcgidp, int *page_memcg_tagp,
*workingsetp = workingset;
}
+bool shadow_from_memcg(void *shadow, struct mem_cgroup *memcg)
+{
+ int memcgid, page_memcg_tag;
+ struct pglist_data *pgdat;
+ unsigned long eviction;
+ bool workingset;
+ int memcg_tag;
+
+ unpack_shadow(shadow, &memcgid, &page_memcg_tag,
+ &pgdat, &eviction, &workingset);
+ memcg_tag = mem_cgroup_id(memcg);
+ memcg_tag &= (1UL << PAGE_MEMCG_TAG_SHIFT) - 1;
+
+ return memcg_tag == page_memcg_tag;
+}
+
static void advance_inactive_age(struct mem_cgroup *memcg, pg_data_t *pgdat,
bool file)
{
diff --git a/mm/zswap.c b/mm/zswap.c
index 55094e6..92f1463 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -848,7 +848,7 @@ static int zswap_get_swap_cache_page(swp_entry_t entry,
bool page_was_allocated;
*retpage = __read_swap_cache_async(entry, GFP_KERNEL,
- NULL, 0, &page_was_allocated);
+ NULL, 0, &page_was_allocated, false);
if (page_was_allocated)
return ZSWAP_SWAPCACHE_NEW;
if (!*retpage)
--
2.7.4
