Hello, On Wed, Jul 31, 2013 at 12:41:55AM +0400, Cyrill Gorcunov wrote: > Andy Lutomirski reported that in case if a page with _PAGE_SOFT_DIRTY > bit set get swapped out, the bit is getting lost and no longer > available when pte read back. > > To resolve this we introduce _PTE_SWP_SOFT_DIRTY bit which is > saved in pte entry for the page being swapped out. When such page > is to be read back from a swap cache we check for bit presence > and if it's there we clear it and restore the former _PAGE_SOFT_DIRTY > bit back. > > One of the problem was to find a place in pte entry where we can > save the _PTE_SWP_SOFT_DIRTY bit while page is in swap. The > _PAGE_PSE was chosen for that, it doesn't intersect with swap > entry format stored in pte. > > Reported-by: Andy Lutomirski <luto@xxxxxxxxxxxxxx> > Signed-off-by: Cyrill Gorcunov <gorcunov@xxxxxxxxxx> > Cc: Pavel Emelyanov <xemul@xxxxxxxxxxxxx> > Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx> > Cc: Matt Mackall <mpm@xxxxxxxxxxx> > Cc: Xiao Guangrong <xiaoguangrong@xxxxxxxxxxxxxxxxxx> > Cc: Marcelo Tosatti <mtosatti@xxxxxxxxxx> > Cc: KOSAKI Motohiro <kosaki.motohiro@xxxxxxxxx> > Cc: Stephen Rothwell <sfr@xxxxxxxxxxxxxxxx> > Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx> > Cc: "Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxxxxxxx> > --- > arch/x86/include/asm/pgtable.h | 15 +++++++++++++++ > arch/x86/include/asm/pgtable_types.h | 13 +++++++++++++ > fs/proc/task_mmu.c | 21 +++++++++++++++------ > include/asm-generic/pgtable.h | 15 +++++++++++++++ > include/linux/swapops.h | 2 ++ > mm/memory.c | 2 ++ > mm/rmap.c | 6 +++++- > mm/swapfile.c | 19 +++++++++++++++++-- > 8 files changed, 84 insertions(+), 9 deletions(-) > > Index: linux-2.6.git/arch/x86/include/asm/pgtable.h > =================================================================== > --- linux-2.6.git.orig/arch/x86/include/asm/pgtable.h > +++ linux-2.6.git/arch/x86/include/asm/pgtable.h > @@ -314,6 +314,21 @@ static inline pmd_t pmd_mksoft_dirty(pmd > return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY); > } > > +static inline pte_t pte_swp_mksoft_dirty(pte_t pte) > +{ > + return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY); > +} > + > +static inline int pte_swp_soft_dirty(pte_t pte) > +{ > + return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY; > +} > + > +static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) > +{ > + return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); > +} > + > /* > * Mask out unsupported bits in a present pgprot. Non-present pgprots > * can use those bits for other purposes, so leave them be. > Index: linux-2.6.git/arch/x86/include/asm/pgtable_types.h > =================================================================== > --- linux-2.6.git.orig/arch/x86/include/asm/pgtable_types.h > +++ linux-2.6.git/arch/x86/include/asm/pgtable_types.h > @@ -67,6 +67,19 @@ > #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) > #endif > > +/* > + * Tracking soft dirty bit when a page goes to a swap is tricky. > + * We need a bit which can be stored in pte _and_ not conflict > + * with swap entry format. On x86 bits 6 and 7 are *not* involved > + * into swap entry computation, but bit 6 is used for nonlinear > + * file mapping, so we borrow bit 7 for soft dirty tracking. > + */ > +#ifdef CONFIG_MEM_SOFT_DIRTY > +#define _PAGE_SWP_SOFT_DIRTY _PAGE_PSE > +#else > +#define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0)) > +#endif > + > #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) > #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) > #else > Index: linux-2.6.git/fs/proc/task_mmu.c > =================================================================== > --- linux-2.6.git.orig/fs/proc/task_mmu.c > +++ linux-2.6.git/fs/proc/task_mmu.c > @@ -730,8 +730,14 @@ static inline void clear_soft_dirty(stru > * of how soft-dirty works. > */ > pte_t ptent = *pte; > - ptent = pte_wrprotect(ptent); > - ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY); > + > + if (pte_present(ptent)) { > + ptent = pte_wrprotect(ptent); > + ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY); > + } else if (is_swap_pte(ptent)) { > + ptent = pte_swp_clear_soft_dirty(ptent); > + } > + > set_pte_at(vma->vm_mm, addr, pte, ptent); > #endif > } > @@ -752,14 +758,15 @@ static int clear_refs_pte_range(pmd_t *p > pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); > for (; addr != end; pte++, addr += PAGE_SIZE) { > ptent = *pte; > - if (!pte_present(ptent)) > - continue; > > if (cp->type == CLEAR_REFS_SOFT_DIRTY) { > clear_soft_dirty(vma, addr, pte); > continue; > } > > + if (!pte_present(ptent)) > + continue; > + > page = vm_normal_page(vma, addr, ptent); > if (!page) > continue; > @@ -930,8 +937,10 @@ static void pte_to_pagemap_entry(pagemap > flags = PM_PRESENT; > page = vm_normal_page(vma, addr, pte); > } else if (is_swap_pte(pte)) { > - swp_entry_t entry = pte_to_swp_entry(pte); > - > + swp_entry_t entry; > + if (pte_swp_soft_dirty(pte)) > + flags2 |= __PM_SOFT_DIRTY; > + entry = pte_to_swp_entry(pte); > frame = swp_type(entry) | > (swp_offset(entry) << MAX_SWAPFILES_SHIFT); > flags = PM_SWAP; > Index: linux-2.6.git/include/asm-generic/pgtable.h > =================================================================== > --- linux-2.6.git.orig/include/asm-generic/pgtable.h > +++ linux-2.6.git/include/asm-generic/pgtable.h > @@ -417,6 +417,21 @@ static inline pmd_t pmd_mksoft_dirty(pmd > { > return pmd; > } > + > +static inline pte_t pte_swp_mksoft_dirty(pte_t pte) > +{ > + return pte; > +} > + > +static inline int pte_swp_soft_dirty(pte_t pte) > +{ > + return 0; > +} > + > +static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) > +{ > + return pte; > +} > #endif > > #ifndef __HAVE_PFNMAP_TRACKING > Index: linux-2.6.git/include/linux/swapops.h > =================================================================== > --- linux-2.6.git.orig/include/linux/swapops.h > +++ linux-2.6.git/include/linux/swapops.h > @@ -67,6 +67,8 @@ static inline swp_entry_t pte_to_swp_ent > swp_entry_t arch_entry; > > BUG_ON(pte_file(pte)); > + if (pte_swp_soft_dirty(pte)) > + pte = pte_swp_clear_soft_dirty(pte); Why do you remove soft-dirty flag whenever pte_to_swp_entry is called? Isn't there any problem if we use mincore? > arch_entry = __pte_to_swp_entry(pte); > return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); > } > Index: linux-2.6.git/mm/memory.c > =================================================================== > --- linux-2.6.git.orig/mm/memory.c > +++ linux-2.6.git/mm/memory.c > @@ -3115,6 +3115,8 @@ static int do_swap_page(struct mm_struct > exclusive = 1; > } > flush_icache_page(vma, page); > + if (pte_swp_soft_dirty(orig_pte)) > + pte = pte_mksoft_dirty(pte); > set_pte_at(mm, address, page_table, pte); > if (page == swapcache) > do_page_add_anon_rmap(page, vma, address, exclusive); > Index: linux-2.6.git/mm/rmap.c > =================================================================== > --- linux-2.6.git.orig/mm/rmap.c > +++ linux-2.6.git/mm/rmap.c > @@ -1236,6 +1236,7 @@ int try_to_unmap_one(struct page *page, > swp_entry_to_pte(make_hwpoison_entry(page))); > } else if (PageAnon(page)) { > swp_entry_t entry = { .val = page_private(page) }; > + pte_t swp_pte; > > if (PageSwapCache(page)) { > /* > @@ -1264,7 +1265,10 @@ int try_to_unmap_one(struct page *page, > BUG_ON(TTU_ACTION(flags) != TTU_MIGRATION); > entry = make_migration_entry(page, pte_write(pteval)); > } > - set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); > + swp_pte = swp_entry_to_pte(entry); > + if (pte_soft_dirty(pteval)) > + swp_pte = pte_swp_mksoft_dirty(swp_pte); > + set_pte_at(mm, address, pte, swp_pte); > BUG_ON(pte_file(*pte)); > } else if (IS_ENABLED(CONFIG_MIGRATION) && > (TTU_ACTION(flags) == TTU_MIGRATION)) { > Index: linux-2.6.git/mm/swapfile.c > =================================================================== > --- linux-2.6.git.orig/mm/swapfile.c > +++ linux-2.6.git/mm/swapfile.c > @@ -866,6 +866,21 @@ unsigned int count_swap_pages(int type, > } > #endif /* CONFIG_HIBERNATION */ > > +static inline int maybe_same_pte(pte_t pte, pte_t swp_pte) Nitpick. If maybe_same_pte is used widely, it looks good to me but it's used for only swapoff at the moment so I think pte_swap_same would be better name. > +{ > +#ifdef CONFIG_MEM_SOFT_DIRTY > + /* > + * When pte keeps soft dirty bit the pte generated > + * from swap entry does not has it, still it's same > + * pte from logical point of view. > + */ > + pte_t swp_pte_dirty = pte_swp_mksoft_dirty(swp_pte); > + return pte_same(pte, swp_pte) || pte_same(pte, swp_pte_dirty); > +#else > + return pte_same(pte, swp_pte); > +#endif > +} > + > /* > * No need to decide whether this PTE shares the swap entry with others, > * just let do_wp_page work it out if a write is requested later - to > @@ -892,7 +907,7 @@ static int unuse_pte(struct vm_area_stru > } > > pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); > - if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) { > + if (unlikely(!maybe_same_pte(*pte, swp_entry_to_pte(entry)))) { > mem_cgroup_cancel_charge_swapin(memcg); > ret = 0; > goto out; > @@ -947,7 +962,7 @@ static int unuse_pte_range(struct vm_are > * swapoff spends a _lot_ of time in this loop! > * Test inline before going to call unuse_pte. > */ > - if (unlikely(pte_same(*pte, swp_pte))) { > + if (unlikely(maybe_same_pte(*pte, swp_pte))) { > pte_unmap(pte); > ret = unuse_pte(vma, pmd, addr, entry, page); > if (ret) > > -- > To unsubscribe, send a message with 'unsubscribe linux-mm' in > the body to majordomo@xxxxxxxxx. For more info on Linux MM, > see: http://www.linux-mm.org/ . > Don't email: <a href=mailto:"dont@xxxxxxxxx"> email@xxxxxxxxx </a> -- Kind regards, Minchan Kim -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx"> email@xxxxxxxxx </a>