Value of anon_vma:
print *((struct anon_vma *)0xffff8820833ed940)
$2 = {
root = 0x0,
rwsem = {
count = 0,
wait_lock = {
raw_lock = {
{
head_tail = 0,
tickets = {
head = 0,
tail = 0
}
}
}
},
wait_list = {
next = 0x0,
prev = 0x0
}
},
refcount = {
counter = 0
},
rb_root = {
rb_node = 0x0
}
}
crash>
-----Original Message-----
From: linux-kernel-owner@xxxxxxxxxxxxxxx [mailto:linux-kernel-owner@xxxxxxxxxxxxxxx] On Behalf Of Dashi DS1 Cao
Sent: Thursday, December 22, 2016 7:53 PM
To: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: linux-mm@xxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx; Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Subject: RE: A small window for a race condition in mm/rmap.c:page_lock_anon_vma_read
I've used another dump with similar backtrace.
PID: 246 TASK: ffff881fd27df300 CPU: 0 COMMAND: "kswapd0"
#0 [ffff881fcfb23748] machine_kexec at ffffffff81051e9b
#1 [ffff881fcfb237a8] crash_kexec at ffffffff810f27e2
#2 [ffff881fcfb23878] oops_end at ffffffff8163f448
#3 [ffff881fcfb238a0] no_context at ffffffff8162f561
#4 [ffff881fcfb238f0] __bad_area_nosemaphore at ffffffff8162f5f7
#5 [ffff881fcfb23938] bad_area_nosemaphore at ffffffff8162f761
#6 [ffff881fcfb23948] __do_page_fault at ffffffff816421ce
#7 [ffff881fcfb239a8] do_page_fault at ffffffff81642363
#8 [ffff881fcfb239d0] page_fault at ffffffff8163e648
[exception RIP: down_read_trylock+9]
RIP: ffffffff810aa9f9 RSP: ffff881fcfb23a88 RFLAGS: 00010202
RAX: 0000000000000000 RBX: ffff8820833ed940 RCX: 0000000000000000
RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000008
RBP: ffff881fcfb23a88 R8: ffffea00779b3a60 R9: ffff883fd0d7b070
R10: 000000000000000e R11: ffffea00049e9580 R12: ffff8820833ed941
R13: ffffea00779b3a40 R14: 0000000000000008 R15: ffffea00779b3a40
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
#9 [ffff881fcfb23a90] page_lock_anon_vma_read at ffffffff811a3365
#10 [ffff881fcfb23ac0] page_referenced at ffffffff811a35e7
#11 [ffff881fcfb23b38] shrink_active_list at ffffffff8117e8cc
#12 [ffff881fcfb23bf0] shrink_lruvec at ffffffff8117ef8d
#13 [ffff881fcfb23cf0] shrink_zone at ffffffff8117f2a6
#14 [ffff881fcfb23d48] balance_pgdat at ffffffff8118054c
#15 [ffff881fcfb23e20] kswapd at ffffffff81180813
#16 [ffff881fcfb23ec8] kthread at ffffffff810a5b8f
#17 [ffff881fcfb23f50] ret_from_fork at ffffffff81646a98
crash> print *((struct page *)0xffffea00779b3a40j)
$1 = {
flags = 13510794587668552,
mapping = 0xffff8820833ed941,
{
{
index = 34194823743,
freelist = 0x7f62b9a3f,
pfmemalloc = 63,
pmd_huge_pte = 0x7f62b9a3f
},
{
counters = 8589934592,
{
{
_mapcount = {
counter = 0
},
{
inuse = 0,
objects = 0,
frozen = 0
},
units = 0
},
_count = {
counter = 2
}
}
}
},
{
lru = {
next = 0xdead000000100100,
prev = 0xdead000000200200
},
{
next = 0xdead000000100100,
pages = 2097664,
pobjects = -559087616
},
list = {
next = 0xdead000000100100,
prev = 0xdead000000200200
},
slab_page = 0xdead000000100100
},
{
private = 0,
ptl = {
{
rlock = {
raw_lock = {
{
head_tail = 0,
tickets = {
head = 0,
tail = 0
}
}
}
}
}
},
slab_cache = 0x0,
first_page = 0x0
}
}
crash> disassemble page_lock_anon_vma_read
Dump of assembler code for function page_lock_anon_vma_read:
0xffffffff811a3310 <+0>: nopl 0x0(%rax,%rax,1)
0xffffffff811a3315 <+5>: push %rbp
0xffffffff811a3316 <+6>: mov %rsp,%rbp
0xffffffff811a3319 <+9>: push %r14
0xffffffff811a331b <+11>: push %r13
0xffffffff811a331d <+13>: mov %rdi,%r13
0xffffffff811a3320 <+16>: push %r12
0xffffffff811a3322 <+18>: push %rbx
0xffffffff811a3323 <+19>: mov 0x8(%rdi),%r12
0xffffffff811a3327 <+23>: mov %r12,%rax
0xffffffff811a332a <+26>: and $0x3,%eax
0xffffffff811a332d <+29>: cmp $0x1,%rax
0xffffffff811a3331 <+33>: je 0xffffffff811a3348 <page_lock_anon_vma_read+56>
0xffffffff811a3333 <+35>: xor %ebx,%ebx
0xffffffff811a3335 <+37>: mov %rbx,%rax
0xffffffff811a3338 <+40>: pop %rbx
0xffffffff811a3339 <+41>: pop %r12
0xffffffff811a333b <+43>: pop %r13
0xffffffff811a333d <+45>: pop %r14
0xffffffff811a333f <+47>: pop %rbp
0xffffffff811a3340 <+48>: retq
0xffffffff811a3341 <+49>: nopl 0x0(%rax)
0xffffffff811a3348 <+56>: mov 0x18(%rdi),%eax
0xffffffff811a334b <+59>: test %eax,%eax
0xffffffff811a334d <+61>: js 0xffffffff811a3333 <page_lock_anon_vma_read+35>
0xffffffff811a334f <+63>: mov -0x1(%r12),%r14
0xffffffff811a3354 <+68>: lea -0x1(%r12),%rbx
0xffffffff811a3359 <+73>: add $0x8,%r14
0xffffffff811a335d <+77>: mov %r14,%rdi
0xffffffff811a3360 <+80>: callq 0xffffffff810aa9f0 <down_read_trylock>
0xffffffff811a3365 <+85>: test %eax,%eax
0xffffffff811a3367 <+87>: je 0xffffffff811a3380 <page_lock_anon_vma_read+112>
0xffffffff811a3369 <+89>: mov 0x18(%r13),%eax
0xffffffff811a336d <+93>: test %eax,%eax
0xffffffff811a336f <+95>: jns 0xffffffff811a3335 <page_lock_anon_vma_read+37>
0xffffffff811a3371 <+97>: mov %r14,%rdi
0xffffffff811a3374 <+100>: xor %ebx,%ebx
0xffffffff811a3376 <+102>: callq 0xffffffff810aaa50 <up_read>
0xffffffff811a337b <+107>: jmp 0xffffffff811a3335 <page_lock_anon_vma_read+37>
0xffffffff811a337d <+109>: nopl (%rax)
0xffffffff811a3380 <+112>: mov 0x28(%rbx),%edx
0xffffffff811a3383 <+115>: test %edx,%edx
0xffffffff811a3385 <+117>: je 0xffffffff811a3333 <page_lock_anon_vma_read+35>
0xffffffff811a3387 <+119>: lea 0x1(%rdx),%ecx
0xffffffff811a338a <+122>: lea 0x27(%r12),%rsi
0xffffffff811a338f <+127>: mov %edx,%eax
0xffffffff811a3391 <+129>: lock cmpxchg %ecx,0x27(%r12)
0xffffffff811a3398 <+136>: cmp %edx,%eax
0xffffffff811a339a <+138>: mov %eax,%ecx
0xffffffff811a339c <+140>: jne 0xffffffff811a3402 <page_lock_anon_vma_read+242>
0xffffffff811a339e <+142>: mov 0x18(%r13),%eax
0xffffffff811a33a2 <+146>: test %eax,%eax
0xffffffff811a33a4 <+148>: js 0xffffffff811a33e2 <page_lock_anon_vma_read+210>
0xffffffff811a33a6 <+150>: mov -0x1(%r12),%rax
0xffffffff811a33ab <+155>: lea 0x8(%rax),%rdi
0xffffffff811a33af <+159>: callq 0xffffffff8163ad30 <down_read>
0xffffffff811a33b4 <+164>: lock decl 0x27(%r12)
0xffffffff811a33ba <+170>: sete %al
0xffffffff811a33bd <+173>: test %al,%al
0xffffffff811a33bf <+175>: je 0xffffffff811a3335 <page_lock_anon_vma_read+37>
0xffffffff811a33c5 <+181>: mov -0x1(%r12),%rdi
0xffffffff811a33ca <+186>: add $0x8,%rdi
0xffffffff811a33ce <+190>: callq 0xffffffff810aaa50 <up_read>
0xffffffff811a33d3 <+195>: mov %rbx,%rdi
0xffffffff811a33d6 <+198>: xor %ebx,%ebx
0xffffffff811a33d8 <+200>: callq 0xffffffff811a2dd0 <__put_anon_vma>
0xffffffff811a33dd <+205>: jmpq 0xffffffff811a3335 <page_lock_anon_vma_read+37>
0xffffffff811a33e2 <+210>: lock decl 0x27(%r12)
0xffffffff811a33e8 <+216>: sete %al
0xffffffff811a33eb <+219>: test %al,%al
0xffffffff811a33ed <+221>: je 0xffffffff811a3333 <page_lock_anon_vma_read+35>
0xffffffff811a33f3 <+227>: mov %rbx,%rdi
0xffffffff811a33f6 <+230>: xor %ebx,%ebx
0xffffffff811a33f8 <+232>: callq 0xffffffff811a2dd0 <__put_anon_vma>
0xffffffff811a33fd <+237>: jmpq 0xffffffff811a3335 <page_lock_anon_vma_read+37>
0xffffffff811a3402 <+242>: test %ecx,%ecx
0xffffffff811a3404 <+244>: je 0xffffffff811a3333 <page_lock_anon_vma_read+35>
0xffffffff811a340a <+250>: lea 0x1(%rcx),%edx
0xffffffff811a340d <+253>: mov %ecx,%eax
0xffffffff811a340f <+255>: lock cmpxchg %edx,(%rsi)
0xffffffff811a3413 <+259>: cmp %eax,%ecx
0xffffffff811a3415 <+261>: je 0xffffffff811a339e <page_lock_anon_vma_read+142>
0xffffffff811a3417 <+263>: mov %eax,%ecx
0xffffffff811a3419 <+265>: jmp 0xffffffff811a3402 <page_lock_anon_vma_read+242>
End of assembler dump.
crash>
Dashi Cao
-----Original Message-----
From: Michal Hocko [mailto:mhocko@xxxxxxxxxx]
Sent: Wednesday, December 21, 2016 10:44 PM
To: Dashi DS1 Cao <caods1@xxxxxxxxxx>
Cc: linux-mm@xxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx; Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Subject: Re: A small window for a race condition in mm/rmap.c:page_lock_anon_vma_read
anon_vma locking is clever^Wsubtle as hell. CC Peter...
On Tue 20-12-16 09:32:27, Dashi DS1 Cao wrote:
> I've collected four crash dumps with similar backtrace.
>
> PID: 247 TASK: ffff881fcfad8000 CPU: 14 COMMAND: "kswapd1"
> #0 [ffff881fcfad7978] machine_kexec at ffffffff81051e9b
> #1 [ffff881fcfad79d8] crash_kexec at ffffffff810f27e2
> #2 [ffff881fcfad7aa8] oops_end at ffffffff8163f448
> #3 [ffff881fcfad7ad0] die at ffffffff8101859b
> #4 [ffff881fcfad7b00] do_general_protection at ffffffff8163ed3e
> #5 [ffff881fcfad7b30] general_protection at ffffffff8163e5e8
> [exception RIP: down_read_trylock+9]
> RIP: ffffffff810aa9f9 RSP: ffff881fcfad7be0 RFLAGS: 00010286
> RAX: 0000000000000000 RBX: ffff882b47ddadc0 RCX: 0000000000000000
> RDX: 0000000000000000 RSI: 0000000000000000 RDI:
> 91550b2b32f5a3e8
rdi is obviously a mess - smells like a string. So either sombody has overwritten root_anon_vma or this is really a use after free...
> RBP: ffff881fcfad7be0 R8: ffffea00ecc28860 R9: ffff883fcffeae28
> R10: ffffffff81a691a0 R11: 0000000000000001 R12: ffff882b47ddadc1
> R13: ffffea00ecc28840 R14: 91550b2b32f5a3e8 R15: ffffea00ecc28840
> ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0000
> #6 [ffff881fcfad7be8] page_lock_anon_vma_read at ffffffff811a3365
> #7 [ffff881fcfad7c18] page_referenced at ffffffff811a35e7
> #8 [ffff881fcfad7c90] shrink_active_list at ffffffff8117e8cc
> #9 [ffff881fcfad7d48] balance_pgdat at ffffffff81180288
> #10 [ffff881fcfad7e20] kswapd at ffffffff81180813
> #11 [ffff881fcfad7ec8] kthread at ffffffff810a5b8f
> #12 [ffff881fcfad7f50] ret_from_fork at ffffffff81646a98
>
> I suspect my customer hits into a small window of a race condition in mm/rmap.c: page_lock_anon_vma_read.
> struct anon_vma *page_lock_anon_vma_read(struct page *page) {
> struct anon_vma *anon_vma = NULL;
> struct anon_vma *root_anon_vma;
> unsigned long anon_mapping;
>
> rcu_read_lock();
> anon_mapping = (unsigned long)READ_ONCE(page->mapping);
> if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
> goto out;
> if (!page_mapped(page))
> goto out;
>
> anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
> root_anon_vma = READ_ONCE(anon_vma->root);
Could you dump the anon_vma and struct page as well?
> if (down_read_trylock(&root_anon_vma->rwsem)) {
> /*
> * If the page is still mapped, then this anon_vma is still
> * its anon_vma, and holding the mutex ensures that it will
> * not go away, see anon_vma_free().
> */
> if (!page_mapped(page)) {
> up_read(&root_anon_vma->rwsem);
> anon_vma = NULL;
> }
> goto out;
> }
> ...
> }
>
> Between the time the two "page_mapped(page)" are checked, the address
> (anon_mapping - PAGE_MAPPING_ANON) is unmapped! However it seems that
> anon_vma->root could still be read in but the value is wild. So the
> kernel crashes in down_read_trylock. But it's weird that all the
> "struct page" has its member "_mapcount" still with value 0, not -1,
> in the four crashes.
--
Michal Hocko
SUSE Labs
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href