On 5/6/21 12:55 PM, Al Viro wrote:
> On Thu, May 06, 2021 at 11:19:03AM -0600, Jens Axboe wrote:
>
>> Doing a quick profile, on the latter run with ->write_iter() we're
>> spending 8% of the time in _copy_from_iter(), and 4% in
>> new_sync_write(). That's obviously not there at all for the first case.
>> Both have about 4% in eventfd_write(). Non-iter case spends 1% in
>> copy_from_user().
>>
>> Finally with your branch pulled in as well, iow using ->write_iter() for
>> eventfd and your iov changes:
>>
>> Executed in 485.26 millis fish external
>> usr time 103.09 millis 70.00 micros 103.03 millis
>> sys time 382.18 millis 83.00 micros 382.09 millis
>>
>> Executed in 485.16 millis fish external
>> usr time 104.07 millis 69.00 micros 104.00 millis
>> sys time 381.09 millis 94.00 micros 381.00 millis
>>
>> and there's no real difference there. We're spending less time in
>> _copy_from_iter() (8% -> 6%) and less time in new_sync_write(), but
>> doesn't seem to manifest itself in reduced runtime.
>
> Interesting... do you have instruction-level profiles for _copy_from_iter()
> and new_sync_write() on the last of those trees?
Attached output of perf annotate <func> for that last run.
--
Jens Axboe
Percent | Source code & Disassembly of vmlinux for cycles (72 samples, percent: local period)
---------------------------------------------------------------------------------------------------
:
:
:
: Disassembly of section .text:
:
: ffffffff812cef20 <new_sync_write>:
: new_sync_write():
: inc_syscr(current);
: return ret;
: }
:
: static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
: {
0.00 : ffffffff812cef20: callq ffffffff8103a8a0 <__fentry__>
0.00 : ffffffff812cef25: push %rbp
0.00 : ffffffff812cef26: mov %rdx,%r8
5.55 : ffffffff812cef29: mov %rsp,%rbp
0.00 : ffffffff812cef2c: push %r12
0.00 : ffffffff812cef2e: push %rbx
0.00 : ffffffff812cef2f: mov %rcx,%r12
0.00 : ffffffff812cef32: sub $0x68,%rsp
: struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
0.00 : ffffffff812cef36: mov %rdx,-0x70(%rbp)
: iocb_flags():
: }
:
: static inline int iocb_flags(struct file *file)
: {
: int res = 0;
: if (file->f_flags & O_APPEND)
0.00 : ffffffff812cef3a: mov 0x40(%rdi),%edx
: new_sync_write():
: {
8.33 : ffffffff812cef3d: mov %rdi,%rbx
: struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
0.00 : ffffffff812cef40: mov %rsi,-0x78(%rbp)
: iocb_flags():
0.00 : ffffffff812cef44: mov %edx,%eax
0.00 : ffffffff812cef46: shr $0x6,%eax
0.00 : ffffffff812cef49: and $0x10,%eax
: res |= IOCB_APPEND;
: if (file->f_flags & O_DIRECT)
: res |= IOCB_DIRECT;
0.00 : ffffffff812cef4c: mov %eax,%ecx
0.00 : ffffffff812cef4e: or $0x20000,%ecx
0.00 : ffffffff812cef54: test $0x40,%dh
6.94 : ffffffff812cef57: cmovne %ecx,%eax
: if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host))
0.00 : ffffffff812cef5a: test $0x10,%dh
0.00 : ffffffff812cef5d: jne ffffffff812cef77 <new_sync_write+0x57>
0.00 : ffffffff812cef5f: mov 0xd0(%rdi),%rcx
0.00 : ffffffff812cef66: mov (%rcx),%rcx
0.00 : ffffffff812cef69: mov 0x28(%rcx),%rsi
0.00 : ffffffff812cef6d: testb $0x10,0x50(%rsi)
13.89 : ffffffff812cef71: je ffffffff812cf04c <new_sync_write+0x12c>
: res |= IOCB_DSYNC;
0.00 : ffffffff812cef77: or $0x2,%eax
: if (file->f_flags & __O_SYNC)
: res |= IOCB_SYNC;
0.00 : ffffffff812cef7a: mov %eax,%ecx
0.00 : ffffffff812cef7c: or $0x4,%ecx
0.00 : ffffffff812cef7f: and $0x100000,%edx
: file_write_hint():
: if (file->f_write_hint != WRITE_LIFE_NOT_SET)
0.00 : ffffffff812cef85: mov 0x34(%rbx),%edx
: iocb_flags():
: res |= IOCB_SYNC;
0.00 : ffffffff812cef88: cmovne %ecx,%eax
: file_write_hint():
: if (file->f_write_hint != WRITE_LIFE_NOT_SET)
0.00 : ffffffff812cef8b: test %edx,%edx
6.97 : ffffffff812cef8d: jne ffffffff812cf03c <new_sync_write+0x11c>
: return file_inode(file)->i_write_hint;
0.00 : ffffffff812cef93: mov 0x20(%rbx),%rdx
0.00 : ffffffff812cef97: movzbl 0x87(%rdx),%edx
: get_current():
:
: DECLARE_PER_CPU(struct task_struct *, current_task);
:
: static __always_inline struct task_struct *get_current(void)
: {
: return this_cpu_read_stable(current_task);
0.00 : ffffffff812cef9e: mov %gs:0x126c0,%rcx
: get_current_ioprio():
: * If the calling process has set an I/O priority, use that. Otherwise, return
: * the default I/O priority.
: */
: static inline int get_current_ioprio(void)
: {
: struct io_context *ioc = current->io_context;
0.00 : ffffffff812cefa7: mov 0x860(%rcx),%rsi
:
: if (ioc)
0.00 : ffffffff812cefae: xor %ecx,%ecx
0.00 : ffffffff812cefb0: test %rsi,%rsi
0.00 : ffffffff812cefb3: je ffffffff812cefb9 <new_sync_write+0x99>
: return ioc->ioprio;
0.00 : ffffffff812cefb5: movzwl 0x14(%rsi),%ecx
: init_sync_kiocb():
: *kiocb = (struct kiocb) {
0.00 : ffffffff812cefb9: shl $0x10,%ecx
12.50 : ffffffff812cefbc: movzwl %dx,%edx
0.00 : ffffffff812cefbf: movq $0x0,-0x38(%rbp)
0.00 : ffffffff812cefc7: movq $0x0,-0x30(%rbp)
0.00 : ffffffff812cefcf: or %ecx,%edx
0.00 : ffffffff812cefd1: movq $0x0,-0x28(%rbp)
0.00 : ffffffff812cefd9: movq $0x0,-0x18(%rbp)
0.00 : ffffffff812cefe1: mov %rbx,-0x40(%rbp)
0.00 : ffffffff812cefe5: mov %eax,-0x20(%rbp)
6.93 : ffffffff812cefe8: mov %edx,-0x1c(%rbp)
: new_sync_write():
: struct kiocb kiocb;
: struct iov_iter iter;
: ssize_t ret;
:
: init_sync_kiocb(&kiocb, filp);
: kiocb.ki_pos = (ppos ? *ppos : 0);
0.00 : ffffffff812cefeb: test %r12,%r12
0.00 : ffffffff812cefee: je ffffffff812cf05b <new_sync_write+0x13b>
: iov_iter_init(&iter, WRITE, &iov, 1, len);
0.00 : ffffffff812ceff0: mov $0x1,%esi
0.00 : ffffffff812ceff5: lea -0x68(%rbp),%rdi
0.00 : ffffffff812ceff9: mov $0x1,%ecx
0.00 : ffffffff812ceffe: lea -0x78(%rbp),%rdx
: kiocb.ki_pos = (ppos ? *ppos : 0);
0.00 : ffffffff812cf002: mov (%r12),%rax
0.00 : ffffffff812cf006: mov %rax,-0x38(%rbp)
: iov_iter_init(&iter, WRITE, &iov, 1, len);
8.33 : ffffffff812cf00a: callq ffffffff814c45e0 <iov_iter_init>
: call_write_iter():
: return file->f_op->write_iter(kio, iter);
12.51 : ffffffff812cf00f: mov 0x28(%rbx),%rax
0.00 : ffffffff812cf013: lea -0x68(%rbp),%rsi
0.00 : ffffffff812cf017: lea -0x40(%rbp),%rdi
0.00 : ffffffff812cf01b: callq *0x28(%rax)
: new_sync_write():
:
: ret = call_write_iter(filp, &kiocb, &iter);
: BUG_ON(ret == -EIOCBQUEUED);
0.00 : ffffffff812cf01e: cmp $0xfffffffffffffdef,%rax
0.00 : ffffffff812cf024: je ffffffff812cf089 <new_sync_write+0x169>
: if (ret > 0 && ppos)
0.00 : ffffffff812cf026: test %rax,%rax
0.00 : ffffffff812cf029: jle ffffffff812cf033 <new_sync_write+0x113>
: *ppos = kiocb.ki_pos;
0.00 : ffffffff812cf02b: mov -0x38(%rbp),%rdx
12.49 : ffffffff812cf02f: mov %rdx,(%r12)
: return ret;
: }
0.00 : ffffffff812cf033: add $0x68,%rsp
0.00 : ffffffff812cf037: pop %rbx
0.00 : ffffffff812cf038: pop %r12
0.00 : ffffffff812cf03a: pop %rbp
0.00 : ffffffff812cf03b: retq
: ki_hint_validate():
: if (hint <= max_hint)
0.00 : ffffffff812cf03c: xor %ecx,%ecx
0.00 : ffffffff812cf03e: cmp $0xffff,%edx
0.00 : ffffffff812cf044: cmova %ecx,%edx
0.00 : ffffffff812cf047: jmpq ffffffff812cef9e <new_sync_write+0x7e>
: iocb_flags():
: if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host))
5.55 : ffffffff812cf04c: testb $0x1,0xc(%rcx)
0.00 : ffffffff812cf050: je ffffffff812cef7a <new_sync_write+0x5a>
0.00 : ffffffff812cf056: jmpq ffffffff812cef77 <new_sync_write+0x57>
: new_sync_write():
: iov_iter_init(&iter, WRITE, &iov, 1, len);
0.00 : ffffffff812cf05b: mov $0x1,%esi
0.00 : ffffffff812cf060: lea -0x68(%rbp),%rdi
0.00 : ffffffff812cf064: mov $0x1,%ecx
0.00 : ffffffff812cf069: lea -0x78(%rbp),%rdx
0.00 : ffffffff812cf06d: callq ffffffff814c45e0 <iov_iter_init>
: call_write_iter():
: return file->f_op->write_iter(kio, iter);
0.00 : ffffffff812cf072: mov 0x28(%rbx),%rax
0.00 : ffffffff812cf076: lea -0x68(%rbp),%rsi
0.00 : ffffffff812cf07a: lea -0x40(%rbp),%rdi
0.00 : ffffffff812cf07e: callq *0x28(%rax)
: new_sync_write():
: BUG_ON(ret == -EIOCBQUEUED);
0.00 : ffffffff812cf081: cmp $0xfffffffffffffdef,%rax
0.00 : ffffffff812cf087: jne ffffffff812cf033 <new_sync_write+0x113>
0.00 : ffffffff812cf089: ud2
Percent | Source code & Disassembly of vmlinux for cycles (113 samples, percent: local period)
----------------------------------------------------------------------------------------------------
:
:
:
: Disassembly of section .text:
:
: ffffffff814c6aa0 <_copy_from_iter>:
: _copy_from_iter():
: }
: EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
: #endif /* CONFIG_ARCH_HAS_COPY_MC */
:
: size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
: {
0.00 : ffffffff814c6aa0: push %rbp
7.07 : ffffffff814c6aa1: mov %rdx,%rax
0.00 : ffffffff814c6aa4: mov %rsp,%rbp
0.00 : ffffffff814c6aa7: push %r15
0.00 : ffffffff814c6aa9: push %r14
0.00 : ffffffff814c6aab: push %r13
3.54 : ffffffff814c6aad: push %r12
0.00 : ffffffff814c6aaf: push %rbx
0.00 : ffffffff814c6ab0: sub $0x50,%rsp
0.00 : ffffffff814c6ab4: mov %rdx,-0x78(%rbp)
: iov_iter_type():
: };
: };
:
: static inline enum iter_type iov_iter_type(const struct iov_iter *i)
: {
: return i->iter_type;
0.89 : ffffffff814c6ab8: movzbl (%rdx),%edx
: _copy_from_iter():
0.00 : ffffffff814c6abb: mov %rdi,-0x68(%rbp)
: if (unlikely(iov_iter_is_pipe(i))) {
0.00 : ffffffff814c6abf: cmp $0x3,%dl
0.00 : ffffffff814c6ac2: je ffffffff814c6bd6 <_copy_from_iter+0x136>
0.00 : ffffffff814c6ac8: mov %rax,%rdi
: WARN_ON(1);
: return 0;
: }
: if (iter_is_iovec(i))
: might_fault();
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6acb: mov 0x10(%rax),%rax
0.00 : ffffffff814c6acf: cmp %rsi,%rax
0.00 : ffffffff814c6ad2: cmovbe %rax,%rsi
2.65 : ffffffff814c6ad6: mov %rsi,%r13
0.00 : ffffffff814c6ad9: test %rsi,%rsi
3.52 : ffffffff814c6adc: je ffffffff814c6bdd <_copy_from_iter+0x13d>
1.76 : ffffffff814c6ae2: test %dl,%dl
0.00 : ffffffff814c6ae4: jne ffffffff814c6be2 <_copy_from_iter+0x142>
0.00 : ffffffff814c6aea: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6aee: mov 0x8(%rdi),%r14
0.00 : ffffffff814c6af2: xor %r15d,%r15d
0.00 : ffffffff814c6af5: mov -0x68(%rbp),%rdi
25.58 : ffffffff814c6af9: lea 0x10(%rax),%r12
0.00 : ffffffff814c6afd: jmp ffffffff814c6b0e <_copy_from_iter+0x6e>
0.00 : ffffffff814c6aff: mov -0x68(%rbp),%rax
0.00 : ffffffff814c6b03: lea (%rax,%r15,1),%rdi
0.00 : ffffffff814c6b07: add $0x10,%r12
: {
0.00 : ffffffff814c6b0b: xor %r14d,%r14d
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6b0e: mov -0x8(%r12),%rcx
1.09 : ffffffff814c6b13: lea -0x10(%r12),%rax
0.00 : ffffffff814c6b18: mov %r12,-0x60(%rbp)
0.00 : ffffffff814c6b1c: mov %rax,-0x70(%rbp)
0.00 : ffffffff814c6b20: mov %rcx,%rbx
0.00 : ffffffff814c6b23: sub %r14,%rbx
1.76 : ffffffff814c6b26: cmp %r13,%rbx
0.00 : ffffffff814c6b29: cmova %r13,%rbx
0.00 : ffffffff814c6b2d: test %rbx,%rbx
0.00 : ffffffff814c6b30: je ffffffff814c6b07 <_copy_from_iter+0x67>
0.00 : ffffffff814c6b32: mov -0x10(%r12),%rsi
0.00 : ffffffff814c6b37: mov %rbx,%rax
0.00 : ffffffff814c6b3a: add %r14,%rsi
: __chk_range_not_ok():
: */
: if (__builtin_constant_p(size))
: return unlikely(addr > limit - size);
:
: /* Arbitrary sizes? Be careful about overflow */
: addr += size;
0.00 : ffffffff814c6b3d: add %rsi,%rax
4.42 : ffffffff814c6b40: jb ffffffff814c6bd1 <_copy_from_iter+0x131>
: copyin():
: if (access_ok(from, n)) {
0.00 : ffffffff814c6b46: movabs $0x7ffffffff000,%rdx
0.00 : ffffffff814c6b50: cmp %rdx,%rax
3.52 : ffffffff814c6b53: ja ffffffff814c6bd1 <_copy_from_iter+0x131>
: copy_user_generic():
: /*
: * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
: * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
: * Otherwise, use copy_user_generic_unrolled.
: */
: alternative_call_2(copy_user_generic_unrolled,
0.00 : ffffffff814c6b55: mov %ebx,%edx
0.00 : ffffffff814c6b57: callq ffffffff81523880 <copy_user_generic_unrolled>
: _copy_from_iter():
: iterate_and_advance(i, bytes, base, len, off,
6.18 : ffffffff814c6b5c: mov -0x8(%r12),%rcx
: copy_user_generic():
: X86_FEATURE_ERMS,
: ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
: "=d" (len)),
: "1" (to), "2" (from), "3" (len)
: : "memory", "rcx", "r8", "r9", "r10", "r11");
: return ret;
0.00 : ffffffff814c6b61: mov %eax,%eax
: _copy_from_iter():
0.00 : ffffffff814c6b63: cltq
0.00 : ffffffff814c6b65: mov %rbx,%rdx
0.00 : ffffffff814c6b68: sub %rbx,%r13
0.00 : ffffffff814c6b6b: sub %rax,%rdx
0.00 : ffffffff814c6b6e: add %rax,%r13
0.00 : ffffffff814c6b71: add %rdx,%r15
3.53 : ffffffff814c6b74: add %r14,%rdx
0.00 : ffffffff814c6b77: cmp %rcx,%rdx
0.00 : ffffffff814c6b7a: jb ffffffff814c6bc4 <_copy_from_iter+0x124>
0.00 : ffffffff814c6b7c: test %r13,%r13
0.00 : ffffffff814c6b7f: jne ffffffff814c6aff <_copy_from_iter+0x5f>
0.00 : ffffffff814c6b85: mov -0x78(%rbp),%rcx
2.66 : ffffffff814c6b89: mov -0x60(%rbp),%rdi
2.65 : ffffffff814c6b8d: mov %rdi,%rax
0.00 : ffffffff814c6b90: sub 0x18(%rcx),%rax
11.54 : ffffffff814c6b94: mov %r13,0x8(%rcx)
0.00 : ffffffff814c6b98: mov %rdi,0x18(%rcx)
12.36 : ffffffff814c6b9c: mov %rcx,%rdi
0.00 : ffffffff814c6b9f: sar $0x4,%rax
0.00 : ffffffff814c6ba3: sub %rax,0x20(%rcx)
0.00 : ffffffff814c6ba7: mov 0x10(%rcx),%rax
0.00 : ffffffff814c6bab: sub %r15,%rax
0.00 : ffffffff814c6bae: mov %rax,0x10(%rdi)
: copyin(addr + off, base, len),
: memcpy(addr + off, base, len)
: )
:
: return bytes;
: }
3.53 : ffffffff814c6bb2: add $0x50,%rsp
0.00 : ffffffff814c6bb6: mov %r15,%rax
0.00 : ffffffff814c6bb9: pop %rbx
0.00 : ffffffff814c6bba: pop %r12
0.00 : ffffffff814c6bbc: pop %r13
0.00 : ffffffff814c6bbe: pop %r14
0.00 : ffffffff814c6bc0: pop %r15
1.76 : ffffffff814c6bc2: pop %rbp
0.00 : ffffffff814c6bc3: retq
0.00 : ffffffff814c6bc4: mov -0x70(%rbp),%rax
0.00 : ffffffff814c6bc8: mov %rdx,%r13
0.00 : ffffffff814c6bcb: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6bcf: jmp ffffffff814c6b85 <_copy_from_iter+0xe5>
: copyin():
0.00 : ffffffff814c6bd1: mov %rbx,%rax
0.00 : ffffffff814c6bd4: jmp ffffffff814c6b63 <_copy_from_iter+0xc3>
: _copy_from_iter():
: WARN_ON(1);
0.00 : ffffffff814c6bd6: ud2
: return 0;
0.00 : ffffffff814c6bd8: xor %r15d,%r15d
0.00 : ffffffff814c6bdb: jmp ffffffff814c6bb2 <_copy_from_iter+0x112>
0.00 : ffffffff814c6bdd: xor %r15d,%r15d
0.00 : ffffffff814c6be0: jmp ffffffff814c6bb2 <_copy_from_iter+0x112>
: iterate_and_advance(i, bytes, base, len, off,
0.00 : ffffffff814c6be2: cmp $0x2,%dl
0.00 : ffffffff814c6be5: je ffffffff814c6e09 <_copy_from_iter+0x369>
0.00 : ffffffff814c6beb: cmp $0x1,%dl
0.00 : ffffffff814c6bee: je ffffffff814c6d6b <_copy_from_iter+0x2cb>
0.00 : ffffffff814c6bf4: mov %rsi,%r15
0.00 : ffffffff814c6bf7: cmp $0x4,%dl
0.00 : ffffffff814c6bfa: jne ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6bfc: mov 0x8(%rdi),%rax
0.00 : ffffffff814c6c00: add 0x20(%rdi),%rax
0.00 : ffffffff814c6c04: movl $0x0,-0x48(%rbp)
0.00 : ffffffff814c6c0b: movq $0x3,-0x40(%rbp)
0.00 : ffffffff814c6c13: movq $0x0,-0x38(%rbp)
0.00 : ffffffff814c6c1b: movq $0x0,-0x30(%rbp)
0.00 : ffffffff814c6c23: mov %eax,%ebx
0.00 : ffffffff814c6c25: shr $0xc,%rax
0.00 : ffffffff814c6c29: mov %rax,%rcx
0.00 : ffffffff814c6c2c: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6c30: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6c34: and $0xfff,%ebx
0.00 : ffffffff814c6c3a: mov %rcx,-0x50(%rbp)
0.00 : ffffffff814c6c3e: mov %rax,-0x58(%rbp)
0.00 : ffffffff814c6c42: mov $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6c49: lea -0x58(%rbp),%rdi
0.00 : ffffffff814c6c4d: xor %r15d,%r15d
0.00 : ffffffff814c6c50: callq ffffffff8151fe40 <xas_find>
0.00 : ffffffff814c6c55: mov %rax,%r14
0.00 : ffffffff814c6c58: test %rax,%rax
0.00 : ffffffff814c6c5b: je ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6c61: mov %ebx,%r12d
: xas_retry():
: * Context: Any context.
: * Return: true if the operation needs to be retried.
: */
: static inline bool xas_retry(struct xa_state *xas, const void *entry)
: {
: if (xa_is_zero(entry))
0.00 : ffffffff814c6c64: cmp $0x406,%r14
0.00 : ffffffff814c6c6b: je ffffffff814c6d1c <_copy_from_iter+0x27c>
: return true;
: if (!xa_is_retry(entry))
0.00 : ffffffff814c6c71: cmp $0x402,%r14
0.00 : ffffffff814c6c78: je ffffffff814c6f7f <_copy_from_iter+0x4df>
: _copy_from_iter():
0.00 : ffffffff814c6c7e: test $0x1,%r14b
0.00 : ffffffff814c6c82: jne ffffffff814c6f78 <_copy_from_iter+0x4d8>
0.00 : ffffffff814c6c88: mov %r14,%rdi
0.00 : ffffffff814c6c8b: callq ffffffff81296c00 <PageHuge>
0.00 : ffffffff814c6c90: mov %eax,%ebx
0.00 : ffffffff814c6c92: test %eax,%eax
0.00 : ffffffff814c6c94: jne ffffffff814c6f30 <_copy_from_iter+0x490>
0.00 : ffffffff814c6c9a: mov -0x60(%rbp),%rdi
0.00 : ffffffff814c6c9e: mov 0x20(%r14),%rax
0.00 : ffffffff814c6ca2: mov %edi,%ecx
0.00 : ffffffff814c6ca4: sub %eax,%ecx
0.00 : ffffffff814c6ca6: cmp %rdi,%rax
0.00 : ffffffff814c6ca9: cmovb %ecx,%ebx
0.00 : ffffffff814c6cac: jmp ffffffff814c6d00 <_copy_from_iter+0x260>
0.00 : ffffffff814c6cae: mov %r12d,%eax
0.00 : ffffffff814c6cb1: mov $0x1000,%edx
0.00 : ffffffff814c6cb6: movslq %ebx,%rsi
0.00 : ffffffff814c6cb9: mov -0x68(%rbp),%rcx
0.00 : ffffffff814c6cbd: sub %rax,%rdx
0.00 : ffffffff814c6cc0: cmp %r13,%rdx
0.00 : ffffffff814c6cc3: cmova %r13,%rdx
0.00 : ffffffff814c6cc7: shl $0x6,%rsi
0.00 : ffffffff814c6ccb: add %r14,%rsi
: lowmem_page_address():
: */
: #include <linux/vmstat.h>
:
: static __always_inline void *lowmem_page_address(const struct page *page)
: {
: return page_to_virt(page);
0.00 : ffffffff814c6cce: sub 0xebda73(%rip),%rsi # ffffffff82384748 <vmemmap_base>
: _copy_from_iter():
0.00 : ffffffff814c6cd5: mov %rdx,%r12
0.00 : ffffffff814c6cd8: lea (%rcx,%r15,1),%rdi
0.00 : ffffffff814c6cdc: add %r12,%r15
: lowmem_page_address():
0.00 : ffffffff814c6cdf: sar $0x6,%rsi
0.00 : ffffffff814c6ce3: shl $0xc,%rsi
0.00 : ffffffff814c6ce7: add 0xebda6a(%rip),%rsi # ffffffff82384758 <page_offset_base>
: _copy_from_iter():
0.00 : ffffffff814c6cee: add %rax,%rsi
: memcpy():
: if (q_size < size)
: __read_overflow2();
: }
: if (p_size < size || q_size < size)
: fortify_panic(__func__);
: return __underlying_memcpy(p, q, size);
0.00 : ffffffff814c6cf1: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6cf6: sub %r12,%r13
0.00 : ffffffff814c6cf9: je ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6cfb: inc %ebx
0.00 : ffffffff814c6cfd: xor %r12d,%r12d
: constant_test_bit():
: }
:
: static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
: {
: return ((1UL << (nr & (BITS_PER_LONG-1))) &
: (addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
0.00 : ffffffff814c6d00: mov (%r14),%rax
0.00 : ffffffff814c6d03: shr $0x10,%rax
0.00 : ffffffff814c6d07: and $0x1,%eax
: thp_nr_pages():
: */
: static inline int thp_nr_pages(struct page *page)
: {
: VM_BUG_ON_PGFLAGS(PageTail(page), page);
: if (PageHead(page))
: return HPAGE_PMD_NR;
0.00 : ffffffff814c6d0a: cmp $0x1,%al
0.00 : ffffffff814c6d0c: sbb %eax,%eax
0.00 : ffffffff814c6d0e: and $0xfffffe01,%eax
0.00 : ffffffff814c6d13: add $0x200,%eax
: _copy_from_iter():
0.00 : ffffffff814c6d18: cmp %eax,%ebx
0.00 : ffffffff814c6d1a: jl ffffffff814c6cae <_copy_from_iter+0x20e>
: xas_next_entry():
: *
: * Return: The next present entry after the one currently referred to by @xas.
: */
: static inline void *xas_next_entry(struct xa_state *xas, unsigned long max)
: {
: struct xa_node *node = xas->xa_node;
0.00 : ffffffff814c6d1c: mov -0x40(%rbp),%rdi
: xas_not_node():
: return ((unsigned long)node & 3) || !node;
0.00 : ffffffff814c6d20: test $0x3,%dil
0.00 : ffffffff814c6d24: setne %cl
0.00 : ffffffff814c6d27: test %rdi,%rdi
0.00 : ffffffff814c6d2a: sete %al
0.00 : ffffffff814c6d2d: or %al,%cl
0.00 : ffffffff814c6d2f: je ffffffff814c6ecc <_copy_from_iter+0x42c>
: xas_next_entry():
: return xas_find(xas, max);
: if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
: return xas_find(xas, max);
: entry = xa_entry(xas->xa, node, xas->xa_offset + 1);
: if (unlikely(xa_is_internal(entry)))
: return xas_find(xas, max);
0.00 : ffffffff814c6d35: mov $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6d3c: lea -0x58(%rbp),%rdi
0.00 : ffffffff814c6d40: callq ffffffff8151fe40 <xas_find>
0.00 : ffffffff814c6d45: mov %rax,%r14
: _copy_from_iter():
0.00 : ffffffff814c6d48: test %rax,%rax
0.00 : ffffffff814c6d4b: jne ffffffff814c6c64 <_copy_from_iter+0x1c4>
: __rcu_read_unlock():
: }
:
: static inline void __rcu_read_unlock(void)
: {
: preempt_enable();
: rcu_read_unlock_strict();
0.00 : ffffffff814c6d51: callq ffffffff810e12c0 <rcu_read_unlock_strict>
: _copy_from_iter():
0.00 : ffffffff814c6d56: mov -0x78(%rbp),%rax
0.00 : ffffffff814c6d5a: mov -0x78(%rbp),%rdi
0.00 : ffffffff814c6d5e: add %r15,0x8(%rax)
0.00 : ffffffff814c6d62: mov 0x10(%rax),%rax
0.00 : ffffffff814c6d66: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6d6b: mov 0x18(%rdi),%rax
0.00 : ffffffff814c6d6f: xor %r15d,%r15d
0.00 : ffffffff814c6d72: mov 0x8(%rdi),%rbx
0.00 : ffffffff814c6d76: mov -0x68(%rbp),%rdi
0.00 : ffffffff814c6d7a: lea 0x10(%rax),%r12
0.00 : ffffffff814c6d7e: mov %r15,%rax
0.00 : ffffffff814c6d81: mov %r12,%r15
0.00 : ffffffff814c6d84: mov %rax,%r12
0.00 : ffffffff814c6d87: jmp ffffffff814c6d97 <_copy_from_iter+0x2f7>
0.00 : ffffffff814c6d89: mov -0x68(%rbp),%rax
0.00 : ffffffff814c6d8d: lea (%rax,%r12,1),%rdi
0.00 : ffffffff814c6d91: add $0x10,%r15
0.00 : ffffffff814c6d95: xor %ebx,%ebx
0.00 : ffffffff814c6d97: mov -0x8(%r15),%r14
0.00 : ffffffff814c6d9b: lea -0x10(%r15),%rax
0.00 : ffffffff814c6d9f: mov %r15,-0x60(%rbp)
0.00 : ffffffff814c6da3: mov %rax,-0x70(%rbp)
0.00 : ffffffff814c6da7: sub %rbx,%r14
0.00 : ffffffff814c6daa: cmp %r13,%r14
0.00 : ffffffff814c6dad: cmova %r13,%r14
0.00 : ffffffff814c6db1: test %r14,%r14
0.00 : ffffffff814c6db4: je ffffffff814c6d91 <_copy_from_iter+0x2f1>
0.00 : ffffffff814c6db6: mov -0x10(%r15),%rsi
: memcpy():
0.00 : ffffffff814c6dba: mov %r14,%rdx
: _copy_from_iter():
0.00 : ffffffff814c6dbd: add %r14,%r12
0.00 : ffffffff814c6dc0: sub %r14,%r13
0.00 : ffffffff814c6dc3: add %rbx,%rsi
: memcpy():
0.00 : ffffffff814c6dc6: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6dcb: lea (%rbx,%r14,1),%rcx
0.00 : ffffffff814c6dcf: cmp %rcx,-0x8(%r15)
0.00 : ffffffff814c6dd3: ja ffffffff814c6eb9 <_copy_from_iter+0x419>
0.00 : ffffffff814c6dd9: test %r13,%r13
0.00 : ffffffff814c6ddc: jne ffffffff814c6d89 <_copy_from_iter+0x2e9>
0.00 : ffffffff814c6dde: mov %r12,%r15
0.00 : ffffffff814c6de1: mov -0x78(%rbp),%rdi
0.00 : ffffffff814c6de5: mov -0x60(%rbp),%rcx
0.00 : ffffffff814c6de9: mov %rcx,%rax
0.00 : ffffffff814c6dec: sub 0x18(%rdi),%rax
0.00 : ffffffff814c6df0: mov %r13,0x8(%rdi)
0.00 : ffffffff814c6df4: mov %rcx,0x18(%rdi)
0.00 : ffffffff814c6df8: sar $0x4,%rax
0.00 : ffffffff814c6dfc: sub %rax,0x20(%rdi)
0.00 : ffffffff814c6e00: mov 0x10(%rdi),%rax
0.00 : ffffffff814c6e04: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6e09: mov 0x18(%rdi),%r14
0.00 : ffffffff814c6e0d: mov 0x8(%rdi),%r12d
0.00 : ffffffff814c6e11: xor %r15d,%r15d
0.00 : ffffffff814c6e14: mov 0xc(%r14),%eax
0.00 : ffffffff814c6e18: mov 0x8(%r14),%edx
0.00 : ffffffff814c6e1c: mov $0x1000,%esi
0.00 : ffffffff814c6e21: mov -0x68(%rbp),%rdi
0.00 : ffffffff814c6e25: add %r12d,%eax
0.00 : ffffffff814c6e28: sub %r12d,%edx
0.00 : ffffffff814c6e2b: mov %eax,%ecx
0.00 : ffffffff814c6e2d: and $0xfff,%ecx
0.00 : ffffffff814c6e33: cmp %r13,%rdx
0.00 : ffffffff814c6e36: cmova %r13,%rdx
0.00 : ffffffff814c6e3a: sub %rcx,%rsi
0.00 : ffffffff814c6e3d: cmp %rsi,%rdx
0.00 : ffffffff814c6e40: cmovbe %rdx,%rsi
0.00 : ffffffff814c6e44: shr $0xc,%eax
0.00 : ffffffff814c6e47: add %r15,%rdi
0.00 : ffffffff814c6e4a: mov %rsi,%rbx
0.00 : ffffffff814c6e4d: mov %eax,%esi
0.00 : ffffffff814c6e4f: shl $0x6,%rsi
0.00 : ffffffff814c6e53: add (%r14),%rsi
: memcpy():
0.00 : ffffffff814c6e56: mov %rbx,%rdx
: _copy_from_iter():
0.00 : ffffffff814c6e59: add %rbx,%r15
: lowmem_page_address():
0.00 : ffffffff814c6e5c: sub 0xebd8e5(%rip),%rsi # ffffffff82384748 <vmemmap_base>
: _copy_from_iter():
0.00 : ffffffff814c6e63: add %ebx,%r12d
: lowmem_page_address():
0.00 : ffffffff814c6e66: sar $0x6,%rsi
0.00 : ffffffff814c6e6a: shl $0xc,%rsi
0.00 : ffffffff814c6e6e: add 0xebd8e3(%rip),%rsi # ffffffff82384758 <page_offset_base>
: _copy_from_iter():
0.00 : ffffffff814c6e75: add %rcx,%rsi
: memcpy():
0.00 : ffffffff814c6e78: callq ffffffff81a22620 <__memcpy>
: _copy_from_iter():
0.00 : ffffffff814c6e7d: cmp %r12d,0x8(%r14)
0.00 : ffffffff814c6e81: jne ffffffff814c6e8a <_copy_from_iter+0x3ea>
0.00 : ffffffff814c6e83: add $0x10,%r14
0.00 : ffffffff814c6e87: xor %r12d,%r12d
0.00 : ffffffff814c6e8a: sub %rbx,%r13
0.00 : ffffffff814c6e8d: jne ffffffff814c6e14 <_copy_from_iter+0x374>
0.00 : ffffffff814c6e8f: mov -0x78(%rbp),%rcx
0.00 : ffffffff814c6e93: mov %r12d,%eax
0.00 : ffffffff814c6e96: mov %rax,0x8(%rcx)
0.00 : ffffffff814c6e9a: mov %r14,%rax
0.00 : ffffffff814c6e9d: sub 0x18(%rcx),%rax
0.00 : ffffffff814c6ea1: mov %rcx,%rdi
0.00 : ffffffff814c6ea4: mov %r14,0x18(%rcx)
0.00 : ffffffff814c6ea8: sar $0x4,%rax
0.00 : ffffffff814c6eac: sub %rax,0x20(%rcx)
0.00 : ffffffff814c6eb0: mov 0x10(%rcx),%rax
0.00 : ffffffff814c6eb4: jmpq ffffffff814c6bab <_copy_from_iter+0x10b>
0.00 : ffffffff814c6eb9: mov -0x70(%rbp),%rax
0.00 : ffffffff814c6ebd: mov %r12,%r15
0.00 : ffffffff814c6ec0: mov %rcx,%r13
0.00 : ffffffff814c6ec3: mov %rax,-0x60(%rbp)
0.00 : ffffffff814c6ec7: jmpq ffffffff814c6de1 <_copy_from_iter+0x341>
: xas_next_entry():
: if (unlikely(xas_not_node(node) || node->shift ||
0.00 : ffffffff814c6ecc: cmpb $0x0,(%rdi)
0.00 : ffffffff814c6ecf: jne ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6ed5: mov -0x50(%rbp),%rsi
0.00 : ffffffff814c6ed9: movzbl -0x46(%rbp),%r9d
0.00 : ffffffff814c6ede: mov %rsi,%r8
0.00 : ffffffff814c6ee1: mov %r9,%rax
0.00 : ffffffff814c6ee4: and $0x3f,%r8d
0.00 : ffffffff814c6ee8: cmp %r8,%r9
0.00 : ffffffff814c6eeb: jne ffffffff814c6d35 <_copy_from_iter+0x295>
: if (unlikely(xas->xa_index >= max))
0.00 : ffffffff814c6ef1: cmp $0xffffffffffffffff,%rsi
0.00 : ffffffff814c6ef5: je ffffffff814c6f60 <_copy_from_iter+0x4c0>
: if (unlikely(xas->xa_offset == XA_CHUNK_MASK))
0.00 : ffffffff814c6ef7: cmp $0x3f,%al
0.00 : ffffffff814c6ef9: je ffffffff814c6f4b <_copy_from_iter+0x4ab>
: entry = xa_entry(xas->xa, node, xas->xa_offset + 1);
0.00 : ffffffff814c6efb: movzbl %al,%r8d
: xa_entry():
: return rcu_dereference_check(node->slots[offset],
0.00 : ffffffff814c6eff: add $0x5,%r8
0.00 : ffffffff814c6f03: mov 0x8(%rdi,%r8,8),%r14
: xa_is_internal():
: return ((unsigned long)entry & 3) == 2;
0.00 : ffffffff814c6f08: mov %r14,%r8
0.00 : ffffffff814c6f0b: and $0x3,%r8d
: xas_next_entry():
: if (unlikely(xa_is_internal(entry)))
0.00 : ffffffff814c6f0f: cmp $0x2,%r8
0.00 : ffffffff814c6f13: je ffffffff814c6f37 <_copy_from_iter+0x497>
: xas->xa_offset++;
0.00 : ffffffff814c6f15: inc %eax
: xas->xa_index++;
0.00 : ffffffff814c6f17: inc %rsi
: } while (!entry);
0.00 : ffffffff814c6f1a: mov $0x1,%ecx
0.00 : ffffffff814c6f1f: test %r14,%r14
0.00 : ffffffff814c6f22: je ffffffff814c6ef1 <_copy_from_iter+0x451>
0.00 : ffffffff814c6f24: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f27: mov %rsi,-0x50(%rbp)
0.00 : ffffffff814c6f2b: jmpq ffffffff814c6c64 <_copy_from_iter+0x1c4>
: _copy_from_iter():
0.00 : ffffffff814c6f30: ud2
0.00 : ffffffff814c6f32: jmpq ffffffff814c6d51 <_copy_from_iter+0x2b1>
0.00 : ffffffff814c6f37: test %cl,%cl
0.00 : ffffffff814c6f39: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f3f: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f42: mov %rsi,-0x50(%rbp)
0.00 : ffffffff814c6f46: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f4b: test %cl,%cl
0.00 : ffffffff814c6f4d: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f53: movb $0x3f,-0x46(%rbp)
0.00 : ffffffff814c6f57: mov %rsi,-0x50(%rbp)
: xas_next_entry():
: return xas_find(xas, max);
0.00 : ffffffff814c6f5b: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f60: test %cl,%cl
0.00 : ffffffff814c6f62: je ffffffff814c6d35 <_copy_from_iter+0x295>
0.00 : ffffffff814c6f68: mov %al,-0x46(%rbp)
0.00 : ffffffff814c6f6b: movq $0xffffffffffffffff,-0x50(%rbp)
: return xas_find(xas, max);
0.00 : ffffffff814c6f73: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
: _copy_from_iter():
0.00 : ffffffff814c6f78: ud2
0.00 : ffffffff814c6f7a: jmpq ffffffff814c6d51 <_copy_from_iter+0x2b1>
: xas_reset():
: xas->xa_node = XAS_RESTART;
0.00 : ffffffff814c6f7f: movq $0x3,-0x40(%rbp)
: xas_not_node():
: return ((unsigned long)node & 3) || !node;
0.00 : ffffffff814c6f87: jmpq ffffffff814c6d35 <_copy_from_iter+0x295>
