在 2025/1/17 12:43, Xin Li 写道:
On 1/16/2025 8:19 PM, Ethan Zhao wrote:
在 2025/1/17 9:21, H. Peter Anvin 写道:
On 1/16/25 16:37, Ethan Zhao wrote:
hpa suggested to introduce "switch_likely" for this kind of
optimization
on a switch statement, which is also easier to read. I measured
it with
a user space focus test, it does improve performance a lot. But
obviously there are still a lot of work to do.
Find a way to instruct compiler to pick the right hot branch
meanwhile make folks
reading happy... yup, a lot of work.
It's not that complicated, believe it or not.
/*
* switch(v) biased for speed in the case v == l
*
* Note: gcc is quite sensitive to the exact form of this
* expression.
*/
#define switch_likely(v,l) \
switch((__typeof__(v))__builtin_expect((v),(l)))
I tried this macro as following, but got something really *weird*
from gcc.
+#define switch_likely(v,l) \
+ switch((__typeof__(v))__builtin_expect((v),(l)))
+
__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
+ unsigned short etype = regs->fred_ss.type & 0xf;
/* Invalidate orig_ax so that syscall_get_nr() works
correctly */
regs->orig_ax = -1;
- switch (regs->fred_ss.type) {
+ switch_likely ((etype == EVENT_TYPE_EXTINT || etype ==
EVENT_TYPE_OTHER), etype) {
Just swap the 2 arguments, and it should be:
+ switch_likely (etype, EVENT_TYPE_OTHER) {
after swapped the parameters as following:
+#define switch_likely(v,l) \
+ switch((__typeof__(v))__builtin_expect((v),(l)))
+
__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
+ unsigned short etype = regs->fred_ss.type & 0xf;
/* Invalidate orig_ax so that syscall_get_nr() works correctly */
regs->orig_ax = -1;
- switch (regs->fred_ss.type) {
+ switch_likely (etype, (EVENT_TYPE_EXTINT == etype ||
EVENT_TYPE_OTHER == etype)) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
@@ -256,11 +260,12 @@ __visible noinstr void fred_entry_from_user(struct
pt_regs *regs)
__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
+ unsigned short etype = regs->fred_ss.type & 0xf;
/* Invalidate orig_ax so that syscall_get_nr() works correctly */
regs->orig_ax = -1;
- switch (regs->fred_ss.type) {
+ switch_likely (etype, EVENT_TYPE_EXTINT == etype) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
Good news -- the gcc works normal, generated right asm code,
Bad news -- it fails back to binary search method to do matching,
ignored our hints.
$objdump -d vmlinux.o | awk '/<fred_entry_from_kernel>:/{c=65} c&&c--'
00000000000016b0 <fred_entry_from_kernel>:
16b0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax
16b7: 48 8b 77 78 mov 0x78(%rdi),%rsi
16bb: 55 push %rbp
16bc: 48 c7 47 78 ff ff ff movq $0xffffffffffffffff,0x78(%rdi)
16c3: ff
16c4: 83 e0 0f and $0xf,%eax
16c7: 48 89 e5 mov %rsp,%rbp
16ca: 3c 03 cmp $0x3,%al
16cc: 74 3c je 170a <fred_entry_from_kernel+0x5a>
16ce: 76 13 jbe 16e3 <fred_entry_from_kernel+0x33>
16d0: 3c 05 cmp $0x5,%al
16d2: 74 41 je 1715 <fred_entry_from_kernel+0x65>
16d4: 3c 06 cmp $0x6,%al
16d6: 75 27 jne 16ff <fred_entry_from_kernel+0x4f>
16d8: e8 73 fe ff ff callq 1550 <fred_swexc>
16dd: 5d pop %rbp
16de: e9 00 00 00 00 jmpq 16e3 <fred_entry_from_kernel+0x33>
16e3: 84 c0 test %al,%al
16e5: 74 42 je 1729 <fred_entry_from_kernel+0x79>
16e7: 3c 02 cmp $0x2,%al
16e9: 75 14 jne 16ff <fred_entry_from_kernel+0x4f>
16eb: 80 bf a4 00 00 00 02 cmpb $0x2,0xa4(%rdi)
16f2: 75 0b jne 16ff <fred_entry_from_kernel+0x4f>
16f4: e8 00 00 00 00 callq 16f9 <fred_entry_from_kernel+0x49>
16f9: 5d pop %rbp
16fa: e9 00 00 00 00 jmpq 16ff <fred_entry_from_kernel+0x4f>
16ff: e8 1c fc ff ff callq 1320 <fred_bad_type>
1704: 5d pop %rbp
1705: e9 00 00 00 00 jmpq 170a <fred_entry_from_kernel+0x5a>
170a: e8 21 fd ff ff callq 1430 <fred_hwexc>
170f: 5d pop %rbp
1710: e9 00 00 00 00 jmpq 1715 <fred_entry_from_kernel+0x65>
1715: 80 bf a4 00 00 00 01 cmpb $0x1,0xa4(%rdi)
171c: 75 e1 jne 16ff <fred_entry_from_kernel+0x4f>
171e: e8 00 00 00 00 callq 1723 <fred_entry_from_kernel+0x73>
1723: 5d pop %rbp
1724: e9 00 00 00 00 jmpq 1729 <fred_entry_from_kernel+0x79>
1729: e8 12 fb ff ff callq 1240 <fred_extint>
172e: 5d pop %rbp
172f: e9 00 00 00 00 jmpq 1734 <fred_entry_from_kernel+0x84>
1734: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1)
173b: 00 00 00 00
173f: 90 nop
$ objdump -d vmlinux.o | awk '/<fred_entry_from_user>:/{c=65} c&&c--'
00000000000015a0 <fred_entry_from_user>:
15a0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax
15a7: 48 8b 77 78 mov 0x78(%rdi),%rsi
15ab: 55 push %rbp
15ac: 48 c7 47 78 ff ff ff movq $0xffffffffffffffff,0x78(%rdi)
15b3: ff
15b4: 83 e0 0f and $0xf,%eax
15b7: 48 89 e5 mov %rsp,%rbp
15ba: 3c 04 cmp $0x4,%al
15bc: 74 78 je 1636 <fred_entry_from_user+0x96>
15be: 77 15 ja 15d5 <fred_entry_from_user+0x35>
15c0: 3c 02 cmp $0x2,%al
15c2: 74 53 je 1617 <fred_entry_from_user+0x77>
15c4: 77 65 ja 162b <fred_entry_from_user+0x8b>
15c6: 84 c0 test %al,%al
15c8: 75 42 jne 160c <fred_entry_from_user+0x6c>
15ca: e8 71 fc ff ff callq 1240 <fred_extint>
15cf: 5d pop %rbp
15d0: e9 00 00 00 00 jmpq 15d5 <fred_entry_from_user+0x35>
15d5: 3c 06 cmp $0x6,%al
15d7: 74 7c je 1655 <fred_entry_from_user+0xb5>
15d9: 72 66 jb 1641 <fred_entry_from_user+0xa1>
15db: 3c 07 cmp $0x7,%al
15dd: 75 2d jne 160c <fred_entry_from_user+0x6c>
15df: 8b 87 a4 00 00 00 mov 0xa4(%rdi),%eax
15e5: 25 ff 00 00 02 and $0x20000ff,%eax
15ea: 3d 01 00 00 02 cmp $0x2000001,%eax
15ef: 75 6f jne 1660 <fred_entry_from_user+0xc0>
15f1: 48 8b 77 50 mov 0x50(%rdi),%rsi
15f5: 48 c7 47 50 da ff ff movq $0xffffffffffffffda,0x50(%rdi)
15fc: ff
15fd: 48 89 77 78 mov %rsi,0x78(%rdi)
1601: e8 00 00 00 00 callq 1606 <fred_entry_from_user+0x66>
1606: 5d pop %rbp
1607: e9 00 00 00 00 jmpq 160c <fred_entry_from_user+0x6c>
160c: e8 0f fd ff ff callq 1320 <fred_bad_type>
1611: 5d pop %rbp
1612: e9 00 00 00 00 jmpq 1617 <fred_entry_from_user+0x77>
1617: 80 bf a4 00 00 00 02 cmpb $0x2,0xa4(%rdi)
161e: 75 ec jne 160c <fred_entry_from_user+0x6c>
1620: e8 00 00 00 00 callq 1625 <fred_entry_from_user+0x85>
1625: 5d pop %rbp
1626: e9 00 00 00 00 jmpq 162b <fred_entry_from_user+0x8b>
162b: e8 00 fe ff ff callq 1430 <fred_hwexc>
1630: 5d pop %rbp
1631: e9 00 00 00 00 jmpq 1636 <fred_entry_from_user+0x96>
1636: e8 85 fc ff ff callq 12c0 <fred_intx>
163b: 5d pop %rbp
163c: e9 00 00 00 00 jmpq 1641 <fred_entry_from_user+0xa1>
1641: 80 bf a4 00 00 00 01 cmpb $0x1,0xa4(%rdi)
1648: 75 c2 jne 160c <fred_entry_from_user+0x6c>
164a: e8 00 00 00 00 callq 164f <fred_entry_from_user+0xaf>
164f: 5d pop %rbp
1650: e9 00 00 00 00 jmpq 1655 <fred_entry_from_user+0xb5>
1655: e8 f6 fe ff ff callq 1550 <fred_swexc>
165a: 5d pop %rbp
165b: e9 00 00 00 00 jmpq 1660 <fred_entry_from_user+0xc0>
1660: 83 f8 02 cmp $0x2,%eax
1663: 75 24 jne 1689 <fred_entry_from_user+0xe9>
1665: 80 3d 00 00 00 00 00 cmpb $0x0,0x0(%rip) #
166c <fred_entry_from_user+0xcc>
166c: 74 1b je 1689 <fred_entry_from_user+0xe9>
166e: 48 8b 47 50 mov 0x50(%rdi),%rax
1672: 48 c7 47 50 da ff ff movq $0xffffffffffffffda,0x50(%rdi)
1679: ff
167a: 48 89 47 78 mov %rax,0x78(%rdi)
In short, seems that __builtin_expect not work with switch(), at least for
gcc version 8.5.0 20210514(RHEL).
Thanks，
Ethan
Probably also check __builtin_expect on
https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html.
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
@@ -256,11 +260,12 @@ __visible noinstr void
fred_entry_from_user(struct pt_regs *regs)
__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;
+ unsigned short etype = regs->fred_ss.type & 0xf;
/* Invalidate orig_ax so that syscall_get_nr() works
correctly */
regs->orig_ax = -1;
- switch (regs->fred_ss.type) {
+ switch_likely (etype == EVENT_TYPE_EXTINT, etype) {
case EVENT_TYPE_EXTINT:
return fred_extint(regs);
case EVENT_TYPE_NMI:
Got the asm code as following:
objdump -d vmlinux.o | awk '/<fred_entry_from_user>:/{c=65} c&&c--'
00000000000015a0 <fred_entry_from_user>:
15a0: 0f b6 87 a6 00 00 00 movzbl 0xa6(%rdi),%eax
15a7: 48 8b 77 78 mov 0x78(%rdi),%rsi
15ab: 55 push %rbp
15ac: 48 c7 47 78 ff ff ff movq
$0xffffffffffffffff,0x78(%rdi)
15b3: ff
15b4: 48 89 e5 mov %rsp,%rbp
15b7: 66 83 e0 0f and $0xf,%ax
15bb: 74 11 je 15ce <fred_entry_from_user+0x2e>
15bd: 66 83 f8 07 cmp $0x7,%ax
15c1: 74 0b je 15ce <fred_entry_from_user+0x2e>
15c3: e8 78 fc ff ff callq 1240 <fred_extint>
15c8: 5d pop %rbp
15c9: e9 00 00 00 00 jmpq 15ce <fred_entry_from_user+0x2e>
15ce: e8 4d fd ff ff callq 1320 <fred_bad_type>
15d3: 5d pop %rbp
15d4: e9 00 00 00 00 jmpq 15d9 <fred_entry_from_user+0x39>
15d9: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
00000000000015e0 <__pfx_fred_entry_from_kernel>:
15e0: 90 nop
15e1: 90 nop
00000000000015f0 <fred_entry_from_kernel>:
15f0: 55 push %rbp
15f1: 48 8b 77 78 mov 0x78(%rdi),%rsi
15f5: 48 c7 47 78 ff ff ff movq
$0xffffffffffffffff,0x78(%rdi)
15fc: ff
15fd: 48 89 e5 mov %rsp,%rbp
1600: f6 87 a6 00 00 00 0f testb $0xf,0xa6(%rdi)
1607: 75 0b jne 1614 <fred_entry_from_kernel+0x24>
1609: e8 12 fd ff ff callq 1320 <fred_bad_type>
160e: 5d pop %rbp
160f: e9 00 00 00 00 jmpq 1614
<fred_entry_from_kernel+0x24>
1614: e8 27 fc ff ff callq 1240 <fred_extint>
1619: 5d pop %rbp
161a: e9 00 00 00 00 jmpq 161f
<fred_entry_from_kernel+0x2f>
161f: 90 nop
0000000000001620 <__pfx___fred_entry_from_kvm>:
1620: 90 nop
1621: 90 nop
Even the fred_entry_from_kernel() asm code doesn't look right.
*gcc version 8.5.0 20210514 (Red Hat 8.5.0-10) (GCC)*
**
*Did I screw up something ?*
**
*Thanks,*
*Ethan*
-hpa
--
"firm, enduring, strong, and long-lived"
