On Fri, Aug 19, 2016 at 3:11 PM, Linus Torvalds
<torvalds@xxxxxxxxxxxxxxxxxxxx> wrote:
>>
>> (I have some experimental patches that actually use "asm goto" in
>> "unsafe_put_user()" to get that nice code generation, but they only
>> work if your gcc version supports "asm goto", which some older
>> versions of gcc does not)
>
> Since you actually are looking at the user access stuff, I'll just put
> them here.
Here's an updated patch that applies on current git and that actually
uses this for filldir() (but not signal handling).
It turns out that on Skylake, which supports SMAP, the clac/stac
instructions are quite slow, and doing them for each access makes
things insanely much slower than it could be. And "filldir" does the
user accesses one by one (except for the name copying), and is
actually somewhat common under some loads (ie the "find . -name XYZ"
kind of thing).
Anyway, the asm coming out of gcc looks nasty, because it has all the
ugly section stuiff and fixups for SMAP not existing on some CPU's
etc. So the resulting fs/readdir.s file is hard to read. But if you
look at the disassembly at the object file that hides all that (and
shows what the end result actually is), the actual filldir user
accesses end up looking beautiful, with no extra code anywhere. An
exception just goes to the EFAULT handling directly.
Sadly, unsafe_get_user() looking as good does require gcc improvements
that aren't imminent.
This patch is untested, although the earlier original pre-rebased
version of it actually got a fair amount of testing on my machine
(including the filldir use)
Linus
arch/x86/include/asm/uaccess.h | 80 ++++++++++++++++++++----------------------
fs/readdir.c | 22 ++++++------
2 files changed, 51 insertions(+), 51 deletions(-)
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index a0ae610b9280..8d6b299522f1 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -204,19 +204,14 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
#ifdef CONFIG_X86_32
-#define __put_user_asm_u64(x, addr, err, errret) \
- asm volatile("\n" \
- "1: movl %%eax,0(%2)\n" \
- "2: movl %%edx,4(%2)\n" \
- "3:" \
- ".section .fixup,\"ax\"\n" \
- "4: movl %3,%0\n" \
- " jmp 3b\n" \
- ".previous\n" \
- _ASM_EXTABLE(1b, 4b) \
- _ASM_EXTABLE(2b, 4b) \
- : "=r" (err) \
- : "A" (x), "r" (addr), "i" (errret), "0" (err))
+#define __put_user_goto_u64(x, addr, label) \
+ asm volatile("\n" \
+ "1: movl %%eax,0(%2)\n" \
+ "2: movl %%edx,4(%2)\n" \
+ _ASM_EXTABLE(1b, %2l) \
+ _ASM_EXTABLE(2b, %2l) \
+ : : "A" (x), "r" (addr) \
+ : : label)
#define __put_user_asm_ex_u64(x, addr) \
asm volatile("\n" \
@@ -231,8 +226,8 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
asm volatile("call __put_user_8" : "=a" (__ret_pu) \
: "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
#else
-#define __put_user_asm_u64(x, ptr, retval, errret) \
- __put_user_asm(x, ptr, retval, "q", "", "er", errret)
+#define __put_user_goto_u64(x, ptr, label) \
+ __put_user_goto(x, ptr, "q", "", "er", label)
#define __put_user_asm_ex_u64(x, addr) \
__put_user_asm_ex(x, addr, "q", "", "er")
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
@@ -293,23 +288,21 @@ extern void __put_user_8(void);
__builtin_expect(__ret_pu, 0); \
})
-#define __put_user_size(x, ptr, size, retval, errret) \
+#define __put_user_size(x, ptr, size, label) \
do { \
- retval = 0; \
__chk_user_ptr(ptr); \
switch (size) { \
case 1: \
- __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
+ __put_user_goto(x, ptr, "b", "b", "iq", label); \
break; \
case 2: \
- __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
+ __put_user_goto(x, ptr, "w", "w", "ir", label); \
break; \
case 4: \
- __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
+ __put_user_goto(x, ptr, "l", "k", "ir", label); \
break; \
case 8: \
- __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
- errret); \
+ __put_user_goto_u64((__typeof__(*ptr))(x), ptr, label); \
break; \
default: \
__put_user_bad(); \
@@ -438,9 +431,12 @@ do { \
#define __put_user_nocheck(x, ptr, size) \
({ \
- int __pu_err; \
+ __label__ __pu_label; \
+ int __pu_err = -EFAULT; \
__uaccess_begin(); \
- __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
+ __put_user_size((x), (ptr), (size), __pu_label); \
+ __pu_err = 0; \
+__pu_label: \
__uaccess_end(); \
__builtin_expect(__pu_err, 0); \
})
@@ -465,17 +461,23 @@ struct __large_struct { unsigned long buf[100]; };
* we do not write to any memory gcc knows about, so there are no
* aliasing issues.
*/
-#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
- asm volatile("\n" \
- "1: mov"itype" %"rtype"1,%2\n" \
- "2:\n" \
- ".section .fixup,\"ax\"\n" \
- "3: mov %3,%0\n" \
- " jmp 2b\n" \
- ".previous\n" \
- _ASM_EXTABLE(1b, 3b) \
- : "=r"(err) \
- : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
+#define __put_user_goto(x, addr, itype, rtype, ltype, label) \
+ asm volatile goto("\n" \
+ "1: mov"itype" %"rtype"0,%1\n" \
+ _ASM_EXTABLE(1b, %l2) \
+ : : ltype(x), "m" (__m(addr)) \
+ : : label)
+
+#define __put_user_failed(x, addr, itype, rtype, ltype, errret) \
+ ({ __label__ __puflab; \
+ int __pufret = errret; \
+ __put_user_goto(x,addr,itype,rtype,ltype,__puflab); \
+ __pufret = 0; \
+ __puflab: __pufret; })
+
+#define __put_user_asm(x, addr, retval, itype, rtype, ltype, errret) do { \
+ retval = __put_user_failed(x, addr, itype, rtype, ltype, errret); \
+} while (0)
#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
asm volatile("1: mov"itype" %"rtype"0,%1\n" \
@@ -814,12 +816,8 @@ copy_to_user(void __user *to, const void *from, unsigned long n)
#define user_access_begin() __uaccess_begin()
#define user_access_end() __uaccess_end()
-#define unsafe_put_user(x, ptr, err_label) \
-do { \
- int __pu_err; \
- __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \
- if (unlikely(__pu_err)) goto err_label; \
-} while (0)
+#define unsafe_put_user(x, ptr, label) \
+ __put_user_size((x), (ptr), sizeof(*(ptr)), label)
#define unsafe_get_user(x, ptr, err_label) \
do { \
diff --git a/fs/readdir.c b/fs/readdir.c
index 9d0212c374d6..03324f54c0e9 100644
--- a/fs/readdir.c
+++ b/fs/readdir.c
@@ -184,25 +184,27 @@ static int filldir(struct dir_context *ctx, const char *name, int namlen,
if (dirent) {
if (signal_pending(current))
return -EINTR;
- if (__put_user(offset, &dirent->d_off))
- goto efault;
}
+
+ user_access_begin();
+ if (dirent)
+ unsafe_put_user(offset, &dirent->d_off, efault_end);
dirent = buf->current_dir;
- if (__put_user(d_ino, &dirent->d_ino))
- goto efault;
- if (__put_user(reclen, &dirent->d_reclen))
- goto efault;
+ unsafe_put_user(d_ino, &dirent->d_ino, efault_end);
+ unsafe_put_user(reclen, &dirent->d_reclen, efault_end);
+ unsafe_put_user(0, dirent->d_name + namlen, efault_end);
+ unsafe_put_user(d_type, (char __user *) dirent + reclen - 1, efault_end);
+ user_access_end();
+
if (copy_to_user(dirent->d_name, name, namlen))
goto efault;
- if (__put_user(0, dirent->d_name + namlen))
- goto efault;
- if (__put_user(d_type, (char __user *) dirent + reclen - 1))
- goto efault;
buf->previous = dirent;
dirent = (void __user *)dirent + reclen;
buf->current_dir = dirent;
buf->count -= reclen;
return 0;
+efault_end:
+ user_access_end();
efault:
buf->error = -EFAULT;
return -EFAULT;
