On Thu, Aug 29, 2019 at 05:03:53PM +0530, Viresh Kumar wrote:
> From: Will Deacon <will.deacon@xxxxxxx>
>
> commit 84624087dd7e3b482b7b11c170ebc1f329b3a218 upstream.
>
> access_ok isn't an expensive operation once the addr_limit for the current
> thread has been loaded into the cache. Given that the initial access_ok
> check preceding a sequence of __{get,put}_user operations will take
> the brunt of the miss, we can make the __* variants identical to the
> full-fat versions, which brings with it the benefits of address masking.
>
> The likely cost in these sequences will be from toggling PAN/UAO, which
> we can address later by implementing the *_unsafe versions.
>
> Reviewed-by: Robin Murphy <robin.murphy@xxxxxxx>
> Signed-off-by: Will Deacon <will.deacon@xxxxxxx>
> Signed-off-by: Catalin Marinas <catalin.marinas@xxxxxxx>
> [ v4.4: Fixed conflicts around {__get_user|__put_user}_unaligned macros ]
> Signed-off-by: Viresh Kumar <viresh.kumar@xxxxxxxxxx>
> ---
Reviewed-by: Mark Rutland <mark.rutland@xxxxxxx> [v4.4 backport]
Mark.
> arch/arm64/include/asm/uaccess.h | 62 ++++++++++++++++++--------------
> 1 file changed, 36 insertions(+), 26 deletions(-)
>
> diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
> index fc11c50af558..a34324436ce1 100644
> --- a/arch/arm64/include/asm/uaccess.h
> +++ b/arch/arm64/include/asm/uaccess.h
> @@ -200,30 +200,35 @@ do { \
> CONFIG_ARM64_PAN)); \
> } while (0)
>
> -#define __get_user(x, ptr) \
> +#define __get_user_check(x, ptr, err) \
> ({ \
> - int __gu_err = 0; \
> - __get_user_err((x), (ptr), __gu_err); \
> - __gu_err; \
> + __typeof__(*(ptr)) __user *__p = (ptr); \
> + might_fault(); \
> + if (access_ok(VERIFY_READ, __p, sizeof(*__p))) { \
> + __p = uaccess_mask_ptr(__p); \
> + __get_user_err((x), __p, (err)); \
> + } else { \
> + (x) = 0; (err) = -EFAULT; \
> + } \
> })
>
> #define __get_user_error(x, ptr, err) \
> ({ \
> - __get_user_err((x), (ptr), (err)); \
> + __get_user_check((x), (ptr), (err)); \
> (void)0; \
> })
>
> -#define __get_user_unaligned __get_user
> -
> -#define get_user(x, ptr) \
> +#define __get_user(x, ptr) \
> ({ \
> - __typeof__(*(ptr)) __user *__p = (ptr); \
> - might_fault(); \
> - access_ok(VERIFY_READ, __p, sizeof(*__p)) ? \
> - __p = uaccess_mask_ptr(__p), __get_user((x), __p) : \
> - ((x) = 0, -EFAULT); \
> + int __gu_err = 0; \
> + __get_user_check((x), (ptr), __gu_err); \
> + __gu_err; \
> })
>
> +#define __get_user_unaligned __get_user
> +
> +#define get_user __get_user
> +
> #define __put_user_asm(instr, reg, x, addr, err) \
> asm volatile( \
> "1: " instr " " reg "1, [%2]\n" \
> @@ -266,30 +271,35 @@ do { \
> CONFIG_ARM64_PAN)); \
> } while (0)
>
> -#define __put_user(x, ptr) \
> +#define __put_user_check(x, ptr, err) \
> ({ \
> - int __pu_err = 0; \
> - __put_user_err((x), (ptr), __pu_err); \
> - __pu_err; \
> + __typeof__(*(ptr)) __user *__p = (ptr); \
> + might_fault(); \
> + if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) { \
> + __p = uaccess_mask_ptr(__p); \
> + __put_user_err((x), __p, (err)); \
> + } else { \
> + (err) = -EFAULT; \
> + } \
> })
>
> #define __put_user_error(x, ptr, err) \
> ({ \
> - __put_user_err((x), (ptr), (err)); \
> + __put_user_check((x), (ptr), (err)); \
> (void)0; \
> })
>
> -#define __put_user_unaligned __put_user
> -
> -#define put_user(x, ptr) \
> +#define __put_user(x, ptr) \
> ({ \
> - __typeof__(*(ptr)) __user *__p = (ptr); \
> - might_fault(); \
> - access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ? \
> - __p = uaccess_mask_ptr(__p), __put_user((x), __p) : \
> - -EFAULT; \
> + int __pu_err = 0; \
> + __put_user_check((x), (ptr), __pu_err); \
> + __pu_err; \
> })
>
> +#define __put_user_unaligned __put_user
> +
> +#define put_user __put_user
> +
> extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);
> extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);
> extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);
> --
> 2.21.0.rc0.269.g1a574e7a288b
>
