From: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Date: Wed, 23 May 2012 14:01:26 -0700
> On Wed, May 23, 2012 at 1:36 PM, David Miller <davem@xxxxxxxxxxxxx> wrote:
>> 1) In the loop, use the test:
>>
>> (x + 0xfefefeff) & ~(x | 0x7f7f7f7f)
>>
>> It's the same effective cost as the current test (on sparc
>> it would be ADD, OR, ANDNCC).
>>
>> We make sure to calculate the "x | 0x7f7f7f7f" part into
>> a variable which is not clobbered by the rest of the test.
>>
>> This is so we can reuse it in #2.
>>
>> 2) Once we find a word containing the zero byte, do a:
>>
>> ~(((x & 0x7f7f7f7f) + 0x7f7f7f7f) | x | 0x7f7f7f7f)
>>
>> and that "x | 0x7f7f7f7f" part is already calculated and thus
>> can be cribbed the place we left it in #1 above.
>>
>> And now we'll have exactly a 0x80 where there is a zero byte,
>> and no bleeding of 0x80 values into adjacent byte positions.
>>
>> Once we have that we can just test that mask directly for the
>> zero byte location search code.
>
> Sounds likely, and you only have two different constants to worry about.
Ok, now confirmed, the following patch generates the best code I've
been able to get out of GCC thus far, and it's tested too.
Therefore, I'll push this out to my sparc tree.
Next, I'll try to add the abstractions et al. necessary to put this
under the top-level lib/ so others can use it too.
diff --git a/arch/sparc/lib/usercopy.c b/arch/sparc/lib/usercopy.c
index 851cb75..78c363f 100644
--- a/arch/sparc/lib/usercopy.c
+++ b/arch/sparc/lib/usercopy.c
@@ -17,29 +17,20 @@ static inline unsigned long has_zero(unsigned long a)
return ((a - REPEAT_BYTE(0x01)) & ~a) & REPEAT_BYTE(0x80);
}
-static inline long find_zero(unsigned long c)
+static inline long find_zero(unsigned long mask)
{
+ long byte = 0;
#ifdef CONFIG_64BIT
- if (!(c & 0xff00000000000000UL))
- return 0;
- if (!(c & 0x00ff000000000000UL))
- return 1;
- if (!(c & 0x0000ff0000000000UL))
- return 2;
- if (!(c & 0x000000ff00000000UL))
- return 3;
-#define __OFF 4
-#else
-#define __OFF 0
+ if (mask >> 32)
+ mask >>= 32;
+ else
+ byte = 4;
#endif
- if (!(c & 0xff000000))
- return __OFF + 0;
- if (!(c & 0x00ff0000))
- return __OFF + 1;
- if (!(c & 0x0000ff00))
- return __OFF + 2;
- return __OFF + 3;
-#undef __OFF
+ if (mask >> 16)
+ mask >>= 16;
+ else
+ byte += 2;
+ return (mask >> 8) ? byte : byte + 1;
}
/*
@@ -50,6 +41,8 @@ static inline long find_zero(unsigned long c)
*/
static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max)
{
+ const unsigned long high_bits = REPEAT_BYTE(0xfe) + 1;
+ const unsigned long low_bits = REPEAT_BYTE(0x7f);
long res = 0;
/*
@@ -63,14 +56,19 @@ static inline long do_strncpy_from_user(char *dst, const char __user *src, long
goto byte_at_a_time;
while (max >= sizeof(unsigned long)) {
- unsigned long c;
+ unsigned long c, v, rhs;
/* Fall back to byte-at-a-time if we get a page fault */
if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
break;
+ rhs = c | low_bits;
+ v = (c + high_bits) & ~rhs;
*(unsigned long *)(dst+res) = c;
- if (has_zero(c))
- return res + find_zero(c);
+ if (v) {
+ v = (c & low_bits) + low_bits;;
+ v = ~(v | rhs);
+ return res + find_zero(v);
+ }
res += sizeof(unsigned long);
max -= sizeof(unsigned long);
}
--
To unsubscribe from this list: send the line "unsubscribe linux-arch" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
