On Thu, Aug 19, 2021 at 02:21:32PM +0100, Will Deacon wrote:
> I suppose the sanitisers might contribute a fair bit to the cmpxchg() loops
> in the allmodconfig case, but looking at how defconfig affects mm/rmap.o the
> *big* difference there seems to be due to the dec_and_test() operation:
>
> atomic_dec_and_test() looks like:
>
> lock decl (%rdi)
> sete %al
> retq
>
> whereas refcount_dec_and_test() looks like:
>
> push %r12
> mov $0xffffffff,%eax
> lock xadd %eax,(%rdi)
> cmp $0x1,%eax
> je 97d
> xor %r12d,%r12d
> test %eax,%eax
> jle 989
> mov %r12d,%eax
> pop %r12
> retq
> mov $0x1,%r12d
> mov %r12d,%eax
> pop %r12
> retq
> mov $0x3,%esi
> callq 993
> mov %r12d,%eax
> pop %r12
> retq
>
> which is a monster by comparison!
>
> This is because refcount_dec_and_test() wraps __refcount_sub_and_test(),
> which is necessary so that we can look at the old value:
>
> (1) If it was equal to 1, then ensure we have acquire semantics and
> return true.
>
> (2) If it was < 0 or the new value written is < 0, then saturate (UAF)
>
> However, talking to Peter and Boris, we may be able to achieve the same
> thing by looking at the flags after a DECL instruction:
>
> * We can implement (1) by checking if we hit zero (ZF=1)
> * We can implement (2) by checking if the new value is < 0 (SF=1).
> We then need to catch the case where the old value was < 0 but the
> new value is 0. I think this is (SF=0 && OF=1).
>
> So maybe the second check is actually SF != OF? I could benefit from some
> x86 expertise here, but hopefully you get the idea.
Right, so the first condition is ZF=1, we hit zero.
The second condition is SF=1, the result is negative.
I'm not sure we need OF, if we hit that condition we've already lost.
But it's easy enough to add I suppose.
> We could do something similar for refcount_dec() and refcount_inc(),
> although commit a435b9a14356 ("locking/refcount: Provide __refcount API
> to obtain the old value") add versions which return the old value,
> which we can't reconstruct from the flags. Since nobody is using these
> variants, I suggest we remove them.
dhowells asked for them... he was very adamand they were required. David?
> Thoughts?
Something like so?
---
arch/x86/include/asm/refcount.h | 29 +++++++++++++++++++++++++++++
include/linux/refcount.h | 6 +++++-
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h
new file mode 100644
index 000000000000..bbc0d4835bbb
--- /dev/null
+++ b/arch/x86/include/asm/refcount.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_REFCOUNT_H
+#define _ASM_X86_REFCOUNT_H
+
+#define refcount_dec_and_test refcount_dec_and_test
+static inline bool refcount_dec_and_test(refcount_t *r)
+{
+ asm_volatile_goto (LOCK_PREFIX "decl %[var]\n\t"
+ "jz %l[cc_zero]\n\t"
+ "js %l[cc_fail]\n\t"
+ "jo %l[cc_fail]"
+ : : [var] "m" (r->refs.counter)
+ : "memory" : cc_zero, cc_fail);
+
+ if (0) {
+cc_zero:
+ smp_acquire__after_ctrl_dep();
+ return true;
+ }
+
+ if (0) {
+cc_fail:
+ refcount_warn_saturate(r, REFCOUNT_SUB_UAF);
+ }
+
+ return false;
+}
+
+#endif /* _ASM_X86_REFCOUNT_H */
diff --git a/include/linux/refcount.h b/include/linux/refcount.h
index b8a6e387f8f9..42e987549d1f 100644
--- a/include/linux/refcount.h
+++ b/include/linux/refcount.h
@@ -117,7 +117,7 @@ typedef struct refcount_struct {
#define REFCOUNT_SATURATED (INT_MIN / 2)
enum refcount_saturation_type {
- REFCOUNT_ADD_NOT_ZERO_OVF,
+ REFCOUNT_ADD_NOT_ZERO_OVF = 0,
REFCOUNT_ADD_OVF,
REFCOUNT_ADD_UAF,
REFCOUNT_SUB_UAF,
@@ -126,6 +126,8 @@ enum refcount_saturation_type {
void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t);
+#include <asm/refcount.h>
+
/**
* refcount_set - set a refcount's value
* @r: the refcount
@@ -328,10 +330,12 @@ static inline __must_check bool __refcount_dec_and_test(refcount_t *r, int *oldp
*
* Return: true if the resulting refcount is 0, false otherwise
*/
+#ifndef refcount_dec_and_test
static inline __must_check bool refcount_dec_and_test(refcount_t *r)
{
return __refcount_dec_and_test(r, NULL);
}
+#endif
static inline void __refcount_dec(refcount_t *r, int *oldp)
{
