list_for_each_entry() selects either the correct value (pos) or a safe
value for the additional mispredicted iteration (NULL) for the list
iterator.
list_for_each_entry() calls select_nospec(), which performs
a branch-less select.
On x86, this select is performed via a cmov. Otherwise, it's performed
via various shift/mask/etc. operations.
Kasper Acknowledgements: Jakob Koschel, Brian Johannesmeyer, Kaveh
Razavi, Herbert Bos, Cristiano Giuffrida from the VUSec group at VU
Amsterdam.
Co-developed-by: Brian Johannesmeyer <bjohannesmeyer@xxxxxxxxx>
Signed-off-by: Brian Johannesmeyer <bjohannesmeyer@xxxxxxxxx>
Signed-off-by: Jakob Koschel <jakobkoschel@xxxxxxxxx>
---
arch/x86/include/asm/barrier.h | 12 ++++++++++++
include/linux/list.h | 3 ++-
include/linux/nospec.h | 16 ++++++++++++++++
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 35389b2af88e..722797ad74e2 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -48,6 +48,18 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
/* Override the default implementation from linux/nospec.h. */
#define array_index_mask_nospec array_index_mask_nospec
+/* Override the default implementation from linux/nospec.h. */
+#define select_nospec(cond, exptrue, expfalse) \
+({ \
+ typeof(exptrue) _out = (exptrue); \
+ \
+ asm volatile("test %1, %1\n\t" \
+ "cmove %2, %0" \
+ : "+r" (_out) \
+ : "r" (cond), "r" (expfalse)); \
+ _out; \
+})
+
/* Prevent speculative execution past this barrier. */
#define barrier_nospec() alternative("", "lfence", X86_FEATURE_LFENCE_RDTSC)
diff --git a/include/linux/list.h b/include/linux/list.h
index dd6c2041d09c..1a1b39fdd122 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -636,7 +636,8 @@ static inline void list_splice_tail_init(struct list_head *list,
*/
#define list_for_each_entry(pos, head, member) \
for (pos = list_first_entry(head, typeof(*pos), member); \
- !list_entry_is_head(pos, head, member); \
+ ({ bool _cond = !list_entry_is_head(pos, head, member); \
+ pos = select_nospec(_cond, pos, NULL); _cond; }); \
pos = list_next_entry(pos, member))
/**
diff --git a/include/linux/nospec.h b/include/linux/nospec.h
index c1e79f72cd89..ca8ed81e4f9e 100644
--- a/include/linux/nospec.h
+++ b/include/linux/nospec.h
@@ -67,4 +67,20 @@ int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
/* Speculation control for seccomp enforced mitigation */
void arch_seccomp_spec_mitigate(struct task_struct *task);
+/**
+ * select_nospec - select a value without using a branch; equivalent to:
+ * cond ? exptrue : expfalse;
+ */
+#ifndef select_nospec
+#define select_nospec(cond, exptrue, expfalse) \
+({ \
+ unsigned long _t = (unsigned long) (exptrue); \
+ unsigned long _f = (unsigned long) (expfalse); \
+ unsigned long _c = (unsigned long) (cond); \
+ OPTIMIZER_HIDE_VAR(_c); \
+ unsigned long _m = -((_c | -_c) >> (BITS_PER_LONG - 1)); \
+ (typeof(exptrue)) ((_t & _m) | (_f & ~_m)); \
+})
+#endif
+
#endif /* _LINUX_NOSPEC_H */
--
2.25.1
