On 12/7/20 8:07 AM, Brendan Jackman wrote:
The BPF_FETCH field can be set in bpf_insn.imm, for BPF_ATOMIC
instructions, in order to have the previous value of the
atomically-modified memory location loaded into the src register
after an atomic op is carried out.
Suggested-by: Yonghong Song <yhs@xxxxxx>
Signed-off-by: Brendan Jackman <jackmanb@xxxxxxxxxx>
---
arch/x86/net/bpf_jit_comp.c | 4 ++++
include/linux/filter.h | 1 +
include/uapi/linux/bpf.h | 3 +++
kernel/bpf/core.c | 13 +++++++++++++
kernel/bpf/disasm.c | 7 +++++++
kernel/bpf/verifier.c | 33 ++++++++++++++++++++++++---------
tools/include/linux/filter.h | 11 +++++++++++
tools/include/uapi/linux/bpf.h | 3 +++
8 files changed, 66 insertions(+), 9 deletions(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
[...]
index f345f12c1ff8..4e0100ba52c2 100644
--- a/tools/include/linux/filter.h
+++ b/tools/include/linux/filter.h
@@ -173,6 +173,7 @@
* Atomic operations:
*
* BPF_ADD *(uint *) (dst_reg + off16) += src_reg
+ * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg);
*/
#define BPF_ATOMIC64(OP, DST, SRC, OFF) \
@@ -201,6 +202,16 @@
.off = OFF, \
.imm = BPF_ADD })
+/* Atomic memory add with fetch, src_reg = atomic_fetch_add(dst_reg + off, src_reg); */
+
+#define BPF_ATOMIC_FETCH_ADD(SIZE, DST, SRC, OFF) \
+ ((struct bpf_insn) { \
+ .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \
+ .dst_reg = DST, \
+ .src_reg = SRC, \
+ .off = OFF, \
+ .imm = BPF_ADD | BPF_FETCH })
Not sure whether it is a good idea or not to fold this into BPF_ATOMIC
macro. At least you can define BPF_ATOMIC macro and
#define BPF_ATOMIC_FETCH_ADD(SIZE, DST, SRC, OFF) \
BPF_ATOMIC(SIZE, DST, SRC, OFF, BPF_ADD | BPF_FETCH)
to avoid too many code duplications?
+
/* Memory store, *(uint *) (dst_reg + off16) = imm32 */
#define BPF_ST_MEM(SIZE, DST, OFF, IMM) \
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 98161e2d389f..d5389119291e 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -44,6 +44,9 @@
#define BPF_CALL 0x80 /* function call */
#define BPF_EXIT 0x90 /* function return */
+/* atomic op type fields (stored in immediate) */
+#define BPF_FETCH 0x01 /* fetch previous value into src reg */
+
/* Register numbers */
enum {
BPF_REG_0 = 0,
