Add 3 sock_ops operators, namely BPF_SOCK_OPS_DATA_SEND_CB,
BPF_SOCK_OPS_DATA_RECV_CB, and BPF_SOCK_OPS_DATA_ACKED_CB. A flag
BPF_SOCK_OPS_DATA_EVENT_CB_FLAG is provided to minimize the performance
impact. The flag must be explicitly set to enable these callbacks.
If the flag is enabled, bpf sock_ops program will be called every time a
tcp data packet is sent, received, and acked.
BPF_SOCK_OPS_DATA_SEND_CB: call bpf after a data packet is sent.
BPF_SOCK_OPS_DATA_RECV_CB: call bpf after a data packet is receviced.
BPF_SOCK_OPS_DATA_ACKED_CB: call bpf after a valid ack packet is
processed (some sent data are ackknowledged).
We use these callbacks for fine-grained tcp monitoring, which collects
and analyses every tcp request/response event information. The whole
system has been described in SIGMOD'18 (see
https://dl.acm.org/doi/pdf/10.1145/3183713.3190659 for details). To
achieve this with bpf, we require hooks for data events that call
sock_ops bpf (1) when any data packet is sent/received/acked, and (2)
after critical tcp state variables have been updated (e.g., snd_una,
snd_nxt, rcv_nxt). However, existing sock_ops operators cannot meet our
requirements.
Besides, these hooks also help to debug tcp when data send/recv/acked.
Signed-off-by: Philo Lu <lulie@xxxxxxxxxxxxxxxxx>
---
include/net/tcp.h | 9 +++++++++
include/uapi/linux/bpf.h | 14 +++++++++++++-
net/ipv4/tcp_input.c | 4 ++++
net/ipv4/tcp_output.c | 2 ++
4 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/include/net/tcp.h b/include/net/tcp.h
index d2f0736b76b8..73eda03fdda5 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2660,6 +2660,15 @@ static inline void tcp_bpf_rtt(struct sock *sk)
tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
}
+/* op must be one of BPF_SOCK_OPS_DATA_SEND_CB, BPF_SOCK_OPS_DATA_RECV_CB,
+ * or BPF_SOCK_OPS_DATA_ACKED_CB.
+ */
+static inline void tcp_bpf_data_event(struct sock *sk, int op)
+{
+ if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_DATA_EVENT_CB_FLAG))
+ tcp_call_bpf(sk, op, 0, NULL);
+}
+
#if IS_ENABLED(CONFIG_SMC)
extern struct static_key_false tcp_have_smc;
#endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 7cf8bcf9f6a2..2154a6235901 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -3016,6 +3016,7 @@ union bpf_attr {
* * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission)
* * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change)
* * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT)
+ * * **BPF_SOCK_OPS_DATA_EVENT_CB_FLAG** (data packet send/recv/acked)
*
* Therefore, this function can be used to clear a callback flag by
* setting the appropriate bit to zero. e.g. to disable the RTO
@@ -6755,8 +6756,10 @@ enum {
* options first before the BPF program does.
*/
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG = (1<<6),
+ /* Call bpf when data send/recv/acked. */
+ BPF_SOCK_OPS_DATA_EVENT_CB_FLAG = (1<<7),
/* Mask of all currently supported cb flags */
- BPF_SOCK_OPS_ALL_CB_FLAGS = 0x7F,
+ BPF_SOCK_OPS_ALL_CB_FLAGS = 0xFF,
};
/* List of known BPF sock_ops operators.
@@ -6869,6 +6872,15 @@ enum {
* by the kernel or the
* earlier bpf-progs.
*/
+ BPF_SOCK_OPS_DATA_SEND_CB, /* Calls BPF program when a
+ * data packet is sent. Pure ack is ignored.
+ */
+ BPF_SOCK_OPS_DATA_RECV_CB, /* Calls BPF program when a
+ * data packet is received. Pure ack is ignored.
+ */
+ BPF_SOCK_OPS_DATA_ACKED_CB, /* Calls BPF program when sent
+ * data are acknowledged.
+ */
};
/* List of TCP states. There is a build check in net/ipv4/tcp.c to detect
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index bcb55d98004c..72c6192e7cd0 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -824,6 +824,8 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb)
now = tcp_jiffies32;
+ tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_RECV_CB);
+
if (!icsk->icsk_ack.ato) {
/* The _first_ data packet received, initialize
* delayed ACK engine.
@@ -3454,6 +3456,8 @@ static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb,
flag |= FLAG_SET_XMIT_TIMER; /* set TLP or RTO timer */
}
+ tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_ACKED_CB);
+
if (icsk->icsk_ca_ops->pkts_acked) {
struct ack_sample sample = { .pkts_acked = pkts_acked,
.rtt_us = sack->rate->rtt_us };
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index eb13a55d660c..ddd6a9c2150f 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -2821,6 +2821,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
/* Send one loss probe per tail loss episode. */
if (push_one != 2)
tcp_schedule_loss_probe(sk, false);
+
+ tcp_bpf_data_event(sk, BPF_SOCK_OPS_DATA_SEND_CB);
return false;
}
return !tp->packets_out && !tcp_write_queue_empty(sk);
