5.10-stable review patch. If anyone has any objections, please let me know.
------------------
From: Arjun Roy <arjunroy@xxxxxxxxxx>
[ Upstream commit 98917cf0d6eda01e8c3c34d35398d46b247b6fd3 ]
Refactor frag-is-remappable test for tcp receive zerocopy. This is
part of a patch set that introduces short-circuited hybrid copies
for small receive operations, which results in roughly 33% fewer
syscalls for small RPC scenarios.
Signed-off-by: Arjun Roy <arjunroy@xxxxxxxxxx>
Signed-off-by: Eric Dumazet <edumazet@xxxxxxxxxx>
Signed-off-by: Soheil Hassas Yeganeh <soheil@xxxxxxxxxx>
Signed-off-by: Jakub Kicinski <kuba@xxxxxxxxxx>
Stable-dep-of: 577e4432f3ac ("tcp: add sanity checks to rx zerocopy")
Signed-off-by: Sasha Levin <sashal@xxxxxxxxxx>
---
net/ipv4/tcp.c | 34 ++++++++++++++++++++++++++--------
1 file changed, 26 insertions(+), 8 deletions(-)
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 27e0d3dfc5bd..18541527abce 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1778,6 +1778,26 @@ static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
return frag;
}
+static bool can_map_frag(const skb_frag_t *frag)
+{
+ return skb_frag_size(frag) == PAGE_SIZE && !skb_frag_off(frag);
+}
+
+static int find_next_mappable_frag(const skb_frag_t *frag,
+ int remaining_in_skb)
+{
+ int offset = 0;
+
+ if (likely(can_map_frag(frag)))
+ return 0;
+
+ while (offset < remaining_in_skb && !can_map_frag(frag)) {
+ offset += skb_frag_size(frag);
+ ++frag;
+ }
+ return offset;
+}
+
static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
struct sk_buff *skb, u32 copylen,
u32 *offset, u32 *seq)
@@ -1903,6 +1923,8 @@ static int tcp_zerocopy_receive(struct sock *sk,
ret = 0;
curr_addr = address;
while (length + PAGE_SIZE <= zc->length) {
+ int mappable_offset;
+
if (zc->recv_skip_hint < PAGE_SIZE) {
u32 offset_frag;
@@ -1930,15 +1952,11 @@ static int tcp_zerocopy_receive(struct sock *sk,
if (!frags || offset_frag)
break;
}
- if (skb_frag_size(frags) != PAGE_SIZE || skb_frag_off(frags)) {
- int remaining = zc->recv_skip_hint;
- while (remaining && (skb_frag_size(frags) != PAGE_SIZE ||
- skb_frag_off(frags))) {
- remaining -= skb_frag_size(frags);
- frags++;
- }
- zc->recv_skip_hint -= remaining;
+ mappable_offset = find_next_mappable_frag(frags,
+ zc->recv_skip_hint);
+ if (mappable_offset) {
+ zc->recv_skip_hint = mappable_offset;
break;
}
pages[pg_idx] = skb_frag_page(frags);
--
2.43.0
