When we attach a bpf program to cgroup/getsockopt any other getsockopt()
syscall starts incurring kzalloc/kfree cost. While, in general, it's
not an issue, sometimes it is, like in the case of TCP_ZEROCOPY_RECEIVE.
TCP_ZEROCOPY_RECEIVE (ab)uses getsockopt system call to implement
fastpath for incoming TCP, we don't want to have extra allocations in
there.
Let add a small buffer on the stack and use it for small (majority)
{s,g}etsockopt values. I've started with 128 bytes to cover
the options we care about (TCP_ZEROCOPY_RECEIVE which is 32 bytes
currently, with some planned extension to 64 + some headroom
for the future).
It seems natural to do the same for setsockopt, but it's a bit more
involved when the BPF program modifies the data (where we have to
kmalloc). The assumption is that for the majority of setsockopt
calls (which are doing pure BPF options or apply policy) this
will bring some benefit as well.
Signed-off-by: Stanislav Fomichev <sdf@xxxxxxxxxx>
---
include/linux/filter.h | 3 +++
kernel/bpf/cgroup.c | 41 +++++++++++++++++++++++++++++++++++++++--
2 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 29c27656165b..362eb0d7af5d 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1281,6 +1281,8 @@ struct bpf_sysctl_kern {
u64 tmp_reg;
};
+#define BPF_SOCKOPT_KERN_BUF_SIZE 128
+
struct bpf_sockopt_kern {
struct sock *sk;
u8 *optval;
@@ -1289,6 +1291,7 @@ struct bpf_sockopt_kern {
s32 optname;
s32 optlen;
s32 retval;
+ u8 buf[BPF_SOCKOPT_KERN_BUF_SIZE];
};
int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len);
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 6ec088a96302..0cb5d4376844 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -1310,6 +1310,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
max_optlen = PAGE_SIZE;
}
+ if (max_optlen <= sizeof(ctx->buf)) {
+ /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
+ * bytes avoid the cost of kzalloc.
+ */
+ ctx->optval = ctx->buf;
+ ctx->optval_end = ctx->optval + max_optlen;
+ return max_optlen;
+ }
+
ctx->optval = kzalloc(max_optlen, GFP_USER);
if (!ctx->optval)
return -ENOMEM;
@@ -1321,9 +1330,31 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
{
+ if (ctx->optval == ctx->buf)
+ return;
kfree(ctx->optval);
}
+static void *sockopt_export_buf(struct bpf_sockopt_kern *ctx)
+{
+ void *p;
+
+ if (ctx->optval != ctx->buf)
+ return ctx->optval;
+
+ /* We've used bpf_sockopt_kern->buf as an intermediary storage,
+ * but the BPF program indicates that we need to pass this
+ * data to the kernel setsockopt handler. No way to export
+ * on-stack buf, have to allocate a new buffer. The caller
+ * is responsible for the kfree().
+ */
+ p = kzalloc(ctx->optlen, GFP_USER);
+ if (!p)
+ return ERR_PTR(-ENOMEM);
+ memcpy(p, ctx->optval, ctx->optlen);
+ return p;
+}
+
int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
int *optname, char __user *optval,
int *optlen, char **kernel_optval)
@@ -1389,8 +1420,14 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
* use original userspace data.
*/
if (ctx.optlen != 0) {
- *optlen = ctx.optlen;
- *kernel_optval = ctx.optval;
+ void *buf = sockopt_export_buf(&ctx);
+
+ if (!IS_ERR(buf)) {
+ *optlen = ctx.optlen;
+ *kernel_optval = buf;
+ } else {
+ ret = PTR_ERR(buf);
+ }
}
}
--
2.29.2.729.g45daf8777d-goog
