COMPILE-TESTED ONLY!
Cc: John Fastabend <john.fastabend@xxxxxxxxx>
Cc: David Ahern <dsahern@xxxxxxxxx>
Cc: Martin KaFai Lau <martin.lau@xxxxxxxxx>
Cc: Jakub Kicinski <kuba@xxxxxxxxxx>
Cc: Willem de Bruijn <willemb@xxxxxxxxxx>
Cc: Jesper Dangaard Brouer <brouer@xxxxxxxxxx>
Cc: Anatoly Burakov <anatoly.burakov@xxxxxxxxx>
Cc: Alexander Lobakin <alexandr.lobakin@xxxxxxxxx>
Cc: Magnus Karlsson <magnus.karlsson@xxxxxxxxx>
Cc: Maryam Tahhan <mtahhan@xxxxxxxxxx>
Cc: xdp-hints@xxxxxxxxxxxxxxx
Cc: netdev@xxxxxxxxxxxxxxx
Signed-off-by: Stanislav Fomichev <sdf@xxxxxxxxxx>
---
.../net/ethernet/mellanox/mlx4/en_netdev.c | 1 +
drivers/net/ethernet/mellanox/mlx4/en_rx.c | 40 +++++++++++++++++++
include/linux/mlx4/device.h | 7 ++++
3 files changed, 48 insertions(+)
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index ca4b93a01034..3ef7995950dc 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -2890,6 +2890,7 @@ static const struct net_device_ops mlx4_netdev_ops_master = {
.ndo_features_check = mlx4_en_features_check,
.ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate,
.ndo_bpf = mlx4_xdp,
+ .ndo_unroll_kfunc = mlx4_unroll_kfunc,
};
struct mlx4_en_bond {
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 467356633172..83c89a1db3b8 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -33,6 +33,7 @@
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
+#include <linux/bpf_patch.h>
#include <linux/mlx4/cq.h>
#include <linux/slab.h>
#include <linux/mlx4/qp.h>
@@ -663,8 +664,43 @@ static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
struct mlx4_xdp_buff {
struct xdp_buff xdp;
+ struct mlx4_cqe *cqe;
+ struct mlx4_en_dev *mdev;
+ u64 r0;
};
+struct mlx4_xdp_buff *mxl4_xdp_rx_timestamp(struct mlx4_xdp_buff *ctx)
+{
+ unsigned int seq;
+ u64 timestamp;
+ u64 nsec;
+
+ timestamp = mlx4_en_get_cqe_ts(ctx->cqe);
+
+ do {
+ seq = read_seqbegin(&ctx->mdev->clock_lock);
+ nsec = timecounter_cyc2time(&ctx->mdev->clock, timestamp);
+ } while (read_seqretry(&ctx->mdev->clock_lock, seq));
+
+ ctx->r0 = (u64)ns_to_ktime(nsec);
+ return ctx;
+}
+
+void mlx4_unroll_kfunc(const struct bpf_prog *prog, u32 func_id,
+ struct bpf_patch *patch)
+{
+ if (func_id == xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_EXPORT_TO_SKB)) {
+ return xdp_metadata_export_to_skb(prog, patch);
+ } else if (func_id == xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP_SUPPORTED)) {
+ /* return true; */
+ bpf_patch_append(patch, BPF_MOV64_IMM(BPF_REG_0, 1));
+ } else if (func_id == xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP)) {
+ xdp_kfunc_call_preserving_r1(patch,
+ offsetof(struct mlx4_xdp_buff, r0),
+ mxl4_xdp_rx_timestamp);
+ }
+}
+
int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
@@ -783,6 +819,10 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
xdp_prepare_buff(&mxbuf.xdp, va - frags[0].page_offset,
frags[0].page_offset, length, false);
orig_data = mxbuf.xdp.data;
+ if (unlikely(ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL)) {
+ mxbuf.cqe = cqe;
+ mxbuf.mdev = priv->mdev;
+ }
act = bpf_prog_run_xdp(xdp_prog, &mxbuf.xdp);
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 6646634a0b9d..a0e4d490b2fb 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -1585,4 +1585,11 @@ static inline int mlx4_get_num_reserved_uar(struct mlx4_dev *dev)
/* The first 128 UARs are used for EQ doorbells */
return (128 >> (PAGE_SHIFT - dev->uar_page_shift));
}
+
+struct bpf_prog;
+struct bpf_insn;
+struct bpf_patch;
+
+void mlx4_unroll_kfunc(const struct bpf_prog *prog, u32 func_id,
+ struct bpf_patch *patch);
#endif /* MLX4_DEVICE_H */
--
2.38.1.431.g37b22c650d-goog
