On Wed, Sep 23, 2020 at 10:32:54AM +0800, zhenwei pi wrote:
> Collect PCI passthrough net device stats from kernel by netlink
> API.
>
> Currently, libvirt can not get PCI passthrough net device stats,
> run command:
> #virsh domifstat instance --interface=52:54:00:2d:b2:35
> error: Failed to get interface stats instance 52:54:00:2d:b2:35
> error: internal error: Interface name not provided
>
> The PCI device(usually SR-IOV virtual function device) is detached
> while it's used in PCI passthrough mode. And we can not parse this
> device from /proc/net/dev any more.
>
> In this patch, libvirt check net device is VF of not firstly, then
> query virNetDevVFInterfaceStats(new API).
> virNetDevVFInterfaceStats parses VFs info of all PFs, compares MAC
> address until the two MAC addresses match.
> '#ip -s link show' can get the same result. Instead of parsing the
> output result, implement this feature by libnl API.
>
> Notice that this feature deponds on driver of PF.
> Test on Mellanox ConnectX-4 Lx, it works well.
> Also test on Intel Corporation 82599ES, it works, but only get 0.
> (ip-link command get the same result).
>
> Signed-off-by: zhenwei pi <pizhenwei@xxxxxxxxxxxxx>
> ---
> src/libvirt_private.syms | 1 +
> src/qemu/qemu_driver.c | 3 ++
> src/util/virnetdev.c | 137 +++++++++++++++++++++++++++++++++++++++++++++++
> src/util/virnetdev.h | 5 ++
> 4 files changed, 146 insertions(+)
>
> diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
> index bdbe3431b8..bcc40b8d69 100644
> --- a/src/libvirt_private.syms
> +++ b/src/libvirt_private.syms
> @@ -2585,6 +2585,7 @@ virNetDevSetRcvMulti;
> virNetDevSetupControl;
> virNetDevSysfsFile;
> virNetDevValidateConfig;
> +virNetDevVFInterfaceStats;
>
>
> # util/virnetdevbandwidth.h
> diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
> index ae715c01d7..f554010c40 100644
> --- a/src/qemu/qemu_driver.c
> +++ b/src/qemu/qemu_driver.c
> @@ -10196,6 +10196,9 @@ qemuDomainInterfaceStats(virDomainPtr dom,
> if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_VHOSTUSER) {
> if (virNetDevOpenvswitchInterfaceStats(net->ifname, stats) < 0)
> goto cleanup;
> + } else if (virDomainNetGetActualType(net) == VIR_DOMAIN_NET_TYPE_HOSTDEV) {
> + if (virNetDevVFInterfaceStats(&net->mac, stats) < 0)
> + goto cleanup;
> } else {
> if (virNetDevTapInterfaceStats(net->ifname, stats,
> !virDomainNetTypeSharesHostView(net)) < 0)
> diff --git a/src/util/virnetdev.c b/src/util/virnetdev.c
> index e1a4cc2bef..377f25aae7 100644
> --- a/src/util/virnetdev.c
> +++ b/src/util/virnetdev.c
> @@ -1489,6 +1489,7 @@ static struct nla_policy ifla_vf_policy[IFLA_VF_MAX+1] = {
> .maxlen = sizeof(struct ifla_vf_mac) },
> [IFLA_VF_VLAN] = { .type = NLA_UNSPEC,
> .maxlen = sizeof(struct ifla_vf_vlan) },
> + [IFLA_VF_STATS] = { .type = NLA_NESTED },
> };
>
>
> @@ -2265,6 +2266,132 @@ virNetDevSetNetConfig(const char *linkdev, int vf,
> return 0;
> }
>
> +static struct nla_policy ifla_vfstats_policy[IFLA_VF_STATS_MAX+1] = {
> + [IFLA_VF_STATS_RX_PACKETS] = { .type = NLA_U64 },
> + [IFLA_VF_STATS_TX_PACKETS] = { .type = NLA_U64 },
> + [IFLA_VF_STATS_RX_BYTES] = { .type = NLA_U64 },
> + [IFLA_VF_STATS_TX_BYTES] = { .type = NLA_U64 },
> + [IFLA_VF_STATS_BROADCAST] = { .type = NLA_U64 },
> + [IFLA_VF_STATS_MULTICAST] = { .type = NLA_U64 },
> +};
> +
> +static int
> +virNetDevParseVfStats(struct nlattr **tb, virMacAddrPtr mac,
> + virDomainInterfaceStatsPtr stats)
> +{
> + int ret = -1, len;
> + struct ifla_vf_mac *vf_lladdr;
> + struct nlattr *nla, *t[IFLA_VF_MAX+1];
> + struct nlattr *stb[IFLA_VF_STATS_MAX+1];
> +
> + if (tb == NULL || mac == NULL || stats == NULL) {
> + return -1;
> + }
> +
> + if (!tb[IFLA_VFINFO_LIST])
> + return -1;
> +
> + len = nla_len(tb[IFLA_VFINFO_LIST]);
> +
> + for (nla = nla_data(tb[IFLA_VFINFO_LIST]); nla_ok(nla, len);
> + nla = nla_next(nla, &len)) {
> + ret = nla_parse(t, IFLA_VF_MAX, nla_data(nla), nla_len(nla),
> + ifla_vf_policy);
> + if (ret < 0)
> + return -1;
> +
> + if (t[IFLA_VF_MAC] == NULL) {
> + continue;
> + }
> +
> + vf_lladdr = nla_data(t[IFLA_VF_MAC]);
> + if (virMacAddrCmpRaw(mac, vf_lladdr->mac)) {
> + continue;
> + }
> +
> + if (t[IFLA_VF_STATS]) {
> + ret = nla_parse_nested(stb, IFLA_VF_STATS_MAX,
> + t[IFLA_VF_STATS],
> + ifla_vfstats_policy);
> + if (ret < 0)
> + return -1;
> +
> + stats->rx_bytes = nla_get_u64(stb[IFLA_VF_STATS_RX_BYTES]);
> + stats->tx_bytes = nla_get_u64(stb[IFLA_VF_STATS_TX_BYTES]);
> + stats->rx_packets = nla_get_u64(stb[IFLA_VF_STATS_RX_PACKETS]);
> + stats->tx_packets = nla_get_u64(stb[IFLA_VF_STATS_TX_PACKETS]);
> + }
> + return 0;
> + }
> +
> + return ret;
> +}
> +
> +/**
> + * virNetDevVFInterfaceStats:
> + * @mac: MAC address of the VF interface
> + * @stats: returns stats of the VF interface
> + *
> + * Get the VF interface from kernel by netlink.
> + * Returns 0 on success, -1 on failure.
> + */
> +int
> +virNetDevVFInterfaceStats(virMacAddrPtr mac,
> + virDomainInterfaceStatsPtr stats)
> +{
> + FILE *fp;
> + char line[256], *colon, *ifname;
> + int rc = -1;
> + void *nlData = NULL;
> + struct nlattr *tb[IFLA_MAX + 1] = {NULL, };
> + char *sysfsDevicePath = NULL;
> +
> + fp = fopen("/proc/net/dev", "r");
> + if (!fp) {
> + virReportSystemError(errno, "%s",
> + _("Could not open /proc/net/dev"));
> + return -1;
> + }
> +
> + /* get all PCI net devices, and parse VFs list from netlink API.
> + * compare MAC address, collect device stats if matching.
> + */
> + while (fgets(line, sizeof(line), fp)) {
> + /* The line looks like:
> + * " eth0:..."
> + * Split it at the colon. and strip blank from head.
> + */
> + colon = strchr(line, ':');
> + if (!colon)
> + continue;
> + *colon = '\0';
> + ifname = line;
> + while ((*ifname == ' ') && (ifname < colon))
> + ifname++;
It seems we're only parsing /proc/net/dev to get a list of
device names, ignoring the rest of the data in that file.
It would be simpler if we just iterate over /sys/class/net
surely, as the filename as the nic names meaning we don't
need to parse anything.
> +
> + if (virNetDevSysfsFile(&sysfsDevicePath, ifname, "device") < 0)
> + break;
> +
> + if (virNetDevIsPCIDevice(sysfsDevicePath)) {
> + rc = virNetlinkDumpLink(ifname, -1, &nlData, tb, 0, 0);
> + if (rc < 0) {
> + rc = -1;
> + goto cleanup;
> + }
> +
> + rc = virNetDevParseVfStats(tb, mac, stats);
> + VIR_FREE(nlData);
> + if (rc == 0)
> + goto cleanup;
> + }
> + VIR_FREE(sysfsDevicePath);
> + }
> +
> + cleanup:
> + VIR_FREE(sysfsDevicePath);
> + VIR_FORCE_FCLOSE(fp);
> + return rc;
> +}
Regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
