Re: [PATCH kvmtool 2/2] Add GICv2m support

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hi,

good stuff, thanks for doing this!

On 03/11/17 11:38, Jean-Philippe Brucker wrote:
> GICv2m is a small extension to the GICv2 architecture, specified in the
> Server Base System Architecture (SBSA). It adds a set of register to
> converts MSIs into SPIs, effectively enabling MSI support for pre-GICv3
> platforms.

I was wondering if it would be worth to split this up, and introduce the
msi_routing_ops in a separate patch first, possibly even the changes to
pci.c.

> Implement a GICv2m emulation entirely in userspace. Add a thin translation
> layer in irq.c to catch the MSI->SPI routing setup of the guest, and then
> transform irqfd injection of MSI into the associated SPI. There shouldn't
> be any significant runtime overhead compared to gicv3-its.

As others have pointed out already, these patches seem to rely on some
other patches.
However by ignoring the two hunks in non-yet-existing functions in irq.c
and adding #include <stdbool.h> to include/kvm/irq.h I could compile and
run it on a Juno.

> The device can be enabled by passing "--irqchip gicv2m" to kvmtool.

Might be worth to mention that for actually using MSIs in general one
needs "--force-pci" on the command line as well.

Given the compile fixes mentioned above, this boots with MSIs on my
Juno. Testing 32 bit in a minute.

Cheers,
Andre.

> 
> Signed-off-by: Jean-Philippe Brucker <jean-philippe.brucker@xxxxxxx>
> ---
>  Makefile                                 |   2 +-
>  arm/aarch32/include/asm/kvm.h            |   2 +
>  arm/aarch64/include/asm/kvm.h            |   2 +
>  arm/gic.c                                |  13 +++
>  arm/gicv2m.c                             | 153 +++++++++++++++++++++++++++++++
>  arm/include/arm-common/gic.h             |   3 +
>  arm/include/arm-common/kvm-config-arch.h |   2 +-
>  include/kvm/irq.h                        |  11 +++
>  irq.c                                    |  68 +++++++++++---
>  virtio/pci.c                             |   4 +-
>  10 files changed, 244 insertions(+), 16 deletions(-)
>  create mode 100644 arm/gicv2m.c
> 
> diff --git a/Makefile b/Makefile
> index 6414b5853947..93dc0673571d 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -153,7 +153,7 @@ ifeq ($(ARCH), powerpc)
>  endif
>  
>  # ARM
> -OBJS_ARM_COMMON		:= arm/fdt.o arm/gic.o arm/ioport.o \
> +OBJS_ARM_COMMON		:= arm/fdt.o arm/gic.o arm/gicv2m.o arm/ioport.o \
>  			   arm/kvm.o arm/kvm-cpu.o arm/pci.o arm/timer.o \
>  			   arm/pmu.o
>  HDRS_ARM_COMMON		:= arm/include
> diff --git a/arm/aarch32/include/asm/kvm.h b/arm/aarch32/include/asm/kvm.h
> index 6ebd3e6a1fd1..022066730dc7 100644
> --- a/arm/aarch32/include/asm/kvm.h
> +++ b/arm/aarch32/include/asm/kvm.h
> @@ -84,6 +84,8 @@ struct kvm_regs {
>  #define KVM_VGIC_V2_DIST_SIZE		0x1000
>  #define KVM_VGIC_V2_CPU_SIZE		0x2000
>  
> +#define KVM_VGIC_V2M_SIZE		0x1000
> +
>  /* Supported VGICv3 address types  */
>  #define KVM_VGIC_V3_ADDR_TYPE_DIST	2
>  #define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
> diff --git a/arm/aarch64/include/asm/kvm.h b/arm/aarch64/include/asm/kvm.h
> index c2860358ae3e..7d14507bd329 100644
> --- a/arm/aarch64/include/asm/kvm.h
> +++ b/arm/aarch64/include/asm/kvm.h
> @@ -84,6 +84,8 @@ struct kvm_regs {
>  #define KVM_VGIC_V2_DIST_SIZE		0x1000
>  #define KVM_VGIC_V2_CPU_SIZE		0x2000
>  
> +#define KVM_VGIC_V2M_SIZE		0x1000
> +
>  /* Supported VGICv3 address types  */
>  #define KVM_VGIC_V3_ADDR_TYPE_DIST	2
>  #define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
> diff --git a/arm/gic.c b/arm/gic.c
> index e53f932e28c3..46cfa245e590 100644
> --- a/arm/gic.c
> +++ b/arm/gic.c
> @@ -32,6 +32,8 @@ int irqchip_parser(const struct option *opt, const char *arg, int unset)
>  
>  	if (!strcmp(arg, "gicv2")) {
>  		*type = IRQCHIP_GICV2;
> +	} else if (!strcmp(arg, "gicv2m")) {
> +		*type = IRQCHIP_GICV2M;
>  	} else if (!strcmp(arg, "gicv3")) {
>  		*type = IRQCHIP_GICV3;
>  	} else if (!strcmp(arg, "gicv3-its")) {
> @@ -134,6 +136,8 @@ static int gic__create_msi_frame(struct kvm *kvm, enum irqchip_type type,
>  				 u64 msi_frame_addr)
>  {
>  	switch (type) {
> +	case IRQCHIP_GICV2M:
> +		return gic__create_gicv2m_frame(kvm, msi_frame_addr);
>  	case IRQCHIP_GICV3_ITS:
>  		return gic__create_its_frame(kvm, msi_frame_addr);
>  	default:	/* No MSI frame needed */
> @@ -165,6 +169,7 @@ static int gic__create_device(struct kvm *kvm, enum irqchip_type type)
>  	};
>  
>  	switch (type) {
> +	case IRQCHIP_GICV2M:
>  	case IRQCHIP_GICV2:
>  		gic_device.type = KVM_DEV_TYPE_ARM_VGIC_V2;
>  		dist_attr.attr  = KVM_VGIC_V2_ADDR_TYPE_DIST;
> @@ -183,6 +188,7 @@ static int gic__create_device(struct kvm *kvm, enum irqchip_type type)
>  	gic_fd = gic_device.fd;
>  
>  	switch (type) {
> +	case IRQCHIP_GICV2M:
>  	case IRQCHIP_GICV2:
>  		err = ioctl(gic_fd, KVM_SET_DEVICE_ATTR, &cpu_if_attr);
>  		break;
> @@ -243,6 +249,10 @@ int gic__create(struct kvm *kvm, enum irqchip_type type)
>  	int err;
>  
>  	switch (type) {
> +	case IRQCHIP_GICV2M:
> +		gic_msi_size = KVM_VGIC_V2M_SIZE;
> +		gic_msi_base = ARM_GIC_DIST_BASE - gic_msi_size;
> +		break;
>  	case IRQCHIP_GICV2:
>  		break;
>  	case IRQCHIP_GICV3_ITS:
> @@ -325,6 +335,9 @@ void gic__generate_fdt_nodes(void *fdt, enum irqchip_type type)
>  	};
>  
>  	switch (type) {
> +	case IRQCHIP_GICV2M:
> +		msi_compatible = "arm,gic-v2m-frame";
> +		/* fall-through */
>  	case IRQCHIP_GICV2:
>  		compatible = "arm,cortex-a15-gic";
>  		reg_prop[2] = cpu_to_fdt64(ARM_GIC_CPUI_BASE);
> diff --git a/arm/gicv2m.c b/arm/gicv2m.c
> new file mode 100644
> index 000000000000..d7e6398a13a8
> --- /dev/null
> +++ b/arm/gicv2m.c
> @@ -0,0 +1,153 @@
> +#include <errno.h>
> +#include <stdlib.h>
> +
> +#include "kvm/irq.h"
> +#include "kvm/kvm.h"
> +#include "kvm/util.h"
> +
> +#include "arm-common/gic.h"
> +
> +#define GICV2M_MSI_TYPER	0x008
> +#define GICV2M_MSI_SETSPI	0x040
> +#define GICV2M_MSI_IIDR		0xfcc
> +
> +#define GICV2M_SPI_MASK		0x3ff
> +#define GICV2M_MSI_TYPER_VAL(start, nr)	\
> +	(((start) & GICV2M_SPI_MASK) << 16 | ((nr) & GICV2M_SPI_MASK))
> +
> +struct gicv2m_chip {
> +	int	first_spi;
> +	int	num_spis;
> +	int	*spis;
> +	u64	base;
> +	u64	size;
> +};
> +
> +static struct gicv2m_chip v2m;
> +
> +/*
> + * MSI routing is setup lazily, when the guest writes the MSI tables. The guest
> + * writes which SPI is associated to an MSI vector into the message data field.
> + * The IRQ code notifies us of any change to MSI routing via this callback.
> + * Store the MSI->SPI translation for later.
> + *
> + * Data is the GIC interrupt ID, that includes SGIs and PPIs. SGIs at 0-15, PPIs
> + * are 16-31 and SPIs are 32-1019. What we're saving for later is the MSI's GSI
> + * number, a logical ID used by KVM for routing. The GSI of an SPI is implicitly
> + * defined by KVM to be its pin number (SPI index), and the GSI of an MSI is
> + * allocated by kvmtool.
> + */
> +static int gicv2m_update_routing(struct kvm *kvm,
> +				 struct kvm_irq_routing_entry *entry)
> +{
> +	int spi;
> +
> +	if (entry->type != KVM_IRQ_ROUTING_MSI)
> +		return -EINVAL;
> +
> +	if (!entry->u.msi.address_hi && !entry->u.msi.address_lo)
> +		return 0;
> +
> +	spi = entry->u.msi.data & GICV2M_SPI_MASK;
> +	if (spi < v2m.first_spi || spi >= v2m.first_spi + v2m.num_spis) {
> +		pr_err("invalid SPI number %d", spi);
> +		return -EINVAL;
> +	}
> +
> +	v2m.spis[spi - v2m.first_spi] = entry->gsi;
> +
> +	return 0;
> +}
> +
> +/*
> + * Find SPI bound to the given MSI and return the associated GSI.
> + */
> +static int gicv2m_translate_gsi(struct kvm *kvm, u32 gsi)
> +{
> +	int i;
> +
> +	for (i = 0; i < v2m.num_spis; i++) {
> +		if (v2m.spis[i] == (int)gsi)
> +			return i + v2m.first_spi - KVM_IRQ_OFFSET;
> +	}
> +
> +	/* Not an MSI */
> +	return gsi;
> +}
> +
> +static bool gicv2m_can_signal_msi(struct kvm *kvm)
> +{
> +	return true;
> +}
> +
> +/*
> + * Instead of setting up MSI routes, virtual devices can also trigger them
> + * manually (like a direct write to MSI_SETSPI). In this case, trigger the SPI
> + * directly.
> + */
> +static int gicv2m_signal_msi(struct kvm *kvm, struct kvm_msi *msi)
> +{
> +	int spi = msi->data & GICV2M_SPI_MASK;
> +
> +	if (spi < v2m.first_spi || spi >= v2m.first_spi + v2m.num_spis) {
> +		pr_err("invalid SPI number %d", spi);
> +		return -EINVAL;
> +	}
> +
> +	kvm__irq_trigger(kvm, spi);
> +	return 0;
> +}
> +
> +static struct msi_routing_ops gicv2m_routing = {
> +	.update_route	= gicv2m_update_routing,
> +	.translate_gsi	= gicv2m_translate_gsi,
> +	.can_signal_msi	= gicv2m_can_signal_msi,
> +	.signal_msi	= gicv2m_signal_msi,
> +};
> +
> +static void gicv2m_mmio_callback(struct kvm_cpu *vcpu, u64 addr, u8 *data,
> +				  u32 len, u8 is_write, void *ptr)
> +{
> +	if (is_write)
> +		return;
> +
> +	addr -= v2m.base;
> +
> +	switch (addr) {
> +		case GICV2M_MSI_TYPER:
> +			*(u32 *)data = GICV2M_MSI_TYPER_VAL(v2m.first_spi,
> +							    v2m.num_spis);
> +			break;
> +		case GICV2M_MSI_IIDR:
> +			*(u32 *)data = 0x0;
> +			break;
> +	}
> +}
> +
> +int gic__create_gicv2m_frame(struct kvm *kvm, u64 base)
> +{
> +	int i;
> +	int irq = irq__alloc_line();
> +
> +	v2m = (struct gicv2m_chip) {
> +		.first_spi	= irq,	/* Includes GIC_SPI_IRQ_BASE */
> +		.num_spis	= 64,	/* arbitrary */
> +		.base		= base,
> +		.size		= KVM_VGIC_V2M_SIZE,
> +	};
> +
> +	v2m.spis = calloc(v2m.num_spis, sizeof(int));
> +	if (!v2m.spis)
> +		return -ENOMEM;
> +
> +	v2m.spis[0] = -1;
> +	for (i = 1; i < v2m.num_spis; i++) {
> +		irq__alloc_line();
> +		v2m.spis[i] = -1;
> +	}
> +
> +	msi_routing_ops = &gicv2m_routing;
> +
> +	return kvm__register_mmio(kvm, base, KVM_VGIC_V2M_SIZE, false,
> +				  gicv2m_mmio_callback, kvm);
> +}
> diff --git a/arm/include/arm-common/gic.h b/arm/include/arm-common/gic.h
> index 0c279aca8a5e..560fea2a2c5a 100644
> --- a/arm/include/arm-common/gic.h
> +++ b/arm/include/arm-common/gic.h
> @@ -23,6 +23,7 @@
>  
>  enum irqchip_type {
>  	IRQCHIP_GICV2,
> +	IRQCHIP_GICV2M,
>  	IRQCHIP_GICV3,
>  	IRQCHIP_GICV3_ITS,
>  };
> @@ -39,4 +40,6 @@ void gic__del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd);
>  #define irq__add_irqfd gic__add_irqfd
>  #define irq__del_irqfd gic__del_irqfd
>  
> +int gic__create_gicv2m_frame(struct kvm *kvm, u64 msi_frame_addr);
> +
>  #endif /* ARM_COMMON__GIC_H */
> diff --git a/arm/include/arm-common/kvm-config-arch.h b/arm/include/arm-common/kvm-config-arch.h
> index f18b2c88fa79..6a196f1852de 100644
> --- a/arm/include/arm-common/kvm-config-arch.h
> +++ b/arm/include/arm-common/kvm-config-arch.h
> @@ -28,7 +28,7 @@ int irqchip_parser(const struct option *opt, const char *arg, int unset);
>  		    "Force virtio devices to use PCI as their default "		\
>  		    "transport"),						\
>          OPT_CALLBACK('\0', "irqchip", &(cfg)->irqchip,				\
> -		     "[gicv2|gicv3|gicv3-its]",					\
> +		     "[gicv2|gicv2m|gicv3|gicv3-its]",				\
>  		     "Type of interrupt controller to emulate in the guest",	\
>  		     irqchip_parser, NULL),
>  
> diff --git a/include/kvm/irq.h b/include/kvm/irq.h
> index b24385b13f66..530ce3f9133d 100644
> --- a/include/kvm/irq.h
> +++ b/include/kvm/irq.h
> @@ -11,6 +11,14 @@
>  
>  struct kvm;
>  
> +struct msi_routing_ops {
> +	int (*update_route)(struct kvm *kvm, struct kvm_irq_routing_entry *);
> +	bool (*can_signal_msi)(struct kvm *kvm);
> +	int (*signal_msi)(struct kvm *kvm, struct kvm_msi *msi);
> +	int (*translate_gsi)(struct kvm *kvm, u32 gsi);
> +};
> +
> +extern struct msi_routing_ops *msi_routing_ops;
>  extern struct kvm_irq_routing *irq_routing;
>  extern int next_gsi;
>  
> @@ -24,6 +32,9 @@ int irq__allocate_routing_entry(void);
>  int irq__add_msix_route(struct kvm *kvm, struct msi_msg *msg, u32 device_id);
>  void irq__update_msix_route(struct kvm *kvm, u32 gsi, struct msi_msg *msg);
>  
> +bool irq__can_signal_msi(struct kvm *kvm);
> +int irq__signal_msi(struct kvm *kvm, struct kvm_msi *msi);
> +
>  /*
>   * The function takes two eventfd arguments, trigger_fd and resample_fd. If
>   * resample_fd is <= 0, resampling is disabled and the IRQ is edge-triggered
> diff --git a/irq.c b/irq.c
> index 25fa572761fa..cdcf99233baa 100644
> --- a/irq.c
> +++ b/irq.c
> @@ -13,6 +13,9 @@ static int allocated_gsis = 0;
>  
>  int next_gsi;
>  
> +struct msi_routing_ops irq__default_routing_ops;
> +struct msi_routing_ops *msi_routing_ops = &irq__default_routing_ops;
> +
>  struct kvm_irq_routing *irq_routing = NULL;
>  
>  int irq__alloc_line(void)
> @@ -66,9 +69,42 @@ static bool check_for_irq_routing(struct kvm *kvm)
>  	return has_irq_routing > 0;
>  }
>  
> +static int irq__update_msix_routes(struct kvm *kvm,
> +				   struct kvm_irq_routing_entry *entry)
> +{
> +	return ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, irq_routing);
> +}
> +
> +static bool irq__default_can_signal_msi(struct kvm *kvm)
> +{
> +	return kvm__supports_extension(kvm, KVM_CAP_SIGNAL_MSI);
> +}
> +
> +static int irq__default_signal_msi(struct kvm *kvm, struct kvm_msi *msi)
> +{
> +	return ioctl(kvm->vm_fd, KVM_SIGNAL_MSI, msi);
> +}
> +
> +struct msi_routing_ops irq__default_routing_ops = {
> +	.update_route	= irq__update_msix_routes,
> +	.signal_msi	= irq__default_signal_msi,
> +	.can_signal_msi	= irq__default_can_signal_msi,
> +};
> +
> +bool irq__can_signal_msi(struct kvm *kvm)
> +{
> +	return msi_routing_ops->can_signal_msi(kvm);
> +}
> +
> +int irq__signal_msi(struct kvm *kvm, struct kvm_msi *msi)
> +{
> +	return msi_routing_ops->signal_msi(kvm, msi);
> +}
> +
>  int irq__add_msix_route(struct kvm *kvm, struct msi_msg *msg, u32 device_id)
>  {
>  	int r;
> +	struct kvm_irq_routing_entry *entry;
>  
>  	if (!check_for_irq_routing(kvm))
>  		return -ENXIO;
> @@ -77,22 +113,23 @@ int irq__add_msix_route(struct kvm *kvm, struct msi_msg *msg, u32 device_id)
>  	if (r)
>  		return r;
>  
> -	irq_routing->entries[irq_routing->nr] =
> -		(struct kvm_irq_routing_entry) {
> -			.gsi = next_gsi,
> -			.type = KVM_IRQ_ROUTING_MSI,
> -			.u.msi.address_hi = msg->address_hi,
> -			.u.msi.address_lo = msg->address_lo,
> -			.u.msi.data = msg->data,
> -		};
> +	entry = &irq_routing->entries[irq_routing->nr];
> +	*entry = (struct kvm_irq_routing_entry) {
> +		.gsi = next_gsi,
> +		.type = KVM_IRQ_ROUTING_MSI,
> +		.u.msi.address_hi = msg->address_hi,
> +		.u.msi.address_lo = msg->address_lo,
> +		.u.msi.data = msg->data,
> +	};
>  
>  	if (kvm->msix_needs_devid) {
> -		irq_routing->entries[irq_routing->nr].flags = KVM_MSI_VALID_DEVID;
> -		irq_routing->entries[irq_routing->nr].u.msi.devid = device_id;
> +		entry->flags = KVM_MSI_VALID_DEVID;
> +		entry->u.msi.devid = device_id;
>  	}
> +
>  	irq_routing->nr++;
>  
> -	r = ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, irq_routing);
> +	r = msi_routing_ops->update_route(kvm, entry);
>  	if (r)
>  		return r;
>  
> @@ -129,7 +166,7 @@ void irq__update_msix_route(struct kvm *kvm, u32 gsi, struct msi_msg *msg)
>  	if (!changed)
>  		return;
>  
> -	if (ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, irq_routing) == -1)
> +	if (msi_routing_ops->update_route(kvm, &irq_routing->entries[i]))
>  		die_perror("KVM_SET_GSI_ROUTING");
>  }
>  
> @@ -143,6 +180,10 @@ int irq__common_add_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd,
>  		.resamplefd	= resample_fd,
>  	};
>  
> +	/* If we emulate MSI routing, translate the MSI to the corresponding IRQ */
> +	if (msi_routing_ops->translate_gsi)
> +		irqfd.gsi = msi_routing_ops->translate_gsi(kvm, gsi);
> +
>  	return ioctl(kvm->vm_fd, KVM_IRQFD, &irqfd);
>  }
>  
> @@ -154,6 +195,9 @@ void irq__common_del_irqfd(struct kvm *kvm, unsigned int gsi, int trigger_fd)
>  		.flags		= KVM_IRQFD_FLAG_DEASSIGN,
>  	};
>  
> +	if (msi_routing_ops->translate_gsi)
> +		irqfd.gsi = msi_routing_ops->translate_gsi(kvm, gsi);
> +
>  	ioctl(kvm->vm_fd, KVM_IRQFD, &irqfd);
>  }
>  
> diff --git a/virtio/pci.c b/virtio/pci.c
> index 46473b00029b..4ce111108100 100644
> --- a/virtio/pci.c
> +++ b/virtio/pci.c
> @@ -350,7 +350,7 @@ static void virtio_pci__signal_msi(struct kvm *kvm, struct virtio_pci *vpci,
>  		msi.devid = vpci->dev_hdr.dev_num << 3;
>  	}
>  
> -	ioctl(kvm->vm_fd, KVM_SIGNAL_MSI, &msi);
> +	irq__signal_msi(kvm, &msi);
>  }
>  
>  int virtio_pci__signal_vq(struct kvm *kvm, struct virtio_device *vdev, u32 vq)
> @@ -488,7 +488,7 @@ int virtio_pci__init(struct kvm *kvm, void *dev, struct virtio_device *vdev,
>  	vpci->pci_hdr.msix.pba_offset = cpu_to_le32(2 | PCI_IO_SIZE);
>  	vpci->config_vector = 0;
>  
> -	if (kvm__supports_extension(kvm, KVM_CAP_SIGNAL_MSI))
> +	if (irq__can_signal_msi(kvm))
>  		vpci->features |= VIRTIO_PCI_F_SIGNAL_MSI;
>  
>  	r = device__register(&vpci->dev_hdr);
> 



[Index of Archives]     [KVM ARM]     [KVM ia64]     [KVM ppc]     [Virtualization Tools]     [Spice Development]     [Libvirt]     [Libvirt Users]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite Questions]     [Linux Kernel]     [Linux SCSI]     [XFree86]

  Powered by Linux