On Mon, 9 Oct 2023 11:32:52 +0200
Nico Boehr <nrb@xxxxxxxxxxxxx> wrote:
> The shadow gmap tracks memory of nested guests (guest-3). In certain
> scenarios, the shadow gmap needs to be rebuilt, which is a costly operation
> since it involves a SIE exit into guest-1 for every entry in the respective
> shadow level.
>
> Add kvm stat counters when new shadow structures are created at various
> levels. Also add a counter gmap_shadow_create when a completely fresh
> shadow gmap is created as well as a counter gmap_shadow_reuse when an
> existing gmap is being reused.
>
> Note that when several levels are shadowed at once, counters on all
> affected levels will be increased.
>
> Also note that not all page table levels need to be present and a ASCE
> can directly point to e.g. a segment table. In this case, a new segment
> table will always be equivalent to a new shadow gmap and hence will be
> counted as gmap_shadow_create and not as gmap_shadow_segment.
>
> Signed-off-by: Nico Boehr <nrb@xxxxxxxxxxxxx>
> Reviewed-by: David Hildenbrand <david@xxxxxxxxxx>
Reviewed-by: Claudio Imbrenda <imbrenda@xxxxxxxxxxxxx>
> ---
> arch/s390/include/asm/kvm_host.h | 7 +++++++
> arch/s390/kvm/gaccess.c | 7 +++++++
> arch/s390/kvm/kvm-s390.c | 9 ++++++++-
> arch/s390/kvm/vsie.c | 5 ++++-
> 4 files changed, 26 insertions(+), 2 deletions(-)
>
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 427f9528a7b6..67a298b6cf6e 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -777,6 +777,13 @@ struct kvm_vm_stat {
> u64 inject_service_signal;
> u64 inject_virtio;
> u64 aen_forward;
> + u64 gmap_shadow_create;
> + u64 gmap_shadow_reuse;
> + u64 gmap_shadow_r1_entry;
> + u64 gmap_shadow_r2_entry;
> + u64 gmap_shadow_r3_entry;
> + u64 gmap_shadow_sg_entry;
> + u64 gmap_shadow_pg_entry;
> };
>
> struct kvm_arch_memory_slot {
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index 6d6bc19b37dc..ff8349d17b33 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -1382,6 +1382,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> unsigned long *pgt, int *dat_protection,
> int *fake)
> {
> + struct kvm *kvm;
> struct gmap *parent;
> union asce asce;
> union vaddress vaddr;
> @@ -1390,6 +1391,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>
> *fake = 0;
> *dat_protection = 0;
> + kvm = sg->private;
> parent = sg->parent;
> vaddr.addr = saddr;
> asce.val = sg->orig_asce;
> @@ -1450,6 +1452,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_r1_entry++;
> }
> fallthrough;
> case ASCE_TYPE_REGION2: {
> @@ -1478,6 +1481,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_r2_entry++;
> }
> fallthrough;
> case ASCE_TYPE_REGION3: {
> @@ -1515,6 +1519,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_r3_entry++;
> }
> fallthrough;
> case ASCE_TYPE_SEGMENT: {
> @@ -1548,6 +1553,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
> if (rc)
> return rc;
> + kvm->stat.gmap_shadow_sg_entry++;
> }
> }
> /* Return the parent address of the page table */
> @@ -1618,6 +1624,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
> pte.p |= dat_protection;
> if (!rc)
> rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
> + vcpu->kvm->stat.gmap_shadow_pg_entry++;
> ipte_unlock(vcpu->kvm);
> mmap_read_unlock(sg->mm);
> return rc;
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index b3f17e014cab..b42493110d76 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -66,7 +66,14 @@ const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
> STATS_DESC_COUNTER(VM, inject_pfault_done),
> STATS_DESC_COUNTER(VM, inject_service_signal),
> STATS_DESC_COUNTER(VM, inject_virtio),
> - STATS_DESC_COUNTER(VM, aen_forward)
> + STATS_DESC_COUNTER(VM, aen_forward),
> + STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
> + STATS_DESC_COUNTER(VM, gmap_shadow_create),
> + STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
> + STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
> + STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
> + STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
> + STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
> };
>
> const struct kvm_stats_header kvm_vm_stats_header = {
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index 61499293c2ac..02dcbe82a8e5 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -1214,8 +1214,10 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
> * we're holding has been unshadowed. If the gmap is still valid,
> * we can safely reuse it.
> */
> - if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat))
> + if (vsie_page->gmap && gmap_shadow_valid(vsie_page->gmap, asce, edat)) {
> + vcpu->kvm->stat.gmap_shadow_reuse++;
> return 0;
> + }
>
> /* release the old shadow - if any, and mark the prefix as unmapped */
> release_gmap_shadow(vsie_page);
> @@ -1223,6 +1225,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
> if (IS_ERR(gmap))
> return PTR_ERR(gmap);
> gmap->private = vcpu->kvm;
> + vcpu->kvm->stat.gmap_shadow_create++;
> WRITE_ONCE(vsie_page->gmap, gmap);
> return 0;
> }
