Re: [PATCH] drm/amdkfd: Provide SMI events watch

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Tue, Mar 17, 2020 at 1:57 PM Amber Lin <Amber.Lin@xxxxxxx> wrote:
>
> When the compute is malfunctioning or performance drops, the system admin
> will use SMI (System Management Interface) tool to monitor/diagnostic what
> went wrong. This patch provides an event watch interface for the user
> space to register events they are interested. After the event is
> registered, the user can use annoymous file descriptor's pull function
> with wait-time specified to wait for the event to happen. Once the event
> happens, the user can use read() to retrieve information related to the
> event.
>
> VM fault event is done in this patch.
>
> Signed-off-by: Amber Lin <Amber.Lin@xxxxxxx>

Can you provide a link to the userspace tools that make use of this interface?

Thanks,

Alex

> ---
>  drivers/gpu/drm/amd/amdkfd/Makefile              |   3 +-
>  drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c |   2 +
>  drivers/gpu/drm/amd/amdkfd/kfd_chardev.c         |  38 ++++++
>  drivers/gpu/drm/amd/amdkfd/kfd_device.c          |   1 +
>  drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c  |   2 +
>  drivers/gpu/drm/amd/amdkfd/kfd_priv.h            |  10 ++
>  drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c      | 143 +++++++++++++++++++++++
>  drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h      |  41 +++++++
>  include/uapi/linux/kfd_ioctl.h                   |  27 ++++-
>  9 files changed, 265 insertions(+), 2 deletions(-)
>  create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
>  create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
> index 6147462..cc98b4a 100644
> --- a/drivers/gpu/drm/amd/amdkfd/Makefile
> +++ b/drivers/gpu/drm/amd/amdkfd/Makefile
> @@ -53,7 +53,8 @@ AMDKFD_FILES  := $(AMDKFD_PATH)/kfd_module.o \
>                 $(AMDKFD_PATH)/kfd_int_process_v9.o \
>                 $(AMDKFD_PATH)/kfd_dbgdev.o \
>                 $(AMDKFD_PATH)/kfd_dbgmgr.o \
> -               $(AMDKFD_PATH)/kfd_crat.o
> +               $(AMDKFD_PATH)/kfd_crat.o \
> +               $(AMDKFD_PATH)/kfd_smi_events.o
>
>  ifneq ($(CONFIG_AMD_IOMMU_V2),)
>  AMDKFD_FILES += $(AMDKFD_PATH)/kfd_iommu.o
> diff --git a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
> index 9f59ba9..24b4717 100644
> --- a/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
> +++ b/drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c
> @@ -24,6 +24,7 @@
>  #include "kfd_events.h"
>  #include "cik_int.h"
>  #include "amdgpu_amdkfd.h"
> +#include "kfd_smi_events.h"
>
>  static bool cik_event_interrupt_isr(struct kfd_dev *dev,
>                                         const uint32_t *ih_ring_entry,
> @@ -107,6 +108,7 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev,
>                 ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
>                 struct kfd_vm_fault_info info;
>
> +               kfd_smi_event_update_vmfault(dev, pasid);
>                 kfd_process_vm_fault(dev->dqm, pasid);
>
>                 memset(&info, 0, sizeof(info));
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> index f8fa03a..8e92956 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
> @@ -39,6 +39,7 @@
>  #include "kfd_device_queue_manager.h"
>  #include "kfd_dbgmgr.h"
>  #include "amdgpu_amdkfd.h"
> +#include "kfd_smi_events.h"
>
>  static long kfd_ioctl(struct file *, unsigned int, unsigned long);
>  static int kfd_open(struct inode *, struct file *);
> @@ -1243,6 +1244,40 @@ static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
>         return ret;
>  }
>
> +/* Handle requests for watching SMI events */
> +static int kfd_ioctl_smi_events(struct file *filep,
> +                               struct kfd_process *p, void *data)
> +{
> +       struct kfd_ioctl_smi_events_args *args = data;
> +       struct kfd_dev *dev;
> +       int ret = 0;
> +
> +       dev = kfd_device_by_id(args->gpu_id);
> +       if (!dev)
> +               return -EINVAL;
> +
> +       switch (args->op) {
> +       case KFD_SMI_EVENTS_REGISTER:
> +               ret = kfd_smi_event_register(dev, args->events);
> +               if (ret >= 0) {
> +                       /* When the registration is successful, it returns the
> +                        * annoymous inode. Pass it to the user in data1
> +                        */
> +                       args->data1 = ret;
> +                       ret = 0;
> +               }
> +               break;
> +       case KFD_SMI_EVENTS_UNREGISTER:
> +               kfd_smi_event_unregister(dev, args->events);
> +               break;
> +       default:
> +               ret = -EINVAL;
> +               break;
> +       }
> +
> +       return ret;
> +}
> +
>  bool kfd_dev_is_large_bar(struct kfd_dev *dev)
>  {
>         struct kfd_local_mem_info mem_info;
> @@ -1827,6 +1862,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
>
>         AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
>                         kfd_ioctl_alloc_queue_gws, 0),
> +
> +       AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
> +                       kfd_ioctl_smi_events, 0),
>  };
>
>  #define AMDKFD_CORE_IOCTL_COUNT        ARRAY_SIZE(amdkfd_ioctls)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index 7866cd06..450368c 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -532,6 +532,7 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
>         kfd->device_info = device_info;
>         kfd->pdev = pdev;
>         kfd->init_complete = false;
> +       kfd->smi_events.events = 0;
>         kfd->kfd2kgd = f2g;
>         atomic_set(&kfd->compute_profile, 0);
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> index e05d75e..151e83e 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
> @@ -24,6 +24,7 @@
>  #include "kfd_events.h"
>  #include "soc15_int.h"
>  #include "kfd_device_queue_manager.h"
> +#include "kfd_smi_events.h"
>
>  static bool event_interrupt_isr_v9(struct kfd_dev *dev,
>                                         const uint32_t *ih_ring_entry,
> @@ -117,6 +118,7 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev,
>                 info.prot_read  = ring_id & 0x10;
>                 info.prot_write = ring_id & 0x20;
>
> +               kfd_smi_event_update_vmfault(dev, pasid);
>                 kfd_process_vm_fault(dev->dqm, pasid);
>                 kfd_signal_vm_fault_event(dev, pasid, &info);
>         }
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 43b888b..fdb51de 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -223,6 +223,13 @@ struct kfd_vmid_info {
>         uint32_t vmid_num_kfd;
>  };
>
> +struct kfd_smi_events {
> +       uint64_t events;
> +       struct kfifo fifo;
> +       wait_queue_head_t wait_queue;
> +       uint32_t max_events;
> +};
> +
>  struct kfd_dev {
>         struct kgd_dev *kgd;
>
> @@ -309,6 +316,9 @@ struct kfd_dev {
>
>         /* Global GWS resource shared b/t processes*/
>         void *gws;
> +
> +       /* if this device is in SMI events watch */
> +       struct kfd_smi_events smi_events;
>  };
>
>  enum kfd_mempool {
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
> new file mode 100644
> index 0000000..ba9d036
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
> @@ -0,0 +1,143 @@
> +/*
> + * Copyright 2020 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#include <linux/poll.h>
> +#include <linux/wait.h>
> +#include <linux/anon_inodes.h>
> +#include <uapi/linux/kfd_ioctl.h>
> +#include "amdgpu_vm.h"
> +#include "kfd_priv.h"
> +#include "kfd_smi_events.h"
> +
> +static DEFINE_MUTEX(kfd_smi_mutex);
> +
> +struct mutex *kfd_get_smi_mutex(void)
> +{
> +       return &kfd_smi_mutex;
> +}
> +
> +static __poll_t kfd_smi_ev_poll(struct file *, struct poll_table_struct *);
> +static ssize_t kfd_smi_ev_read(struct file *, char __user *, size_t, loff_t *);
> +static int kfd_smi_ev_release(struct inode *, struct file *);
> +
> +static const char kfd_smi_name[] = "kfd_smi_ev";
> +
> +static const struct file_operations kfd_smi_ev_fops = {
> +       .owner = THIS_MODULE,
> +       .poll = kfd_smi_ev_poll,
> +       .read = kfd_smi_ev_read,
> +       .release = kfd_smi_ev_release
> +};
> +
> +static __poll_t kfd_smi_ev_poll(struct file *filep,
> +                               struct poll_table_struct *wait)
> +{
> +       struct kfd_dev *dev = filep->private_data;
> +       struct kfd_smi_events *ev = &dev->smi_events;
> +
> +       __poll_t mask = 0;
> +
> +       poll_wait(filep, &ev->wait_queue, wait);
> +       mask |= !kfifo_is_empty(&ev->fifo) ? POLLIN | POLLRDNORM : mask;
> +
> +       return mask;
> +}
> +
> +static ssize_t kfd_smi_ev_read(struct file *filep, char __user *user,
> +                              size_t size, loff_t *offset)
> +{
> +       int ret, copied = 0;
> +       struct kfd_dev *dev = filep->private_data;
> +
> +       ret = kfifo_to_user(&dev->smi_events.fifo, user, size, &copied);
> +       if (ret || !copied) {
> +               pr_debug("kfd smi-events: Fail to read fd (%i) (%i)\n",
> +                               ret, copied);
> +               return ret ? ret : -EAGAIN;
> +       }
> +
> +       return copied;
> +}
> +
> +static int kfd_smi_ev_release(struct inode *inode, struct file *filep)
> +{
> +       struct kfd_dev *dev = filep->private_data;
> +
> +       kfifo_free(&dev->smi_events.fifo);
> +       return 0;
> +}
> +
> +void kfd_smi_event_update_vmfault(struct kfd_dev *kdev, uint16_t pasid)
> +{
> +       struct kfd_smi_vmfault_fifo fifo_out;
> +       struct amdgpu_device *adev = (struct amdgpu_device *)kdev->kgd;
> +       struct amdgpu_task_info task_info;
> +
> +       if (!kdev->smi_events.events)
> +               return;
> +
> +       if (!(kdev->smi_events.events & KFD_SMI_EV_VMFAULT))
> +               return;
> +
> +       memset(&task_info, 0, sizeof(struct amdgpu_task_info));
> +       amdgpu_vm_get_task_info(adev, pasid, &task_info);
> +
> +       fifo_out.group = 0;
> +       fifo_out.event = KFD_SMI_EV_VMFAULT;
> +       fifo_out.gpu_id = kdev->id;
> +       fifo_out.pid = task_info.pid;
> +       strcpy(fifo_out.task_name, task_info.task_name);
> +       kfifo_in(&kdev->smi_events.fifo, &fifo_out, sizeof(fifo_out));
> +       wake_up_all(&kdev->smi_events.wait_queue);
> +}
> +
> +void kfd_smi_event_unregister(struct kfd_dev *dev, uint64_t events)
> +{
> +       mutex_lock(kfd_get_smi_mutex());
> +       dev->smi_events.events &= ~events;
> +       mutex_unlock(kfd_get_smi_mutex());
> +}
> +
> +int kfd_smi_event_register(struct kfd_dev *dev, uint64_t events)
> +{
> +       int ret;
> +
> +       mutex_lock(kfd_get_smi_mutex());
> +       dev->smi_events.events |= events;
> +       mutex_unlock(kfd_get_smi_mutex());
> +
> +       /* We use the lower 32 bits for now. Each bit represents one event. If
> +        * featured events are increased to more than 32, we'll use the upper
> +        * bits as groups so the total number of events can be up to 32*32.
> +        */
> +       dev->smi_events.max_events = 32;
> +       ret = kfifo_alloc(&dev->smi_events.fifo, dev->smi_events.max_events,
> +                        GFP_KERNEL);
> +       if (ret) {
> +               pr_err("fail to allocate kfifo\n");
> +               return ret;
> +       }
> +       init_waitqueue_head(&dev->smi_events.wait_queue);
> +
> +       return anon_inode_getfd(kfd_smi_name, &kfd_smi_ev_fops,
> +                               (void *)dev, 0);
> +}
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
> new file mode 100644
> index 0000000..2e320d3
> --- /dev/null
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.h
> @@ -0,0 +1,41 @@
> +/*
> + * Copyright 2020 Advanced Micro Devices, Inc.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
> + * OTHER DEALINGS IN THE SOFTWARE.
> + */
> +
> +#ifndef KFD_SMI_EVENTS_H_INCLUDED
> +#define KFD_SMI_EVENTS_H_INCLUDED
> +
> +int kfd_smi_event_register(struct kfd_dev *dev, uint64_t events);
> +void kfd_smi_event_unregister(struct kfd_dev *dev, uint64_t events);
> +void kfd_smi_event_update_vmfault(struct kfd_dev *dev, uint16_t pasid);
> +
> +/* All FIFO must start with "uint32_t group" and "uint32_t event" so the user
> + * can read the first 8 bytes to determine the next read length.
> + */
> +struct kfd_smi_vmfault_fifo {
> +       uint32_t group;
> +       uint32_t event;
> +       unsigned int gpu_id;
> +       pid_t pid;
> +       char task_name[TASK_COMM_LEN];
> +};
> +
> +#endif
> diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
> index 4f66764..6ce7c69 100644
> --- a/include/uapi/linux/kfd_ioctl.h
> +++ b/include/uapi/linux/kfd_ioctl.h
> @@ -442,6 +442,28 @@ struct kfd_ioctl_import_dmabuf_args {
>         __u32 dmabuf_fd;        /* to KFD */
>  };
>
> +/*
> + * KFD SMI(System Management Interface) events
> + */
> +enum kfd_smi_events_op {
> +       KFD_SMI_EVENTS_REGISTER = 1,
> +       KFD_SMI_EVENTS_UNREGISTER
> +};
> +
> +/* Event ID (mask) */
> +#define KFD_SMI_EV_VMFAULT     0x00000001
> +
> +struct kfd_ioctl_smi_events_args {
> +       __u32 op;       /* to KFD */
> +       /* upper 32 bits: group. lower 32 bits: event IDs */
> +       __u64 events;   /* to KFD */
> +       __u32 gpu_id;   /* to KFD */
> +       pid_t pid;      /* to KFD */
> +       __u32 data1;    /* from KFD */
> +       __u32 data2;
> +       __u32 data3;
> +};
> +
>  /* Register offset inside the remapped mmio page
>   */
>  enum kfd_mmio_remap {
> @@ -546,7 +568,10 @@ enum kfd_mmio_remap {
>  #define AMDKFD_IOC_ALLOC_QUEUE_GWS             \
>                 AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args)
>
> +#define AMDKFD_IOC_SMI_EVENTS                  \
> +               AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
> +
>  #define AMDKFD_COMMAND_START           0x01
> -#define AMDKFD_COMMAND_END             0x1F
> +#define AMDKFD_COMMAND_END             0x20
>
>  #endif
> --
> 2.7.4
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@xxxxxxxxxxxxxxxxxxxxx
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/amd-gfx



[Index of Archives]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]

  Powered by Linux