[PATCH 16/25] drm/amdkfd: Implement KFD process eviction/restore

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Wed, Feb 7, 2018 at 3:32 AM, Felix Kuehling <Felix.Kuehling at amd.com> wrote:
> When the TTM memory manager in KGD evicts BOs, all user mode queues
> potentially accessing these BOs must be evicted temporarily. Once
> user mode queues are evicted, the eviction fence is signaled,
> allowing the migration of the BO to proceed.
>
> A delayed worker is scheduled to restore all the BOs belonging to
> the evicted process and restart its queues.
>
> During suspend/resume of the GPU we also evict all processes to allow
> KGD to save BOs in system memory, since VRAM will be lost.
>
> v2:
> * Account for eviction when updating of q->is_active in MQD manager
>
> Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan at amd.com>
> Signed-off-by: Felix Kuehling <Felix.Kuehling at amd.com>
> ---
>  drivers/gpu/drm/amd/amdkfd/kfd_device.c            |  65 +++++-
>  .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  | 219 ++++++++++++++++++++-
>  .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |   9 +
>  drivers/gpu/drm/amd/amdkfd/kfd_module.c            |   2 +
>  drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c   |   9 +-
>  drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |   6 +-
>  drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |  32 ++-
>  drivers/gpu/drm/amd/amdkfd/kfd_process.c           | 213 ++++++++++++++++++++
>  8 files changed, 547 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> index 4ac2d61..334669996 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
> @@ -33,6 +33,7 @@
>  #include "kfd_iommu.h"
>
>  #define MQD_SIZE_ALIGNED 768
> +static atomic_t kfd_device_suspended = ATOMIC_INIT(0);
>
>  #ifdef KFD_SUPPORT_IOMMU_V2
>  static const struct kfd_device_info kaveri_device_info = {
> @@ -469,6 +470,10 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
>         if (!kfd->init_complete)
>                 return;
>
> +       /* For first KFD device suspend all the KFD processes */
> +       if (atomic_inc_return(&kfd_device_suspended) == 1)
> +               kfd_suspend_all_processes();
> +
>         kfd->dqm->ops.stop(kfd->dqm);
>
>         kfd_iommu_suspend(kfd);
> @@ -476,11 +481,21 @@ void kgd2kfd_suspend(struct kfd_dev *kfd)
>
>  int kgd2kfd_resume(struct kfd_dev *kfd)
>  {
> +       int ret, count;
> +
>         if (!kfd->init_complete)
>                 return 0;
>
> -       return kfd_resume(kfd);
> +       ret = kfd_resume(kfd);
> +       if (ret)
> +               return ret;
> +
> +       count = atomic_dec_return(&kfd_device_suspended);
> +       WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
> +       if (count == 0)
> +               ret = kfd_resume_all_processes();
>
> +       return ret;
>  }
>
>  static int kfd_resume(struct kfd_dev *kfd)
> @@ -526,6 +541,54 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
>         spin_unlock(&kfd->interrupt_lock);
>  }
>
> +/** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
> + *   prepare for safe eviction of KFD BOs that belong to the specified
> + *   process.
> + *
> + * @mm: mm_struct that identifies the specified KFD process
> + * @fence: eviction fence attached to KFD process BOs
> + *
> + */
> +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
> +                                              struct dma_fence *fence)
> +{
> +       struct kfd_process *p;
> +       unsigned long active_time;
> +       unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
> +
> +       if (!fence)
> +               return -EINVAL;
> +
> +       if (dma_fence_is_signaled(fence))
> +               return 0;
> +
> +       p = kfd_lookup_process_by_mm(mm);
> +       if (!p)
> +               return -ENODEV;
> +
> +       if (fence->seqno == p->last_eviction_seqno)
> +               goto out;
> +
> +       p->last_eviction_seqno = fence->seqno;
> +
> +       /* Avoid KFD process starvation. Wait for at least
> +        * PROCESS_ACTIVE_TIME_MS before evicting the process again
> +        */
> +       active_time = get_jiffies_64() - p->last_restore_timestamp;
> +       if (delay_jiffies > active_time)
> +               delay_jiffies -= active_time;
> +       else
> +               delay_jiffies = 0;
> +
> +       /* During process initialization eviction_work.dwork is initialized
> +        * to kfd_evict_bo_worker
> +        */
> +       schedule_delayed_work(&p->eviction_work, delay_jiffies);
> +out:
> +       kfd_unref_process(p);
> +       return 0;
> +}
> +
>  static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
>                                 unsigned int chunk_size)
>  {
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> index b7d0639..b3b6dab 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
> @@ -21,10 +21,11 @@
>   *
>   */
>
> +#include <linux/ratelimit.h>
> +#include <linux/printk.h>
>  #include <linux/slab.h>
>  #include <linux/list.h>
>  #include <linux/types.h>
> -#include <linux/printk.h>
>  #include <linux/bitops.h>
>  #include <linux/sched.h>
>  #include "kfd_priv.h"
> @@ -180,6 +181,14 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
>                         goto out_unlock;
>         }
>         q->properties.vmid = qpd->vmid;
> +       /*
> +        * Eviction state logic: we only mark active queues as evicted
> +        * to avoid the overhead of restoring inactive queues later
> +        */
> +       if (qpd->evicted)
> +               q->properties.is_evicted = (q->properties.queue_size > 0 &&
> +                                           q->properties.queue_percent > 0 &&
> +                                           q->properties.queue_address != 0);
>
>         q->properties.tba_addr = qpd->tba_addr;
>         q->properties.tma_addr = qpd->tma_addr;
> @@ -377,15 +386,29 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
>  {
>         int retval;
>         struct mqd_manager *mqd;
> +       struct kfd_process_device *pdd;
>         bool prev_active = false;
>
>         mutex_lock(&dqm->lock);
> +       pdd = kfd_get_process_device_data(q->device, q->process);
> +       if (!pdd) {
> +               retval = -ENODEV;
> +               goto out_unlock;
> +       }
>         mqd = dqm->ops.get_mqd_manager(dqm,
>                         get_mqd_type_from_queue_type(q->properties.type));
>         if (!mqd) {
>                 retval = -ENOMEM;
>                 goto out_unlock;
>         }
> +       /*
> +        * Eviction state logic: we only mark active queues as evicted
> +        * to avoid the overhead of restoring inactive queues later
> +        */
> +       if (pdd->qpd.evicted)
> +               q->properties.is_evicted = (q->properties.queue_size > 0 &&
> +                                           q->properties.queue_percent > 0 &&
> +                                           q->properties.queue_address != 0);
>
>         /* Save previous activity state for counters */
>         prev_active = q->properties.is_active;
> @@ -457,6 +480,187 @@ static struct mqd_manager *get_mqd_manager(
>         return mqd;
>  }
>
> +static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
> +                                       struct qcm_process_device *qpd)
> +{
> +       struct queue *q;
> +       struct mqd_manager *mqd;
> +       struct kfd_process_device *pdd;
> +       int retval = 0;
> +
> +       mutex_lock(&dqm->lock);
> +       if (qpd->evicted++ > 0) /* already evicted, do nothing */
> +               goto out;
> +
> +       pdd = qpd_to_pdd(qpd);
> +       pr_info_ratelimited("Evicting PASID %u queues\n",
> +                           pdd->process->pasid);
> +
> +       /* unactivate all active queues on the qpd */
> +       list_for_each_entry(q, &qpd->queues_list, list) {
> +               if (!q->properties.is_active)
> +                       continue;
> +               mqd = dqm->ops.get_mqd_manager(dqm,
> +                       get_mqd_type_from_queue_type(q->properties.type));
> +               if (!mqd) { /* should not be here */
> +                       pr_err("Cannot evict queue, mqd mgr is NULL\n");
> +                       retval = -ENOMEM;
> +                       goto out;
> +               }
> +               q->properties.is_evicted = true;
> +               q->properties.is_active = false;
> +               retval = mqd->destroy_mqd(mqd, q->mqd,
> +                               KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
> +                               KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
> +               if (retval)
> +                       goto out;
> +               dqm->queue_count--;
> +       }
> +
> +out:
> +       mutex_unlock(&dqm->lock);
> +       return retval;
> +}
> +
> +static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
> +                                     struct qcm_process_device *qpd)
> +{
> +       struct queue *q;
> +       struct kfd_process_device *pdd;
> +       int retval = 0;
> +
> +       mutex_lock(&dqm->lock);
> +       if (qpd->evicted++ > 0) /* already evicted, do nothing */
> +               goto out;
> +
> +       pdd = qpd_to_pdd(qpd);
> +       pr_info_ratelimited("Evicting PASID %u queues\n",
> +                           pdd->process->pasid);
> +
> +       /* unactivate all active queues on the qpd */
> +       list_for_each_entry(q, &qpd->queues_list, list) {
> +               if (!q->properties.is_active)
> +                       continue;
> +               q->properties.is_evicted = true;
> +               q->properties.is_active = false;
> +               dqm->queue_count--;
> +       }
> +       retval = execute_queues_cpsch(dqm,
> +                               qpd->is_debug ?
> +                               KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
> +                               KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
> +
> +out:
> +       mutex_unlock(&dqm->lock);
> +       return retval;
> +}
> +
> +static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
> +                                         struct qcm_process_device *qpd)
> +{
> +       struct queue *q;
> +       struct mqd_manager *mqd;
> +       struct kfd_process_device *pdd;
> +       uint32_t pd_base;
> +       int retval = 0;
> +
> +       pdd = qpd_to_pdd(qpd);
> +       /* Retrieve PD base */
> +       pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
> +
> +       mutex_lock(&dqm->lock);
> +       if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
> +               goto out;
> +       if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
> +               qpd->evicted--;
> +               goto out;
> +       }
> +
> +       pr_info_ratelimited("Restoring PASID %u queues\n",
> +                           pdd->process->pasid);
> +
> +       /* Update PD Base in QPD */
> +       qpd->page_table_base = pd_base;
> +       pr_debug("Updated PD address to 0x%08x\n", pd_base);
> +
> +       if (!list_empty(&qpd->queues_list)) {
> +               dqm->dev->kfd2kgd->set_vm_context_page_table_base(
> +                               dqm->dev->kgd,
> +                               qpd->vmid,
> +                               qpd->page_table_base);
> +               kfd_flush_tlb(pdd);
> +       }
> +
> +       /* activate all active queues on the qpd */
> +       list_for_each_entry(q, &qpd->queues_list, list) {
> +               if (!q->properties.is_evicted)
> +                       continue;
> +               mqd = dqm->ops.get_mqd_manager(dqm,
> +                       get_mqd_type_from_queue_type(q->properties.type));
> +               if (!mqd) { /* should not be here */
> +                       pr_err("Cannot restore queue, mqd mgr is NULL\n");
> +                       retval = -ENOMEM;
> +                       goto out;
> +               }
> +               q->properties.is_evicted = false;
> +               q->properties.is_active = true;
> +               retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
> +                                      q->queue, &q->properties,
> +                                      q->process->mm);
> +               if (retval)
> +                       goto out;
> +               dqm->queue_count++;
> +       }
> +       qpd->evicted = 0;
> +out:
> +       mutex_unlock(&dqm->lock);
> +       return retval;
> +}
> +
> +static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
> +                                       struct qcm_process_device *qpd)
> +{
> +       struct queue *q;
> +       struct kfd_process_device *pdd;
> +       uint32_t pd_base;
> +       int retval = 0;
> +
> +       pdd = qpd_to_pdd(qpd);
> +       /* Retrieve PD base */
> +       pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
> +
> +       mutex_lock(&dqm->lock);
> +       if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
> +               goto out;
> +       if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
> +               qpd->evicted--;
> +               goto out;
> +       }
> +
> +       pr_info_ratelimited("Restoring PASID %u queues\n",
> +                           pdd->process->pasid);
> +
> +       /* Update PD Base in QPD */
> +       qpd->page_table_base = pd_base;
> +       pr_debug("Updated PD address to 0x%08x\n", pd_base);
> +
> +       /* activate all active queues on the qpd */
> +       list_for_each_entry(q, &qpd->queues_list, list) {
> +               if (!q->properties.is_evicted)
> +                       continue;
> +               q->properties.is_evicted = false;
> +               q->properties.is_active = true;
> +               dqm->queue_count++;
> +       }
> +       retval = execute_queues_cpsch(dqm,
> +                               KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
> +       if (!retval)
> +               qpd->evicted = 0;
> +out:
> +       mutex_unlock(&dqm->lock);
> +       return retval;
> +}
> +
>  static int register_process(struct device_queue_manager *dqm,
>                                         struct qcm_process_device *qpd)
>  {
> @@ -853,6 +1057,14 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
>                 retval = -ENOMEM;
>                 goto out;
>         }
> +       /*
> +        * Eviction state logic: we only mark active queues as evicted
> +        * to avoid the overhead of restoring inactive queues later
> +        */
> +       if (qpd->evicted)
> +               q->properties.is_evicted = (q->properties.queue_size > 0 &&
> +                                           q->properties.queue_percent > 0 &&
> +                                           q->properties.queue_address != 0);
>
>         dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
>
> @@ -1291,6 +1503,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
>                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
>                 dqm->ops.set_trap_handler = set_trap_handler;
>                 dqm->ops.process_termination = process_termination_cpsch;
> +               dqm->ops.evict_process_queues = evict_process_queues_cpsch;
> +               dqm->ops.restore_process_queues = restore_process_queues_cpsch;
>                 break;
>         case KFD_SCHED_POLICY_NO_HWS:
>                 /* initialize dqm for no cp scheduling */
> @@ -1307,6 +1521,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
>                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
>                 dqm->ops.set_trap_handler = set_trap_handler;
>                 dqm->ops.process_termination = process_termination_nocpsch;
> +               dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
> +               dqm->ops.restore_process_queues =
> +                       restore_process_queues_nocpsch;
>                 break;
>         default:
>                 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> index 68be0aa..412beff 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
> @@ -79,6 +79,10 @@ struct device_process_node {
>   *
>   * @process_termination: Clears all process queues belongs to that device.
>   *
> + * @evict_process_queues: Evict all active queues of a process
> + *
> + * @restore_process_queues: Restore all evicted queues queues of a process
> + *
>   */
>
>  struct device_queue_manager_ops {
> @@ -129,6 +133,11 @@ struct device_queue_manager_ops {
>
>         int (*process_termination)(struct device_queue_manager *dqm,
>                         struct qcm_process_device *qpd);
> +
> +       int (*evict_process_queues)(struct device_queue_manager *dqm,
> +                                   struct qcm_process_device *qpd);
> +       int (*restore_process_queues)(struct device_queue_manager *dqm,
> +                                     struct qcm_process_device *qpd);
>  };
>
>  struct device_queue_manager_asic_ops {
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
> index 3ac72be..65574c6 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
> @@ -43,6 +43,8 @@ static const struct kgd2kfd_calls kgd2kfd = {
>         .interrupt      = kgd2kfd_interrupt,
>         .suspend        = kgd2kfd_suspend,
>         .resume         = kgd2kfd_resume,
> +       .schedule_evict_and_restore_process =
> +                         kgd2kfd_schedule_evict_and_restore_process,
>  };
>
>  int sched_policy = KFD_SCHED_POLICY_HWS;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
> index fbe3f83..c00c325 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
> @@ -202,7 +202,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
>
>         q->is_active = (q->queue_size > 0 &&
>                         q->queue_address != 0 &&
> -                       q->queue_percent > 0);
> +                       q->queue_percent > 0 &&
> +                       !q->is_evicted);
>
>         return 0;
>  }
> @@ -245,7 +246,8 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
>
>         q->is_active = (q->queue_size > 0 &&
>                         q->queue_address != 0 &&
> -                       q->queue_percent > 0);
> +                       q->queue_percent > 0 &&
> +                       !q->is_evicted);
>
>         return 0;
>  }
> @@ -377,7 +379,8 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
>
>         q->is_active = (q->queue_size > 0 &&
>                         q->queue_address != 0 &&
> -                       q->queue_percent > 0);
> +                       q->queue_percent > 0 &&
> +                       !q->is_evicted);
>
>         return 0;
>  }
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
> index 58221c1..89e4242 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
> @@ -198,7 +198,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
>
>         q->is_active = (q->queue_size > 0 &&
>                         q->queue_address != 0 &&
> -                       q->queue_percent > 0);
> +                       q->queue_percent > 0 &&
> +                       !q->is_evicted);
>
>         return 0;
>  }
> @@ -342,7 +343,8 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
>
>         q->is_active = (q->queue_size > 0 &&
>                         q->queue_address != 0 &&
> -                       q->queue_percent > 0);
> +                       q->queue_percent > 0 &&
> +                       !q->is_evicted);
>
>         return 0;
>  }
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 56c2e36..cac7aa2 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -335,7 +335,11 @@ enum kfd_queue_format {
>   * @is_interop: Defines if this is a interop queue. Interop queue means that
>   * the queue can access both graphics and compute resources.
>   *
> - * @is_active: Defines if the queue is active or not.
> + * @is_evicted: Defines if the queue is evicted. Only active queues
> + * are evicted, rendering them inactive.
> + *
> + * @is_active: Defines if the queue is active or not. @is_active and
> + * @is_evicted are protected by the DQM lock.
>   *
>   * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
>   * of the queue.
> @@ -357,6 +361,7 @@ struct queue_properties {
>         uint32_t __iomem *doorbell_ptr;
>         uint32_t doorbell_off;
>         bool is_interop;
> +       bool is_evicted;
>         bool is_active;
>         /* Not relevant for user mode queues in cp scheduling */
>         unsigned int vmid;
> @@ -460,6 +465,7 @@ struct qcm_process_device {
>         unsigned int queue_count;
>         unsigned int vmid;
>         bool is_debug;
> +       unsigned int evicted; /* eviction counter, 0=active */
>
>         /* This flag tells if we should reset all wavefronts on
>          * process termination
> @@ -486,6 +492,17 @@ struct qcm_process_device {
>         uint64_t tma_addr;
>  };
>
> +/* KFD Memory Eviction */
> +
> +/* Approx. wait time before attempting to restore evicted BOs */
> +#define PROCESS_RESTORE_TIME_MS 100
> +/* Approx. back off time if restore fails due to lack of memory */
> +#define PROCESS_BACK_OFF_TIME_MS 100
> +/* Approx. time before evicting the process again */
> +#define PROCESS_ACTIVE_TIME_MS 10
> +
> +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
> +                                              struct dma_fence *fence);
>
>  enum kfd_pdd_bound {
>         PDD_UNBOUND = 0,
> @@ -600,6 +617,16 @@ struct kfd_process {
>          * during restore
>          */
>         struct dma_fence *ef;
> +
> +       /* Work items for evicting and restoring BOs */
> +       struct delayed_work eviction_work;
> +       struct delayed_work restore_work;
> +       /* seqno of the last scheduled eviction */
> +       unsigned int last_eviction_seqno;
> +       /* Approx. the last timestamp (in jiffies) when the process was
> +        * restored after an eviction
> +        */
> +       unsigned long last_restore_timestamp;
>  };
>
>  #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
> @@ -629,7 +656,10 @@ void kfd_process_destroy_wq(void);
>  struct kfd_process *kfd_create_process(struct file *filep);
>  struct kfd_process *kfd_get_process(const struct task_struct *);
>  struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
> +struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
>  void kfd_unref_process(struct kfd_process *p);
> +void kfd_suspend_all_processes(void);
> +int kfd_resume_all_processes(void);
>
>  struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
>                                                 struct kfd_process *p);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index cf4fa25..18b2b86 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -55,6 +55,9 @@ static struct kfd_process *create_process(const struct task_struct *thread,
>                                         struct file *filep);
>  static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep);
>
> +static void evict_process_worker(struct work_struct *work);
> +static void restore_process_worker(struct work_struct *work);
> +
>
>  void kfd_process_create_wq(void)
>  {
> @@ -230,6 +233,9 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
>         mutex_unlock(&kfd_processes_mutex);
>         synchronize_srcu(&kfd_processes_srcu);
>
> +       cancel_delayed_work_sync(&p->eviction_work);
> +       cancel_delayed_work_sync(&p->restore_work);
> +
>         mutex_lock(&p->mutex);
>
>         /* Iterate over all process device data structures and if the
> @@ -351,6 +357,10 @@ static struct kfd_process *create_process(const struct task_struct *thread,
>         if (err != 0)
>                 goto err_init_apertures;
>
> +       INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
> +       INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
> +       process->last_restore_timestamp = get_jiffies_64();
> +
>         err = kfd_process_init_cwsr(process, filep);
>         if (err)
>                 goto err_init_cwsr;
> @@ -402,6 +412,7 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
>         INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
>         pdd->qpd.dqm = dev->dqm;
>         pdd->qpd.pqm = &p->pqm;
> +       pdd->qpd.evicted = 0;
>         pdd->process = p;
>         pdd->bound = PDD_UNBOUND;
>         pdd->already_dequeued = false;
> @@ -490,6 +501,208 @@ struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
>         return ret_p;
>  }
>
> +/* This increments the process->ref counter. */
> +struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
> +{
> +       struct kfd_process *p;
> +
> +       int idx = srcu_read_lock(&kfd_processes_srcu);
> +
> +       p = find_process_by_mm(mm);
> +       if (p)
> +               kref_get(&p->ref);
> +
> +       srcu_read_unlock(&kfd_processes_srcu, idx);
> +
> +       return p;
> +}
> +
> +/* process_evict_queues - Evict all user queues of a process
> + *
> + * Eviction is reference-counted per process-device. This means multiple
> + * evictions from different sources can be nested safely.
> + */
> +static int process_evict_queues(struct kfd_process *p)
> +{
> +       struct kfd_process_device *pdd;
> +       int r = 0;
> +       unsigned int n_evicted = 0;
> +
> +       list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
> +               r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
> +                                                           &pdd->qpd);
> +               if (r) {
> +                       pr_err("Failed to evict process queues\n");
> +                       goto fail;
> +               }
> +               n_evicted++;
> +       }
> +
> +       return r;
> +
> +fail:
> +       /* To keep state consistent, roll back partial eviction by
> +        * restoring queues
> +        */
> +       list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
> +               if (n_evicted == 0)
> +                       break;
> +               if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
> +                                                             &pdd->qpd))
> +                       pr_err("Failed to restore queues\n");
> +
> +               n_evicted--;
> +       }
> +
> +       return r;
> +}
> +
> +/* process_restore_queues - Restore all user queues of a process */
> +static  int process_restore_queues(struct kfd_process *p)
> +{
> +       struct kfd_process_device *pdd;
> +       int r, ret = 0;
> +
> +       list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
> +               r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
> +                                                             &pdd->qpd);
> +               if (r) {
> +                       pr_err("Failed to restore process queues\n");
> +                       if (!ret)
> +                               ret = r;
> +               }
> +       }
> +
> +       return ret;
> +}
> +
> +static void evict_process_worker(struct work_struct *work)
> +{
> +       int ret;
> +       struct kfd_process *p;
> +       struct delayed_work *dwork;
> +
> +       dwork = to_delayed_work(work);
> +
> +       /* Process termination destroys this worker thread. So during the
> +        * lifetime of this thread, kfd_process p will be valid
> +        */
> +       p = container_of(dwork, struct kfd_process, eviction_work);
> +       WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
> +                 "Eviction fence mismatch\n");
> +
> +       /* Narrow window of overlap between restore and evict work
> +        * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
> +        * unreserves KFD BOs, it is possible to evicted again. But
> +        * restore has few more steps of finish. So lets wait for any
> +        * previous restore work to complete
> +        */
> +       flush_delayed_work(&p->restore_work);
> +
> +       pr_debug("Started evicting pasid %d\n", p->pasid);
> +       ret = process_evict_queues(p);
> +       if (!ret) {
> +               dma_fence_signal(p->ef);
> +               dma_fence_put(p->ef);
> +               p->ef = NULL;
> +               schedule_delayed_work(&p->restore_work,
> +                               msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
> +
> +               pr_debug("Finished evicting pasid %d\n", p->pasid);
> +       } else
> +               pr_err("Failed to evict queues of pasid %d\n", p->pasid);
> +}
> +
> +static void restore_process_worker(struct work_struct *work)
> +{
> +       struct delayed_work *dwork;
> +       struct kfd_process *p;
> +       struct kfd_process_device *pdd;
> +       int ret = 0;
> +
> +       dwork = to_delayed_work(work);
> +
> +       /* Process termination destroys this worker thread. So during the
> +        * lifetime of this thread, kfd_process p will be valid
> +        */
> +       p = container_of(dwork, struct kfd_process, restore_work);
> +
> +       /* Call restore_process_bos on the first KGD device. This function
> +        * takes care of restoring the whole process including other devices.
> +        * Restore can fail if enough memory is not available. If so,
> +        * reschedule again.
> +        */
> +       pdd = list_first_entry(&p->per_device_data,
> +                              struct kfd_process_device,
> +                              per_device_list);
> +
> +       pr_debug("Started restoring pasid %d\n", p->pasid);
> +
> +       /* Setting last_restore_timestamp before successful restoration.
> +        * Otherwise this would have to be set by KGD (restore_process_bos)
> +        * before KFD BOs are unreserved. If not, the process can be evicted
> +        * again before the timestamp is set.
> +        * If restore fails, the timestamp will be set again in the next
> +        * attempt. This would mean that the minimum GPU quanta would be
> +        * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
> +        * functions)
> +        */
> +
> +       p->last_restore_timestamp = get_jiffies_64();
> +       ret = pdd->dev->kfd2kgd->restore_process_bos(p->kgd_process_info,
> +                                                    &p->ef);
> +       if (ret) {
> +               pr_debug("Failed to restore BOs of pasid %d, retry after %d ms\n",
> +                        p->pasid, PROCESS_BACK_OFF_TIME_MS);
> +               ret = schedule_delayed_work(&p->restore_work,
> +                               msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
> +               WARN(!ret, "reschedule restore work failed\n");
> +               return;
> +       }
> +
> +       ret = process_restore_queues(p);
> +       if (!ret)
> +               pr_debug("Finished restoring pasid %d\n", p->pasid);
> +       else
> +               pr_err("Failed to restore queues of pasid %d\n", p->pasid);
> +}
> +
> +void kfd_suspend_all_processes(void)
> +{
> +       struct kfd_process *p;
> +       unsigned int temp;
> +       int idx = srcu_read_lock(&kfd_processes_srcu);
> +
> +       hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
> +               cancel_delayed_work_sync(&p->eviction_work);
> +               cancel_delayed_work_sync(&p->restore_work);
> +
> +               if (process_evict_queues(p))
> +                       pr_err("Failed to suspend process %d\n", p->pasid);
> +               dma_fence_signal(p->ef);
> +               dma_fence_put(p->ef);
> +               p->ef = NULL;
> +       }
> +       srcu_read_unlock(&kfd_processes_srcu, idx);
> +}
> +
> +int kfd_resume_all_processes(void)
> +{
> +       struct kfd_process *p;
> +       unsigned int temp;
> +       int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
> +
> +       hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
> +               if (!schedule_delayed_work(&p->restore_work, 0)) {
> +                       pr_err("Restore process %d failed during resume\n",
> +                              p->pasid);
> +                       ret = -EFAULT;
> +               }
> +       }
> +       srcu_read_unlock(&kfd_processes_srcu, idx);
> +       return ret;
> +}
> +
>  int kfd_reserved_mem_mmap(struct kfd_process *process,
>                           struct vm_area_struct *vma)
>  {
> --
> 2.7.4
>

This patch is:
Acked-by: Oded Gabbay <oded.gabbay at gmail.com>


[Index of Archives]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]

  Powered by Linux