From: Xiaogang Chen <xiaogang.chen@xxxxxxx>
The purpose of this patch is having kfd driver function as expected during AMD
gpu device plug/unplug.
When an AMD gpu device got unplug kfd driver stops all queues from this device.
If there are user processes still ref the render node this device is marked as
invalid. kfd driver will return error for following requests to the device from
all existing user processes. Existing user processes can still use remaining
gpu devices during/after unplug event.
After all refs to the device have been closed from user space kfd driver gpu
node topology is updated by removing correspodent kfd nodes.
At same time user space can use remaining gpu devices that are valid. When all
AMD gpu devices got removed kfd driver will not allow open /dev/kfd request.
Unplugged AMD gpu devices can be re-plugged. kfd driver will use added devices
and function as usual.
Signed-off-by: Xiaogang Chen<Xiaogang.Chen@xxxxxxx>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 7 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 3 +-
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 6 ++
drivers/gpu/drm/amd/amdkfd/kfd_device.c | 59 +++++++++++++++++++
drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c | 6 ++
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 7 +++
drivers/gpu/drm/amd/amdkfd/kfd_process.c | 24 +++++++-
.../amd/amdkfd/kfd_process_queue_manager.c | 26 ++++++++
9 files changed, 141 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index b545940e512b..651ae0775f80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -248,6 +248,11 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
kgd2kfd_interrupt(adev->kfd.dev, ih_ring_entry);
}
+void amdgpu_amdkfd_teardown_kfd_device(struct kfd_dev *kfd)
+{
+ kgd2kfd_teardown_kfd_device(kfd);
+}
+
void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm)
{
if (adev->kfd.dev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 7e0a22072536..bd241f569b79 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -152,6 +152,7 @@ struct amdkfd_process_info {
int amdgpu_amdkfd_init(void);
void amdgpu_amdkfd_fini(void);
+void amdgpu_amdkfd_teardown_kfd_device(struct kfd_dev *kfd);
void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);
int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);
@@ -431,6 +432,7 @@ int kgd2kfd_check_and_lock_kfd(void);
void kgd2kfd_unlock_kfd(void);
int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id);
int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id);
+void kgd2kfd_teardown_kfd_device(struct kfd_dev *kfd);
#else
static inline int kgd2kfd_init(void)
{
@@ -511,5 +513,10 @@ static inline int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)
{
return 0;
}
+
+void kgd2kfd_teardown_processes(void)
+{
+}
+
#endif
#endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 1e47655e02c6..4529d7a88b98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3315,7 +3315,8 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
- amdgpu_amdkfd_suspend(adev, false);
+ if (adev->kfd.dev)
+ amdgpu_amdkfd_teardown_kfd_device(adev->kfd.dev);
/* Workaroud for ASICs need to disable SMC first */
amdgpu_device_smu_fini_early(adev);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index a1f191a5984b..962a3a15a7ee 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -704,6 +704,9 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
for (i = 0; i < p->n_pdds; i++) {
struct kfd_process_device *pdd = p->pdds[i];
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
pAperture =
&args->process_apertures[args->num_of_nodes];
pAperture->gpu_id = pdd->dev->id;
@@ -779,6 +782,9 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp,
for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
struct kfd_process_device *pdd = p->pdds[i];
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
pa[i].gpu_id = pdd->dev->id;
pa[i].lds_base = pdd->lds_base;
pa[i].lds_limit = pdd->lds_limit;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index fad1c8f2bc83..5b786c72db64 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -893,6 +893,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
svm_range_set_max_pages(kfd->adev);
kfd->init_complete = true;
+ kfd->valid = true;
dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
kfd->adev->pdev->device);
@@ -919,6 +920,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
void kgd2kfd_device_exit(struct kfd_dev *kfd)
{
+ struct kfd_process *p;
+ unsigned int i, j;
+ unsigned int temp;
+
if (kfd->init_complete) {
/* Cleanup KFD nodes */
kfd_cleanup_nodes(kfd, kfd->num_nodes);
@@ -929,6 +934,20 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem);
}
+ /* now this kfd_dev has been completely removed from kfd driver
+ * before kfree kfd iterate all existing kfd processes. if kfd process
+ * uses any kfd node from this kfd set its ref to NULL
+ */
+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+ for (i = 0; i < kfd->num_nodes; i++)
+ for (j = 0; j < p->n_pdds; j++) {
+ if (kfd->nodes[i] == p->pdds[j]->dev) {
+ p->pdds[j]->dev = NULL;
+ break;
+ }
+ }
+ }
+
kfree(kfd);
}
@@ -1485,6 +1504,46 @@ int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)
return node->dqm->ops.halt(node->dqm);
}
+/* tear down this kfd deve */
+void kgd2kfd_teardown_kfd_device(struct kfd_dev *kfd)
+{
+ struct kfd_process *p;
+ struct kfd_node *dev;
+ unsigned int temp;
+ unsigned int i, j;
+ bool found;
+
+ kfd->valid = false;
+ /* stop queues from kfd nodes in this kfd dev */
+ for (i = 0; i < kfd->num_nodes; i++) {
+ dev = kfd->nodes[i];
+ dev->dqm->ops.stop(dev->dqm);
+ }
+
+ /* iterate all existing kfd processes, signal a gpu device is being teared
+ * down if any kfd process uses kfd node from this kfd dev by
+ * KFD_EVENT_TYPE_HW_EXCEPTION event
+ */
+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+ found = false;
+ for (i = 0; i < kfd->num_nodes; i++) {
+ for (j = 0; j < p->n_pdds; j++)
+ if (kfd->nodes[i] == p->pdds[j]->dev) {
+ found = true;
+ break;
+ }
+
+ if (found)
+ break;
+ }
+
+ if (found)
+ kfd_signal_hw_exception_event(p->pasid);
+ }
+
+ return;
+}
+
#if defined(CONFIG_DEBUG_FS)
/* This function will send a package to HIQ to hang the HWS
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index dbcb60eb54b2..b8dd80ee17be 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -378,6 +378,12 @@ int kfd_init_apertures(struct kfd_process *process)
continue;
}
+ /* kfd device that this kfd node belogns is not valid */
+ if (!dev->kfd->valid) {
+ id++;
+ continue;
+ }
+
pdd = kfd_create_process_device_data(dev, process);
if (!pdd) {
dev_err(dev->adev->dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 6a5bf88cc232..97e7692ce569 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -371,6 +371,9 @@ struct kfd_dev {
/* bitmap for dynamic doorbell allocation from doorbell object */
unsigned long *doorbell_bitmap;
+
+ /* this kfd_dev valid or not */
+ bool valid;
};
enum kfd_mempool {
@@ -1055,6 +1058,10 @@ int kfd_process_restore_queues(struct kfd_process *p);
void kfd_suspend_all_processes(void);
int kfd_resume_all_processes(void);
+static inline bool is_kfd_process_device_valid(struct kfd_process_device *pdd) {
+ return (pdd && pdd->dev && pdd->dev->kfd && pdd->dev->kfd->valid);
+}
+
struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *process,
uint32_t gpu_id);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index d07acf1b2f93..2f4aff81dc64 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1536,6 +1536,12 @@ static struct kfd_process *create_process(const struct task_struct *thread)
if (err != 0)
goto err_init_apertures;
+ /* no any kfd_process_device can be created */
+ if (!process->n_pdds) {
+ err = -ENODEV;
+ goto err_init_apertures;
+ }
+
/* Check XNACK support after PDDs are created in kfd_init_apertures */
process->xnack_enabled = kfd_process_xnack_mode(process, false);
@@ -1595,9 +1601,13 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
{
int i;
- for (i = 0; i < p->n_pdds; i++)
+ for (i = 0; i < p->n_pdds; i++) {
+ if (!is_kfd_process_device_valid(p->pdds[i]))
+ continue;
+
if (p->pdds[i]->dev == dev)
return p->pdds[i];
+ }
return NULL;
}
@@ -1863,6 +1873,9 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
struct kfd_process_device *pdd = p->pdds[i];
struct device *dev = pdd->dev->adev->dev;
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
trigger);
@@ -1893,6 +1906,9 @@ int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
if (n_evicted == 0)
break;
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
@@ -1916,6 +1932,9 @@ int kfd_process_restore_queues(struct kfd_process *p)
struct kfd_process_device *pdd = p->pdds[i];
struct device *dev = pdd->dev->adev->dev;
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
@@ -2255,6 +2274,9 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
for (i = 0; i < p->n_pdds; i++) {
struct kfd_process_device *pdd = p->pdds[i];
+ if (!is_kfd_process_device_valid(pdd))
+ continue;
+
if (pdd->user_gpu_id == gpu_id)
return pdd;
}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index c76db22a1000..d2f95be81216 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -604,7 +604,9 @@ int pqm_update_mqd(struct process_queue_manager *pqm,
unsigned int qid, struct mqd_update_info *minfo)
{
int retval;
+ struct kfd_node *dev = NULL;
struct process_queue_node *pqn;
+ struct kfd_process_device *pdd;
pqn = get_queue_by_qid(pqm, qid);
if (!pqn) {
@@ -612,6 +614,17 @@ int pqm_update_mqd(struct process_queue_manager *pqm,
return -EFAULT;
}
+ if (pqn->q)
+ dev = pqn->q->device;
+ if (WARN_ON(!dev))
+ return -ENODEV;
+
+ pdd = kfd_get_process_device_data(dev, pqm->process);
+ if (!pdd) {
+ pr_err("Process device data doesn't exist\n");
+ return -EINVAL;
+ }
+
/* CUs are masked for debugger requirements so deny user mask */
if (pqn->q->properties.is_dbg_wa && minfo && minfo->cu_mask.ptr)
return -EBUSY;
@@ -671,6 +684,8 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
u32 *save_area_used_size)
{
struct process_queue_node *pqn;
+ struct kfd_process_device *pdd;
+ struct kfd_node *dev = NULL;
pqn = get_queue_by_qid(pqm, qid);
if (!pqn) {
@@ -679,6 +694,17 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
return -EFAULT;
}
+ if (pqn->q)
+ dev = pqn->q->device;
+ if (WARN_ON(!dev))
+ return -ENODEV;
+
+ pdd = kfd_get_process_device_data(dev, pqm->process);
+ if (!pdd) {
+ pr_err("Process device data doesn't exist\n");
+ return -EINVAL;
+ }
+
return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
pqn->q,
ctl_stack,
--
2.25.1
