Dump contents of queue control stacks on CRIU dump and restore them
during CRIU restore.
(rajneesh: rebased to 5.11 and fixed merge conflict)
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@xxxxxxx>
Signed-off-by: David Yat Sin <david.yatsin@xxxxxxx>
---
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 2 +-
drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c | 2 +-
.../drm/amd/amdkfd/kfd_device_queue_manager.c | 30 +++++++----
.../drm/amd/amdkfd/kfd_device_queue_manager.h | 10 ++--
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h | 8 ++-
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c | 17 ++++--
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c | 18 +++++--
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 27 ++++++++--
.../gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c | 17 ++++--
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 8 +--
.../amd/amdkfd/kfd_process_queue_manager.c | 52 ++++++++++++-------
11 files changed, 136 insertions(+), 55 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index fad17f9b13f6..dd0ecfe76c2b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -313,7 +313,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
dev->id);
err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, NULL,
- NULL, &doorbell_offset_in_process);
+ NULL, NULL, &doorbell_offset_in_process);
if (err != 0)
goto err_create_queue;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index c6c0cd47e7f7..3c29e60b967f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -185,7 +185,7 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
properties.type = KFD_QUEUE_TYPE_DIQ;
status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
- &properties, &qid, NULL, NULL, NULL);
+ &properties, &qid, NULL, NULL, NULL, NULL);
if (status) {
pr_err("Failed to create DIQ\n");
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 14199e467e96..5943dcf1720f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -319,7 +319,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
- const void *restore_mqd)
+ const void *restore_mqd,
+ const void *restore_ctl_stack)
{
struct mqd_manager *mqd_mgr;
int retval;
@@ -380,8 +381,9 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
}
if (qd)
- mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
- &q->gart_mqd_addr, &q->properties, restore_mqd);
+ mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
+ &q->properties, restore_mqd,
+ restore_ctl_stack, qd->ctl_stack_size);
else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
@@ -1322,7 +1324,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
- const void *restore_mqd)
+ const void *restore_mqd, const void *restore_ctl_stack)
{
int retval;
struct mqd_manager *mqd_mgr;
@@ -1370,8 +1372,9 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
q->properties.is_evicted = !!qpd->evicted;
if (qd)
- mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
- &q->gart_mqd_addr, &q->properties, restore_mqd);
+ mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
+ &q->properties, restore_mqd, restore_ctl_stack,
+ qd->ctl_stack_size);
else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
&q->gart_mqd_addr, &q->properties);
@@ -1759,19 +1762,28 @@ static int get_wave_state(struct device_queue_manager *dqm,
static void get_queue_dump_info(struct device_queue_manager *dqm,
const struct queue *q,
- u32 *mqd_size)
+ u32 *mqd_size,
+ u32 *ctl_stack_size)
{
struct mqd_manager *mqd_mgr;
enum KFD_MQD_TYPE mqd_type =
get_mqd_type_from_queue_type(q->properties.type);
+ dqm_lock(dqm);
mqd_mgr = dqm->mqd_mgrs[mqd_type];
*mqd_size = mqd_mgr->mqd_size;
+ *ctl_stack_size = 0;
+
+ if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_dump_info)
+ mqd_mgr->get_dump_info(mqd_mgr, q->mqd, ctl_stack_size);
+
+ dqm_unlock(dqm);
}
static int dump_mqd(struct device_queue_manager *dqm,
const struct queue *q,
- void *mqd)
+ void *mqd,
+ void *ctl_stack)
{
struct mqd_manager *mqd_mgr;
int r = 0;
@@ -1791,7 +1803,7 @@ static int dump_mqd(struct device_queue_manager *dqm,
goto dqm_unlock;
}
- mqd_mgr->dump_mqd(mqd_mgr, q->mqd, mqd);
+ mqd_mgr->dump_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
dqm_unlock:
dqm_unlock(dqm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index ae4170aece6d..9d7d1308df71 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -88,7 +88,8 @@ struct device_queue_manager_ops {
struct queue *q,
struct qcm_process_device *qpd,
const struct kfd_criu_queue_priv_data *qd,
- const void *restore_mqd);
+ const void *restore_mqd,
+ const void *restore_ctl_stack);
int (*destroy_queue)(struct device_queue_manager *dqm,
struct qcm_process_device *qpd,
@@ -138,12 +139,13 @@ struct device_queue_manager_ops {
u32 *save_area_used_size);
void (*get_queue_dump_info)(struct device_queue_manager *dqm,
- const struct queue *q,
- u32 *mqd_size);
+ const struct queue *q, u32 *mqd_size,
+ u32 *ctl_stack_size);
int (*dump_mqd)(struct device_queue_manager *dqm,
const struct queue *q,
- void *mqd);
+ void *mqd,
+ void *ctl_stack);
};
struct device_queue_manager_asic_ops {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 497e6f874352..bb91b95b4970 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -98,12 +98,16 @@ struct mqd_manager {
u32 *ctl_stack_used_size,
u32 *save_area_used_size);
- void (*dump_mqd)(struct mqd_manager *mm, void *mqd, void *mqd_dst);
+ void (*get_dump_info)(struct mqd_manager *mm, void *mqd, uint32_t *ctl_stack_size);
+
+ void (*dump_mqd)(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst);
void (*restore_mqd)(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *p,
- const void *mqd_src);
+ const void *mqd_src,
+ const void *ctl_stack_src,
+ const u32 ctl_stack_size);
#if defined(CONFIG_DEBUG_FS)
int (*debugfs_show_mqd)(struct seq_file *m, void *data);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 1d000252080c..bf32c67b723a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -275,7 +275,13 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
pipe_id, queue_id);
}
-static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void get_dump_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+ /* Control stack is stored in user mode */
+ *ctl_stack_size = 0;
+}
+
+static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct cik_mqd *m;
@@ -287,7 +293,8 @@ static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct cik_mqd *m;
@@ -309,7 +316,7 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
qp->is_active = 0;
}
-static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct cik_sdma_rlc_registers *m;
@@ -321,7 +328,8 @@ static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct cik_sdma_rlc_registers *m;
@@ -451,6 +459,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
mqd->update_mqd = update_mqd;
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
+ mqd->get_dump_info = get_dump_info;
mqd->dump_mqd = dump_mqd;
mqd->restore_mqd = restore_mqd;
mqd->mqd_size = sizeof(struct cik_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 0066a2cf5672..6dd06ee98b60 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -283,7 +283,13 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
-static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void get_dump_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+ /* Control stack is stored in user mode */
+ *ctl_stack_size = 0;
+}
+
+static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v10_compute_mqd *m;
@@ -295,7 +301,8 @@ static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct v10_compute_mqd *m;
@@ -405,7 +412,7 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
}
-static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v10_sdma_mqd *m;
@@ -417,7 +424,9 @@ static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src,
+ const u32 ctl_stack_size)
{
uint64_t addr;
struct v10_sdma_mqd *m;
@@ -478,6 +487,7 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
mqd->is_occupied = is_occupied;
mqd->mqd_size = sizeof(struct v10_compute_mqd);
mqd->get_wave_state = get_wave_state;
+ mqd->get_dump_info = get_dump_info;
mqd->dump_mqd = dump_mqd;
mqd->restore_mqd = restore_mqd;
#if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 5b6beb69dfc2..db9f138e1135 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -338,21 +338,34 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
-static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void get_dump_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+ struct v9_mqd *m = get_mqd(mqd);
+
+ *ctl_stack_size = m->cp_hqd_cntl_stack_size;
+}
+
+static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v9_mqd *m;
+ /* Control stack is located one page after MQD. */
+ void *ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
m = get_mqd(mqd);
memcpy(mqd_dst, m, sizeof(struct v9_mqd));
+ memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size);
}
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
- struct queue_properties *qp, const void *mqd_src)
+ struct queue_properties *qp,
+ const void *mqd_src,
+ const void *ctl_stack_src, u32 ctl_stack_size)
{
uint64_t addr;
struct v9_mqd *m;
+ void *ctl_stack;
m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;
@@ -364,6 +377,9 @@ static void restore_mqd(struct mqd_manager *mm, void **mqd,
*gart_addr = addr;
/* Control stack is located one page after MQD. */
+ ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
+ memcpy(ctl_stack, ctl_stack_src, ctl_stack_size);
+
m->cp_hqd_pq_doorbell_control =
qp->doorbell_off <<
CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
@@ -460,7 +476,7 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
}
-static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct v9_sdma_mqd *m;
@@ -471,7 +487,9 @@ static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
- struct queue_properties *qp, const void *mqd_src)
+ struct queue_properties *qp,
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct v9_sdma_mqd *m;
@@ -530,6 +548,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
+ mqd->get_dump_info = get_dump_info;
mqd->dump_mqd = dump_mqd;
mqd->restore_mqd = restore_mqd;
mqd->mqd_size = sizeof(struct v9_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index ae5e3edec92e..88f320abe850 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -303,7 +303,13 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
return 0;
}
-static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void get_dump_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+ /* Control stack is stored in user mode */
+ *ctl_stack_size = 0;
+}
+
+static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct vi_mqd *m;
@@ -315,7 +321,8 @@ static void dump_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct vi_mqd *m;
@@ -429,7 +436,7 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
}
-static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
{
struct vi_sdma_mqd *m;
@@ -441,7 +448,8 @@ static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
struct queue_properties *qp,
- const void *mqd_src)
+ const void *mqd_src,
+ const void *ctl_stack_src, const u32 ctl_stack_size)
{
uint64_t addr;
struct vi_sdma_mqd *m;
@@ -500,6 +508,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
mqd->destroy_mqd = destroy_mqd;
mqd->is_occupied = is_occupied;
mqd->get_wave_state = get_wave_state;
+ mqd->get_dump_info = get_dump_info;
mqd->dump_mqd = dump_mqd;
mqd->restore_mqd = restore_mqd;
mqd->mqd_size = sizeof(struct vi_mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 45be2daf49b3..b282aa9f9235 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1120,6 +1120,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
unsigned int *qid,
const struct kfd_criu_queue_priv_data *q_data,
const void *restore_mqd,
+ const void *restore_ctl_stack,
uint32_t *p_doorbell_offset_in_process);
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
@@ -1142,11 +1143,12 @@ int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
uint64_t fence_value,
unsigned int timeout_ms);
int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
- unsigned int qid,
- u32 *mqd_size);
+ unsigned int qid,
+ u32 *mqd_size, u32 *ctl_stack_size);
int pqm_dump_mqd(struct process_queue_manager *pqm,
unsigned int qid,
- void *dst_mqd);
+ void *dst_mqd,
+ void *dst_ctl_stack);
/* Packet Manager */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index b67d5dd3dda9..3b7747b7f388 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -209,6 +209,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
unsigned int *qid,
const struct kfd_criu_queue_priv_data *q_data,
const void *restore_mqd,
+ const void *restore_ctl_stack,
uint32_t *p_doorbell_offset_in_process)
{
int retval;
@@ -273,7 +274,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
goto err_create_queue;
pqn->q = q;
pqn->kq = NULL;
- retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);
+ retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
+ restore_mqd, restore_ctl_stack);
print_queue(q);
break;
@@ -293,7 +295,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
goto err_create_queue;
pqn->q = q;
pqn->kq = NULL;
- retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);
+ retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
+ restore_mqd, restore_ctl_stack);
print_queue(q);
break;
case KFD_QUEUE_TYPE_DIQ:
@@ -532,13 +535,15 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
static int get_queue_data_sizes(struct kfd_process_device *pdd,
struct queue *q,
uint32_t *cu_mask_size,
- uint32_t *mqd_size)
+ uint32_t *mqd_size,
+ uint32_t *ctl_stack_size)
{
int ret;
*cu_mask_size = sizeof(uint32_t) * (q->properties.cu_mask_count / 32);
- ret = pqm_get_queue_dump_info(&pdd->process->pqm, q->properties.queue_id, mqd_size);
+ ret = pqm_get_queue_dump_info(&pdd->process->pqm, q->properties.queue_id, mqd_size,
+ ctl_stack_size);
if (ret)
pr_err("Failed to get queue dump info (%d)\n", ret);
@@ -562,13 +567,15 @@ int kfd_process_get_queue_info(struct kfd_process *p, uint32_t *num_queues, uint
q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
u32 cu_mask_size;
u32 mqd_size;
+ u32 ctl_stack_size;
int ret;
- ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size);
+ ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size,
+ &ctl_stack_size);
if (ret)
return ret;
- data_sizes += cu_mask_size + mqd_size;
+ data_sizes += cu_mask_size + mqd_size + ctl_stack_size;
q_index++;
} else {
pr_err("Unsupported queue type (%d)\n", q->properties.type);
@@ -588,11 +595,12 @@ static int criu_dump_queue(struct kfd_process_device *pdd,
void *private_data)
{
struct kfd_criu_queue_priv_data *q_data = (struct kfd_criu_queue_priv_data *) private_data;
- uint8_t *cu_mask, *mqd;
+ uint8_t *cu_mask, *mqd, *ctl_stack;
int ret;
cu_mask = (void *)(q_data + 1);
mqd = cu_mask + q_data->cu_mask_size;
+ ctl_stack = mqd + q_data->mqd_size;
q_bucket->gpu_id = pdd->dev->id;
q_data->type = q->properties.type;
@@ -622,7 +630,7 @@ static int criu_dump_queue(struct kfd_process_device *pdd,
if (q_data->cu_mask_size)
memcpy(cu_mask, q->properties.cu_mask, q_data->cu_mask_size);
- ret = pqm_dump_mqd(&pdd->process->pqm, q->properties.queue_id, mqd);
+ ret = pqm_dump_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
if (ret) {
pr_err("Failed dump queue_mqd (%d)\n", ret);
return ret;
@@ -650,6 +658,7 @@ static int criu_dump_queues_device(struct kfd_process_device *pdd,
uint64_t q_data_size;
uint32_t cu_mask_size;
uint32_t mqd_size;
+ uint32_t ctl_stack_size;
if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
q->properties.type != KFD_QUEUE_TYPE_SDMA &&
@@ -661,11 +670,11 @@ static int criu_dump_queues_device(struct kfd_process_device *pdd,
memset(&q_bucket, 0, sizeof(q_bucket));
- ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size);
+ ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size, &ctl_stack_size);
if (ret)
return ret;
- q_data_size = sizeof(*q_data) + cu_mask_size + mqd_size;
+ q_data_size = sizeof(*q_data) + cu_mask_size + mqd_size + ctl_stack_size;
/* Increase local buffer space if needed */
if (q_private_data_size < q_data_size) {
@@ -681,8 +690,10 @@ static int criu_dump_queues_device(struct kfd_process_device *pdd,
q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
+ /* data stored in this order: priv_data, cu_mask, mqd, ctl_stack */
q_data->cu_mask_size = cu_mask_size;
q_data->mqd_size = mqd_size;
+ q_data->ctl_stack_size = ctl_stack_size;
ret = criu_dump_queue(pdd, q, &q_bucket, q_data);
if (ret)
@@ -739,8 +750,8 @@ int kfd_criu_dump_queues(struct kfd_process *p, struct kfd_ioctl_criu_dumper_arg
return -EINVAL;
}
- /* Queue private data size for each queue can vary in size as it also includes cu_mask, mqd.
- * First queue private data starts after all queue_buckets
+ /* Queue private data size for each queue can vary in size as it also includes cu_mask, mqd
+ * and ctl_stack. First queue private data starts after all queue_buckets
*/
queue_buckets = (struct kfd_criu_queue_bucket *)args->objects;
@@ -805,16 +816,17 @@ static int criu_restore_queue(struct kfd_process *p,
void *private_data)
{
struct kfd_criu_queue_priv_data *q_data = (struct kfd_criu_queue_priv_data *) private_data;
- uint8_t *cu_mask, *mqd;
+ uint8_t *cu_mask, *mqd, *ctl_stack;
struct queue_properties qp;
unsigned int queue_id;
int ret = 0;
pr_debug("Restoring Queue: gpu_id:%x queue_id:%u\n", q_bucket->gpu_id, q_data->q_id);
- /* data stored in this order: cu_mask, mqd */
+ /* data stored in this order: cu_mask, mqd, ctl_stack */
cu_mask = (void *)(q_data + 1);
mqd = cu_mask + q_data->cu_mask_size;
+ ctl_stack = mqd + q_data->mqd_size;
memset(&qp, 0, sizeof(qp));
ret = set_queue_properties_from_criu(&qp, q_bucket, q_data, cu_mask);
@@ -823,7 +835,7 @@ static int criu_restore_queue(struct kfd_process *p,
print_queue_properties(&qp);
- ret = pqm_create_queue(&p->pqm, dev, NULL, &qp, &queue_id, q_data, mqd, NULL);
+ ret = pqm_create_queue(&p->pqm, dev, NULL, &qp, &queue_id, q_data, mqd, ctl_stack, NULL);
if (ret) {
pr_err("Failed to create new queue err:%d\n", ret);
ret = -EINVAL;
@@ -922,7 +934,8 @@ int kfd_criu_restore_queues(struct kfd_process *p, struct kfd_ioctl_criu_restore
int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
unsigned int qid,
- u32 *mqd_size)
+ u32 *mqd_size,
+ u32 *ctl_stack_size)
{
struct process_queue_node *pqn;
@@ -938,12 +951,13 @@ int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
}
pqn->q->device->dqm->ops.get_queue_dump_info(pqn->q->device->dqm,
- pqn->q, mqd_size);
+ pqn->q, mqd_size,
+ ctl_stack_size);
return 0;
}
int pqm_dump_mqd(struct process_queue_manager *pqm,
- unsigned int qid, void *mqd)
+ unsigned int qid, void *mqd, void *ctl_stack)
{
struct process_queue_node *pqn;
@@ -959,7 +973,7 @@ int pqm_dump_mqd(struct process_queue_manager *pqm,
}
return pqn->q->device->dqm->ops.dump_mqd(pqn->q->device->dqm,
- pqn->q, mqd);
+ pqn->q, mqd, ctl_stack);
}
#if defined(CONFIG_DEBUG_FS)
--
2.17.1
