Re: [RFC 01/11] soc: mediatek: MediaTek Command Queue (CMDQ) driver

[CK Hu <ck.hu@xxxxxxxxxxxx>]

Hi, Ulrich:

This patch looks old-fashioned. Please follow up with [1].

[1]
http://lists.infradead.org/pipermail/linux-mediatek/2017-January/007654.html

Regards,
CK

On Fri, 2017-09-29 at 15:09 +0200, Ulrich Hecht wrote:
> Ported from chromeos-3.18 kernel.
> 
> Signed-off-by: Ulrich Hecht <ulrich.hecht+renesas@xxxxxxxxx>
> ---
>  drivers/soc/mediatek/Kconfig    |   10 +
>  drivers/soc/mediatek/Makefile   |    1 +
>  drivers/soc/mediatek/mtk-cmdq.c | 2814 +++++++++++++++++++++++++++++++++++++++
>  include/soc/mediatek/cmdq.h     |  211 +++
>  4 files changed, 3036 insertions(+)
>  create mode 100644 drivers/soc/mediatek/mtk-cmdq.c
>  create mode 100644 include/soc/mediatek/cmdq.h
> 
> diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
> index 609bb34..ef271e0 100644
> --- a/drivers/soc/mediatek/Kconfig
> +++ b/drivers/soc/mediatek/Kconfig
> @@ -1,6 +1,16 @@
>  #
>  # MediaTek SoC drivers
>  #
> +config MTK_CMDQ
> +	bool "MediaTek CMDQ Support"
> +	depends on ARCH_MEDIATEK || COMPILE_TEST
> +	select MTK_INFRACFG
> +	help
> +	  Say yes here to add support for the MediaTek Command Queue (CMDQ)
> +	  driver. The CMDQ is used to help read/write registers with critical
> +	  time limitation, such as updating display configuration during the
> +	  vblank.
> +
>  config MTK_INFRACFG
>  	bool "MediaTek INFRACFG Support"
>  	depends on ARCH_MEDIATEK || COMPILE_TEST
> diff --git a/drivers/soc/mediatek/Makefile b/drivers/soc/mediatek/Makefile
> index 12998b0..f7397ef 100644
> --- a/drivers/soc/mediatek/Makefile
> +++ b/drivers/soc/mediatek/Makefile
> @@ -1,3 +1,4 @@
> +obj-$(CONFIG_MTK_CMDQ) += mtk-cmdq.o
>  obj-$(CONFIG_MTK_INFRACFG) += mtk-infracfg.o
>  obj-$(CONFIG_MTK_PMIC_WRAP) += mtk-pmic-wrap.o
>  obj-$(CONFIG_MTK_SCPSYS) += mtk-scpsys.o
> diff --git a/drivers/soc/mediatek/mtk-cmdq.c b/drivers/soc/mediatek/mtk-cmdq.c
> new file mode 100644
> index 0000000..a8bfb5c
> --- /dev/null
> +++ b/drivers/soc/mediatek/mtk-cmdq.c
> @@ -0,0 +1,2814 @@
> +/*
> + * Copyright (c) 2015 MediaTek Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <linux/clk.h>
> +#include <linux/clk-provider.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/errno.h>
> +#include <linux/interrupt.h>
> +#include <linux/iopoll.h>
> +#include <linux/kernel.h>
> +#include <linux/kthread.h>
> +#include <linux/ktime.h>
> +#include <linux/module.h>
> +#include <linux/mutex.h>
> +#include <linux/of_irq.h>
> +#include <linux/platform_device.h>
> +#include <linux/slab.h>
> +#include <linux/spinlock.h>
> +#include <linux/workqueue.h>
> +#include <soc/mediatek/cmdq.h>
> +
> +/*
> + * Please calculate this value for each platform.
> + * task number = vblank time / ((task cmds * cmd ticks) / GCE freq)
> + */
> +#define CMDQ_MAX_TASK_IN_THREAD		70
> +
> +#define CMDQ_INITIAL_CMD_BLOCK_SIZE	PAGE_SIZE
> +#define CMDQ_CMD_BUF_POOL_BUF_SIZE	PAGE_SIZE
> +#define CMDQ_CMD_BUF_POOL_BUF_NUM	140 /* 2 * 70 = 140 */
> +#define CMDQ_INST_SIZE			8 /* instruction is 64-bit */
> +
> +/*
> + * cmdq_thread cookie value is from 0 to CMDQ_MAX_COOKIE_VALUE.
> + * And, this value also be used as MASK.
> + */
> +#define CMDQ_MAX_COOKIE_VALUE		0xffff
> +#define CMDQ_COOKIE_MASK		CMDQ_MAX_COOKIE_VALUE
> +
> +#define CMDQ_DEFAULT_TIMEOUT_MS		1000
> +#define CMDQ_ACQUIRE_THREAD_TIMEOUT_MS	5000
> +#define CMDQ_PREALARM_TIMEOUT_NS	200000000
> +
> +#define CMDQ_INVALID_THREAD		-1
> +
> +#define CMDQ_DRIVER_DEVICE_NAME		"mtk_cmdq"
> +
> +#define CMDQ_CLK_NAME			"gce"
> +
> +#define CMDQ_CURR_IRQ_STATUS_OFFSET	0x010
> +#define CMDQ_CURR_LOADED_THR_OFFSET	0x018
> +#define CMDQ_THR_SLOT_CYCLES_OFFSET	0x030
> +#define CMDQ_THR_EXEC_CYCLES_OFFSET	0x034
> +#define CMDQ_THR_TIMEOUT_TIMER_OFFSET	0x038
> +#define CMDQ_BUS_CONTROL_TYPE_OFFSET	0x040
> +
> +#define CMDQ_SYNC_TOKEN_ID_OFFSET	0x060
> +#define CMDQ_SYNC_TOKEN_VAL_OFFSET	0x064
> +#define CMDQ_SYNC_TOKEN_UPD_OFFSET	0x068
> +
> +#define CMDQ_GPR_SHIFT			0x004
> +#define CMDQ_GPR_OFFSET			0x080
> +
> +#define CMDQ_THR_SHIFT			0x080
> +#define CMDQ_THR_WARM_RESET_OFFSET	0x100
> +#define CMDQ_THR_ENABLE_TASK_OFFSET	0x104
> +#define CMDQ_THR_SUSPEND_TASK_OFFSET	0x108
> +#define CMDQ_THR_CURR_STATUS_OFFSET	0x10c
> +#define CMDQ_THR_IRQ_STATUS_OFFSET	0x110
> +#define CMDQ_THR_IRQ_ENABLE_OFFSET	0x114
> +#define CMDQ_THR_CURR_ADDR_OFFSET	0x120
> +#define CMDQ_THR_END_ADDR_OFFSET	0x124
> +#define CMDQ_THR_EXEC_CNT_OFFSET	0x128
> +#define CMDQ_THR_WAIT_TOKEN_OFFSET	0x130
> +#define CMDQ_THR_CFG_OFFSET		0x140
> +#define CMDQ_THR_INST_CYCLES_OFFSET	0x150
> +#define CMDQ_THR_INST_THRESX_OFFSET	0x154
> +#define CMDQ_THR_STATUS_OFFSET		0x18c
> +
> +#define CMDQ_SYNC_TOKEN_SET		BIT(16)
> +#define CMDQ_IRQ_MASK			0xffff
> +
> +#define CMDQ_THR_ENABLED		0x1
> +#define CMDQ_THR_DISABLED		0x0
> +#define CMDQ_THR_SUSPEND		0x1
> +#define CMDQ_THR_RESUME			0x0
> +#define CMDQ_THR_STATUS_SUSPENDED	BIT(1)
> +#define CMDQ_THR_WARM_RESET		BIT(0)
> +#define CMDQ_THR_SLOT_CYCLES		0x3200
> +#define CMDQ_THR_NO_TIMEOUT		0x0
> +#define CMDQ_THR_PRIORITY		3
> +#define CMDQ_THR_IRQ_DONE		0x1
> +#define CMDQ_THR_IRQ_ERROR		0x12
> +#define CMDQ_THR_IRQ_EN			0x13 /* done + error */
> +#define CMDQ_THR_IRQ_MASK		0x13
> +#define CMDQ_THR_EXECUTING		BIT(31)
> +#define CMDQ_THR_IS_WAITING		BIT(31)
> +
> +#define CMDQ_ARG_A_MASK			0xffffff
> +#define CMDQ_ARG_A_WRITE_MASK		0xffff
> +#define CMDQ_ARG_A_SUBSYS_MASK		0x1f0000
> +#define CMDQ_SUBSYS_MASK		0x1f
> +
> +#define CMDQ_OP_CODE_SHIFT		24
> +#define CMDQ_SUBSYS_SHIFT		16
> +
> +#define CMDQ_JUMP_BY_OFFSET		0x10000000
> +#define CMDQ_JUMP_BY_PA			0x10000001
> +#define CMDQ_JUMP_PASS			CMDQ_INST_SIZE
> +
> +#define CMDQ_WFE_UPDATE			BIT(31)
> +#define CMDQ_WFE_WAIT			BIT(15)
> +#define CMDQ_WFE_WAIT_VALUE		0x1
> +
> +#define CMDQ_MARK_NON_SUSPENDABLE	BIT(21) /* 53 - 32 = 21 */
> +#define CMDQ_MARK_NOT_ADD_COUNTER	BIT(16) /* 48 - 32 = 16 */
> +#define CMDQ_MARK_PREFETCH_MARKER	BIT(20)
> +#define CMDQ_MARK_PREFETCH_MARKER_EN	BIT(17)
> +#define CMDQ_MARK_PREFETCH_EN		BIT(16)
> +
> +#define CMDQ_EOC_IRQ_EN			BIT(0)
> +
> +#define CMDQ_ENABLE_MASK		BIT(0)
> +
> +#define CMDQ_OP_CODE_MASK		0xff000000
> +
> +enum cmdq_thread_index {
> +	CMDQ_THR_DISP_DSI0 = 0,		/* main: dsi0 */
> +	CMDQ_THR_DISP_DPI0,		/* sub: dpi0 */
> +	CMDQ_MAX_THREAD_COUNT,		/* max */
> +};
> +
> +struct cmdq_command {
> +	struct cmdq		*cqctx;
> +	/* bit flag of used engines */
> +	u64			engine_flag;
> +	/*
> +	 * pointer of instruction buffer
> +	 * This must point to an 64-bit aligned u32 array
> +	 */
> +	u32			*va_base;
> +	/* size of instruction buffer, in bytes. */
> +	size_t			block_size;
> +};
> +
> +enum cmdq_code {
> +	/* These are actual HW op code. */
> +	CMDQ_CODE_MOVE = 0x02,
> +	CMDQ_CODE_WRITE = 0x04,
> +	CMDQ_CODE_JUMP = 0x10,
> +	CMDQ_CODE_WFE = 0x20,	/* wait for event (and clear) */
> +	CMDQ_CODE_CLEAR_EVENT = 0x21,	/* clear event */
> +	CMDQ_CODE_EOC = 0x40,	/* end of command */
> +};
> +
> +enum cmdq_task_state {
> +	TASK_STATE_IDLE,	/* free task */
> +	TASK_STATE_BUSY,	/* task running on a thread */
> +	TASK_STATE_KILLED,	/* task process being killed */
> +	TASK_STATE_ERROR,	/* task execution error */
> +	TASK_STATE_DONE,	/* task finished */
> +	TASK_STATE_WAITING,	/* allocated but waiting for available thread */
> +};
> +
> +struct cmdq_cmd_buf {
> +	atomic_t		used;
> +	void			*va;
> +	dma_addr_t		pa;
> +};
> +
> +struct cmdq_task_cb {
> +	/* called by isr */
> +	cmdq_async_flush_cb	isr_cb;
> +	void			*isr_data;
> +	/* called by releasing task */
> +	cmdq_async_flush_cb	done_cb;
> +	void			*done_data;
> +};
> +
> +struct cmdq_task {
> +	struct cmdq		*cqctx;
> +	struct list_head	list_entry;
> +
> +	/* state for task life cycle */
> +	enum cmdq_task_state	task_state;
> +	/* virtual address of command buffer */
> +	u32			*va_base;
> +	/* physical address of command buffer */
> +	dma_addr_t		mva_base;
> +	/* size of allocated command buffer */
> +	size_t			buf_size;
> +	/* It points to a cmdq_cmd_buf if this task use command buffer pool. */
> +	struct cmdq_cmd_buf	*cmd_buf;
> +
> +	u64			engine_flag;
> +	size_t			command_size;
> +	u32			num_cmd; /* 2 * number of commands */
> +	int			reorder;
> +	/* HW thread ID; CMDQ_INVALID_THREAD if not running */
> +	int			thread;
> +	/* flag of IRQ received */
> +	int			irq_flag;
> +	/* callback functions */
> +	struct cmdq_task_cb	cb;
> +	/* work item when auto release is used */
> +	struct work_struct	auto_release_work;
> +
> +	ktime_t			submit; /* submit time */
> +
> +	pid_t			caller_pid;
> +	char			caller_name[TASK_COMM_LEN];
> +};
> +
> +struct cmdq_thread {
> +	u32			task_count;
> +	u32			wait_cookie;
> +	u32			next_cookie;
> +	struct cmdq_task	*cur_task[CMDQ_MAX_TASK_IN_THREAD];
> +};
> +
> +struct cmdq {
> +	struct device		*dev;
> +
> +	void __iomem		*base;
> +	u32			irq;
> +
> +	/*
> +	 * task information
> +	 * task_cache: struct cmdq_task object cache
> +	 * task_free_list: unused free tasks
> +	 * task_active_list: active tasks
> +	 * task_consume_wait_queue_item: task consumption work item
> +	 * task_auto_release_wq: auto-release workqueue
> +	 * task_consume_wq: task consumption workqueue (for queued tasks)
> +	 */
> +	struct kmem_cache	*task_cache;
> +	struct list_head	task_free_list;
> +	struct list_head	task_active_list;
> +	struct list_head	task_wait_list;
> +	struct work_struct	task_consume_wait_queue_item;
> +	struct workqueue_struct	*task_auto_release_wq;
> +	struct workqueue_struct	*task_consume_wq;
> +
> +	struct cmdq_thread	thread[CMDQ_MAX_THREAD_COUNT];
> +
> +	/* mutex, spinlock, flag */
> +	struct mutex		task_mutex;	/* for task list */
> +	struct mutex		clock_mutex;	/* for clock operation */
> +	spinlock_t		thread_lock;	/* for cmdq hardware thread */
> +	int			thread_usage;
> +	spinlock_t		exec_lock;	/* for exec task */
> +
> +	/* command buffer pool */
> +	struct cmdq_cmd_buf	cmd_buf_pool[CMDQ_CMD_BUF_POOL_BUF_NUM];
> +
> +	/*
> +	 * notification
> +	 * wait_queue: for task done
> +	 * thread_dispatch_queue: for thread acquiring
> +	 */
> +	wait_queue_head_t	wait_queue[CMDQ_MAX_THREAD_COUNT];
> +	wait_queue_head_t	thread_dispatch_queue;
> +
> +	/* ccf */
> +	struct clk		*clock;
> +};
> +
> +struct cmdq_event_item {
> +	enum cmdq_event	event;
> +	const char	*name;
> +};
> +
> +struct cmdq_subsys {
> +	u32		base_addr;
> +	int		id;
> +	const char	*name;
> +};
> +
> +static const struct cmdq_event_item cmdq_events[] = {
> +	/* Display start of frame(SOF) events */
> +	{CMDQ_EVENT_DISP_OVL0_SOF, "CMDQ_EVENT_DISP_OVL0_SOF"},
> +	{CMDQ_EVENT_DISP_OVL1_SOF, "CMDQ_EVENT_DISP_OVL1_SOF"},
> +	{CMDQ_EVENT_DISP_RDMA0_SOF, "CMDQ_EVENT_DISP_RDMA0_SOF"},
> +	{CMDQ_EVENT_DISP_RDMA1_SOF, "CMDQ_EVENT_DISP_RDMA1_SOF"},
> +	{CMDQ_EVENT_DISP_RDMA2_SOF, "CMDQ_EVENT_DISP_RDMA2_SOF"},
> +	{CMDQ_EVENT_DISP_WDMA0_SOF, "CMDQ_EVENT_DISP_WDMA0_SOF"},
> +	{CMDQ_EVENT_DISP_WDMA1_SOF, "CMDQ_EVENT_DISP_WDMA1_SOF"},
> +	/* Display end of frame(EOF) events */
> +	{CMDQ_EVENT_DISP_OVL0_EOF, "CMDQ_EVENT_DISP_OVL0_EOF"},
> +	{CMDQ_EVENT_DISP_OVL1_EOF, "CMDQ_EVENT_DISP_OVL1_EOF"},
> +	{CMDQ_EVENT_DISP_RDMA0_EOF, "CMDQ_EVENT_DISP_RDMA0_EOF"},
> +	{CMDQ_EVENT_DISP_RDMA1_EOF, "CMDQ_EVENT_DISP_RDMA1_EOF"},
> +	{CMDQ_EVENT_DISP_RDMA2_EOF, "CMDQ_EVENT_DISP_RDMA2_EOF"},
> +	{CMDQ_EVENT_DISP_WDMA0_EOF, "CMDQ_EVENT_DISP_WDMA0_EOF"},
> +	{CMDQ_EVENT_DISP_WDMA1_EOF, "CMDQ_EVENT_DISP_WDMA1_EOF"},
> +	/* Mutex end of frame(EOF) events */
> +	{CMDQ_EVENT_MUTEX0_STREAM_EOF, "CMDQ_EVENT_MUTEX0_STREAM_EOF"},
> +	{CMDQ_EVENT_MUTEX1_STREAM_EOF, "CMDQ_EVENT_MUTEX1_STREAM_EOF"},
> +	{CMDQ_EVENT_MUTEX2_STREAM_EOF, "CMDQ_EVENT_MUTEX2_STREAM_EOF"},
> +	{CMDQ_EVENT_MUTEX3_STREAM_EOF, "CMDQ_EVENT_MUTEX3_STREAM_EOF"},
> +	{CMDQ_EVENT_MUTEX4_STREAM_EOF, "CMDQ_EVENT_MUTEX4_STREAM_EOF"},
> +	/* Display underrun events */
> +	{CMDQ_EVENT_DISP_RDMA0_UNDERRUN, "CMDQ_EVENT_DISP_RDMA0_UNDERRUN"},
> +	{CMDQ_EVENT_DISP_RDMA1_UNDERRUN, "CMDQ_EVENT_DISP_RDMA1_UNDERRUN"},
> +	{CMDQ_EVENT_DISP_RDMA2_UNDERRUN, "CMDQ_EVENT_DISP_RDMA2_UNDERRUN"},
> +	/* Keep this at the end of HW events */
> +	{CMDQ_MAX_HW_EVENT_COUNT, "CMDQ_MAX_HW_EVENT_COUNT"},
> +	/* This is max event and also can be used as mask. */
> +	{CMDQ_SYNC_TOKEN_MAX, "CMDQ_SYNC_TOKEN_MAX"},
> +	/* Invalid event */
> +	{CMDQ_SYNC_TOKEN_INVALID, "CMDQ_SYNC_TOKEN_INVALID"},
> +};
> +
> +static const struct cmdq_subsys g_subsys[] = {
> +	{0x1400, 1, "MMSYS"},
> +	{0x1401, 2, "DISP"},
> +	{0x1402, 3, "DISP"},
> +};
> +
> +static const char *cmdq_event_get_name(enum cmdq_event event)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(cmdq_events); i++)
> +		if (cmdq_events[i].event == event)
> +			return cmdq_events[i].name;
> +
> +	return "CMDQ_EVENT_UNKNOWN";
> +}
> +
> +static void cmdq_event_reset(struct cmdq *cqctx)
> +{
> +	int i;
> +
> +	/* set all defined HW events to 0 */
> +	for (i = 0; i < ARRAY_SIZE(cmdq_events); i++) {
> +		if (cmdq_events[i].event >= CMDQ_MAX_HW_EVENT_COUNT)
> +			break;
> +		writel(cmdq_events[i].event,
> +		       cqctx->base + CMDQ_SYNC_TOKEN_UPD_OFFSET);
> +	}
> +}
> +
> +static int cmdq_subsys_base_addr_to_id(u32 base_addr)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
> +		if (g_subsys[i].base_addr == base_addr)
> +			return g_subsys[i].id;
> +	}
> +
> +	return -EFAULT;
> +}
> +
> +static u32 cmdq_subsys_id_to_base_addr(int id)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(g_subsys); i++) {
> +		if (g_subsys[i].id == id)
> +			return g_subsys[i].base_addr;
> +	}
> +
> +	return 0;
> +}
> +
> +static const char *cmdq_subsys_base_addr_to_name(u32 base_addr)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(g_subsys); i++)
> +		if (g_subsys[i].base_addr == base_addr)
> +			return g_subsys[i].name;
> +
> +	return NULL;
> +}
> +
> +static int cmdq_eng_get_thread(u64 flag)
> +{
> +	if (flag & BIT_ULL(CMDQ_ENG_DISP_DSI0))
> +		return CMDQ_THR_DISP_DSI0;
> +	else /* CMDQ_ENG_DISP_DPI0 */
> +		return CMDQ_THR_DISP_DPI0;
> +}
> +
> +static const char *cmdq_event_get_module(enum cmdq_event event)
> +{
> +	const char *module;
> +
> +	switch (event) {
> +	case CMDQ_EVENT_DISP_RDMA0_SOF:
> +	case CMDQ_EVENT_DISP_RDMA1_SOF:
> +	case CMDQ_EVENT_DISP_RDMA2_SOF:
> +	case CMDQ_EVENT_DISP_RDMA0_EOF:
> +	case CMDQ_EVENT_DISP_RDMA1_EOF:
> +	case CMDQ_EVENT_DISP_RDMA2_EOF:
> +	case CMDQ_EVENT_DISP_RDMA0_UNDERRUN:
> +	case CMDQ_EVENT_DISP_RDMA1_UNDERRUN:
> +	case CMDQ_EVENT_DISP_RDMA2_UNDERRUN:
> +		module = "DISP_RDMA";
> +		break;
> +	case CMDQ_EVENT_DISP_WDMA0_SOF:
> +	case CMDQ_EVENT_DISP_WDMA1_SOF:
> +	case CMDQ_EVENT_DISP_WDMA0_EOF:
> +	case CMDQ_EVENT_DISP_WDMA1_EOF:
> +		module = "DISP_WDMA";
> +		break;
> +	case CMDQ_EVENT_DISP_OVL0_SOF:
> +	case CMDQ_EVENT_DISP_OVL1_SOF:
> +	case CMDQ_EVENT_DISP_OVL0_EOF:
> +	case CMDQ_EVENT_DISP_OVL1_EOF:
> +		module = "DISP_OVL";
> +		break;
> +	case CMDQ_EVENT_MUTEX0_STREAM_EOF ... CMDQ_EVENT_MUTEX4_STREAM_EOF:
> +		module = "DISP";
> +		break;
> +	default:
> +		module = "CMDQ";
> +		break;
> +	}
> +
> +	return module;
> +}
> +
> +static u32 cmdq_thread_get_cookie(struct cmdq *cqctx, int tid)
> +{
> +	return readl(cqctx->base + CMDQ_THR_EXEC_CNT_OFFSET +
> +		     CMDQ_THR_SHIFT * tid) & CMDQ_COOKIE_MASK;
> +}
> +
> +static int cmdq_cmd_buf_pool_init(struct cmdq *cqctx)
> +{
> +	struct device *dev = cqctx->dev;
> +	int i;
> +	int ret = 0;
> +	struct cmdq_cmd_buf *buf;
> +
> +	for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> +		buf = &cqctx->cmd_buf_pool[i];
> +		buf->va = dma_alloc_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE,
> +					     &buf->pa, GFP_KERNEL);
> +		if (!buf->va) {
> +			dev_err(dev, "failed to alloc cmdq_cmd_buf\n");
> +			ret = -ENOMEM;
> +			goto fail_alloc;
> +		}
> +	}
> +
> +	return 0;
> +
> +fail_alloc:
> +	for (i -= 1; i >= 0 ; i--) {
> +		buf = &cqctx->cmd_buf_pool[i];
> +		dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
> +				  buf->pa);
> +	}
> +
> +	return ret;
> +}
> +
> +static void cmdq_cmd_buf_pool_uninit(struct cmdq *cqctx)
> +{
> +	struct device *dev = cqctx->dev;
> +	int i;
> +	struct cmdq_cmd_buf *buf;
> +
> +	for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> +		buf = &cqctx->cmd_buf_pool[i];
> +		dma_free_coherent(dev, CMDQ_CMD_BUF_POOL_BUF_SIZE, buf->va,
> +				  buf->pa);
> +		if (atomic_read(&buf->used))
> +			dev_err(dev,
> +				"cmdq_cmd_buf[%d] va:0x%p still in use\n",
> +				i, buf->va);
> +	}
> +}
> +
> +static struct cmdq_cmd_buf *cmdq_cmd_buf_pool_get(struct cmdq *cqctx)
> +{
> +	int i;
> +	struct cmdq_cmd_buf *buf;
> +
> +	for (i = 0; i < ARRAY_SIZE(cqctx->cmd_buf_pool); i++) {
> +		buf = &cqctx->cmd_buf_pool[i];
> +		if (!atomic_cmpxchg(&buf->used, 0, 1))
> +			return buf;
> +	}
> +
> +	return NULL;
> +}
> +
> +static void cmdq_cmd_buf_pool_put(struct cmdq_cmd_buf *buf)
> +{
> +	atomic_set(&buf->used, 0);
> +}
> +
> +static int cmdq_subsys_from_phys_addr(struct cmdq *cqctx, u32 cmdq_phys_addr)
> +{
> +	u32 base_addr = cmdq_phys_addr >> 16;
> +	int subsys = cmdq_subsys_base_addr_to_id(base_addr);
> +
> +	if (subsys < 0)
> +		dev_err(cqctx->dev,
> +			"unknown subsys: error=%d, phys=0x%08x\n",
> +			subsys, cmdq_phys_addr);
> +
> +	return subsys;
> +}
> +
> +/*
> + * It's a kmemcache creator for cmdq_task to initialize variables
> + * without command buffer.
> + */
> +static void cmdq_task_ctor(void *param)
> +{
> +	struct cmdq_task *task = param;
> +
> +	memset(task, 0, sizeof(*task));
> +	INIT_LIST_HEAD(&task->list_entry);
> +	task->task_state = TASK_STATE_IDLE;
> +	task->thread = CMDQ_INVALID_THREAD;
> +}
> +
> +static void cmdq_task_free_command_buffer(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +
> +	if (!task->va_base)
> +		return;
> +
> +	if (task->cmd_buf)
> +		cmdq_cmd_buf_pool_put(task->cmd_buf);
> +	else
> +		dma_free_coherent(dev, task->buf_size, task->va_base,
> +				  task->mva_base);
> +
> +	task->va_base = NULL;
> +	task->mva_base = 0;
> +	task->buf_size = 0;
> +	task->command_size = 0;
> +	task->num_cmd = 0;
> +	task->cmd_buf = NULL;
> +}
> +
> +/*
> + * Ensure size of command buffer in the given cmdq_task.
> + * Existing buffer data will be copied to new buffer.
> + * This buffer is guaranteed to be physically continuous.
> + * returns -ENOMEM if cannot allocate new buffer
> + */
> +static int cmdq_task_realloc_command_buffer(struct cmdq_task *task, size_t size)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	void *new_buf = NULL;
> +	dma_addr_t new_mva_base;
> +	size_t cmd_size;
> +	u32 num_cmd;
> +	struct cmdq_cmd_buf *cmd_buf = NULL;
> +
> +	if (task->va_base && task->buf_size >= size)
> +		return 0;
> +
> +	/* try command pool first */
> +	if (size <= CMDQ_CMD_BUF_POOL_BUF_SIZE) {
> +		cmd_buf = cmdq_cmd_buf_pool_get(cqctx);
> +		if (cmd_buf) {
> +			new_buf = cmd_buf->va;
> +			new_mva_base = cmd_buf->pa;
> +			memset(new_buf, 0, CMDQ_CMD_BUF_POOL_BUF_SIZE);
> +		}
> +	}
> +
> +	if (!new_buf) {
> +		new_buf = dma_alloc_coherent(dev, size, &new_mva_base,
> +					     GFP_KERNEL);
> +		if (!new_buf) {
> +			dev_err(dev, "realloc cmd buffer of size %zu failed\n",
> +				size);
> +			return -ENOMEM;
> +		}
> +	}
> +
> +	/* copy and release old buffer */
> +	if (task->va_base)
> +		memcpy(new_buf, task->va_base, task->buf_size);
> +
> +	/*
> +	 * we should keep track of num_cmd and cmd_size
> +	 * since they are cleared in free command buffer
> +	 */
> +	num_cmd = task->num_cmd;
> +	cmd_size = task->command_size;
> +	cmdq_task_free_command_buffer(task);
> +
> +	/* attach the new buffer */
> +	task->va_base = new_buf;
> +	task->mva_base = new_mva_base;
> +	task->buf_size = cmd_buf ? CMDQ_CMD_BUF_POOL_BUF_SIZE : size;
> +	task->num_cmd = num_cmd;
> +	task->command_size = cmd_size;
> +	task->cmd_buf = cmd_buf;
> +
> +	return 0;
> +}
> +
> +/* allocate and initialize struct cmdq_task and its command buffer */
> +static struct cmdq_task *cmdq_task_create(struct cmdq *cqctx)
> +{
> +	struct device *dev = cqctx->dev;
> +	struct cmdq_task *task;
> +	int status;
> +
> +	task = kmem_cache_alloc(cqctx->task_cache, GFP_KERNEL);
> +	task->cqctx = cqctx;
> +	status = cmdq_task_realloc_command_buffer(
> +			task, CMDQ_INITIAL_CMD_BLOCK_SIZE);
> +	if (status < 0) {
> +		dev_err(dev, "allocate command buffer failed\n");
> +		kmem_cache_free(cqctx->task_cache, task);
> +		return NULL;
> +	}
> +	return task;
> +}
> +
> +static int cmdq_dev_init(struct platform_device *pdev, struct cmdq *cqctx)
> +{
> +	struct device *dev = &pdev->dev;
> +	struct device_node *node = dev->of_node;
> +	struct resource *res;
> +
> +	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
> +	cqctx->base = devm_ioremap_resource(dev, res);
> +	if (IS_ERR(cqctx->base)) {
> +		dev_err(dev, "failed to ioremap gce\n");
> +		return PTR_ERR(cqctx->base);
> +	}
> +
> +	cqctx->irq = irq_of_parse_and_map(node, 0);
> +	if (!cqctx->irq) {
> +		dev_err(dev, "failed to get irq\n");
> +		return -EINVAL;
> +	}
> +
> +	dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n",
> +		dev, cqctx->base, cqctx->irq);
> +	return 0;
> +}
> +
> +static void cmdq_task_release_unlocked(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +
> +	/* This func should be inside cqctx->task_mutex mutex */
> +	lockdep_assert_held(&cqctx->task_mutex);
> +
> +	task->task_state = TASK_STATE_IDLE;
> +	task->thread = CMDQ_INVALID_THREAD;
> +
> +	cmdq_task_free_command_buffer(task);
> +
> +	/*
> +	 * move from active/waiting list to free list
> +	 * todo: shrink free list
> +	 */
> +	list_move_tail(&task->list_entry, &cqctx->task_free_list);
> +}
> +
> +static void cmdq_task_release_internal(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +
> +	mutex_lock(&cqctx->task_mutex);
> +	cmdq_task_release_unlocked(task);
> +	mutex_unlock(&cqctx->task_mutex);
> +}
> +
> +static struct cmdq_task *cmdq_core_find_free_task(struct cmdq *cqctx)
> +{
> +	struct cmdq_task *task;
> +
> +	mutex_lock(&cqctx->task_mutex);
> +
> +	/*
> +	 * Pick from free list first;
> +	 * create one if there is no free entry.
> +	 */
> +	if (list_empty(&cqctx->task_free_list)) {
> +		task = cmdq_task_create(cqctx);
> +	} else {
> +		task = list_first_entry(&cqctx->task_free_list,
> +					struct cmdq_task, list_entry);
> +		/* remove from free list */
> +		list_del_init(&task->list_entry);
> +	}
> +
> +	mutex_unlock(&cqctx->task_mutex);
> +
> +	return task;
> +}
> +
> +/* After dropping error task, we have to reorder remaining valid tasks. */
> +static void cmdq_thread_reorder_task_array(struct cmdq_thread *thread,
> +					   int prev_id)
> +{
> +	int i, j;
> +	int next_id, search_id;
> +	int reorder_count = 0;
> +	struct cmdq_task *task;
> +
> +	next_id = prev_id + 1;
> +	for (i = 1; i < (CMDQ_MAX_TASK_IN_THREAD - 1); i++, next_id++) {
> +		if (next_id >= CMDQ_MAX_TASK_IN_THREAD)
> +			next_id = 0;
> +
> +		if (thread->cur_task[next_id])
> +			break;
> +
> +		search_id = next_id + 1;
> +		for (j = (i + 1); j < CMDQ_MAX_TASK_IN_THREAD;
> +		     j++, search_id++) {
> +			if (search_id >= CMDQ_MAX_TASK_IN_THREAD)
> +				search_id = 0;
> +
> +			if (thread->cur_task[search_id]) {
> +				thread->cur_task[next_id] =
> +					thread->cur_task[search_id];
> +				thread->cur_task[search_id] = NULL;
> +				if ((j - i) > reorder_count)
> +					reorder_count = j - i;
> +
> +				break;
> +			}
> +		}
> +
> +		task = thread->cur_task[next_id];
> +		if ((task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_OFFSET) &&
> +		    (task->va_base[task->num_cmd - 2] == CMDQ_JUMP_PASS)) {
> +			/* We reached the last task */
> +			break;
> +		}
> +	}
> +
> +	thread->next_cookie -= reorder_count;
> +}
> +
> +static int cmdq_core_sync_command(struct cmdq_task *task,
> +				  struct cmdq_command *cmd_desc)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	int status;
> +	size_t size;
> +
> +	size = task->command_size + CMDQ_INST_SIZE;
> +	status = cmdq_task_realloc_command_buffer(task, size);
> +	if (status < 0) {
> +		dev_err(dev, "failed to realloc command buffer\n");
> +		dev_err(dev, "task=0x%p, request size=%zu\n", task, size);
> +		return status;
> +	}
> +
> +	/* copy the commands to our DMA buffer */
> +	memcpy(task->va_base, cmd_desc->va_base, cmd_desc->block_size);
> +
> +	/* re-adjust num_cmd according to command_size */
> +	task->num_cmd = task->command_size / sizeof(task->va_base[0]);
> +
> +	return 0;
> +}
> +
> +static struct cmdq_task *cmdq_core_acquire_task(struct cmdq_command *cmd_desc,
> +						struct cmdq_task_cb *cb)
> +{
> +	struct cmdq *cqctx = cmd_desc->cqctx;
> +	struct device *dev = cqctx->dev;
> +	struct cmdq_task *task;
> +
> +	task = cmdq_core_find_free_task(cqctx);
> +	if (!task) {
> +		dev_err(dev, "can't acquire task info\n");
> +		return NULL;
> +	}
> +
> +	/* initialize field values */
> +	task->engine_flag = cmd_desc->engine_flag;
> +	task->task_state = TASK_STATE_WAITING;
> +	task->reorder = 0;
> +	task->thread = CMDQ_INVALID_THREAD;
> +	task->irq_flag = 0x0;
> +	if (cb)
> +		task->cb = *cb;
> +	else
> +		memset(&task->cb, 0, sizeof(task->cb));
> +	task->command_size = cmd_desc->block_size;
> +
> +	/* store caller info for debug */
> +	if (current) {
> +		task->caller_pid = current->pid;
> +		memcpy(task->caller_name, current->comm, sizeof(current->comm));
> +	}
> +
> +	if (cmdq_core_sync_command(task, cmd_desc) < 0) {
> +		dev_err(dev, "fail to sync command\n");
> +		cmdq_task_release_internal(task);
> +		return NULL;
> +	}
> +
> +	/* insert into waiting list to process */
> +	if (task) {
> +		task->submit = ktime_get();
> +		mutex_lock(&cqctx->task_mutex);
> +		list_add_tail(&task->list_entry, &cqctx->task_wait_list);
> +		mutex_unlock(&cqctx->task_mutex);
> +	}
> +
> +	return task;
> +}
> +
> +static int cmdq_clk_enable(struct cmdq *cqctx)
> +{
> +	struct device *dev = cqctx->dev;
> +	int ret = 0;
> +
> +	if (cqctx->thread_usage == 0) {
> +		ret = clk_prepare_enable(cqctx->clock);
> +		if (ret) {
> +			dev_err(dev, "prepare and enable clk:%s fail\n",
> +				CMDQ_CLK_NAME);
> +			return ret;
> +		}
> +		cmdq_event_reset(cqctx);
> +	}
> +	cqctx->thread_usage++;
> +
> +	return ret;
> +}
> +
> +static void cmdq_clk_disable(struct cmdq *cqctx)
> +{
> +	cqctx->thread_usage--;
> +	if (cqctx->thread_usage <= 0)
> +		clk_disable_unprepare(cqctx->clock);
> +}
> +
> +static int cmdq_core_find_free_thread(struct cmdq *cqctx, int tid)
> +{
> +	struct cmdq_thread *thread = cqctx->thread;
> +	u32 next_cookie;
> +
> +	/*
> +	 * make sure the found thread has enough space for the task;
> +	 * cmdq_thread->cur_task has size limitation.
> +	 */
> +	if (thread[tid].task_count >= CMDQ_MAX_TASK_IN_THREAD) {
> +		dev_warn(cqctx->dev, "thread(%d) task count = %d\n",
> +			 tid, thread[tid].task_count);
> +		return CMDQ_INVALID_THREAD;
> +	}
> +
> +	next_cookie = thread[tid].next_cookie % CMDQ_MAX_TASK_IN_THREAD;
> +	if (thread[tid].cur_task[next_cookie]) {
> +		dev_warn(cqctx->dev, "thread(%d) next cookie = %d\n",
> +			 tid, next_cookie);
> +		return CMDQ_INVALID_THREAD;
> +	}
> +
> +	return tid;
> +}
> +
> +static struct cmdq_thread *cmdq_core_acquire_thread(struct cmdq *cqctx,
> +						    int candidate_tid)
> +{
> +	int tid;
> +
> +	tid = cmdq_core_find_free_thread(cqctx, candidate_tid);
> +	if (tid != CMDQ_INVALID_THREAD) {
> +		mutex_lock(&cqctx->clock_mutex);
> +		cmdq_clk_enable(cqctx);
> +		mutex_unlock(&cqctx->clock_mutex);
> +		return &cqctx->thread[tid];
> +	}
> +	return NULL;
> +}
> +
> +static void cmdq_core_release_thread(struct cmdq *cqctx, int tid)
> +{
> +	if (WARN_ON(tid == CMDQ_INVALID_THREAD))
> +		return;
> +
> +	mutex_lock(&cqctx->clock_mutex);
> +	cmdq_clk_disable(cqctx);
> +	mutex_unlock(&cqctx->clock_mutex);
> +}
> +
> +static void cmdq_task_remove_thread(struct cmdq_task *task)
> +{
> +	int tid = task->thread;
> +
> +	task->thread = CMDQ_INVALID_THREAD;
> +	cmdq_core_release_thread(task->cqctx, tid);
> +}
> +
> +static int cmdq_thread_suspend(struct cmdq *cqctx, int tid)
> +{
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	u32 enabled;
> +	u32 status;
> +
> +	/* write suspend bit */
> +	writel(CMDQ_THR_SUSPEND,
> +	       gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +
> +	/* If already disabled, treat as suspended successful. */
> +	enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> +			CMDQ_THR_SHIFT * tid);
> +	if (!(enabled & CMDQ_THR_ENABLED))
> +		return 0;
> +
> +	/* poll suspended status */
> +	if (readl_poll_timeout_atomic(gce_base +
> +				      CMDQ_THR_CURR_STATUS_OFFSET +
> +				      CMDQ_THR_SHIFT * tid,
> +				      status,
> +				      status & CMDQ_THR_STATUS_SUSPENDED,
> +				      0, 10)) {
> +		dev_err(dev, "Suspend HW thread %d failed\n", tid);
> +		return -EFAULT;
> +	}
> +
> +	return 0;
> +}
> +
> +static void cmdq_thread_resume(struct cmdq *cqctx, int tid)
> +{
> +	void __iomem *gce_base = cqctx->base;
> +
> +	writel(CMDQ_THR_RESUME,
> +	       gce_base + CMDQ_THR_SUSPEND_TASK_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +}
> +
> +static int cmdq_thread_reset(struct cmdq *cqctx, int tid)
> +{
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	u32 warm_reset;
> +
> +	writel(CMDQ_THR_WARM_RESET,
> +	       gce_base + CMDQ_THR_WARM_RESET_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +
> +	if (readl_poll_timeout_atomic(gce_base + CMDQ_THR_WARM_RESET_OFFSET +
> +				      CMDQ_THR_SHIFT * tid,
> +				      warm_reset,
> +				      !(warm_reset & CMDQ_THR_WARM_RESET),
> +				      0, 10)) {
> +		dev_err(dev, "Reset HW thread %d failed\n", tid);
> +		return -EFAULT;
> +	}
> +
> +	writel(CMDQ_THR_SLOT_CYCLES, gce_base + CMDQ_THR_SLOT_CYCLES_OFFSET);
> +	return 0;
> +}
> +
> +static int cmdq_thread_disable(struct cmdq *cqctx, int tid)
> +{
> +	void __iomem *gce_base = cqctx->base;
> +
> +	cmdq_thread_reset(cqctx, tid);
> +	writel(CMDQ_THR_DISABLED,
> +	       gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +	return 0;
> +}
> +
> +static u32 *cmdq_task_get_pc_and_inst(const struct cmdq_task *task, int tid,
> +				      u32 insts[2])
> +{
> +	struct cmdq *cqctx;
> +	void __iomem *gce_base;
> +	unsigned long pc_pa;
> +	u8 *pc_va;
> +	u8 *cmd_end;
> +
> +	memset(insts, 0, sizeof(u32) * 2);
> +
> +	if (!task ||
> +	    !task->va_base ||
> +	    tid == CMDQ_INVALID_THREAD) {
> +		pr_err("cmdq get pc failed since invalid param, task 0x%p, task->va_base:0x%p, thread:%d\n",
> +		       task, task->va_base, tid);
> +		return NULL;
> +	}
> +
> +	cqctx = task->cqctx;
> +	gce_base = cqctx->base;
> +
> +	pc_pa = (unsigned long)readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +				     CMDQ_THR_SHIFT * tid);
> +	pc_va = (u8 *)task->va_base + (pc_pa - task->mva_base);
> +	cmd_end = (u8 *)(task->va_base + task->num_cmd - 1);
> +
> +	if (((u8 *)task->va_base <= pc_va) && (pc_va <= cmd_end)) {
> +		if (pc_va < cmd_end) {
> +			/* get arg_a and arg_b */
> +			insts[0] = readl(pc_va);
> +			insts[1] = readl(pc_va + 4);
> +		} else {
> +			/* get arg_a and arg_b of previous cmd */
> +			insts[0] = readl(pc_va - 8);
> +			insts[1] = readl(pc_va - 4);
> +		}
> +	} else {
> +		return NULL;
> +	}
> +
> +	return (u32 *)pc_va;
> +}
> +
> +static const char *cmdq_core_parse_module_from_subsys(u32 arg_a)
> +{
> +	int id = (arg_a & CMDQ_ARG_A_SUBSYS_MASK) >> CMDQ_SUBSYS_SHIFT;
> +	u32 base_addr = cmdq_subsys_id_to_base_addr(id);
> +	const char *module = cmdq_subsys_base_addr_to_name(base_addr);
> +
> +	return module ? module : "CMDQ";
> +}
> +
> +static const char *cmdq_core_parse_op(u32 op_code)
> +{
> +	switch (op_code) {
> +	case CMDQ_CODE_WRITE:
> +		return "WRIT";
> +	case CMDQ_CODE_WFE:
> +		return "SYNC";
> +	case CMDQ_CODE_MOVE:
> +		return "MASK";
> +	case CMDQ_CODE_JUMP:
> +		return "JUMP";
> +	case CMDQ_CODE_EOC:
> +		return "MARK";
> +	}
> +	return NULL;
> +}
> +
> +static void cmdq_core_parse_error(struct cmdq_task *task, int tid,
> +				  const char **module_name, int *flag,
> +				  u32 *inst_a, u32 *inst_b)
> +{
> +	int irq_flag = task->irq_flag;
> +	u32 insts[2] = { 0 };
> +	const char *module;
> +
> +	/*
> +	 * other cases, use instruction to judge
> +	 * because engine flag are not sufficient
> +	 */
> +	if (cmdq_task_get_pc_and_inst(task, tid, insts)) {
> +		u32 op, arg_a, arg_b;
> +
> +		op = insts[1] >> CMDQ_OP_CODE_SHIFT;
> +		arg_a = insts[1] & CMDQ_ARG_A_MASK;
> +		arg_b = insts[0];
> +
> +		switch (op) {
> +		case CMDQ_CODE_WRITE:
> +			module = cmdq_core_parse_module_from_subsys(arg_a);
> +			break;
> +		case CMDQ_CODE_WFE:
> +			/* arg_a is the event id */
> +			module = cmdq_event_get_module((enum cmdq_event)arg_a);
> +			break;
> +		case CMDQ_CODE_MOVE:
> +		case CMDQ_CODE_JUMP:
> +		case CMDQ_CODE_EOC:
> +		default:
> +			module = "CMDQ";
> +			break;
> +		}
> +	} else {
> +		module = "CMDQ";
> +	}
> +
> +	/* fill output parameter */
> +	*module_name = module;
> +	*flag = irq_flag;
> +	*inst_a = insts[1];
> +	*inst_b = insts[0];
> +}
> +
> +static void cmdq_thread_insert_task_by_cookie(struct cmdq_thread *thread,
> +					      struct cmdq_task *task,
> +					      int cookie)
> +{
> +	thread->wait_cookie = cookie;
> +	thread->next_cookie = cookie + 1;
> +	if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
> +		thread->next_cookie = 0;
> +
> +	/* first task, so set to 1 */
> +	thread->task_count = 1;
> +
> +	thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
> +}
> +
> +static int cmdq_thread_remove_task_by_index(struct cmdq_thread *thread,
> +					    int index,
> +					    enum cmdq_task_state new_state)
> +{
> +	struct cmdq_task *task;
> +	struct device *dev;
> +
> +	task = thread->cur_task[index];
> +	if (!task) {
> +		pr_err("%s: remove fail, task:%d on thread:0x%p is NULL\n",
> +		       __func__, index, thread);
> +		return -EINVAL;
> +	}
> +	dev = task->cqctx->dev;
> +
> +	/*
> +	 * note timing to switch a task to done_status(_ERROR, _KILLED, _DONE)
> +	 * is aligned with thread's taskcount change
> +	 * check task status to prevent double clean-up thread's taskcount
> +	 */
> +	if (task->task_state != TASK_STATE_BUSY) {
> +		dev_err(dev, "remove task failed\n");
> +		dev_err(dev, "state:%d. thread:0x%p, task:%d, new_state:%d\n",
> +			task->task_state, thread, index, new_state);
> +		return -EINVAL;
> +	}
> +
> +	if (thread->task_count == 0) {
> +		dev_err(dev, "no task to remove\n");
> +		dev_err(dev, "thread:%d, index:%d\n", task->thread, index);
> +		return -EINVAL;
> +	}
> +
> +	task->task_state = new_state;
> +	thread->cur_task[index] = NULL;
> +	thread->task_count--;
> +
> +	return 0;
> +}
> +
> +static int cmdq_thread_force_remove_task(struct cmdq_task *task, int tid)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	void __iomem *gce_base = cqctx->base;
> +	int status;
> +	int cookie;
> +	struct cmdq_task *exec_task;
> +
> +	status = cmdq_thread_suspend(cqctx, tid);
> +
> +	writel(CMDQ_THR_NO_TIMEOUT,
> +	       gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +
> +	/* The cookie of the task currently being processed */
> +	cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
> +
> +	exec_task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
> +	if (exec_task && exec_task == task) {
> +		dma_addr_t eoc_pa = task->mva_base + task->command_size - 16;
> +
> +		/* The task is executed now, set the PC to EOC for bypass */
> +		writel(eoc_pa,
> +		       gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +		thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = NULL;
> +		task->task_state = TASK_STATE_KILLED;
> +	} else {
> +		int i, j;
> +
> +		j = thread->task_count;
> +		for (i = cookie; j > 0; j--, i++) {
> +			i %= CMDQ_MAX_TASK_IN_THREAD;
> +
> +			exec_task = thread->cur_task[i];
> +			if (!exec_task)
> +				continue;
> +
> +			if ((exec_task->va_base[exec_task->num_cmd - 1] ==
> +			     CMDQ_JUMP_BY_OFFSET) &&
> +			    (exec_task->va_base[exec_task->num_cmd - 2] ==
> +			     CMDQ_JUMP_PASS)) {
> +				/* reached the last task */
> +				break;
> +			}
> +
> +			if (exec_task->va_base[exec_task->num_cmd - 2] ==
> +			    task->mva_base) {
> +				/* fake EOC command */
> +				exec_task->va_base[exec_task->num_cmd - 2] =
> +					CMDQ_EOC_IRQ_EN;
> +				exec_task->va_base[exec_task->num_cmd - 1] =
> +					CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT;
> +
> +				/* bypass the task */
> +				exec_task->va_base[exec_task->num_cmd] =
> +					task->va_base[task->num_cmd - 2];
> +				exec_task->va_base[exec_task->num_cmd + 1] =
> +					task->va_base[task->num_cmd - 1];
> +
> +				i = (i + 1) % CMDQ_MAX_TASK_IN_THREAD;
> +
> +				thread->cur_task[i] = NULL;
> +				task->task_state = TASK_STATE_KILLED;
> +				status = 0;
> +				break;
> +			}
> +		}
> +	}
> +
> +	return status;
> +}
> +
> +static struct cmdq_task *cmdq_thread_search_task_by_pc(
> +		const struct cmdq_thread *thread, u32 pc)
> +{
> +	struct cmdq_task *task;
> +	int i;
> +
> +	for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> +		task = thread->cur_task[i];
> +		if (task &&
> +		    pc >= task->mva_base &&
> +		    pc <= task->mva_base + task->command_size)
> +			break;
> +	}
> +
> +	return task;
> +}
> +
> +/*
> + * Re-fetch thread's command buffer
> + * Use Case:
> + *     If SW modifies command buffer content after SW configed commands to GCE,
> + *     SW should notify GCE to re-fetch commands in order to
> + *     prevent inconsistent command buffer content between DRAM and GCE's SRAM.
> + */
> +static void cmdq_core_invalidate_hw_fetched_buffer(struct cmdq *cqctx,
> +						   int tid)
> +{
> +	void __iomem *pc_va;
> +	u32 pc;
> +
> +	/*
> +	 * Setting HW thread PC will invoke that
> +	 * GCE (CMDQ HW) gives up fetched command buffer,
> +	 * and fetch command from DRAM to GCE's SRAM again.
> +	 */
> +	pc_va = cqctx->base + CMDQ_THR_CURR_ADDR_OFFSET + CMDQ_THR_SHIFT * tid;
> +	pc = readl(pc_va);
> +	writel(pc, pc_va);
> +}
> +
> +static int cmdq_task_insert_into_thread(struct cmdq_task *task,
> +					int tid, int loop)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	struct cmdq_task *prev_task;
> +	int index, prev;
> +
> +	/* find previous task and then link this task behind it */
> +
> +	index = thread->next_cookie % CMDQ_MAX_TASK_IN_THREAD;
> +	prev = (index + CMDQ_MAX_TASK_IN_THREAD - 1) % CMDQ_MAX_TASK_IN_THREAD;
> +
> +	prev_task = thread->cur_task[prev];
> +
> +	/* maybe the job is killed, search a new one */
> +	for (; !prev_task && loop > 1; loop--) {
> +		dev_err(dev,
> +			"prev_task is NULL, prev:%d, loop:%d, index:%d\n",
> +			prev, loop, index);
> +
> +		prev--;
> +		if (prev < 0)
> +			prev = CMDQ_MAX_TASK_IN_THREAD - 1;
> +
> +		prev_task = thread->cur_task[prev];
> +	}
> +
> +	if (!prev_task) {
> +		dev_err(dev,
> +			"invalid prev_task index:%d, loop:%d\n",
> +			index, loop);
> +		return -EFAULT;
> +	}
> +
> +	/* insert this task */
> +	thread->cur_task[index] = task;
> +	/* let previous task jump to this new task */
> +	prev_task->va_base[prev_task->num_cmd - 1] = CMDQ_JUMP_BY_PA;
> +	prev_task->va_base[prev_task->num_cmd - 2] = task->mva_base;
> +
> +	/* re-fetch command buffer again. */
> +	cmdq_core_invalidate_hw_fetched_buffer(cqctx, tid);
> +
> +	return 0;
> +}
> +
> +static bool cmdq_command_is_wfe(u32 *cmd)
> +{
> +	u32 wfe_option = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT | CMDQ_WFE_WAIT_VALUE;
> +	u32 wfe_op = CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT;
> +
> +	return (cmd[0] == wfe_option && (cmd[1] & CMDQ_OP_CODE_MASK) == wfe_op);
> +}
> +
> +/* we assume tasks in the same display thread are waiting the same event. */
> +static void cmdq_task_remove_wfe(struct cmdq_task *task)
> +{
> +	u32 *base = task->va_base;
> +	int i;
> +
> +	/*
> +	 * Replace all WFE commands in the task command queue and
> +	 * replace them with JUMP_PASS.
> +	 */
> +	for (i = 0; i < task->num_cmd; i += 2) {
> +		if (cmdq_command_is_wfe(&base[i])) {
> +			base[i] = CMDQ_JUMP_PASS;
> +			base[i + 1] = CMDQ_JUMP_BY_OFFSET;
> +		}
> +	}
> +}
> +
> +static bool cmdq_thread_is_in_wfe(struct cmdq *cqctx, int tid)
> +{
> +	return readl(cqctx->base + CMDQ_THR_WAIT_TOKEN_OFFSET +
> +		     CMDQ_THR_SHIFT * tid) & CMDQ_THR_IS_WAITING;
> +}
> +
> +static void cmdq_thread_wait_end(struct cmdq *cqctx, int tid,
> +				 unsigned long end_pa)
> +{
> +	void __iomem *gce_base = cqctx->base;
> +	unsigned long curr_pa;
> +
> +	if (readl_poll_timeout_atomic(
> +			gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +			CMDQ_THR_SHIFT * tid,
> +			curr_pa, curr_pa == end_pa, 1, 20)) {
> +		dev_err(cqctx->dev, "GCE thread(%d) cannot run to end.\n", tid);
> +	}
> +}
> +
> +static int cmdq_task_exec_async_impl(struct cmdq_task *task, int tid)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	int status;
> +	struct cmdq_thread *thread;
> +	unsigned long flags;
> +	int loop;
> +	int minimum;
> +	int cookie;
> +
> +	status = 0;
> +	thread = &cqctx->thread[tid];
> +
> +	spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> +	/* update task's thread info */
> +	task->thread = tid;
> +	task->irq_flag = 0;
> +	task->task_state = TASK_STATE_BUSY;
> +
> +	/* case 1. first task for this thread */
> +	if (thread->task_count <= 0) {
> +		if (cmdq_thread_reset(cqctx, tid) < 0) {
> +			spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +			return -EFAULT;
> +		}
> +
> +		writel(CMDQ_THR_NO_TIMEOUT,
> +		       gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +		writel(task->mva_base,
> +		       gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +		writel(task->mva_base + task->command_size,
> +		       gce_base + CMDQ_THR_END_ADDR_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +		writel(CMDQ_THR_PRIORITY,
> +		       gce_base + CMDQ_THR_CFG_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +		writel(CMDQ_THR_IRQ_EN,
> +		       gce_base + CMDQ_THR_IRQ_ENABLE_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +		minimum = cmdq_thread_get_cookie(cqctx, tid);
> +		cmdq_thread_insert_task_by_cookie(
> +				thread, task, (minimum + 1));
> +
> +		/* enable HW thread */
> +		writel(CMDQ_THR_ENABLED,
> +		       gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +	} else {
> +		unsigned long curr_pa, end_pa;
> +
> +		status = cmdq_thread_suspend(cqctx, tid);
> +		if (status < 0) {
> +			spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +			return status;
> +		}
> +
> +		writel(CMDQ_THR_NO_TIMEOUT,
> +		       gce_base + CMDQ_THR_INST_CYCLES_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +		cookie = thread->next_cookie;
> +
> +		curr_pa = (unsigned long)readl(gce_base +
> +					       CMDQ_THR_CURR_ADDR_OFFSET +
> +					       CMDQ_THR_SHIFT * tid);
> +		end_pa = (unsigned long)readl(gce_base +
> +					      CMDQ_THR_END_ADDR_OFFSET +
> +					      CMDQ_THR_SHIFT * tid);
> +
> +		/*
> +		 * case 2. If already exited WFE, wait for current task to end
> +		 * and then jump directly to new task.
> +		 */
> +		if (!cmdq_thread_is_in_wfe(cqctx, tid)) {
> +			cmdq_thread_resume(cqctx, tid);
> +			cmdq_thread_wait_end(cqctx, tid, end_pa);
> +			status = cmdq_thread_suspend(cqctx, tid);
> +			if (status < 0) {
> +				spin_unlock_irqrestore(&cqctx->exec_lock,
> +						       flags);
> +				return status;
> +			}
> +			/* set to task directly */
> +			writel(task->mva_base,
> +			       gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +			       CMDQ_THR_SHIFT * tid);
> +			writel(task->mva_base + task->command_size,
> +			       gce_base + CMDQ_THR_END_ADDR_OFFSET +
> +			       CMDQ_THR_SHIFT * tid);
> +			thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD] = task;
> +			thread->task_count++;
> +
> +		/*
> +		 * case 3. If thread is still in WFE from previous task, clear
> +		 * WFE in new task and append to thread.
> +		 */
> +		} else {
> +			/* Current task that shuld be processed */
> +			minimum = cmdq_thread_get_cookie(cqctx, tid) + 1;
> +			if (minimum > CMDQ_MAX_COOKIE_VALUE)
> +				minimum = 0;
> +
> +			/* Calculate loop count to adjust the tasks' order */
> +			if (minimum <= cookie)
> +				loop = cookie - minimum;
> +			else
> +				/* Counter wrapped */
> +				loop = (CMDQ_MAX_COOKIE_VALUE - minimum + 1) +
> +				       cookie;
> +
> +			if (loop < 0) {
> +				dev_err(dev, "reorder fail:\n");
> +				dev_err(dev, "  task count=%d\n", loop);
> +				dev_err(dev, "  thread=%d\n", tid);
> +				dev_err(dev, "  next cookie=%d\n",
> +					thread->next_cookie);
> +				dev_err(dev, "  (HW) next cookie=%d\n",
> +					minimum);
> +				dev_err(dev, "  task=0x%p\n", task);
> +
> +				spin_unlock_irqrestore(&cqctx->exec_lock,
> +						       flags);
> +				return -EFAULT;
> +			}
> +
> +			if (loop > CMDQ_MAX_TASK_IN_THREAD)
> +				loop %= CMDQ_MAX_TASK_IN_THREAD;
> +
> +			status = cmdq_task_insert_into_thread(task, tid, loop);
> +			if (status < 0) {
> +				spin_unlock_irqrestore(
> +						&cqctx->exec_lock, flags);
> +				dev_err(dev,
> +					"invalid task state for reorder.\n");
> +				return status;
> +			}
> +
> +			cmdq_task_remove_wfe(task);
> +
> +			smp_mb(); /* modify jump before enable thread */
> +
> +			writel(task->mva_base + task->command_size,
> +			       gce_base + CMDQ_THR_END_ADDR_OFFSET +
> +			       CMDQ_THR_SHIFT * tid);
> +			thread->task_count++;
> +		}
> +
> +		thread->next_cookie += 1;
> +		if (thread->next_cookie > CMDQ_MAX_COOKIE_VALUE)
> +			thread->next_cookie = 0;
> +
> +		/* resume HW thread */
> +		cmdq_thread_resume(cqctx, tid);
> +	}
> +
> +	spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +
> +	return status;
> +}
> +
> +static void cmdq_core_handle_error(struct cmdq *cqctx, int tid, int value)
> +{
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	struct cmdq_thread *thread;
> +	struct cmdq_task *task;
> +	int cookie;
> +	int count;
> +	int inner;
> +	int status;
> +	u32 curr_pa, end_pa;
> +
> +	curr_pa = readl(gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +			CMDQ_THR_SHIFT * tid);
> +	end_pa = readl(gce_base + CMDQ_THR_END_ADDR_OFFSET +
> +		       CMDQ_THR_SHIFT * tid);
> +
> +	dev_err(dev, "IRQ: error thread=%d, irq_flag=0x%x\n", tid, value);
> +	dev_err(dev, "IRQ: Thread PC: 0x%08x, End PC:0x%08x\n",
> +		curr_pa, end_pa);
> +
> +	thread = &cqctx->thread[tid];
> +
> +	cookie = cmdq_thread_get_cookie(cqctx, tid);
> +
> +	/*
> +	 * we assume error happens BEFORE EOC
> +	 * because it wouldn't be error if this interrupt is issue by EOC.
> +	 * so we should inc by 1 to locate "current" task
> +	 */
> +	cookie++;
> +
> +	/* set the issued task to error state */
> +	if (thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD]) {
> +		task = thread->cur_task[cookie % CMDQ_MAX_TASK_IN_THREAD];
> +		task->irq_flag = value;
> +		cmdq_thread_remove_task_by_index(
> +				thread, cookie % CMDQ_MAX_TASK_IN_THREAD,
> +				TASK_STATE_ERROR);
> +	} else {
> +		dev_err(dev,
> +			"IRQ: can not find task in %s, pc:0x%08x, end_pc:0x%08x\n",
> +			__func__, curr_pa, end_pa);
> +		if (thread->task_count <= 0) {
> +			/*
> +			 * suspend HW thread first,
> +			 * so that we work in a consistent state
> +			 * outer function should acquire spinlock:
> +			 *   cqctx->exec_lock
> +			 */
> +			status = cmdq_thread_suspend(cqctx, tid);
> +			if (status < 0)
> +				dev_err(dev, "IRQ: suspend HW thread failed!");
> +
> +			cmdq_thread_disable(cqctx, tid);
> +			dev_err(dev,
> +				"IRQ: there is no task for thread (%d) %s\n",
> +				tid, __func__);
> +		}
> +	}
> +
> +	/* set the remain tasks to done state */
> +	if (thread->wait_cookie <= cookie) {
> +		count = cookie - thread->wait_cookie + 1;
> +	} else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
> +			thread->wait_cookie) {
> +		count = 0;
> +	} else {
> +		/* counter wrapped */
> +		count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
> +			(cookie + 1);
> +		dev_err(dev,
> +			"IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
> +			thread->wait_cookie, cookie, count);
> +	}
> +
> +	for (inner = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
> +	     count--, inner++) {
> +		if (inner >= CMDQ_MAX_TASK_IN_THREAD)
> +			inner = 0;
> +
> +		if (thread->cur_task[inner]) {
> +			task = thread->cur_task[inner];
> +			task->irq_flag = 0;	/* don't know irq flag */
> +			/* still call isr_cb to prevent lock */
> +			if (task->cb.isr_cb)
> +				task->cb.isr_cb(task->cb.isr_data);
> +			cmdq_thread_remove_task_by_index(
> +					thread, inner, TASK_STATE_DONE);
> +		}
> +	}
> +
> +	thread->wait_cookie = cookie + 1;
> +	if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
> +		thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
> +			/* min cookie value is 0 */
> +
> +	wake_up(&cqctx->wait_queue[tid]);
> +}
> +
> +static void cmdq_core_handle_done(struct cmdq *cqctx, int tid, int value)
> +{
> +	struct device *dev = cqctx->dev;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	int cookie = cmdq_thread_get_cookie(cqctx, tid);
> +	int count;
> +	int i;
> +	struct cmdq_task *task;
> +
> +	if (thread->wait_cookie <= cookie) {
> +		count = cookie - thread->wait_cookie + 1;
> +	} else if ((cookie + 1) % CMDQ_MAX_COOKIE_VALUE ==
> +			thread->wait_cookie) {
> +		count = 0;
> +	} else {
> +		/* counter wrapped */
> +		count = (CMDQ_MAX_COOKIE_VALUE - thread->wait_cookie + 1) +
> +			(cookie + 1);
> +		dev_err(dev,
> +			"IRQ: counter wrapped: wait cookie:%d, hw cookie:%d, count=%d",
> +			thread->wait_cookie, cookie, count);
> +	}
> +
> +	for (i = (thread->wait_cookie % CMDQ_MAX_TASK_IN_THREAD); count > 0;
> +	     count--, i++) {
> +		if (i >= CMDQ_MAX_TASK_IN_THREAD)
> +			i = 0;
> +
> +		if (thread->cur_task[i]) {
> +			task = thread->cur_task[i];
> +			task->irq_flag = value;
> +			if (task->cb.isr_cb)
> +				task->cb.isr_cb(task->cb.isr_data);
> +			cmdq_thread_remove_task_by_index(
> +					thread, i, TASK_STATE_DONE);
> +		}
> +	}
> +
> +	thread->wait_cookie = cookie + 1;
> +	if (thread->wait_cookie > CMDQ_MAX_COOKIE_VALUE)
> +		thread->wait_cookie -= (CMDQ_MAX_COOKIE_VALUE + 1);
> +			/* min cookie value is 0 */
> +
> +	wake_up(&cqctx->wait_queue[tid]);
> +}
> +
> +static void cmdq_core_handle_irq(struct cmdq *cqctx, int tid)
> +{
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	unsigned long flags = 0L;
> +	int value;
> +	int enabled;
> +	int cookie;
> +
> +	/*
> +	 * normal execution, marks tasks done and remove from thread
> +	 * also, handle "loop CB fail" case
> +	 */
> +	spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> +	/*
> +	 * it is possible for another CPU core
> +	 * to run "release task" right before we acquire the spin lock
> +	 * and thus reset / disable this HW thread
> +	 * so we check both the IRQ flag and the enable bit of this thread
> +	 */
> +	value = readl(gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> +		      CMDQ_THR_SHIFT * tid);
> +	if (!(value & CMDQ_THR_IRQ_MASK)) {
> +		dev_err(dev,
> +			"IRQ: thread %d got interrupt but IRQ flag is 0x%08x\n",
> +			tid, value);
> +		spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +		return;
> +	}
> +
> +	enabled = readl(gce_base + CMDQ_THR_ENABLE_TASK_OFFSET +
> +			CMDQ_THR_SHIFT * tid);
> +	if (!(enabled & CMDQ_THR_ENABLED)) {
> +		dev_err(dev,
> +			"IRQ: thread %d got interrupt already disabled 0x%08x\n",
> +			tid, enabled);
> +		spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +		return;
> +	}
> +
> +	/* read HW cookie here for printing message */
> +	cookie = cmdq_thread_get_cookie(cqctx, tid);
> +
> +	/*
> +	 * Move the reset IRQ before read HW cookie
> +	 * to prevent race condition and save the cost of suspend
> +	 */
> +	writel(~value,
> +	       gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +
> +	if (value & CMDQ_THR_IRQ_ERROR)
> +		cmdq_core_handle_error(cqctx, tid, value);
> +	else if (value & CMDQ_THR_IRQ_DONE)
> +		cmdq_core_handle_done(cqctx, tid, value);
> +
> +	spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +}
> +
> +static int cmdq_task_exec_async(struct cmdq_task *task, int tid)
> +{
> +	struct device *dev = task->cqctx->dev;
> +	int status;
> +
> +	status = cmdq_task_exec_async_impl(task, tid);
> +	if (status >= 0)
> +		return status;
> +
> +	if ((task->task_state == TASK_STATE_KILLED) ||
> +	    (task->task_state == TASK_STATE_ERROR)) {
> +		dev_err(dev, "cmdq_task_exec_async_impl fail\n");
> +		return -EFAULT;
> +	}
> +
> +	return 0;
> +}
> +
> +static void cmdq_core_consume_waiting_list(struct work_struct *work)
> +{
> +	struct list_head *p, *n = NULL;
> +	bool thread_acquired;
> +	ktime_t consume_time;
> +	s64 waiting_time_ns;
> +	bool need_log;
> +	struct cmdq *cqctx;
> +	struct device *dev;
> +	u32 err_bits = 0;
> +
> +	cqctx = container_of(work, struct cmdq,
> +			     task_consume_wait_queue_item);
> +	dev = cqctx->dev;
> +
> +	consume_time = ktime_get();
> +
> +	mutex_lock(&cqctx->task_mutex);
> +
> +	thread_acquired = false;
> +
> +	/* scan and remove (if executed) waiting tasks */
> +	list_for_each_safe(p, n, &cqctx->task_wait_list) {
> +		struct cmdq_task *task;
> +		struct cmdq_thread *thread;
> +		int tid;
> +		int status;
> +
> +		task = list_entry(p, struct cmdq_task, list_entry);
> +		tid = cmdq_eng_get_thread(task->engine_flag);
> +
> +		waiting_time_ns = ktime_to_ns(
> +				ktime_sub(consume_time, task->submit));
> +		need_log = waiting_time_ns >= CMDQ_PREALARM_TIMEOUT_NS;
> +
> +		/*
> +		 * Once waiting occur,
> +		 * skip following tasks to keep order of display tasks.
> +		 */
> +		if (err_bits & BIT(tid))
> +			continue;
> +
> +		/* acquire HW thread */
> +		thread = cmdq_core_acquire_thread(cqctx, tid);
> +		if (!thread) {
> +			/* have to wait, remain in wait list */
> +			dev_warn(dev, "acquire thread(%d) fail, need to wait\n",
> +				 tid);
> +			if (need_log) /* task wait too long */
> +				dev_warn(dev, "waiting:%lldns, task:0x%p\n",
> +					 waiting_time_ns, task);
> +			err_bits |= BIT(tid);
> +			continue;
> +		}
> +
> +		/* some task is ready to run */
> +		thread_acquired = true;
> +
> +		/*
> +		 * start execution
> +		 * remove from wait list and put into active list
> +		 */
> +		list_move_tail(&task->list_entry,
> +			       &cqctx->task_active_list);
> +
> +		/* run task on thread */
> +		status = cmdq_task_exec_async(task, tid);
> +		if (status < 0) {
> +			dev_err(dev, "%s fail, release task 0x%p\n",
> +				__func__, task);
> +			cmdq_task_remove_thread(task);
> +			cmdq_task_release_unlocked(task);
> +			task = NULL;
> +		}
> +	}
> +
> +	if (thread_acquired) {
> +		/*
> +		 * notify some task's sw thread to change their waiting state.
> +		 * (if they have already called cmdq_task_wait_and_release())
> +		 */
> +		wake_up_all(&cqctx->thread_dispatch_queue);
> +	}
> +
> +	mutex_unlock(&cqctx->task_mutex);
> +}
> +
> +static int cmdq_core_submit_task_async(struct cmdq_command *cmd_desc,
> +				       struct cmdq_task **task_out,
> +				       struct cmdq_task_cb *cb)
> +{
> +	struct cmdq *cqctx = cmd_desc->cqctx;
> +
> +	/* creates a new task and put into tail of waiting list */
> +	*task_out = cmdq_core_acquire_task(cmd_desc, cb);
> +
> +	if (!(*task_out))
> +		return -EFAULT;
> +
> +	/*
> +	 * Consume the waiting list.
> +	 * This may or may not execute the task, depending on available threads.
> +	 */
> +	cmdq_core_consume_waiting_list(&cqctx->task_consume_wait_queue_item);
> +
> +	return 0;
> +}
> +
> +static int cmdq_core_release_task(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	int tid = task->thread;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	unsigned long flags;
> +	int status;
> +
> +	if (tid != CMDQ_INVALID_THREAD && thread) {
> +		/* this task is being executed (or queueed) on a hw thread */
> +
> +		/* get sw lock first to ensure atomic access hw */
> +		spin_lock_irqsave(&cqctx->exec_lock, flags);
> +		smp_mb();	/* make sure atomic access hw */
> +
> +		status = cmdq_thread_force_remove_task(task, tid);
> +		if (thread->task_count > 0)
> +			cmdq_thread_resume(cqctx, tid);
> +
> +		spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +		wake_up(&cqctx->wait_queue[tid]);
> +	}
> +
> +	cmdq_task_remove_thread(task);
> +	cmdq_task_release_internal(task);
> +	return 0;
> +}
> +
> +struct cmdq_task_error_report {
> +	bool throw_err;
> +	const char *module;
> +	u32 inst_a;
> +	u32 inst_b;
> +	u32 irq_flag;
> +};
> +
> +static int cmdq_task_handle_error_result(
> +		struct cmdq_task *task, int tid, int wait_q,
> +		struct cmdq_task_error_report *error_report)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	void __iomem *gce_base = cqctx->base;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	int status = 0;
> +	int i;
> +	bool is_err = false;
> +	struct cmdq_task *next_task;
> +	struct cmdq_task *prev_task;
> +	int cookie;
> +	unsigned long thread_pc;
> +
> +	dev_err(dev,
> +		"task(0x%p) state is not TASK_STATE_DONE, but %d.\n",
> +		task, task->task_state);
> +
> +	/*
> +	 * Oops, that task is not done.
> +	 * We have several possible error cases:
> +	 * 1. task still running (hang / timeout)
> +	 * 2. IRQ pending (done or error/timeout IRQ)
> +	 * 3. task's SW thread has been signaled (e.g. SIGKILL)
> +	 */
> +
> +	/*
> +	 * suspend HW thread first,
> +	 * so that we work in a consistent state
> +	 */
> +	status = cmdq_thread_suspend(cqctx, tid);
> +	if (status < 0)
> +		error_report->throw_err = true;
> +
> +	/* The cookie of the task currently being processed */
> +	cookie = cmdq_thread_get_cookie(cqctx, tid) + 1;
> +	thread_pc = (unsigned long)readl(gce_base +
> +					 CMDQ_THR_CURR_ADDR_OFFSET +
> +					 CMDQ_THR_SHIFT * tid);
> +
> +	/* process any pending IRQ */
> +	error_report->irq_flag = readl(
> +			gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> +			CMDQ_THR_SHIFT * tid);
> +	if (error_report->irq_flag & CMDQ_THR_IRQ_ERROR)
> +		cmdq_core_handle_error(cqctx, tid, error_report->irq_flag);
> +	else if (error_report->irq_flag & CMDQ_THR_IRQ_DONE)
> +		cmdq_core_handle_done(cqctx, tid, error_report->irq_flag);
> +
> +	writel(~error_report->irq_flag,
> +	       gce_base + CMDQ_THR_IRQ_STATUS_OFFSET +
> +	       CMDQ_THR_SHIFT * tid);
> +
> +	/* check if this task has finished after handling pending IRQ */
> +	if (task->task_state == TASK_STATE_DONE)
> +		return 0;
> +
> +	/* Then decide we are SW timeout or SIGNALed (not an error) */
> +	if (!wait_q) {
> +		/* SW timeout and no IRQ received */
> +		is_err = true;
> +		dev_err(dev, "SW timeout of task 0x%p on tid %d\n",
> +			task, tid);
> +		error_report->throw_err = true;
> +		cmdq_core_parse_error(task, tid,
> +				      &error_report->module,
> +				      &error_report->irq_flag,
> +				      &error_report->inst_a,
> +				      &error_report->inst_b);
> +		status = -ETIMEDOUT;
> +	} else if (wait_q < 0) {
> +		/*
> +		 * Task is killed.
> +		 * Not an error, but still need to remove.
> +		 */
> +		is_err = false;
> +
> +		if (wait_q == -ERESTARTSYS)
> +			dev_err(dev,
> +				"Task 0x%p KILLED by wait_q = -ERESTARTSYS\n",
> +				task);
> +		else if (wait_q == -EINTR)
> +			dev_err(dev,
> +				"Task 0x%p KILLED by wait_q = -EINTR\n",
> +				task);
> +		else
> +			dev_err(dev,
> +				"Task 0x%p KILLED by wait_q = %d\n",
> +				task, wait_q);
> +
> +		status = wait_q;
> +	}
> +
> +	if (task->task_state == TASK_STATE_BUSY) {
> +		/*
> +		 * if task_state is BUSY,
> +		 * this means we did not reach EOC,
> +		 * did not have error IRQ.
> +		 * - remove the task from thread.cur_task[]
> +		 * - and decrease thread.task_count
> +		 * NOTE: after this,
> +		 * the cur_task will not contain link to task anymore.
> +		 * and task should become TASK_STATE_ERROR
> +		 */
> +
> +		/* we find our place in thread->cur_task[]. */
> +		for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> +			if (thread->cur_task[i] == task) {
> +				/* update task_count and cur_task[] */
> +				cmdq_thread_remove_task_by_index(
> +						thread, i, is_err ?
> +						TASK_STATE_ERROR :
> +						TASK_STATE_KILLED);
> +				break;
> +			}
> +		}
> +	}
> +
> +	next_task = NULL;
> +
> +	/* find task's jump destination or no next task*/
> +	if (task->va_base[task->num_cmd - 1] == CMDQ_JUMP_BY_PA)
> +		next_task = cmdq_thread_search_task_by_pc(
> +				thread,
> +				task->va_base[task->num_cmd - 2]);
> +
> +	/*
> +	 * Then, we try remove task from the chain of thread->cur_task.
> +	 * . if HW PC falls in task range
> +	 * . HW EXEC_CNT += 1
> +	 * . thread.wait_cookie += 1
> +	 * . set HW PC to next task head
> +	 * . if not, find previous task
> +	 *                (whose jump address is task->mva_base)
> +	 * . check if HW PC points is not at the EOC/JUMP end
> +	 * . change jump to fake EOC(no IRQ)
> +	 * . insert jump to next task head and increase cmd buffer size
> +	 * . if there is no next task, set HW End Address
> +	 */
> +	if (task->num_cmd && thread_pc >= task->mva_base &&
> +	    thread_pc <= (task->mva_base + task->command_size)) {
> +		if (next_task) {
> +			/* cookie already +1 */
> +			writel(cookie,
> +			       gce_base + CMDQ_THR_EXEC_CNT_OFFSET +
> +			       CMDQ_THR_SHIFT * tid);
> +			thread->wait_cookie = cookie + 1;
> +			writel(next_task->mva_base,
> +			       gce_base + CMDQ_THR_CURR_ADDR_OFFSET +
> +			       CMDQ_THR_SHIFT * tid);
> +		}
> +	} else {
> +		prev_task = NULL;
> +		for (i = 0; i < CMDQ_MAX_TASK_IN_THREAD; i++) {
> +			u32 *prev_va, *curr_va;
> +			u32 prev_num, curr_num;
> +
> +			prev_task = thread->cur_task[i];
> +			if (!prev_task)
> +				continue;
> +
> +			prev_va = prev_task->va_base;
> +			prev_num = prev_task->num_cmd;
> +			if (!prev_num)
> +				continue;
> +
> +			curr_va = task->va_base;
> +			curr_num = task->num_cmd;
> +
> +			/* find which task JUMP into task */
> +			if (prev_va[prev_num - 2] == task->mva_base &&
> +			    prev_va[prev_num - 1] == CMDQ_JUMP_BY_PA) {
> +				/* Copy Jump instruction */
> +				prev_va[prev_num - 2] =
> +					curr_va[curr_num - 2];
> +				prev_va[prev_num - 1] =
> +					curr_va[curr_num - 1];
> +
> +				if (next_task)
> +					cmdq_thread_reorder_task_array(
> +							thread, i);
> +
> +				/*
> +				 * Give up fetched command,
> +				 * invoke CMDQ HW to re-fetch command.
> +				 */
> +				cmdq_core_invalidate_hw_fetched_buffer(
> +						cqctx, tid);
> +
> +				break;
> +			}
> +		}
> +	}
> +
> +	return status;
> +}
> +
> +static int cmdq_task_wait_result(struct cmdq_task *task, int tid, int wait_q)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct cmdq_thread *thread = &cqctx->thread[tid];
> +	int status = 0;
> +	unsigned long flags;
> +	struct cmdq_task_error_report error_report = { 0 };
> +
> +	/*
> +	 * Note that although we disable IRQ, HW continues to execute
> +	 * so it's possible to have pending IRQ
> +	 */
> +	spin_lock_irqsave(&cqctx->exec_lock, flags);
> +
> +	if (task->task_state != TASK_STATE_DONE)
> +		status = cmdq_task_handle_error_result(
> +				task, tid, wait_q, &error_report);
> +
> +	if (thread->task_count <= 0)
> +		cmdq_thread_disable(cqctx, tid);
> +	else
> +		cmdq_thread_resume(cqctx, tid);
> +
> +	spin_unlock_irqrestore(&cqctx->exec_lock, flags);
> +
> +	if (error_report.throw_err) {
> +		u32 op = error_report.inst_a >> CMDQ_OP_CODE_SHIFT;
> +
> +		switch (op) {
> +		case CMDQ_CODE_WFE:
> +			dev_err(cqctx->dev,
> +				"%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:WAIT EVENT:%s\n",
> +				error_report.module, error_report.irq_flag,
> +				error_report.inst_a, error_report.inst_b,
> +				cmdq_event_get_name(error_report.inst_a &
> +						    CMDQ_ARG_A_MASK));
> +			break;
> +		default:
> +			dev_err(cqctx->dev,
> +				"%s in CMDQ IRQ:0x%02x, INST:(0x%08x, 0x%08x), OP:%s\n",
> +				error_report.module, error_report.irq_flag,
> +				error_report.inst_a, error_report.inst_b,
> +				cmdq_core_parse_op(op));
> +			break;
> +		}
> +	}
> +
> +	return status;
> +}
> +
> +static int cmdq_task_wait_done(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +	struct device *dev = cqctx->dev;
> +	int wait_q;
> +	int tid;
> +	unsigned long timeout = msecs_to_jiffies(
> +			CMDQ_ACQUIRE_THREAD_TIMEOUT_MS);
> +
> +	/*
> +	 * wait for acquire thread
> +	 * (this is done by cmdq_core_consume_waiting_list);
> +	 */
> +	wait_q = wait_event_timeout(
> +			cqctx->thread_dispatch_queue,
> +			(task->thread != CMDQ_INVALID_THREAD), timeout);
> +
> +	if (!wait_q) {
> +		mutex_lock(&cqctx->task_mutex);
> +
> +		/*
> +		 * it's possible that the task was just consumed now.
> +		 * so check again.
> +		 */
> +		if (task->thread == CMDQ_INVALID_THREAD) {
> +			/*
> +			 * Task may have released,
> +			 * or starved to death.
> +			 */
> +			dev_err(dev,
> +				"task(0x%p) timeout with invalid thread\n",
> +				task);
> +
> +			/*
> +			 * remove from waiting list,
> +			 * so that it won't be consumed in the future
> +			 */
> +			list_del_init(&task->list_entry);
> +
> +			mutex_unlock(&cqctx->task_mutex);
> +			return -EINVAL;
> +		}
> +
> +		/* valid thread, so we keep going */
> +		mutex_unlock(&cqctx->task_mutex);
> +	}
> +
> +	tid = task->thread;
> +	if (tid < 0 || tid >= CMDQ_MAX_THREAD_COUNT) {
> +		dev_err(dev, "invalid thread %d in %s\n", tid, __func__);
> +		return -EINVAL;
> +	}
> +
> +	/* start to wait */
> +	wait_q = wait_event_timeout(task->cqctx->wait_queue[tid],
> +				    (task->task_state != TASK_STATE_BUSY &&
> +				     task->task_state != TASK_STATE_WAITING),
> +				    msecs_to_jiffies(CMDQ_DEFAULT_TIMEOUT_MS));
> +	if (!wait_q)
> +		dev_dbg(dev, "timeout!\n");
> +
> +	/* wake up and continue */
> +	return cmdq_task_wait_result(task, tid, wait_q);
> +}
> +
> +static int cmdq_task_wait_and_release(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx;
> +	int status;
> +
> +	if (!task) {
> +		pr_err("%s err ptr=0x%p\n", __func__, task);
> +		return -EFAULT;
> +	}
> +
> +	if (task->task_state == TASK_STATE_IDLE) {
> +		pr_err("%s task=0x%p is IDLE\n", __func__, task);
> +		return -EFAULT;
> +	}
> +
> +	cqctx = task->cqctx;
> +
> +	/* wait for task finish */
> +	status = cmdq_task_wait_done(task);
> +
> +	/* release */
> +	cmdq_task_remove_thread(task);
> +	cmdq_task_release_internal(task);
> +
> +	return status;
> +}
> +
> +static void cmdq_core_auto_release_work(struct work_struct *work_item)
> +{
> +	struct cmdq_task *task;
> +	int status;
> +	struct cmdq_task_cb cb;
> +
> +	task = container_of(work_item, struct cmdq_task, auto_release_work);
> +	cb = task->cb;
> +	status = cmdq_task_wait_and_release(task);
> +
> +	/* isr fail, so call isr_cb here to prevent lock */
> +	if (status && cb.isr_cb)
> +		cb.isr_cb(cb.isr_data);
> +
> +	if (cb.done_cb)
> +		cb.done_cb(cb.done_data);
> +}
> +
> +static int cmdq_core_auto_release_task(struct cmdq_task *task)
> +{
> +	struct cmdq *cqctx = task->cqctx;
> +
> +	/*
> +	 * the work item is embeded in task already
> +	 * but we need to initialized it
> +	 */
> +	INIT_WORK(&task->auto_release_work, cmdq_core_auto_release_work);
> +	queue_work(cqctx->task_auto_release_wq, &task->auto_release_work);
> +	return 0;
> +}
> +
> +static int cmdq_core_submit_task(struct cmdq_command *cmd_desc)
> +{
> +	struct device *dev = cmd_desc->cqctx->dev;
> +	int status;
> +	struct cmdq_task *task;
> +
> +	status = cmdq_core_submit_task_async(cmd_desc, &task, NULL);
> +	if (status < 0) {
> +		dev_err(dev, "cmdq_core_submit_task_async failed=%d\n", status);
> +		return status;
> +	}
> +
> +	status = cmdq_task_wait_and_release(task);
> +	if (status < 0)
> +		dev_err(dev, "task(0x%p) wait fail\n", task);
> +
> +	return status;
> +}
> +
> +static void cmdq_core_deinitialize(struct platform_device *pdev)
> +{
> +	struct cmdq *cqctx = platform_get_drvdata(pdev);
> +	int i;
> +	struct list_head *lists[] = {
> +		&cqctx->task_free_list,
> +		&cqctx->task_active_list,
> +		&cqctx->task_wait_list
> +	};
> +
> +	/*
> +	 * Directly destroy the auto release WQ
> +	 * since we're going to release tasks anyway.
> +	 */
> +	destroy_workqueue(cqctx->task_auto_release_wq);
> +	cqctx->task_auto_release_wq = NULL;
> +
> +	destroy_workqueue(cqctx->task_consume_wq);
> +	cqctx->task_consume_wq = NULL;
> +
> +	/* release all tasks in both list */
> +	for (i = 0; i < ARRAY_SIZE(lists); i++) {
> +		struct cmdq_task *task, *tmp;
> +
> +		list_for_each_entry_safe(task, tmp, lists[i], list_entry) {
> +			cmdq_task_free_command_buffer(task);
> +			kmem_cache_free(cqctx->task_cache, task);
> +			list_del(&task->list_entry);
> +		}
> +	}
> +
> +	kmem_cache_destroy(cqctx->task_cache);
> +	cqctx->task_cache = NULL;
> +
> +	/* release command buffer pool */
> +	cmdq_cmd_buf_pool_uninit(cqctx);
> +}
> +
> +static irqreturn_t cmdq_irq_handler(int irq, void *dev)
> +{
> +	struct cmdq *cqctx = dev;
> +	int i;
> +	u32 irq_status;
> +	bool handled = false;
> +
> +	irq_status = readl(cqctx->base + CMDQ_CURR_IRQ_STATUS_OFFSET);
> +	irq_status &= CMDQ_IRQ_MASK;
> +	for (i = 0;
> +	     irq_status != CMDQ_IRQ_MASK && i < CMDQ_MAX_THREAD_COUNT;
> +	     i++) {
> +		/* STATUS bit set to 0 means IRQ asserted */
> +		if (irq_status & BIT(i))
> +			continue;
> +
> +		/*
> +		 * We mark irq_status to 1 to denote finished
> +		 * processing, and we can early-exit if no more
> +		 * threads being asserted.
> +		 */
> +		irq_status |= BIT(i);
> +
> +		cmdq_core_handle_irq(cqctx, i);
> +		handled = true;
> +	}
> +
> +	if (!handled)
> +		return IRQ_NONE;
> +
> +	queue_work(cqctx->task_consume_wq,
> +		   &cqctx->task_consume_wait_queue_item);
> +	return IRQ_HANDLED;
> +}
> +
> +static int cmdq_core_initialize(struct platform_device *pdev,
> +				struct cmdq **cqctx)
> +{
> +	struct cmdq *lcqctx; /* local cmdq context */
> +	int i;
> +	int ret = 0;
> +
> +	lcqctx = devm_kzalloc(&pdev->dev, sizeof(*lcqctx), GFP_KERNEL);
> +
> +	/* save dev */
> +	lcqctx->dev = &pdev->dev;
> +
> +	/* initial cmdq device related data */
> +	ret = cmdq_dev_init(pdev, lcqctx);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to init cmdq device\n");
> +		goto fail_dev;
> +	}
> +
> +	/* initial mutex, spinlock */
> +	mutex_init(&lcqctx->task_mutex);
> +	mutex_init(&lcqctx->clock_mutex);
> +	spin_lock_init(&lcqctx->thread_lock);
> +	spin_lock_init(&lcqctx->exec_lock);
> +
> +	/* initial wait queue for notification */
> +	for (i = 0; i < ARRAY_SIZE(lcqctx->wait_queue); i++)
> +		init_waitqueue_head(&lcqctx->wait_queue[i]);
> +	init_waitqueue_head(&lcqctx->thread_dispatch_queue);
> +
> +	/* create task pool */
> +	lcqctx->task_cache = kmem_cache_create(
> +			CMDQ_DRIVER_DEVICE_NAME "_task",
> +			sizeof(struct cmdq_task),
> +			__alignof__(struct cmdq_task),
> +			SLAB_POISON | SLAB_HWCACHE_ALIGN | SLAB_RED_ZONE,
> +			&cmdq_task_ctor);
> +
> +	/* initialize task lists */
> +	INIT_LIST_HEAD(&lcqctx->task_free_list);
> +	INIT_LIST_HEAD(&lcqctx->task_active_list);
> +	INIT_LIST_HEAD(&lcqctx->task_wait_list);
> +	INIT_WORK(&lcqctx->task_consume_wait_queue_item,
> +		  cmdq_core_consume_waiting_list);
> +
> +	/* initialize command buffer pool */
> +	ret = cmdq_cmd_buf_pool_init(lcqctx);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to init command buffer pool\n");
> +		goto fail_cmd_buf_pool;
> +	}
> +
> +	lcqctx->task_auto_release_wq = alloc_ordered_workqueue(
> +			"%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_auto_release");
> +	lcqctx->task_consume_wq = alloc_ordered_workqueue(
> +			"%s", WQ_MEM_RECLAIM | WQ_HIGHPRI, "cmdq_task");
> +
> +	*cqctx = lcqctx;
> +	return ret;
> +
> +fail_cmd_buf_pool:
> +	destroy_workqueue(lcqctx->task_auto_release_wq);
> +	destroy_workqueue(lcqctx->task_consume_wq);
> +	kmem_cache_destroy(lcqctx->task_cache);
> +
> +fail_dev:
> +	return ret;
> +}
> +
> +static int cmdq_rec_realloc_cmd_buffer(struct cmdq_rec *handle, size_t size)
> +{
> +	void *new_buf;
> +
> +	new_buf = krealloc(handle->buf_ptr, size, GFP_KERNEL | __GFP_ZERO);
> +	if (!new_buf)
> +		return -ENOMEM;
> +	handle->buf_ptr = new_buf;
> +	handle->buf_size = size;
> +	return 0;
> +}
> +
> +static int cmdq_rec_stop_running_task(struct cmdq_rec *handle)
> +{
> +	int status;
> +
> +	status = cmdq_core_release_task(handle->running_task_ptr);
> +	handle->running_task_ptr = NULL;
> +	return status;
> +}
> +
> +int cmdq_rec_create(struct device *dev, u64 engine_flag,
> +		    struct cmdq_rec **handle_ptr)
> +{
> +	struct cmdq *cqctx;
> +	struct cmdq_rec *handle;
> +	int ret;
> +
> +	cqctx = dev_get_drvdata(dev);
> +	if (!cqctx) {
> +		dev_err(dev, "cmdq context is NULL\n");
> +		return -EINVAL;
> +	}
> +
> +	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
> +	if (!handle)
> +		return -ENOMEM;
> +
> +	handle->cqctx = dev_get_drvdata(dev);
> +	handle->engine_flag = engine_flag;
> +
> +	ret = cmdq_rec_realloc_cmd_buffer(handle, CMDQ_INITIAL_CMD_BLOCK_SIZE);
> +	if (ret) {
> +		kfree(handle);
> +		return ret;
> +	}
> +
> +	*handle_ptr = handle;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL(cmdq_rec_create);
> +
> +static int cmdq_rec_append_command(struct cmdq_rec *handle,
> +				   enum cmdq_code code,
> +				   u32 arg_a, u32 arg_b)
> +{
> +	struct cmdq *cqctx;
> +	struct device *dev;
> +	int subsys;
> +	u32 *cmd_ptr;
> +	int ret;
> +
> +	cqctx = handle->cqctx;
> +	dev = cqctx->dev;
> +	cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
> +
> +	if (handle->finalized) {
> +		dev_err(dev,
> +			"already finalized record(cannot add more command)");
> +		dev_err(dev, "handle=0x%p, tid=%d\n", handle, current->pid);
> +		return -EBUSY;
> +	}
> +
> +	/* check if we have sufficient buffer size */
> +	if (unlikely(handle->block_size + CMDQ_INST_SIZE > handle->buf_size)) {
> +		ret = cmdq_rec_realloc_cmd_buffer(handle, handle->buf_size * 2);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	/*
> +	 * we must re-calculate current PC
> +	 * because we may already insert MARKER inst.
> +	 */
> +	cmd_ptr = (u32 *)((u8 *)handle->buf_ptr + handle->block_size);
> +
> +	switch (code) {
> +	case CMDQ_CODE_MOVE:
> +		cmd_ptr[0] = arg_b;
> +		cmd_ptr[1] = (CMDQ_CODE_MOVE << CMDQ_OP_CODE_SHIFT) |
> +			     (arg_a & CMDQ_ARG_A_MASK);
> +		break;
> +	case CMDQ_CODE_WRITE:
> +		subsys = cmdq_subsys_from_phys_addr(cqctx, arg_a);
> +		if (subsys < 0) {
> +			dev_err(dev,
> +				"unsupported memory base address 0x%08x\n",
> +				arg_a);
> +			return -EFAULT;
> +		}
> +
> +		cmd_ptr[0] = arg_b;
> +		cmd_ptr[1] = (CMDQ_CODE_WRITE << CMDQ_OP_CODE_SHIFT) |
> +			     (arg_a & CMDQ_ARG_A_WRITE_MASK) |
> +			     ((subsys & CMDQ_SUBSYS_MASK) << CMDQ_SUBSYS_SHIFT);
> +		break;
> +	case CMDQ_CODE_JUMP:
> +		cmd_ptr[0] = arg_b;
> +		cmd_ptr[1] = (CMDQ_CODE_JUMP << CMDQ_OP_CODE_SHIFT) |
> +			     (arg_a & CMDQ_ARG_A_MASK);
> +		break;
> +	case CMDQ_CODE_WFE:
> +		/*
> +		 * bit 0-11: wait_value, 1
> +		 * bit 15: to_wait, true
> +		 * bit 16-27: update_value, 0
> +		 * bit 31: to_update, true
> +		 */
> +		cmd_ptr[0] = CMDQ_WFE_UPDATE | CMDQ_WFE_WAIT |
> +			     CMDQ_WFE_WAIT_VALUE;
> +		cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
> +		break;
> +	case CMDQ_CODE_CLEAR_EVENT:
> +		/*
> +		 * bit 0-11: wait_value, 0
> +		 * bit 15: to_wait, false
> +		 * bit 16-27: update_value, 0
> +		 * bit 31: to_update, true
> +		 */
> +		cmd_ptr[0] = CMDQ_WFE_UPDATE;
> +		cmd_ptr[1] = (CMDQ_CODE_WFE << CMDQ_OP_CODE_SHIFT) | arg_a;
> +		break;
> +	case CMDQ_CODE_EOC:
> +		cmd_ptr[0] = arg_b;
> +		cmd_ptr[1] = (CMDQ_CODE_EOC << CMDQ_OP_CODE_SHIFT) |
> +			     (arg_a & CMDQ_ARG_A_MASK);
> +		break;
> +	default:
> +		return -EFAULT;
> +	}
> +
> +	handle->block_size += CMDQ_INST_SIZE;
> +
> +	return 0;
> +}
> +
> +int cmdq_rec_reset(struct cmdq_rec *handle)
> +{
> +	if (handle->running_task_ptr)
> +		cmdq_rec_stop_running_task(handle);
> +
> +	handle->block_size = 0;
> +	handle->finalized = false;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL(cmdq_rec_reset);
> +
> +int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr)
> +{
> +	return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
> +}
> +EXPORT_SYMBOL(cmdq_rec_write);
> +
> +int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
> +			u32 addr, u32 mask)
> +{
> +	int ret;
> +
> +	if (mask != 0xffffffff) {
> +		ret = cmdq_rec_append_command(handle, CMDQ_CODE_MOVE, 0, ~mask);
> +		if (ret)
> +			return ret;
> +
> +		addr = addr | CMDQ_ENABLE_MASK;
> +	}
> +
> +	return cmdq_rec_append_command(handle, CMDQ_CODE_WRITE, addr, value);
> +}
> +EXPORT_SYMBOL(cmdq_rec_write_mask);
> +
> +int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event)
> +{
> +	if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
> +	    event < 0)
> +		return -EINVAL;
> +
> +	return cmdq_rec_append_command(handle, CMDQ_CODE_WFE, event, 0);
> +}
> +EXPORT_SYMBOL(cmdq_rec_wait);
> +
> +int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event)
> +{
> +	if (event == CMDQ_SYNC_TOKEN_INVALID || event >= CMDQ_SYNC_TOKEN_MAX ||
> +	    event < 0)
> +		return -EINVAL;
> +
> +	return cmdq_rec_append_command(handle, CMDQ_CODE_CLEAR_EVENT, event, 0);
> +}
> +EXPORT_SYMBOL(cmdq_rec_clear_event);
> +
> +static int cmdq_rec_finalize_command(struct cmdq_rec *handle)
> +{
> +	int status;
> +	struct device *dev;
> +	u32 arg_b;
> +
> +	dev = handle->cqctx->dev;
> +
> +	if (!handle->finalized) {
> +		/* insert EOC and generate IRQ for each command iteration */
> +		arg_b = CMDQ_EOC_IRQ_EN;
> +		status = cmdq_rec_append_command(handle, CMDQ_CODE_EOC,
> +						 0, arg_b);
> +		if (status)
> +			return status;
> +
> +		/* JUMP to begin */
> +		status = cmdq_rec_append_command(handle, CMDQ_CODE_JUMP, 0, 8);
> +		if (status)
> +			return status;
> +
> +		handle->finalized = true;
> +	}
> +
> +	return 0;
> +}
> +
> +static int cmdq_rec_fill_cmd_desc(struct cmdq_rec *handle,
> +				  struct cmdq_command *desc)
> +{
> +	int ret;
> +
> +	ret = cmdq_rec_finalize_command(handle);
> +	if (ret)
> +		return ret;
> +
> +	desc->cqctx = handle->cqctx;
> +	desc->engine_flag = handle->engine_flag;
> +	desc->va_base = handle->buf_ptr;
> +	desc->block_size = handle->block_size;
> +
> +	return ret;
> +}
> +
> +int cmdq_rec_flush(struct cmdq_rec *handle)
> +{
> +	int ret;
> +	struct cmdq_command desc;
> +
> +	ret = cmdq_rec_fill_cmd_desc(handle, &desc);
> +	if (ret)
> +		return ret;
> +
> +	return cmdq_core_submit_task(&desc);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush);
> +
> +static int cmdq_rec_flush_async_cb(struct cmdq_rec *handle,
> +				   cmdq_async_flush_cb isr_cb,
> +				   void *isr_data,
> +				   cmdq_async_flush_cb done_cb,
> +				   void *done_data)
> +{
> +	int ret;
> +	struct cmdq_command desc;
> +	struct cmdq_task *task;
> +	struct cmdq_task_cb cb;
> +
> +	ret = cmdq_rec_fill_cmd_desc(handle, &desc);
> +	if (ret)
> +		return ret;
> +
> +	cb.isr_cb = isr_cb;
> +	cb.isr_data = isr_data;
> +	cb.done_cb = done_cb;
> +	cb.done_data = done_data;
> +
> +	ret = cmdq_core_submit_task_async(&desc, &task, &cb);
> +	if (ret)
> +		return ret;
> +
> +	ret = cmdq_core_auto_release_task(task);
> +
> +	return ret;
> +}
> +
> +int cmdq_rec_flush_async(struct cmdq_rec *handle)
> +{
> +	return cmdq_rec_flush_async_cb(handle, NULL, NULL, NULL, NULL);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush_async);
> +
> +int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
> +				  cmdq_async_flush_cb isr_cb,
> +				  void *isr_data,
> +				  cmdq_async_flush_cb done_cb,
> +				  void *done_data)
> +{
> +	return cmdq_rec_flush_async_cb(handle, isr_cb, isr_data,
> +				       done_cb, done_data);
> +}
> +EXPORT_SYMBOL(cmdq_rec_flush_async_callback);
> +
> +void cmdq_rec_destroy(struct cmdq_rec *handle)
> +{
> +	if (handle->running_task_ptr)
> +		cmdq_rec_stop_running_task(handle);
> +
> +	/* free command buffer */
> +	kfree(handle->buf_ptr);
> +	handle->buf_ptr = NULL;
> +
> +	/* free command handle */
> +	kfree(handle);
> +}
> +EXPORT_SYMBOL(cmdq_rec_destroy);
> +
> +static int cmdq_probe(struct platform_device *pdev)
> +{
> +	struct cmdq *cqctx;
> +	int ret;
> +
> +	/* init cmdq context, and save it */
> +	ret = cmdq_core_initialize(pdev, &cqctx);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to init cmdq context\n");
> +		return ret;
> +	}
> +	platform_set_drvdata(pdev, cqctx);
> +
> +	ret = devm_request_irq(&pdev->dev, cqctx->irq, cmdq_irq_handler,
> +			       IRQF_TRIGGER_LOW | IRQF_SHARED,
> +			       CMDQ_DRIVER_DEVICE_NAME, cqctx);
> +	if (ret) {
> +		dev_err(&pdev->dev, "failed to register ISR (%d)\n", ret);
> +		goto fail;
> +	}
> +
> +	cqctx->clock = devm_clk_get(&pdev->dev, CMDQ_CLK_NAME);
> +	if (IS_ERR(cqctx->clock)) {
> +		dev_err(&pdev->dev, "failed to get clk:%s\n", CMDQ_CLK_NAME);
> +		ret = PTR_ERR(cqctx->clock);
> +		goto fail;
> +	}
> +
> +	return ret;
> +
> +fail:
> +	cmdq_core_deinitialize(pdev);
> +	return ret;
> +}
> +
> +static int cmdq_remove(struct platform_device *pdev)
> +{
> +	cmdq_core_deinitialize(pdev);
> +	return 0;
> +}
> +
> +static const struct of_device_id cmdq_of_ids[] = {
> +	{.compatible = "mediatek,mt8173-gce",},
> +	{}
> +};
> +
> +static struct platform_driver cmdq_drv = {
> +	.probe = cmdq_probe,
> +	.remove = cmdq_remove,
> +	.driver = {
> +		.name = CMDQ_DRIVER_DEVICE_NAME,
> +		.owner = THIS_MODULE,
> +		.of_match_table = cmdq_of_ids,
> +	}
> +};
> +
> +builtin_platform_driver(cmdq_drv);
> diff --git a/include/soc/mediatek/cmdq.h b/include/soc/mediatek/cmdq.h
> new file mode 100644
> index 0000000..29931c9
> --- /dev/null
> +++ b/include/soc/mediatek/cmdq.h
> @@ -0,0 +1,211 @@
> +/*
> + * Copyright (c) 2015 MediaTek Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#ifndef __MTK_CMDQ_H__
> +#define __MTK_CMDQ_H__
> +
> +#include <linux/platform_device.h>
> +#include <linux/types.h>
> +
> +enum cmdq_eng {
> +	CMDQ_ENG_DISP_AAL,
> +	CMDQ_ENG_DISP_COLOR0,
> +	CMDQ_ENG_DISP_COLOR1,
> +	CMDQ_ENG_DISP_DPI0,
> +	CMDQ_ENG_DISP_DSI0,
> +	CMDQ_ENG_DISP_DSI1,
> +	CMDQ_ENG_DISP_GAMMA,
> +	CMDQ_ENG_DISP_OD,
> +	CMDQ_ENG_DISP_OVL0,
> +	CMDQ_ENG_DISP_OVL1,
> +	CMDQ_ENG_DISP_PWM0,
> +	CMDQ_ENG_DISP_PWM1,
> +	CMDQ_ENG_DISP_RDMA0,
> +	CMDQ_ENG_DISP_RDMA1,
> +	CMDQ_ENG_DISP_RDMA2,
> +	CMDQ_ENG_DISP_UFOE,
> +	CMDQ_ENG_DISP_WDMA0,
> +	CMDQ_ENG_DISP_WDMA1,
> +	CMDQ_ENG_MAX,
> +};
> +
> +/* events for CMDQ and display */
> +enum cmdq_event {
> +	/* Display start of frame(SOF) events */
> +	CMDQ_EVENT_DISP_OVL0_SOF = 11,
> +	CMDQ_EVENT_DISP_OVL1_SOF = 12,
> +	CMDQ_EVENT_DISP_RDMA0_SOF = 13,
> +	CMDQ_EVENT_DISP_RDMA1_SOF = 14,
> +	CMDQ_EVENT_DISP_RDMA2_SOF = 15,
> +	CMDQ_EVENT_DISP_WDMA0_SOF = 16,
> +	CMDQ_EVENT_DISP_WDMA1_SOF = 17,
> +	/* Display end of frame(EOF) events */
> +	CMDQ_EVENT_DISP_OVL0_EOF = 39,
> +	CMDQ_EVENT_DISP_OVL1_EOF = 40,
> +	CMDQ_EVENT_DISP_RDMA0_EOF = 41,
> +	CMDQ_EVENT_DISP_RDMA1_EOF = 42,
> +	CMDQ_EVENT_DISP_RDMA2_EOF = 43,
> +	CMDQ_EVENT_DISP_WDMA0_EOF = 44,
> +	CMDQ_EVENT_DISP_WDMA1_EOF = 45,
> +	/* Mutex end of frame(EOF) events */
> +	CMDQ_EVENT_MUTEX0_STREAM_EOF = 53,
> +	CMDQ_EVENT_MUTEX1_STREAM_EOF = 54,
> +	CMDQ_EVENT_MUTEX2_STREAM_EOF = 55,
> +	CMDQ_EVENT_MUTEX3_STREAM_EOF = 56,
> +	CMDQ_EVENT_MUTEX4_STREAM_EOF = 57,
> +	/* Display underrun events */
> +	CMDQ_EVENT_DISP_RDMA0_UNDERRUN = 63,
> +	CMDQ_EVENT_DISP_RDMA1_UNDERRUN = 64,
> +	CMDQ_EVENT_DISP_RDMA2_UNDERRUN = 65,
> +	/* Keep this at the end of HW events */
> +	CMDQ_MAX_HW_EVENT_COUNT = 260,
> +	/* This is max event and also can be used as mask. */
> +	CMDQ_SYNC_TOKEN_MAX = 0x1ff,
> +	/* Invalid event */
> +	CMDQ_SYNC_TOKEN_INVALID = -1,
> +};
> +
> +/* called after isr done or task done */
> +typedef int (*cmdq_async_flush_cb)(void *data);
> +
> +struct cmdq_task;
> +struct cmdq;
> +
> +struct cmdq_rec {
> +	struct cmdq		*cqctx;
> +	u64			engine_flag;
> +	size_t			block_size; /* command size */
> +	void			*buf_ptr;
> +	size_t			buf_size;
> +	/* running task after flush */
> +	struct cmdq_task	*running_task_ptr;
> +	bool			finalized;
> +};
> +
> +/**
> + * cmdq_rec_create() - create command queue record handle
> + * @dev:		device
> + * @engine_flag:	command queue engine flag
> + * @handle_ptr:		command queue record handle pointer to retrieve cmdq_rec
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_create(struct device *dev, u64 engine_flag,
> +		    struct cmdq_rec **handle_ptr);
> +
> +/**
> + * cmdq_rec_reset() - reset command queue record commands
> + * @handle:	the command queue record handle
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_reset(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_write() - append write command to the command queue record
> + * @handle:	the command queue record handle
> + * @value:	the specified target register value
> + * @addr:	the specified target register physical address
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_write(struct cmdq_rec *handle, u32 value, u32 addr);
> +
> +/**
> + * cmdq_rec_write_mask() - append write command with mask to the command queue
> + *			   record
> + * @handle:	the command queue record handle
> + * @value:	the specified target register value
> + * @addr:	the specified target register physical address
> + * @mask:	the specified target register mask
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_write_mask(struct cmdq_rec *handle, u32 value,
> +			u32 addr, u32 mask);
> +
> +/**
> + * cmdq_rec_wait() - append wait command to the command queue record
> + * @handle:	the command queue record handle
> + * @event:	the desired event type to "wait and CLEAR"
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_wait(struct cmdq_rec *handle, enum cmdq_event event);
> +
> +/**
> + * cmdq_rec_clear_event() - append clear event command to the command queue
> + *			    record
> + * @handle:	the command queue record handle
> + * @event:	the desired event to be cleared
> + *
> + * Return: 0 for success; else the error code is returned
> + */
> +int cmdq_rec_clear_event(struct cmdq_rec *handle, enum cmdq_event event);
> +
> +/**
> + * cmdq_rec_flush() - trigger CMDQ to execute the recorded commands
> + * @handle:	the command queue record handle
> + *
> + * Return: 0 for success; else the error code is returned
> + *
> + * Trigger CMDQ to execute the recorded commands. Note that this is a
> + * synchronous flush function. When the function returned, the recorded
> + * commands have been done.
> + */
> +int cmdq_rec_flush(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_flush_async() - trigger CMDQ to asynchronously execute the
> + *			    recorded commands
> + * @handle:	the command queue record handle
> + *
> + * Return: 0 for successfully start execution; else the error code is returned
> + *
> + * Trigger CMDQ to asynchronously execute the recorded commands. Note that this
> + * is an ASYNC function. When the function returned, it may or may not be
> + * finished. There is no way to retrieve the result.
> + */
> +int cmdq_rec_flush_async(struct cmdq_rec *handle);
> +
> +/**
> + * cmdq_rec_flush_async_callback() - trigger CMDQ to asynchronously execute
> + *				     the recorded commands and call back after
> + *				     ISR is finished and this flush is finished
> + * @handle:	the command queue record handle
> + * @isr_cb:	called by ISR in the end of CMDQ ISR
> + * @isr_data:	this data will pass back to isr_cb
> + * @done_cb:	called after flush is done
> + * @done_data:	this data will pass back to done_cb
> + *
> + * Return: 0 for success; else the error code is returned
> + *
> + * Trigger CMDQ to asynchronously execute the recorded commands and call back
> + * after ISR is finished and this flush is finished. Note that this is an ASYNC
> + * function. When the function returned, it may or may not be finished. The ISR
> + * callback function is called in the end of ISR, and  the done callback
> + * function is called after all commands are done.
> + */
> +int cmdq_rec_flush_async_callback(struct cmdq_rec *handle,
> +				  cmdq_async_flush_cb isr_cb,
> +				  void *isr_data,
> +				  cmdq_async_flush_cb done_cb,
> +				  void *done_data);
> +
> +/**
> + * cmdq_rec_destroy() - destroy command queue record handle
> + * @handle:	the command queue record handle
> + */
> +void cmdq_rec_destroy(struct cmdq_rec *handle);
> +
> +#endif	/* __MTK_CMDQ_H__ */


_______________________________________________
dri-devel mailing list
dri-devel@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/dri-devel