[PATCH v2 15/16] media: vicodec: set pixelformat V4L2_PIX_FMT_FWHT_STATELESS for stateless encoder

[Dafna Hirschfeld <dafna3@xxxxxxxxx>]

for stateless encoder, set the capture pixelformat
to V4L2_PIX_FMT_FWHT_STATELESS and the pix info to
pixfmt_stateless_fwht

Signed-off-by: Dafna Hirschfeld <dafna3@xxxxxxxxx>
---
 drivers/media/platform/vicodec/vicodec-core.c | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 8780df73a41f..c972c175707c 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -903,8 +903,12 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		if (multiplanar)
 			return -EINVAL;
 		pix = &f->fmt.pix;
-		pix->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				   find_fmt(f->fmt.pix.pixelformat)->id;
+		if (!ctx->is_enc)
+			pix->pixelformat = find_fmt(f->fmt.pix.pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix->pixelformat = V4L2_PIX_FMT_FWHT;
 		pix->colorspace = ctx->state.colorspace;
 		pix->xfer_func = ctx->state.xfer_func;
 		pix->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -914,8 +918,13 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		if (!multiplanar)
 			return -EINVAL;
 		pix_mp = &f->fmt.pix_mp;
-		pix_mp->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				      find_fmt(pix_mp->pixelformat)->id;
+		if (!ctx->is_enc)
+			pix_mp->pixelformat =
+				find_fmt(f->fmt.pix_mp.pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT;
 		pix_mp->colorspace = ctx->state.colorspace;
 		pix_mp->xfer_func = ctx->state.xfer_func;
 		pix_mp->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -1647,22 +1656,22 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	state->stride = q_data->coded_width *
 				info->bytesperline_mult;
 
-	if (ctx->is_stateless) {
+	if (!ctx->is_enc && ctx->is_stateless) {
 		state->ref_stride = state->stride;
 		return 0;
 	}
 	state->ref_stride = q_data->coded_width * info->luma_alpha_step;
 
 	state->ref_frame.buf = kvmalloc(total_planes_size, GFP_KERNEL);
-	state->ref_frame.luma = state->ref_frame.buf;
 	new_comp_frame = kvmalloc(ctx->comp_max_size, GFP_KERNEL);
 
-	if (!state->ref_frame.luma || !new_comp_frame) {
-		kvfree(state->ref_frame.luma);
+	if (!state->ref_frame.buf || !new_comp_frame) {
+		kvfree(state->ref_frame.buf);
 		kvfree(new_comp_frame);
 		vicodec_return_bufs(q, VB2_BUF_STATE_QUEUED);
 		return -ENOMEM;
 	}
+	state->ref_frame.luma = state->ref_frame.buf;
 	/*
 	 * if state->compressed_frame was already allocated then
 	 * it contain data of the first frame of the new resolution
@@ -1964,9 +1973,10 @@ static int vicodec_open(struct file *file)
 			comp_size + sizeof(struct fwht_cframe_hdr);
 	ctx->q_data[V4L2_M2M_DST] = ctx->q_data[V4L2_M2M_SRC];
 	if (ctx->is_enc) {
-		ctx->q_data[V4L2_M2M_DST].info = &pixfmt_fwht;
-		ctx->q_data[V4L2_M2M_DST].sizeimage =
-			comp_size + sizeof(struct fwht_cframe_hdr);
+		ctx->q_data[V4L2_M2M_DST].info = ctx->is_stateless ?
+			&pixfmt_stateless_fwht : &pixfmt_fwht;
+		ctx->q_data[V4L2_M2M_DST].sizeimage = comp_size +
+			sizeof(struct fwht_cframe_hdr);
 	} else {
 		ctx->q_data[V4L2_M2M_DST].info = info;
 		ctx->q_data[V4L2_M2M_DST].sizeimage = raw_size;
-- 
2.17.1