From: Ezequiel Garcia <ezequiel@xxxxxxxxxxxxx> commit ae02d49493b5d32bb3e035fdeb1655346f5e1ea5 upstream. Commit 953aaa1492c53 ("media: rockchip/vpu: Prepare things to support decoders") changed the conditions under S_FMT was allowed for OUTPUT CAPTURE buffers. However, and according to the mem-to-mem stateless decoder specification, in order to support dynamic resolution changes, S_FMT should be allowed even if OUTPUT buffers have been allocated. Relax decoder S_FMT restrictions on OUTPUT buffers, allowing a resolution modification, provided the pixel format stays the same. Tested on RK3288 platforms using ChromiumOS Video Decode/Encode Accelerator Unittests. [hverkuil: fix typo: In other -> In order] Fixes: 953aaa1492c53 ("media: rockchip/vpu: Prepare things to support decoders") Signed-off-by: Ezequiel Garcia <ezequiel@xxxxxxxxxxxxx> Reviewed-by: Boris Brezillon <boris.brezillon@xxxxxxxxxxxxx> Cc: <stable@xxxxxxxxxxxxxxx> # for v5.4 and up Signed-off-by: Hans Verkuil <hverkuil-cisco@xxxxxxxxx> Signed-off-by: Mauro Carvalho Chehab <mchehab@xxxxxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- drivers/staging/media/hantro/hantro_v4l2.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) --- a/drivers/staging/media/hantro/hantro_v4l2.c +++ b/drivers/staging/media/hantro/hantro_v4l2.c @@ -356,20 +356,27 @@ vidioc_s_fmt_out_mplane(struct file *fil { struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; struct hantro_ctx *ctx = fh_to_ctx(priv); + struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); const struct hantro_fmt *formats; unsigned int num_fmts; - struct vb2_queue *vq; int ret; - /* Change not allowed if queue is busy. */ - vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); - if (vb2_is_busy(vq)) - return -EBUSY; + ret = vidioc_try_fmt_out_mplane(file, priv, f); + if (ret) + return ret; if (!hantro_is_encoder_ctx(ctx)) { struct vb2_queue *peer_vq; /* + * In order to support dynamic resolution change, + * the decoder admits a resolution change, as long + * as the pixelformat remains. Can't be done if streaming. + */ + if (vb2_is_streaming(vq) || (vb2_is_busy(vq) && + pix_mp->pixelformat != ctx->src_fmt.pixelformat)) + return -EBUSY; + /* * Since format change on the OUTPUT queue will reset * the CAPTURE queue, we can't allow doing so * when the CAPTURE queue has buffers allocated. @@ -378,12 +385,15 @@ vidioc_s_fmt_out_mplane(struct file *fil V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); if (vb2_is_busy(peer_vq)) return -EBUSY; + } else { + /* + * The encoder doesn't admit a format change if + * there are OUTPUT buffers allocated. + */ + if (vb2_is_busy(vq)) + return -EBUSY; } - ret = vidioc_try_fmt_out_mplane(file, priv, f); - if (ret) - return ret; - formats = hantro_get_formats(ctx, &num_fmts); ctx->vpu_src_fmt = hantro_find_format(formats, num_fmts, pix_mp->pixelformat);