[PATCH 07/10] media: coda: limit frame interval enumeration to supported frame sizes

[Philipp Zabel <p.zabel@xxxxxxxxxxxxxx>]

Let VIDIOC_ENUM_FRAMEINTERVALS return -EINVAL if userspace queries
frame intervals for unsupported frame sizes.

Signed-off-by: Philipp Zabel <p.zabel@xxxxxxxxxxxxxx>
---
 drivers/media/platform/coda/coda-common.c | 33 ++++++++++++++++++-----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
index 943f003c26c4..2966eb1c4d2d 100644
--- a/drivers/media/platform/coda/coda-common.c
+++ b/drivers/media/platform/coda/coda-common.c
@@ -1117,7 +1117,8 @@ static int coda_enum_frameintervals(struct file *file, void *fh,
 				    struct v4l2_frmivalenum *f)
 {
 	struct coda_ctx *ctx = fh_to_ctx(fh);
-	int i;
+	struct coda_q_data *q_data;
+	const struct coda_codec *codec;
 
 	if (f->index)
 		return -EINVAL;
@@ -1126,12 +1127,32 @@ static int coda_enum_frameintervals(struct file *file, void *fh,
 	if (!ctx->vdoa && f->pixel_format == V4L2_PIX_FMT_YUYV)
 		return -EINVAL;
 
-	for (i = 0; i < CODA_MAX_FORMATS; i++) {
-		if (f->pixel_format == ctx->cvd->src_formats[i] ||
-		    f->pixel_format == ctx->cvd->dst_formats[i])
-			break;
+	if (ctx->inst_type == CODA_INST_ENCODER) {
+		if (coda_format_normalize_yuv(f->pixel_format) ==
+		    V4L2_PIX_FMT_YUV420) {
+			q_data = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
+			codec = coda_find_codec(ctx->dev, f->pixel_format,
+						q_data->fourcc);
+		} else {
+			codec = coda_find_codec(ctx->dev, V4L2_PIX_FMT_YUV420,
+						f->pixel_format);
+		}
+	} else {
+		if (coda_format_normalize_yuv(f->pixel_format) ==
+		    V4L2_PIX_FMT_YUV420) {
+			q_data = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
+			codec = coda_find_codec(ctx->dev, q_data->fourcc,
+						f->pixel_format);
+		} else {
+			codec = coda_find_codec(ctx->dev, f->pixel_format,
+						V4L2_PIX_FMT_YUV420);
+		}
 	}
-	if (i == CODA_MAX_FORMATS)
+	if (!codec)
+		return -EINVAL;
+
+	if (f->width < MIN_W || f->width > codec->max_w ||
+	    f->height < MIN_H || f->height > codec->max_h)
 		return -EINVAL;
 
 	f->type = V4L2_FRMIVAL_TYPE_CONTINUOUS;
-- 
2.20.1