Hi Hans,
Thanks for the patch.
It has been successfully tested.
Tested-by: Vincent Abriou <vincent.abriou@xxxxxx>
On 03/06/2017 03:23 PM, Hans Verkuil wrote:
> vivid_try_fmt_vid_cap() called tpg_calc_line_width to calculate the
> sizeimage
> value, but that tpg function uses the current format, not the proposed
> (tried)
> format.
>
> Rewrote this code to calculate this correctly.
>
> The vivid_try_fmt_vid_out() code was completely wrong w.r.t. sizeimage, and
> neither did it take the vdownsampling[] factors into account.
>
> Signed-off-by: Hans Verkuil <hans.verkuil@xxxxxxxxx
> ---
> diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c
> b/drivers/media/platform/vivid/vivid-vid-cap.c
> index a18e6fec219b..01419455e545 100644
> --- a/drivers/media/platform/vivid/vivid-vid-cap.c
> +++ b/drivers/media/platform/vivid/vivid-vid-cap.c
> @@ -616,7 +616,7 @@ int vivid_try_fmt_vid_cap(struct file *file, void
> *priv,
> /* This driver supports custom bytesperline values */
>
> mp->num_planes = fmt->buffers;
> - for (p = 0; p < mp->num_planes; p++) {
> + for (p = 0; p < fmt->buffers; p++) {
> /* Calculate the minimum supported bytesperline value */
> bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;
> /* Calculate the maximum supported bytesperline value */
> @@ -626,10 +626,17 @@ int vivid_try_fmt_vid_cap(struct file *file, void
> *priv,
> pfmt[p].bytesperline = max_bpl;
> if (pfmt[p].bytesperline < bytesperline)
> pfmt[p].bytesperline = bytesperline;
> - pfmt[p].sizeimage = tpg_calc_line_width(&dev->tpg, p,
> pfmt[p].bytesperline) *
> - mp->height + fmt->data_offset[p];
> +
> + pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /
> + fmt->vdownsampling[p] + fmt->data_offset[p];
> +
> memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));
> }
> + for (p = fmt->buffers; p < fmt->planes; p++)
> + pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *
> + (fmt->bit_depth[p] / fmt->vdownsampling[p])) /
> + (fmt->bit_depth[0] / fmt->vdownsampling[0]);
> +
> mp->colorspace = vivid_colorspace_cap(dev);
> if (fmt->color_enc == TGP_COLOR_ENC_HSV)
> mp->hsv_enc = vivid_hsv_enc_cap(dev);
> diff --git a/drivers/media/platform/vivid/vivid-vid-out.c
> b/drivers/media/platform/vivid/vivid-vid-out.c
> index 7ba52ee98371..b3b3b31c873b 100644
> --- a/drivers/media/platform/vivid/vivid-vid-out.c
> +++ b/drivers/media/platform/vivid/vivid-vid-out.c
> @@ -390,22 +390,28 @@ int vivid_try_fmt_vid_out(struct file *file, void
> *priv,
>
> /* This driver supports custom bytesperline values */
>
> - /* Calculate the minimum supported bytesperline value */
> - bytesperline = (mp->width * fmt->bit_depth[0]) >> 3;
> - /* Calculate the maximum supported bytesperline value */
> - max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[0]) >> 3;
> mp->num_planes = fmt->buffers;
> - for (p = 0; p < mp->num_planes; p++) {
> + for (p = 0; p < fmt->buffers; p++) {
> + /* Calculate the minimum supported bytesperline value */
> + bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;
> + /* Calculate the maximum supported bytesperline value */
> + max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[p]) >> 3;
> +
> if (pfmt[p].bytesperline > max_bpl)
> pfmt[p].bytesperline = max_bpl;
> if (pfmt[p].bytesperline < bytesperline)
> pfmt[p].bytesperline = bytesperline;
> - pfmt[p].sizeimage = pfmt[p].bytesperline * mp->height;
> +
> + pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /
> + fmt->vdownsampling[p];
> +
> memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));
> }
> for (p = fmt->buffers; p < fmt->planes; p++)
> - pfmt[0].sizeimage += (pfmt[0].bytesperline * fmt->bit_depth[p]) /
> - (fmt->bit_depth[0] * fmt->vdownsampling[p]);
> + pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *
> + (fmt->bit_depth[p] / fmt->vdownsampling[p])) /
> + (fmt->bit_depth[0] / fmt->vdownsampling[0]);
> +
> mp->xfer_func = V4L2_XFER_FUNC_DEFAULT;
> mp->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
> mp->quantization = V4L2_QUANTIZATION_DEFAULT;
>
