Re: [PATCH v8, 16/17] media: mediatek: vcodec: support stateless VP9 decoding

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Le jeudi 31 mars 2022 à 10:48 +0800, Yunfei Dong a écrit :
> Add support for VP9 decoding using the stateless API,
> as supported by MT8192. And the drivers is lat and core architecture.
> 
> Signed-off-by: George Sun <george.sun@xxxxxxxxxxxx>
> Signed-off-by: Xiaoyong Lu <xiaoyong.lu@xxxxxxxxxxxx>
> Signed-off-by: Yunfei Dong <yunfei.dong@xxxxxxxxxxxx>
> Reviewed-by: AngeloGioacchino Del Regno <angelogioacchino.delregno@xxxxxxxxxxxxx>
> ---
> changed compare with v7:
> Using upstream interface to update vp9 prob tables.
> ---
>  .../media/platform/mediatek/vcodec/Makefile   |    1 +
>  .../vcodec/mtk_vcodec_dec_stateless.c         |   26 +-
>  .../platform/mediatek/vcodec/mtk_vcodec_drv.h |    1 +
>  .../vcodec/vdec/vdec_vp9_req_lat_if.c         | 2072 +++++++++++++++++
>  .../platform/mediatek/vcodec/vdec_drv_if.c    |    4 +
>  .../platform/mediatek/vcodec/vdec_drv_if.h    |    1 +
>  6 files changed, 2102 insertions(+), 3 deletions(-)
>  create mode 100644 drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
> 
> diff --git a/drivers/media/platform/mediatek/vcodec/Makefile b/drivers/media/platform/mediatek/vcodec/Makefile
> index b457daf2d196..93e7a343b5b0 100644
> --- a/drivers/media/platform/mediatek/vcodec/Makefile
> +++ b/drivers/media/platform/mediatek/vcodec/Makefile
> @@ -9,6 +9,7 @@ mtk-vcodec-dec-y := vdec/vdec_h264_if.o \
>  		vdec/vdec_vp8_if.o \
>  		vdec/vdec_vp8_req_if.o \
>  		vdec/vdec_vp9_if.o \
> +		vdec/vdec_vp9_req_lat_if.o \
>  		vdec/vdec_h264_req_if.o \
>  		vdec/vdec_h264_req_common.o \
>  		vdec/vdec_h264_req_multi_if.o \
> diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
> index 3208f834ff80..a4735e67d39e 100644
> --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
> +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_stateless.c
> @@ -91,13 +91,28 @@ static const struct mtk_stateless_control mtk_stateless_controls[] = {
>  			.max = V4L2_MPEG_VIDEO_VP8_PROFILE_3,
>  		},
>  		.codec_type = V4L2_PIX_FMT_VP8_FRAME,
> -	}
> +	},
> +	{
> +		.cfg = {
> +			.id = V4L2_CID_STATELESS_VP9_FRAME,
> +		},
> +		.codec_type = V4L2_PIX_FMT_VP9_FRAME,
> +	},
> +	{
> +		.cfg = {
> +			.id = V4L2_CID_MPEG_VIDEO_VP9_PROFILE,
> +			.min = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
> +			.def = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
> +			.max = V4L2_MPEG_VIDEO_VP9_PROFILE_3,
> +		},
> +		.codec_type = V4L2_PIX_FMT_VP9_FRAME,
> +	},
>  };
>  
>  #define NUM_CTRLS ARRAY_SIZE(mtk_stateless_controls)
>  
> -static struct mtk_video_fmt mtk_video_formats[4];
> -static struct mtk_codec_framesizes mtk_vdec_framesizes[2];
> +static struct mtk_video_fmt mtk_video_formats[5];
> +static struct mtk_codec_framesizes mtk_vdec_framesizes[3];
>  
>  static struct mtk_video_fmt default_out_format;
>  static struct mtk_video_fmt default_cap_format;
> @@ -338,6 +353,7 @@ static void mtk_vcodec_add_formats(unsigned int fourcc,
>  	switch (fourcc) {
>  	case V4L2_PIX_FMT_H264_SLICE:
>  	case V4L2_PIX_FMT_VP8_FRAME:
> +	case V4L2_PIX_FMT_VP9_FRAME:
>  		mtk_video_formats[count_formats].fourcc = fourcc;
>  		mtk_video_formats[count_formats].type = MTK_FMT_DEC;
>  		mtk_video_formats[count_formats].num_planes = 1;
> @@ -385,6 +401,10 @@ static void mtk_vcodec_get_supported_formats(struct mtk_vcodec_ctx *ctx)
>  		mtk_vcodec_add_formats(V4L2_PIX_FMT_VP8_FRAME, ctx);
>  		out_format_count++;
>  	}
> +	if (ctx->dev->dec_capability & MTK_VDEC_FORMAT_VP9_FRAME) {
> +		mtk_vcodec_add_formats(V4L2_PIX_FMT_VP9_FRAME, ctx);
> +		out_format_count++;
> +	}
>  
>  	if (cap_format_count)
>  		default_cap_format = mtk_video_formats[cap_format_count - 1];
> diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h
> index 2ba1c19f07b6..a29041a0b7e0 100644
> --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h
> +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h
> @@ -355,6 +355,7 @@ enum mtk_vdec_format_types {
>  	MTK_VDEC_FORMAT_MT21C = 0x40,
>  	MTK_VDEC_FORMAT_H264_SLICE = 0x100,
>  	MTK_VDEC_FORMAT_VP8_FRAME = 0x200,
> +	MTK_VDEC_FORMAT_VP9_FRAME = 0x400,
>  };
>  
>  /**
> diff --git a/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c b/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
> new file mode 100644
> index 000000000000..d63399085b9b
> --- /dev/null
> +++ b/drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c
> @@ -0,0 +1,2072 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright (c) 2021 MediaTek Inc.
> + * Author: George Sun <george.sun@xxxxxxxxxxxx>
> + */
> +
> +#include <linux/module.h>
> +#include <linux/slab.h>
> +#include <media/videobuf2-dma-contig.h>
> +#include <media/v4l2-vp9.h>
> +
> +#include "../mtk_vcodec_util.h"
> +#include "../mtk_vcodec_dec.h"
> +#include "../mtk_vcodec_intr.h"
> +#include "../vdec_drv_base.h"
> +#include "../vdec_drv_if.h"
> +#include "../vdec_vpu_if.h"
> +
> +/* reset_frame_context defined in VP9 spec */
> +#define VP9_RESET_FRAME_CONTEXT_NONE0 0
> +#define VP9_RESET_FRAME_CONTEXT_NONE1 1
> +#define VP9_RESET_FRAME_CONTEXT_SPEC 2
> +#define VP9_RESET_FRAME_CONTEXT_ALL 3
> +
> +#define VP9_TILE_BUF_SIZE 4096
> +#define VP9_PROB_BUF_SIZE 2560
> +#define VP9_COUNTS_BUF_SIZE 16384
> +
> +#define HDR_FLAG(x) (!!((hdr)->flags & V4L2_VP9_FRAME_FLAG_##x))
> +#define LF_FLAG(x) (!!((lf)->flags & V4L2_VP9_LOOP_FILTER_FLAG_##x))
> +#define SEG_FLAG(x) (!!((seg)->flags & V4L2_VP9_SEGMENTATION_FLAG_##x))
> +#define VP9_BAND_6(band) ((band) == 0 ? 3 : 6)
> +
> +/*
> + * struct vdec_vp9_slice_frame_ctx - vp9 prob tables footprint
> + */
> +struct vdec_vp9_slice_frame_ctx {
> +	struct {
> +		u8 probs[6][3];
> +		u8 padding[2];
> +	} coef_probs[4][2][2][6];
> +
> +	u8 y_mode_prob[4][16];
> +	u8 switch_interp_prob[4][16];
> +	u8 seg[32];  /* ignore */
> +	u8 comp_inter_prob[16];
> +	u8 comp_ref_prob[16];
> +	u8 single_ref_prob[5][2];
> +	u8 single_ref_prob_padding[6];
> +
> +	u8 joint[3];
> +	u8 joint_padding[13];
> +	struct {
> +		u8 sign;
> +		u8 classes[10];
> +		u8 padding[5];
> +	} sign_classes[2];
> +	struct {
> +		u8 class0[1];
> +		u8 bits[10];
> +		u8 padding[5];
> +	} class0_bits[2];
> +	struct {
> +		u8 class0_fp[2][3];
> +		u8 fp[3];
> +		u8 class0_hp;
> +		u8 hp;
> +		u8 padding[5];
> +	} class0_fp_hp[2];
> +
> +	u8 uv_mode_prob[10][16];
> +	u8 uv_mode_prob_padding[2][16];
> +
> +	u8 partition_prob[16][4];
> +
> +	u8 inter_mode_probs[7][4];
> +	u8 skip_probs[4];
> +
> +	u8 tx_p8x8[2][4];
> +	u8 tx_p16x16[2][4];
> +	u8 tx_p32x32[2][4];
> +	u8 intra_inter_prob[8];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_frame_counts - vp9 counts tables footprint
> + */
> +struct vdec_vp9_slice_frame_counts {
> +	union {
> +		struct {
> +			u32 band_0[3];
> +			u32 padding0[1];
> +			u32 band_1_5[5][6];
> +			u32 padding1[2];
> +		} eob_branch[4][2][2];
> +		u32 eob_branch_space[256 * 4];
> +	};
> +
> +	struct {
> +		u32 band_0[3][4];
> +		u32 band_1_5[5][6][4];
> +	} coef_probs[4][2][2];
> +
> +	u32 intra_inter[4][2];
> +	u32 comp_inter[5][2];
> +	u32 comp_inter_padding[2];
> +	u32 comp_ref[5][2];
> +	u32 comp_ref_padding[2];
> +	u32 single_ref[5][2][2];
> +	u32 inter_mode[7][4];
> +	u32 y_mode[4][12];
> +	u32 uv_mode[10][10];
> +	u32 partition[16][4];
> +	u32 switchable_interp[4][4];
> +
> +	u32 tx_p8x8[2][2];
> +	u32 tx_p16x16[2][4];
> +	u32 tx_p32x32[2][4];
> +
> +	u32 skip[3][4];
> +
> +	u32 joint[4];
> +
> +	struct {
> +		u32 sign[2];
> +		u32 class0[2];
> +		u32 classes[12];
> +		u32 bits[10][2];
> +		u32 padding[4];
> +		u32 class0_fp[2][4];
> +		u32 fp[4];
> +		u32 class0_hp[2];
> +		u32 hp[2];
> +	} mvcomp[2];
> +
> +	u32 reserved[126][4];
> +};
> +
> +/**
> + * struct vdec_vp9_slice_counts_map - vp9 counts tables to map
> + *                                    v4l2_vp9_frame_symbol_counts
> + * @skip:	skip counts.
> + * @y_mode:	Y prediction mode counts.
> + * @filter:	interpolation filter counts.
> + * @mv_joint:	motion vector joint counts.
> + * @sign:	motion vector sign counts.
> + * @classes:	motion vector class counts.
> + * @class0:	motion vector class0 bit counts.
> + * @bits:	motion vector bits counts.
> + * @class0_fp:	motion vector class0 fractional bit counts.
> + * @fp:	motion vector fractional bit counts.
> + * @class0_hp:	motion vector class0 high precision fractional bit counts.
> + * @hp:	motion vector high precision fractional bit counts.
> + */
> +struct vdec_vp9_slice_counts_map {
> +	u32 skip[3][2];
> +	u32 y_mode[4][10];
> +	u32 filter[4][3];
> +	u32 sign[2][2];
> +	u32 classes[2][11];
> +	u32 class0[2][2];
> +	u32 bits[2][10][2];
> +	u32 class0_fp[2][2][4];
> +	u32 fp[2][4];
> +	u32 class0_hp[2][2];
> +	u32 hp[2][2];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_uncompressed_header - vp9 uncompressed header syntax
> + *                                             used for decoding
> + */
> +struct vdec_vp9_slice_uncompressed_header {
> +	u8 profile;
> +	u8 last_frame_type;
> +	u8 frame_type;
> +
> +	u8 last_show_frame;
> +	u8 show_frame;
> +	u8 error_resilient_mode;
> +
> +	u8 bit_depth;
> +	u8 padding0[1];
> +	u16 last_frame_width;
> +	u16 last_frame_height;
> +	u16 frame_width;
> +	u16 frame_height;
> +
> +	u8 intra_only;
> +	u8 reset_frame_context;
> +	u8 ref_frame_sign_bias[4];
> +	u8 allow_high_precision_mv;
> +	u8 interpolation_filter;
> +
> +	u8 refresh_frame_context;
> +	u8 frame_parallel_decoding_mode;
> +	u8 frame_context_idx;
> +
> +	/* loop_filter_params */
> +	u8 loop_filter_level;
> +	u8 loop_filter_sharpness;
> +	u8 loop_filter_delta_enabled;
> +	s8 loop_filter_ref_deltas[4];
> +	s8 loop_filter_mode_deltas[2];
> +
> +	/* quantization_params */
> +	u8 base_q_idx;
> +	s8 delta_q_y_dc;
> +	s8 delta_q_uv_dc;
> +	s8 delta_q_uv_ac;
> +
> +	/* segmentation_params */
> +	u8 segmentation_enabled;
> +	u8 segmentation_update_map;
> +	u8 segmentation_tree_probs[7];
> +	u8 padding1[1];
> +	u8 segmentation_temporal_udpate;
> +	u8 segmentation_pred_prob[3];
> +	u8 segmentation_update_data;
> +	u8 segmentation_abs_or_delta_update;
> +	u8 feature_enabled[8];
> +	s16 feature_value[8][4];
> +
> +	/* tile_info */
> +	u8 tile_cols_log2;
> +	u8 tile_rows_log2;
> +	u8 padding2[2];
> +
> +	u16 uncompressed_header_size;
> +	u16 header_size_in_bytes;
> +
> +	/* LAT OUT, CORE IN */
> +	u32 dequant[8][4];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_compressed_header - vp9 compressed header syntax
> + *                                           used for decoding.
> + */
> +struct vdec_vp9_slice_compressed_header {
> +	u8 tx_mode;
> +	u8 ref_mode;
> +	u8 comp_fixed_ref;
> +	u8 comp_var_ref[2];
> +	u8 padding[3];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_tiles - vp9 tile syntax
> + */
> +struct vdec_vp9_slice_tiles {
> +	u32 size[4][64];
> +	u32 mi_rows[4];
> +	u32 mi_cols[64];
> +	u8 actual_rows;
> +	u8 padding[7];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_reference - vp9 reference frame information
> + */
> +struct vdec_vp9_slice_reference {
> +	u16 frame_width;
> +	u16 frame_height;
> +	u8 bit_depth;
> +	u8 subsampling_x;
> +	u8 subsampling_y;
> +	u8 padding;
> +};
> +
> +/*
> + * struct vdec_vp9_slice_frame - vp9 syntax used for decoding
> + */
> +struct vdec_vp9_slice_frame {
> +	struct vdec_vp9_slice_uncompressed_header uh;
> +	struct vdec_vp9_slice_compressed_header ch;
> +	struct vdec_vp9_slice_tiles tiles;
> +	struct vdec_vp9_slice_reference ref[3];
> +};
> +
> +/*
> + * struct vdec_vp9_slice_init_vsi - VSI used to initialize instance
> + */
> +struct vdec_vp9_slice_init_vsi {
> +	unsigned int architecture;
> +	unsigned int reserved;
> +	u64 core_vsi;
> +	/* default frame context's position in MicroP */
> +	u64 default_frame_ctx;
> +};
> +
> +/*
> + * struct vdec_vp9_slice_mem - memory address and size
> + */
> +struct vdec_vp9_slice_mem {
> +	union {
> +		u64 buf;
> +		dma_addr_t dma_addr;
> +	};
> +	union {
> +		size_t size;
> +		dma_addr_t dma_addr_end;
> +		u64 padding;
> +	};
> +};
> +
> +/*
> + * struct vdec_vp9_slice_bs - input buffer for decoding
> + */
> +struct vdec_vp9_slice_bs {
> +	struct vdec_vp9_slice_mem buf;
> +	struct vdec_vp9_slice_mem frame;
> +};
> +
> +/*
> + * struct vdec_vp9_slice_fb - frame buffer for decoding
> + */
> +struct vdec_vp9_slice_fb {
> +	struct vdec_vp9_slice_mem y;
> +	struct vdec_vp9_slice_mem c;
> +};
> +
> +/*
> + * struct vdec_vp9_slice_state - decoding state
> + */
> +struct vdec_vp9_slice_state {
> +	int err;
> +	unsigned int full;
> +	unsigned int timeout;
> +	unsigned int perf;
> +
> +	unsigned int crc[12];
> +};
> +
> +/**
> + * struct vdec_vp9_slice_vsi - exchange decoding information
> + *                             between Main CPU and MicroP
> + *
> + * @bs:	input buffer
> + * @fb:	output buffer
> + * @ref:	3 reference buffers
> + * @mv:	mv working buffer
> + * @seg:	segmentation working buffer
> + * @tile:	tile buffer
> + * @prob:	prob table buffer, used to set/update prob table
> + * @counts:	counts table buffer, used to update prob table
> + * @ube:	general buffer
> + * @trans:	trans buffer position in general buffer
> + * @err_map:	error buffer
> + * @row_info:	row info buffer
> + * @frame:	decoding syntax
> + * @state:	decoding state
> + */
> +struct vdec_vp9_slice_vsi {
> +	/* used in LAT stage */
> +	struct vdec_vp9_slice_bs bs;
> +	/* used in Core stage */
> +	struct vdec_vp9_slice_fb fb;
> +	struct vdec_vp9_slice_fb ref[3];
> +
> +	struct vdec_vp9_slice_mem mv[2];
> +	struct vdec_vp9_slice_mem seg[2];
> +	struct vdec_vp9_slice_mem tile;
> +	struct vdec_vp9_slice_mem prob;
> +	struct vdec_vp9_slice_mem counts;
> +
> +	/* LAT stage's output, Core stage's input */
> +	struct vdec_vp9_slice_mem ube;
> +	struct vdec_vp9_slice_mem trans;
> +	struct vdec_vp9_slice_mem err_map;
> +	struct vdec_vp9_slice_mem row_info;
> +
> +	/* decoding parameters */
> +	struct vdec_vp9_slice_frame frame;
> +
> +	struct vdec_vp9_slice_state state;
> +};
> +
> +/**
> + * struct vdec_vp9_slice_pfc - per-frame context that contains a local vsi.
> + *                             pass it from lat to core
> + *
> + * @vsi:	local vsi. copy to/from remote vsi before/after decoding
> + * @ref_idx:	reference buffer index
> + * @seq:	picture sequence
> + * @state:	decoding state
> + */
> +struct vdec_vp9_slice_pfc {
> +	struct vdec_vp9_slice_vsi vsi;
> +
> +	u64 ref_idx[3];
> +
> +	int seq;
> +
> +	/* LAT/Core CRC */
> +	struct vdec_vp9_slice_state state[2];
> +};
> +
> +/*
> + * enum vdec_vp9_slice_resolution_level
> + */
> +enum vdec_vp9_slice_resolution_level {
> +	VP9_RES_NONE,
> +	VP9_RES_FHD,
> +	VP9_RES_4K,
> +	VP9_RES_8K,
> +};
> +
> +/*
> + * struct vdec_vp9_slice_ref - picture's width & height should kept
> + *                             for later decoding as reference picture
> + */
> +struct vdec_vp9_slice_ref {
> +	unsigned int width;
> +	unsigned int height;
> +};
> +
> +/**
> + * struct vdec_vp9_slice_instance - represent one vp9 instance
> + *
> + * @ctx:		pointer to codec's context
> + * @vpu:		VPU instance
> + * @seq:		global picture sequence
> + * @level:		level of current resolution
> + * @width:		width of last picture
> + * @height:		height of last picture
> + * @frame_type:	frame_type of last picture
> + * @irq:		irq to Main CPU or MicroP
> + * @show_frame:	show_frame of last picture
> + * @dpb:		picture information (width/height) for reference
> + * @mv:		mv working buffer
> + * @seg:		segmentation working buffer
> + * @tile:		tile buffer
> + * @prob:		prob table buffer, used to set/update prob table
> + * @counts:		counts table buffer, used to update prob table
> + * @frame_ctx:		4 frame context according to VP9 Spec
> + * @frame_ctx_helper:	4 frame context according to newest kernel spec
> + * @dirty:		state of each frame context
> + * @init_vsi:		vsi used for initialized VP9 instance
> + * @vsi:		vsi used for decoding/flush ...
> + * @core_vsi:		vsi used for Core stage
> + * @counts_map:	used map to counts_helper
> + * &counts_helper:	counts table according to newest kernel spec

Change & into @ to fix:

drivers/media/platform/mediatek/vcodec/vdec/vdec_vp9_req_lat_if.c:492: warning: Function parameter or member 'counts_helper' not described in 'vdec_vp9_slice_instance'

> + */
> +struct vdec_vp9_slice_instance {
> +	struct mtk_vcodec_ctx *ctx;
> +	struct vdec_vpu_inst vpu;
> +
> +	int seq;
> +
> +	enum vdec_vp9_slice_resolution_level level;
> +
> +	/* for resolution change and get_pic_info */
> +	unsigned int width;
> +	unsigned int height;
> +
> +	/* for last_frame_type */
> +	unsigned int frame_type;
> +	unsigned int irq;
> +
> +	unsigned int show_frame;
> +
> +	/* maintain vp9 reference frame state */
> +	struct vdec_vp9_slice_ref dpb[VB2_MAX_FRAME];
> +
> +	/*
> +	 * normal working buffers
> +	 * mv[0]/seg[0]/tile/prob/counts is used for LAT
> +	 * mv[1]/seg[1] is used for CORE
> +	 */
> +	struct mtk_vcodec_mem mv[2];
> +	struct mtk_vcodec_mem seg[2];
> +	struct mtk_vcodec_mem tile;
> +	struct mtk_vcodec_mem prob;
> +	struct mtk_vcodec_mem counts;
> +
> +	/* 4 prob tables */
> +	struct vdec_vp9_slice_frame_ctx frame_ctx[4];
> +	/*4 helper tables */
> +	struct v4l2_vp9_frame_context frame_ctx_helper;
> +	unsigned char dirty[4];
> +
> +	/* MicroP vsi */
> +	union {
> +		struct vdec_vp9_slice_init_vsi *init_vsi;
> +		struct vdec_vp9_slice_vsi *vsi;
> +	};
> +	struct vdec_vp9_slice_vsi *core_vsi;
> +
> +	struct vdec_vp9_slice_counts_map counts_map;
> +	struct v4l2_vp9_frame_symbol_counts counts_helper;
> +};


[Index of Archives]     [Device Tree Compilter]     [Device Tree Spec]     [Linux Driver Backports]     [Video for Linux]     [Linux USB Devel]     [Linux PCI Devel]     [Linux Audio Users]     [Linux Kernel]     [Linux SCSI]     [XFree86]     [Yosemite Backpacking]


  Powered by Linux