> This commandline cause core dumped:
> ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \
> -hwaccel_output_format vaapi -i input.264 \
> -vf "hwmap=derive_device=qsv,format=qsv" \
> -c:v h264_qsv output.264
>
> reason: We use nb_surfaces to assign surface to handle_pairs_internal
> but handle_pairs_internal is alloced with the size of init_pool_size.
> This lead to access to illegal address.
>
> Now change it to use nb_surfaces to allocate handle_pairs_internal and the
> core dumped error is unseen. Also change D3D11VA to use nb_surfaces
> to align to VAAPI and DXVA2.
>
> Signed-off-by: Wenbin Chen <wenbin.c...@intel.com>
> ---
> libavutil/hwcontext_qsv.c | 13 ++++++-------
> 1 file changed, 6 insertions(+), 7 deletions(-)
>
> diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c
> index c18747f7eb..5a285fd25b 100644
> --- a/libavutil/hwcontext_qsv.c
> +++ b/libavutil/hwcontext_qsv.c
> @@ -1123,8 +1123,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
> case AV_HWDEVICE_TYPE_VAAPI:
> {
> AVVAAPIFramesContext *src_hwctx = src_ctx->hwctx;
> - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> -
> sizeof(*s->handle_pairs_internal));
> + s->handle_pairs_internal = av_calloc(src_hwctx->nb_surfaces,
> sizeof(*s->handle_pairs_internal));
> if (!s->handle_pairs_internal)
> return AVERROR(ENOMEM);
> s->surfaces_internal = av_calloc(src_hwctx->nb_surfaces,
> @@ -1146,15 +1145,15 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
> case AV_HWDEVICE_TYPE_D3D11VA:
> {
> AVD3D11VAFramesContext *src_hwctx = src_ctx->hwctx;
> - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> + s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
>
> sizeof(*s->handle_pairs_internal));
> if (!s->handle_pairs_internal)
> return AVERROR(ENOMEM);
> - s->surfaces_internal = av_calloc(src_ctx->initial_pool_size,
> + s->surfaces_internal = av_calloc(src_ctx->nb_surfaces,
> sizeof(*s->surfaces_internal));
> if (!s->surfaces_internal)
> return AVERROR(ENOMEM);
> - for (i = 0; i < src_ctx->initial_pool_size; i++) {
> + for (i = 0; i < src_ctx->nb_surfaces; i++) {
> qsv_init_surface(dst_ctx, &s->surfaces_internal[i]);
> s->handle_pairs_internal[i].first = (mfxMemId)src_hwctx-
> >texture_infos[i].texture;
> if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
> @@ -1164,7 +1163,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
> }
> s->surfaces_internal[i].Data.MemId = (mfxMemId)&s-
> >handle_pairs_internal[i];
> }
> - dst_hwctx->nb_surfaces = src_ctx->initial_pool_size;
> + dst_hwctx->nb_surfaces = src_ctx->nb_surfaces;
> if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
> dst_hwctx->frame_type |=
> MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET;
> } else {
> @@ -1177,7 +1176,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
> case AV_HWDEVICE_TYPE_DXVA2:
> {
> AVDXVA2FramesContext *src_hwctx = src_ctx->hwctx;
> - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> + s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
>
> sizeof(*s->handle_pairs_internal));
> if (!s->handle_pairs_internal)
> return AVERROR(ENOMEM);
> --
> 2.25.1
ping
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
