This commandline cause core dumped: ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \ -hwaccel_output_format vaapi -i input.264 \ -vf "hwmap=derive_device=qsv,format=qsv" \ -c:v h264_qsv output.264
reason: We use nb_surfaces to assign surface to handle_pairs_internal but handle_pairs_internal is alloced with the size of init_pool_size. This lead to access to illegal address. Now change it to use nb_surfaces to allocate handle_pairs_internal and the core dumped error is unseen. Also change D3D11VA to use nb_surfaces to align to VAAPI and DXVA2. Signed-off-by: Wenbin Chen <wenbin.c...@intel.com> --- libavutil/hwcontext_qsv.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c index c18747f7eb..5a285fd25b 100644 --- a/libavutil/hwcontext_qsv.c +++ b/libavutil/hwcontext_qsv.c @@ -1123,8 +1123,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx, case AV_HWDEVICE_TYPE_VAAPI: { AVVAAPIFramesContext *src_hwctx = src_ctx->hwctx; - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size, - sizeof(*s->handle_pairs_internal)); + s->handle_pairs_internal = av_calloc(src_hwctx->nb_surfaces, sizeof(*s->handle_pairs_internal)); if (!s->handle_pairs_internal) return AVERROR(ENOMEM); s->surfaces_internal = av_calloc(src_hwctx->nb_surfaces, @@ -1146,15 +1145,15 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx, case AV_HWDEVICE_TYPE_D3D11VA: { AVD3D11VAFramesContext *src_hwctx = src_ctx->hwctx; - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size, + s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces, sizeof(*s->handle_pairs_internal)); if (!s->handle_pairs_internal) return AVERROR(ENOMEM); - s->surfaces_internal = av_calloc(src_ctx->initial_pool_size, + s->surfaces_internal = av_calloc(src_ctx->nb_surfaces, sizeof(*s->surfaces_internal)); if (!s->surfaces_internal) return AVERROR(ENOMEM); - for (i = 0; i < src_ctx->initial_pool_size; i++) { + for (i = 0; i < src_ctx->nb_surfaces; i++) { qsv_init_surface(dst_ctx, &s->surfaces_internal[i]); s->handle_pairs_internal[i].first = (mfxMemId)src_hwctx->texture_infos[i].texture; if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) { @@ -1164,7 +1163,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx, } s->surfaces_internal[i].Data.MemId = (mfxMemId)&s->handle_pairs_internal[i]; } - dst_hwctx->nb_surfaces = src_ctx->initial_pool_size; + dst_hwctx->nb_surfaces = src_ctx->nb_surfaces; if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) { dst_hwctx->frame_type |= MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET; } else { @@ -1177,7 +1176,7 @@ static int qsv_frames_derive_to(AVHWFramesContext *dst_ctx, case AV_HWDEVICE_TYPE_DXVA2: { AVDXVA2FramesContext *src_hwctx = src_ctx->hwctx; - s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size, + s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces, sizeof(*s->handle_pairs_internal)); if (!s->handle_pairs_internal) return AVERROR(ENOMEM); -- 2.25.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".