This commandline cause core dumped:
ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \
-hwaccel_output_format vaapi -i input.264 \
-vf "hwmap=derive_device=qsv,format=qsv" \
-c:v h264_qsv output.264
reason: We use nb_surfaces to assign surface to handle_pairs_internal
but handle_pairs_internal is alloced with the size of init_pool_size.
This lead to access to illegal address.
Now change it to use nb_surfaces to allocate handle_pairs_internal and the
core dumped error is unseen. Also change D3D11VA to use nb_surfaces
to align to VAAPI and DXVA2.
Signed-off-by: Wenbin Chen <wenbin.c...@intel.com>
---
libavutil/hwcontext_qsv.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c
index c18747f7eb..5a285fd25b 100644
--- a/libavutil/hwcontext_qsv.c
+++ b/libavutil/hwcontext_qsv.c
@@ -1123,8 +1123,7 @@ static int qsv_frames_derive_to(AVHWFramesContext
*dst_ctx,
case AV_HWDEVICE_TYPE_VAAPI:
{
AVVAAPIFramesContext *src_hwctx = src_ctx->hwctx;
- s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
-
sizeof(*s->handle_pairs_internal));
+ s->handle_pairs_internal = av_calloc(src_hwctx->nb_surfaces,
sizeof(*s->handle_pairs_internal));
if (!s->handle_pairs_internal)
return AVERROR(ENOMEM);
s->surfaces_internal = av_calloc(src_hwctx->nb_surfaces,
@@ -1146,15 +1145,15 @@ static int qsv_frames_derive_to(AVHWFramesContext
*dst_ctx,
case AV_HWDEVICE_TYPE_D3D11VA:
{
AVD3D11VAFramesContext *src_hwctx = src_ctx->hwctx;
- s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
+ s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
sizeof(*s->handle_pairs_internal));
if (!s->handle_pairs_internal)
return AVERROR(ENOMEM);
- s->surfaces_internal = av_calloc(src_ctx->initial_pool_size,
+ s->surfaces_internal = av_calloc(src_ctx->nb_surfaces,
sizeof(*s->surfaces_internal));
if (!s->surfaces_internal)
return AVERROR(ENOMEM);
- for (i = 0; i < src_ctx->initial_pool_size; i++) {
+ for (i = 0; i < src_ctx->nb_surfaces; i++) {
qsv_init_surface(dst_ctx, &s->surfaces_internal[i]);
s->handle_pairs_internal[i].first =
(mfxMemId)src_hwctx->texture_infos[i].texture;
if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
@@ -1164,7 +1163,7 @@ static int qsv_frames_derive_to(AVHWFramesContext
*dst_ctx,
}
s->surfaces_internal[i].Data.MemId =
(mfxMemId)&s->handle_pairs_internal[i];
}
- dst_hwctx->nb_surfaces = src_ctx->initial_pool_size;
+ dst_hwctx->nb_surfaces = src_ctx->nb_surfaces;
if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
dst_hwctx->frame_type |=
MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET;
} else {
@@ -1177,7 +1176,7 @@ static int qsv_frames_derive_to(AVHWFramesContext
*dst_ctx,
case AV_HWDEVICE_TYPE_DXVA2:
{
AVDXVA2FramesContext *src_hwctx = src_ctx->hwctx;
- s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
+ s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
sizeof(*s->handle_pairs_internal));
if (!s->handle_pairs_internal)
return AVERROR(ENOMEM);
--
2.25.1
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
