Andreas Rheinhardt:
> In case trellis is outside of 0..23, an invalid shift and/or a signed
> integer overflow happens; furthermore, it can lead to the request to
> allocate nonsense amounts of memory. So validate first.
>
> Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@gmail.com>
> ---
> libavcodec/g722enc.c | 25 ++++++++++++-------------
> 1 file changed, 12 insertions(+), 13 deletions(-)
>
> diff --git a/libavcodec/g722enc.c b/libavcodec/g722enc.c
> index 9357f170fe..9e2ebf67c5 100644
> --- a/libavcodec/g722enc.c
> +++ b/libavcodec/g722enc.c
> @@ -64,19 +64,6 @@ static av_cold int g722_encode_init(AVCodecContext * avctx)
> c->band[1].scale_factor = 2;
> c->prev_samples_pos = 22;
>
> - if (avctx->trellis) {
> - int frontier = 1 << avctx->trellis;
> - int max_paths = frontier * FREEZE_INTERVAL;
> - int i;
> - for (i = 0; i < 2; i++) {
> - c->paths[i] = av_mallocz_array(max_paths, sizeof(**c->paths));
> - c->node_buf[i] = av_mallocz_array(frontier, 2 *
> sizeof(**c->node_buf));
> - c->nodep_buf[i] = av_mallocz_array(frontier, 2 *
> sizeof(**c->nodep_buf));
> - if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i])
> - return AVERROR(ENOMEM);
> - }
> - }
> -
> if (avctx->frame_size) {
> /* validate frame size */
> if (avctx->frame_size & 1 || avctx->frame_size > MAX_FRAME_SIZE) {
> @@ -110,6 +97,18 @@ static av_cold int g722_encode_init(AVCodecContext *
> avctx)
> avctx->trellis);
> avctx->trellis = new_trellis;
> }
> + if (avctx->trellis) {
> + int frontier = 1 << avctx->trellis;
> + int max_paths = frontier * FREEZE_INTERVAL;
> +
> + for (int i = 0; i < 2; i++) {
> + c->paths[i] = av_calloc(max_paths, sizeof(**c->paths));
> + c->node_buf[i] = av_calloc(frontier, 2 *
> sizeof(**c->node_buf));
> + c->nodep_buf[i] = av_calloc(frontier, 2 *
> sizeof(**c->nodep_buf));
> + if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i])
> + return AVERROR(ENOMEM);
> + }
> + }
> }
>
> ff_g722dsp_init(&c->dsp);
>
Will apply later today unless there are objections.
- Andreas
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
