From fc7b4d7bc56c014752bb55e7792aa5e764d839cc Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Sat, 1 Aug 2020 10:47:07 +0200 Subject: [PATCH 3/5] avcodec/cfhd: improve decompanding quality with reference implementation --- libavcodec/cfhd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/cfhd.c b/libavcodec/cfhd.c index aa6e2ebee7..0327b456ac 100644 --- a/libavcodec/cfhd.c +++ b/libavcodec/cfhd.c @@ -117,8 +117,8 @@ static inline int dequant_and_decompand(int level, int quantisation, int codeboo { if (codebook == 0 || codebook == 1) { int64_t abslevel = abs(level); - if (level < 264) - return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) * + if (abslevel < 256) + return (abslevel + ((768 * abslevel * abslevel * abslevel) / (256 * 256 * 256))) * FFSIGN(level) * quantisation; else return level * quantisation; -- 2.26.2