On 4/21/2025 12:24 PM, Mark Thompson wrote:
Typical checkasm result on Alder Lake:decode_transquant_8_c: 461.1 ( 1.00x) decode_transquant_8_avx2: 97.5 ( 4.73x) decode_transquant_10_c: 483.9 ( 1.00x) decode_transquant_10_avx2: 91.7 ( 5.28x) --- libavcodec/apv_dsp.c | 4 + libavcodec/apv_dsp.h | 2 + libavcodec/x86/Makefile | 2 + libavcodec/x86/apv_dsp.asm | 279 ++++++++++++++++++++++++++++++++++ libavcodec/x86/apv_dsp_init.c | 40 +++++ tests/checkasm/Makefile | 1 + tests/checkasm/apv_dsp.c | 109 +++++++++++++ tests/checkasm/checkasm.c | 3 + tests/checkasm/checkasm.h | 1 + tests/fate/checkasm.mak | 1 + 10 files changed, 442 insertions(+) create mode 100644 libavcodec/x86/apv_dsp.asm create mode 100644 libavcodec/x86/apv_dsp_init.c create mode 100644 tests/checkasm/apv_dsp.c diff --git a/libavcodec/apv_dsp.c b/libavcodec/apv_dsp.c index fe11cd6b94..fd814ef900 100644 --- a/libavcodec/apv_dsp.c +++ b/libavcodec/apv_dsp.c @@ -133,4 +133,8 @@ static void apv_decode_transquant_c(void *output, av_cold void ff_apv_dsp_init(APVDSPContext *dsp) { dsp->decode_transquant = apv_decode_transquant_c; + +#if ARCH_X86_64 + ff_apv_dsp_init_x86_64(dsp); +#endif } diff --git a/libavcodec/apv_dsp.h b/libavcodec/apv_dsp.h index 31645b8581..c63d6a88ee 100644 --- a/libavcodec/apv_dsp.h +++ b/libavcodec/apv_dsp.h @@ -34,4 +34,6 @@ typedef struct APVDSPContext {void ff_apv_dsp_init(APVDSPContext *dsp); +void ff_apv_dsp_init_x86_64(APVDSPContext *dsp);+ #endif /* AVCODEC_APV_DSP_H */ diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile index 5d53515381..821c410a0f 100644 --- a/libavcodec/x86/Makefile +++ b/libavcodec/x86/Makefile @@ -44,6 +44,7 @@ OBJS-$(CONFIG_ADPCM_G722_DECODER) += x86/g722dsp_init.o OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp_init.o OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp_init.o OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp_init.o +OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp_init.o OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsdsp.o OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp_init.o OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp_init.o @@ -149,6 +150,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_DECODER) += x86/g722dsp.o X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp.o X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp.o X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp.o +X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp.o X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsidct.o X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp.o X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp.o diff --git a/libavcodec/x86/apv_dsp.asm b/libavcodec/x86/apv_dsp.asm new file mode 100644 index 0000000000..6b045e989a --- /dev/null +++ b/libavcodec/x86/apv_dsp.asm @@ -0,0 +1,279 @@ +;************************************************************************ +;* This file is part of FFmpeg. +;* +;* FFmpeg is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* FFmpeg is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with FFmpeg; if not, write to the Free Software +;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +SECTION_RODATA 32 + +; Full matrix for row transform. +const tmatrix_row + dw 64, 89, 84, 75, 64, 50, 35, 18 + dw 64, -18, -84, 50, 64, -75, -35, 89 + dw 64, 75, 35, -18, -64, -89, -84, -50 + dw 64, -50, -35, 89, -64, -18, 84, -75 + dw 64, 50, -35, -89, -64, 18, 84, 75 + dw 64, -75, 35, 18, -64, 89, -84, 50 + dw 64, 18, -84, -50, 64, 75, -35, -89 + dw 64, -89, 84, -75, 64, -50, 35, -18 + +; Constant pairs for broadcast in column transform. +const tmatrix_col_even + dw 64, 64, 64, -64 + dw 84, 35, 35, -84 +const tmatrix_col_odd + dw 89, 75, 50, 18 + dw 75, -18, -89, -50 + dw 50, -89, 18, 75 + dw 18, -50, 75, -89 + +; Memory targets for vpbroadcastd (register version requires AVX512). +cextern pd_1 +const sixtyfour + dd 64 + +SECTION .text + +; void ff_apv_decode_transquant_avx2(void *output, +; ptrdiff_t pitch, +; const int16_t *input, +; const int16_t *qmatrix, +; int bit_depth, +; int qp_shift); + +INIT_YMM avx2 + +cglobal apv_decode_transquant, 6, 7, 16, output, pitch, input, qmatrix, bit_depth, qp_shift, tmp + + ; Load input and dequantise + + vpbroadcastd m10, [pd_1] + lea tmpq, [bit_depthq - 2]
lea tmpd, [bit_depthd - 2] The upper 32 bits of the register may have garbage.
+ movd xm8, qp_shiftd
If you declare the function as 5, 7, 16, then qp_shift will not be loaded into a gpr on ABIs where it's on stack (Win64, and x86_32 if it was supported), and then you can do
movd xm8, qp_shiftm
Which will load it directly to the simd register from memory, saving one
instruction in the prologue.
+ movd xm9, tmpd + vpslld m10, m10, xm9 + vpsrld m10, m10, 1 + + ; m8 = scalar qp_shift + ; m9 = scalar bd_shift + ; m10 = vector 1 << (bd_shift - 1) + ; m11 = qmatrix load + +%macro LOAD_AND_DEQUANT 2 ; (xmm input, constant offset) + vpmovsxwd m%1, [inputq + %2] + vpmovsxwd m11, [qmatrixq + %2] + vpmaddwd m%1, m%1, m11 + vpslld m%1, m%1, xm8 + vpaddd m%1, m%1, m10 + vpsrad m%1, m%1, xm9 + vpackssdw m%1, m%1, m%1 +%endmacro + + LOAD_AND_DEQUANT 0, 0x00 + LOAD_AND_DEQUANT 1, 0x10 + LOAD_AND_DEQUANT 2, 0x20 + LOAD_AND_DEQUANT 3, 0x30 + LOAD_AND_DEQUANT 4, 0x40 + LOAD_AND_DEQUANT 5, 0x50 + LOAD_AND_DEQUANT 6, 0x60 + LOAD_AND_DEQUANT 7, 0x70 + + ; mN = row N words 0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7 + + ; Transform columns + ; This applies a 1-D DCT butterfly + + vpunpcklwd m12, m0, m4 + vpunpcklwd m13, m2, m6 + vpunpcklwd m14, m1, m3 + vpunpcklwd m15, m5, m7 + + ; m12 = rows 0 and 4 interleaved + ; m13 = rows 2 and 6 interleaved + ; m14 = rows 1 and 3 interleaved + ; m15 = rows 5 and 7 interleaved + + vpbroadcastd m0, [tmatrix_col_even + 0x00] + vpbroadcastd m1, [tmatrix_col_even + 0x04] + vpbroadcastd m2, [tmatrix_col_even + 0x08] + vpbroadcastd m3, [tmatrix_col_even + 0x0c]
Maybe do lea tmpq, [tmatrix_col_even] vpbroadcastd m0, [tmpq + 0x00] vpbroadcastd m1, [tmpq + 0x04] ...To emit smaller instructions. Same for tmatrix_col_odd and tmatrix_row below.
+ + vpmaddwd m4, m12, m0 + vpmaddwd m5, m12, m1 + vpmaddwd m6, m13, m2 + vpmaddwd m7, m13, m3 + vpaddd m8, m4, m6 + vpaddd m9, m5, m7 + vpsubd m10, m5, m7 + vpsubd m11, m4, m6 + + vpbroadcastd m0, [tmatrix_col_odd + 0x00] + vpbroadcastd m1, [tmatrix_col_odd + 0x04] + vpbroadcastd m2, [tmatrix_col_odd + 0x08] + vpbroadcastd m3, [tmatrix_col_odd + 0x0c] + + vpmaddwd m4, m14, m0 + vpmaddwd m5, m15, m1 + vpmaddwd m6, m14, m2 + vpmaddwd m7, m15, m3 + vpaddd m12, m4, m5 + vpaddd m13, m6, m7 + + vpbroadcastd m0, [tmatrix_col_odd + 0x10] + vpbroadcastd m1, [tmatrix_col_odd + 0x14] + vpbroadcastd m2, [tmatrix_col_odd + 0x18] + vpbroadcastd m3, [tmatrix_col_odd + 0x1c] + + vpmaddwd m4, m14, m0 + vpmaddwd m5, m15, m1 + vpmaddwd m6, m14, m2 + vpmaddwd m7, m15, m3 + vpaddd m14, m4, m5 + vpaddd m15, m6, m7 + + vpaddd m0, m8, m12 + vpaddd m1, m9, m13 + vpaddd m2, m10, m14 + vpaddd m3, m11, m15 + vpsubd m4, m11, m15 + vpsubd m5, m10, m14 + vpsubd m6, m9, m13 + vpsubd m7, m8, m12 + + ; Mid-transform normalisation + ; Note that outputs here are fitted to 16 bits + + vpbroadcastd m8, [sixtyfour] + +%macro NORMALISE 1 + vpaddd m%1, m%1, m8 + vpsrad m%1, m%1, 7 + vpackssdw m%1, m%1, m%1 + vpermq m%1, m%1, q3120 +%endmacro + + NORMALISE 0 + NORMALISE 1 + NORMALISE 2 + NORMALISE 3 + NORMALISE 4 + NORMALISE 5 + NORMALISE 6 + NORMALISE 7 + + ; mN = row N words 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + + ; Transform rows + ; This multiplies the rows directly by the transform matrix, + ; avoiding the need to transpose anything + + mova m12, [tmatrix_row + 0x00] + mova m13, [tmatrix_row + 0x20] + mova m14, [tmatrix_row + 0x40] + mova m15, [tmatrix_row + 0x60] + +%macro TRANS_ROW_STEP 1 + vpmaddwd m8, m%1, m12 + vpmaddwd m9, m%1, m13 + vpmaddwd m10, m%1, m14 + vpmaddwd m11, m%1, m15 + vphaddd m8, m8, m9 + vphaddd m10, m10, m11 + vphaddd m%1, m8, m10 +%endmacro + + TRANS_ROW_STEP 0 + TRANS_ROW_STEP 1 + TRANS_ROW_STEP 2 + TRANS_ROW_STEP 3 + TRANS_ROW_STEP 4 + TRANS_ROW_STEP 5 + TRANS_ROW_STEP 6 + TRANS_ROW_STEP 7 + + ; Renormalise, clip and store output + + vpbroadcastd m14, [pd_1] + mov tmpd, 20 + sub tmpd, bit_depthd + movd xm9, tmpd + dec tmpd + movd xm13, tmpd + movd xm15, bit_depthd + vpslld m8, m14, xm13 + vpslld m12, m14, xm15 + vpsrld m10, m12, 1 + vpsubd m12, m12, m14 + vpxor m11, m11, m11 + + ; m8 = vector 1 << (bd_shift - 1) + ; m9 = scalar bd_shift + ; m10 = vector 1 << (bit_depth - 1) + ; m11 = zero + ; m12 = vector (1 << bit_depth) - 1 + + cmp bit_depthd, 8 + jne store_10 + +%macro NORMALISE_AND_STORE_8 1 + vpaddd m%1, m%1, m8 + vpsrad m%1, m%1, xm9 + vpaddd m%1, m%1, m10 + vextracti128 xm13, m%1, 0 + vextracti128 xm14, m%1, 1 + vpackusdw xm%1, xm13, xm14 + vpackuswb xm%1, xm%1, xm%1
vpaddd m%1, m%1, m10
vextracti128 xm14, m%1, 1
vpackusdw xm%1, xm%1, xm14
vpackuswb xm%1, xm%1, xm%1
vextracti128 with 0 as third argument is the same as a mova for the
lower 128 bits, so it's not needed.
+ movq [outputq], xm%1 + add outputq, pitchq +%endmacro + + NORMALISE_AND_STORE_8 0 + NORMALISE_AND_STORE_8 1 + NORMALISE_AND_STORE_8 2 + NORMALISE_AND_STORE_8 3 + NORMALISE_AND_STORE_8 4 + NORMALISE_AND_STORE_8 5 + NORMALISE_AND_STORE_8 6 + NORMALISE_AND_STORE_8 7 + + RET + +store_10: + +%macro NORMALISE_AND_STORE_10 1 + vpaddd m%1, m%1, m8 + vpsrad m%1, m%1, xm9 + vpaddd m%1, m%1, m10 + vpmaxsd m%1, m%1, m11 + vpminsd m%1, m%1, m12 + vextracti128 xm13, m%1, 0 + vextracti128 xm14, m%1, 1 + vpackusdw xm%1, xm13, xm14
Same.
+ mova [outputq], xm%1
+ add outputq, pitchq
+%endmacro
+
+ NORMALISE_AND_STORE_10 0
+ NORMALISE_AND_STORE_10 1
+ NORMALISE_AND_STORE_10 2
+ NORMALISE_AND_STORE_10 3
+ NORMALISE_AND_STORE_10 4
+ NORMALISE_AND_STORE_10 5
+ NORMALISE_AND_STORE_10 6
+ NORMALISE_AND_STORE_10 7
+
+ RET
diff --git a/libavcodec/x86/apv_dsp_init.c b/libavcodec/x86/apv_dsp_init.c
new file mode 100644
index 0000000000..bc017ce37a
--- /dev/null
+++ b/libavcodec/x86/apv_dsp_init.c
@@ -0,0 +1,40 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+#include "libavutil/attributes.h"
+#include "libavutil/cpu.h"
+#include "libavutil/x86/asm.h"
+#include "libavutil/x86/cpu.h"
+#include "libavcodec/apv_dsp.h"
+
+void ff_apv_decode_transquant_avx2(void *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int bit_depth,
+ int qp_shift);
+
+av_cold void ff_apv_dsp_init_x86_64(APVDSPContext *dsp)
+{
+ int cpu_flags = av_get_cpu_flags();
+
+ if (EXTERNAL_AVX2_FAST(cpu_flags)) {
+ dsp->decode_transquant = ff_apv_decode_transquant_avx2;
+ }
+}
diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile
index d5c50e5599..193c1e4633 100644
--- a/tests/checkasm/Makefile
+++ b/tests/checkasm/Makefile
@@ -28,6 +28,7 @@ AVCODECOBJS-$(CONFIG_AAC_DECODER) += aacpsdsp.o \
sbrdsp.o
AVCODECOBJS-$(CONFIG_AAC_ENCODER) += aacencdsp.o
AVCODECOBJS-$(CONFIG_ALAC_DECODER) += alacdsp.o
+AVCODECOBJS-$(CONFIG_APV_DECODER) += apv_dsp.o
AVCODECOBJS-$(CONFIG_DCA_DECODER) += synth_filter.o
AVCODECOBJS-$(CONFIG_DIRAC_DECODER) += diracdsp.o
AVCODECOBJS-$(CONFIG_EXR_DECODER) += exrdsp.o
diff --git a/tests/checkasm/apv_dsp.c b/tests/checkasm/apv_dsp.c
new file mode 100644
index 0000000000..b3adb8ca06
--- /dev/null
+++ b/tests/checkasm/apv_dsp.c
@@ -0,0 +1,109 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include "checkasm.h"
+
+#include "libavutil/attributes.h"
+#include "libavutil/mem_internal.h"
+#include "libavcodec/apv_dsp.h"
+
+
+static void check_decode_transquant_8(void)
+{
+ LOCAL_ALIGNED_16(int16_t, input, [64]);
+ LOCAL_ALIGNED_16(int16_t, qmatrix, [64]);
+ LOCAL_ALIGNED_16(uint8_t, new_output, [64]);
+ LOCAL_ALIGNED_16(uint8_t, ref_output, [64]);
+
+ declare_func(void,
+ uint8_t *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int bit_depth,
+ int qp_shift);
+
+ for (int i = 0; i < 64; i++) {
+ // Any signed 12-bit integer.
+ input[i] = rnd() % 2048 - 1024;
+
+ // qmatrix input is premultiplied by level_scale, so
+ // range is 1 to 255 * 71. Interesting values are all
+ // at the low end of that, though.
+ qmatrix[i] = rnd() % 16 + 16;
+ }
+
+ call_ref(ref_output, 8, input, qmatrix, 8, 4);
+ call_new(new_output, 8, input, qmatrix, 8, 4);
+
+ if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output)))
+ fail();
+
+ bench_new(new_output, 8, input, qmatrix, 8, 4);
+}
+
+static void check_decode_transquant_10(void)
+{
+ LOCAL_ALIGNED_16( int16_t, input, [64]);
+ LOCAL_ALIGNED_16( int16_t, qmatrix, [64]);
+ LOCAL_ALIGNED_16(uint16_t, new_output, [64]);
+ LOCAL_ALIGNED_16(uint16_t, ref_output, [64]);
+
+ declare_func(void,
+ uint16_t *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int bit_depth,
+ int qp_shift);
+
+ for (int i = 0; i < 64; i++) {
+ // Any signed 14-bit integer.
+ input[i] = rnd() % 16384 - 8192;
+
+ // qmatrix input is premultiplied by level_scale, so
+ // range is 1 to 255 * 71. Interesting values are all
+ // at the low end of that, though.
+ qmatrix[i] = 16; //rnd() % 16 + 16;
+ }
+
+ call_ref(ref_output, 16, input, qmatrix, 10, 4);
+ call_new(new_output, 16, input, qmatrix, 10, 4);
+
+ if (memcmp(new_output, ref_output, 64 * sizeof(*ref_output)))
+ fail();
+
+ bench_new(new_output, 16, input, qmatrix, 10, 4);
+}
+
+void checkasm_check_apv_dsp(void)
+{
+ APVDSPContext dsp;
+
+ ff_apv_dsp_init(&dsp);
+
+ if (check_func(dsp.decode_transquant, "decode_transquant_8"))
+ check_decode_transquant_8();
+
+ if (check_func(dsp.decode_transquant, "decode_transquant_10"))
+ check_decode_transquant_10();
+
+ report("apv_dsp");
+}
diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c
index 412b8b2cd1..3bb82ed0e5 100644
--- a/tests/checkasm/checkasm.c
+++ b/tests/checkasm/checkasm.c
@@ -129,6 +129,9 @@ static const struct {
#if CONFIG_ALAC_DECODER
{ "alacdsp", checkasm_check_alacdsp },
#endif
+ #if CONFIG_APV_DECODER
+ { "apv_dsp", checkasm_check_apv_dsp },
+ #endif
#if CONFIG_AUDIODSP
{ "audiodsp", checkasm_check_audiodsp },
#endif
diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h
index ad239fb2a4..a6b5965e02 100644
--- a/tests/checkasm/checkasm.h
+++ b/tests/checkasm/checkasm.h
@@ -83,6 +83,7 @@ void checkasm_check_ac3dsp(void);
void checkasm_check_aes(void);
void checkasm_check_afir(void);
void checkasm_check_alacdsp(void);
+void checkasm_check_apv_dsp(void);
void checkasm_check_audiodsp(void);
void checkasm_check_av_tx(void);
void checkasm_check_blend(void);
diff --git a/tests/fate/checkasm.mak b/tests/fate/checkasm.mak
index 6d42df148e..720c5fd77e 100644
--- a/tests/fate/checkasm.mak
+++ b/tests/fate/checkasm.mak
@@ -4,6 +4,7 @@ FATE_CHECKASM = fate-checkasm-aacencdsp
\
fate-checkasm-aes \
fate-checkasm-af_afir \
fate-checkasm-alacdsp \
+ fate-checkasm-apv_dsp \
fate-checkasm-audiodsp \
fate-checkasm-av_tx \
fate-checkasm-blockdsp \
OpenPGP_signature.asc
Description: OpenPGP digital signature
_______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
