Original x86 intrinsics code and initial 8bit yasm port by Pierre-Edouard Lepere. 10/12bit yasm ports, refactoring and optimizations by James Almer
Benchmarks of BQTerrace_1920x1080_60_qp22.bin with an Intel Core i5-4200U width 32 40338 decicycles in sao_band_filter_0_8, 2048 runs, 0 skips 8585 decicycles in ff_hevc_sao_band_filter_8_sse2, 2048 runs, 0 skips 4543 decicycles in ff_hevc_sao_band_filter_8_avx2, 2048 runs, 0 skips width 64 136046 decicycles in sao_band_filter_0_8, 16384 runs, 0 skips 29366 decicycles in ff_hevc_sao_band_filter_8_sse2, 16384 runs, 0 skips 15357 decicycles in ff_hevc_sao_band_filter_8_avx2, 16383 runs, 1 skips Signed-off-by: James Almer <jamr...@gmail.com> --- For reference: Original intrinsics code: https://github.com/OpenHEVC/openHEVC/blob/shm6.1/libavcodec/x86/hevc_sao_sse.c Initial 8bit yasm port: https://github.com/OpenHEVC/FFmpeg/blob/rext/libavcodec/x86/hevc_sao.asm x86_32 port is pretty much done and will follow soon (I just need to make sure the macros are not ugly). And having it separate will make reviewing easier. libavcodec/x86/Makefile | 3 +- libavcodec/x86/hevc_sao.asm | 262 ++++++++++++++++++++++++++++++++++++++++++ libavcodec/x86/hevcdsp_init.c | 25 ++++ 3 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 libavcodec/x86/hevc_sao.asm diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile index 7c8e7aa..00dacda 100644 --- a/libavcodec/x86/Makefile +++ b/libavcodec/x86/Makefile @@ -134,7 +134,8 @@ YASM-OBJS-$(CONFIG_DCA_DECODER) += x86/dcadsp.o YASM-OBJS-$(CONFIG_HEVC_DECODER) += x86/hevc_mc.o \ x86/hevc_deblock.o \ x86/hevc_idct.o \ - x86/hevc_res_add.o + x86/hevc_res_add.o \ + x86/hevc_sao.o YASM-OBJS-$(CONFIG_MLP_DECODER) += x86/mlpdsp.o YASM-OBJS-$(CONFIG_PNG_DECODER) += x86/pngdsp.o YASM-OBJS-$(CONFIG_PRORES_DECODER) += x86/proresdsp.o diff --git a/libavcodec/x86/hevc_sao.asm b/libavcodec/x86/hevc_sao.asm new file mode 100644 index 0000000..f64e70e --- /dev/null +++ b/libavcodec/x86/hevc_sao.asm @@ -0,0 +1,262 @@ +;****************************************************************************** +;* SIMD optimized SAO functions for HEVC decoding +;* +;* Copyright (c) 2013 Pierre-Edouard LEPERE +;* Copyright (c) 2014 James Almer +;* +;* This file is part of FFmpeg. +;* +;* FFmpeg is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* FFmpeg is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with FFmpeg; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +%if ARCH_X86_64 +SECTION_RODATA 32 + +pw_mask10: times 16 dw 0x03FF +pw_mask12: times 16 dw 0x0FFF + +SECTION_TEXT + +%macro HEVC_SAO_BAND_FILTER_INIT 1 + movsxd widthq, dword widthm + mov heightd, heightm + pxor m14, m14 +%if %1 > 8 + mova m13, [pw_mask %+ %1] +%endif + and leftq, 31 + movd xm0, leftd + add leftq, 1 + and leftq, 31 + movd xm1, leftd + add leftq, 1 + and leftq, 31 + movd xm2, leftd + add leftq, 1 + and leftq, 31 + movd xm3, leftd + +%if mmsize > 16 + cmp widthq, 16 + je hevc_sao_band_filter_16_%1 %+ SUFFIX +%endif + cmp widthq, 8 + je hevc_sao_band_filter_8_%1 %+ SUFFIX +%endmacro + +%macro HEVC_SAO_BAND_FILTER_INIT_OFFSETS 1 +%if %1 > 8 + shl widthd, 1 +%endif + SPLATW m0, xm0 + SPLATW m1, xm1 + SPLATW m2, xm2 + SPLATW m3, xm3 +%if cpuflag(avx2) + SPLATW m4, [offsetq + 2] + SPLATW m5, [offsetq + 4] + SPLATW m6, [offsetq + 6] + SPLATW m7, [offsetq + 8] +%else + movd m4, [offsetq + 2] + movd m5, [offsetq + 4] + movd m6, [offsetq + 6] + movd m7, [offsetq + 8] + SPLATW m4, m4 + SPLATW m5, m5 + SPLATW m6, m6 + SPLATW m7, m7 +%endif + + add srcq, widthq + add dstq, widthq + neg widthq + mov tmpq, widthq +%endmacro + +%macro HEVC_SAO_BAND_FILTER_COMPUTE 3 + psraw %2, %3, %1-5 + pcmpeqw m10, %2, m0 + pcmpeqw m11, %2, m1 + pcmpeqw m12, %2, m2 + pcmpeqw %2, m3 + pand m10, m4 + pand m11, m5 + pand m12, m6 + pand %2, m7 + por m10, m11 + por m12, %2 + por m10, m12 + paddw %3, m10 +%endmacro + +;void ff_hevc_sao_band_filter_8_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, +; int16_t *sao_offset_val, int sao_left_class, int width, int height); +%macro HEVC_SAO_BAND_FILTER_8 0 +cglobal hevc_sao_band_filter_8, 6, 9, 15, dst, src, dststride, srcstride, offset, left, width, height, tmp + HEVC_SAO_BAND_FILTER_INIT 8 + HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8 + +; width >= mmsize +align 16 +.loop: + movu m13, [srcq+widthq] + punpcklbw m8, m13, m14 + HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m8 + punpckhbw m13, m14 + HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m13 + packuswb m8, m13 + movu [dstq+widthq], m8 + + add widthq, mmsize + jl .loop + + add dstq, dststrideq + add srcq, srcstrideq + mov widthq, tmpq + dec heightd + jg .loop + REP_RET + +%if mmsize > 16 +INIT_XMM cpuname +; width 16, AVX2 only +hevc_sao_band_filter_16_8 %+ SUFFIX + HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8 + +align 16 +.loop: + movu m13, [srcq+widthq] + punpcklbw m8, m13, m14 + HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m8 + punpckhbw m13, m14 + HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m13 + packuswb m8, m13 + movu [dstq+widthq], m8 + + add dstq, dststrideq + add srcq, srcstrideq + dec heightd + jg .loop + REP_RET +%endif + +; width 8 +hevc_sao_band_filter_8_8 %+ SUFFIX + HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8 + +align 16 +.loop: + movq m8, [srcq+widthq] + punpcklbw m8, m14 + HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m8 + packuswb m8, m14 + movq [dstq+widthq], m8 + + add dstq, dststrideq + add srcq, srcstrideq + dec heightd + jg .loop + REP_RET +%endmacro + +;void ff_hevc_sao_band_filter_<depth>_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, +; int16_t *sao_offset_val, int sao_left_class, int width, int height); +%macro HEVC_SAO_BAND_FILTER_16 1 +cglobal hevc_sao_band_filter_%1, 6, 9, 15, dst, src, dststride, srcstride, offset, left, width, height, tmp + HEVC_SAO_BAND_FILTER_INIT %1 + HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1 + +; width >= mmsize +align 16 +.loop: + movu m8, [srcq+widthq] + HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8 + CLIPW m8, m14, m13 + movu [dstq+widthq], m8 + + movu m9, [srcq+widthq+mmsize] + HEVC_SAO_BAND_FILTER_COMPUTE %1, m8, m9 + CLIPW m9, m14, m13 + movu [dstq+widthq+mmsize], m9 + + add widthq, mmsize*2 + jl .loop + + add dstq, dststrideq + add srcq, srcstrideq + mov widthq, tmpq + dec heightd + jg .loop + REP_RET + +%if mmsize > 16 +INIT_XMM cpuname +; width 16, AVX2 only +hevc_sao_band_filter_16_%1 %+ SUFFIX + HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1 + +align 16 +.loop: + movu m8, [srcq+widthq] + HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8 + CLIPW m8, m14, m13 + movu [dstq+widthq], m8 + + movu m9, [srcq+widthq+mmsize] + HEVC_SAO_BAND_FILTER_COMPUTE %1, m8, m9 + CLIPW m9, m14, m13 + movu [dstq+widthq+mmsize], m9 + + add dstq, dststrideq + add srcq, srcstrideq + dec heightd + jg .loop + REP_RET +%endif + +; width 8 +hevc_sao_band_filter_8_%1 %+ SUFFIX + HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1 + +align 16 +.loop: + movu m8, [srcq+widthq] + HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8 + CLIPW m8, m14, m13 + movu [dstq+widthq], m8 + + add dstq, dststrideq + add srcq, srcstrideq + dec heightd + jg .loop + REP_RET +%endmacro + +INIT_XMM sse2 +HEVC_SAO_BAND_FILTER_8 +HEVC_SAO_BAND_FILTER_16 10 +HEVC_SAO_BAND_FILTER_16 12 +%if HAVE_AVX2_EXTERNAL +INIT_YMM avx2 +HEVC_SAO_BAND_FILTER_8 +INIT_YMM avx2 +HEVC_SAO_BAND_FILTER_16 10 +INIT_YMM avx2 +HEVC_SAO_BAND_FILTER_16 12 +%endif +%endif diff --git a/libavcodec/x86/hevcdsp_init.c b/libavcodec/x86/hevcdsp_init.c index eaa97e1..04b9439 100644 --- a/libavcodec/x86/hevcdsp_init.c +++ b/libavcodec/x86/hevcdsp_init.c @@ -478,6 +478,16 @@ mc_bi_w_funcs(qpel_v, 12, sse4); mc_bi_w_funcs(qpel_hv, 12, sse4); #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL +#define SAO_BAND_FILTER_FUNCS(bitd, opt) \ +void ff_hevc_sao_band_filter_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \ + int16_t *sao_offset_val, int sao_left_class, int width, int height) + +SAO_BAND_FILTER_FUNCS(8, sse2); +SAO_BAND_FILTER_FUNCS(10, sse2); +SAO_BAND_FILTER_FUNCS(12, sse2); +SAO_BAND_FILTER_FUNCS(8, avx2); +SAO_BAND_FILTER_FUNCS(10, avx2); +SAO_BAND_FILTER_FUNCS(12, avx2); #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \ PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \ @@ -516,6 +526,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (ARCH_X86_64) { c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2; c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2; + + c->sao_band_filter = ff_hevc_sao_band_filter_8_sse2; } c->idct_dc[1] = ff_hevc_idct8x8_dc_8_sse2; c->idct_dc[2] = ff_hevc_idct16x16_dc_8_sse2; @@ -555,6 +567,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (EXTERNAL_AVX2(cpu_flags)) { c->idct_dc[2] = ff_hevc_idct16x16_dc_8_avx2; c->idct_dc[3] = ff_hevc_idct32x32_dc_8_avx2; + if (ARCH_X86_64) { + c->sao_band_filter = ff_hevc_sao_band_filter_8_avx2; + } c->transform_add[3] = ff_hevc_transform_add32_8_avx2; } @@ -570,6 +585,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (ARCH_X86_64) { c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2; c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2; + + c->sao_band_filter = ff_hevc_sao_band_filter_10_sse2; } c->idct_dc[1] = ff_hevc_idct8x8_dc_10_sse2; @@ -607,6 +624,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) c->idct_dc[2] = ff_hevc_idct16x16_dc_10_avx2; c->idct_dc[3] = ff_hevc_idct32x32_dc_10_avx2; + if (ARCH_X86_64) { + c->sao_band_filter = ff_hevc_sao_band_filter_10_avx2; + } c->transform_add[2] = ff_hevc_transform_add16_10_avx2; c->transform_add[3] = ff_hevc_transform_add32_10_avx2; @@ -623,6 +643,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (ARCH_X86_64) { c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2; c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2; + + c->sao_band_filter = ff_hevc_sao_band_filter_12_sse2; } c->idct_dc[1] = ff_hevc_idct8x8_dc_12_sse2; @@ -655,6 +677,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth) if (EXTERNAL_AVX2(cpu_flags)) { c->idct_dc[2] = ff_hevc_idct16x16_dc_12_avx2; c->idct_dc[3] = ff_hevc_idct32x32_dc_12_avx2; + if (ARCH_X86_64) { + c->sao_band_filter = ff_hevc_sao_band_filter_12_avx2; + } } } } -- 2.2.2 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
