This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit 8fdafc8198c7aac078b4a9c550ac50e32ae3ec2b Author: Niklas Haas <[email protected]> AuthorDate: Tue Jul 14 14:04:01 2026 +0200 Commit: Niklas Haas <[email protected]> CommitDate: Fri Jul 17 13:23:21 2026 +0000 swscale/x86/ops: rewrite packed_shuffle() in terms of uops macro This is mostly the same, with the added complication of now needing to handle the case of clearing to a nonzero value, which we do using `pblendb` on SSE4, and respectively `vpblendb` / `vpblendmb` on AVX-2 and AVX-512. Since `pblendb` hard-codes XMM0 implicitly, we have to flip the order of `m0` and `m1` in the rest of the code as well; though that's not a huge deal. The bigger complication comes from the fact that the macro gives us the size for a size 16 mask, which may need to be rounded up for the lane-aligned AVX-2 / AVX-512 paths, so we need to recreate this logic in the NASM macro itself. Fortunately, it's fairly straightforward. The big upside is that we now gain a fast path for e.g. rgb24 -> rgba, which is arguably more common than rgb24 -> rgb0: rgb24 1920x1080 -> rgba 1920x1080, speedup=7.106x faster It's worth pointing out that, because checkasm sometimes generates weird ops that don't occur in real pixel formats, it's not a guarantee that we will actually implement all paths that the optimizer spits out. This should be improved by a future checkasm refactor; but for now, we need to return ENOTSUP in the case that we encounter a shuffle mask combination that doesn't actually exist. (In practice, this only happens for degenerate no-op tests like 1->1 shuffles) Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops.c | 111 ++++++++++++++++----------------- libswscale/x86/ops_common.asm | 128 +++++++++++++++++++++++++-------------- libswscale/x86/uops_macros.asm.h | 1 + 3 files changed, 140 insertions(+), 100 deletions(-) diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 143bfa5013..a55b201a16 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -488,6 +488,11 @@ SWS_DECL_FUNC(ff_sws_process2_x86); SWS_DECL_FUNC(ff_sws_process3_x86); SWS_DECL_FUNC(ff_sws_process4_x86); +/* Declare packed shuffle functions */ +SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _sse4, NULL, NULL) +SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx2, NULL, NULL) +SWS_FOR_STRUCT(U8, RW_SHUFFLE, DECL_ENTRY, _avx512, NULL, NULL) + static int get_mmsize(void) { const int cpu_flags = av_get_cpu_flags(); @@ -508,35 +513,38 @@ static int movsize(const int bytes, const int mmsize) mmsize; /* movu */ } -static int solve_shuffle(const SwsOpList *ops, SwsCompiledOp *out) +static int translate_shuffle(const SwsUOp *uop, int mmsize, SwsCompiledOp *out) { - uint8_t shuffle[16]; - int mmsize = get_mmsize(); - int read_bytes, write_bytes; - int pixels; - - if (mmsize < 0) - return mmsize; - - /* Solve the shuffle mask for one 128-bit lane only */ - pixels = ff_sws_solve_shuffle(ops, shuffle, 16, 0x80, &read_bytes, &write_bytes); - if (pixels < 0) - return pixels; - /* We can't shuffle across lanes, so restrict the vector size to XMM * whenever the read/write size would be a subset of the full vector */ - if (read_bytes < 16 || write_bytes < 16) + const SwsShuffleUOp *par = &uop->par.shuffle; + const int lane_aligned = par->read_size == par->write_size && + 16 % par->read_size == 0; + if (!lane_aligned) mmsize = 16; - const int num_lanes = mmsize / 16; - const int in_total = num_lanes * read_bytes; - const int out_total = num_lanes * write_bytes; + /* Generate the shuffle mask */ + const int mask_size = lane_aligned ? 16 : mmsize; + int8_t *mask = av_malloc(mask_size); + if (!mask) + return AVERROR(ENOMEM); + + const int groups = ff_sws_shuffle_mask(uop, mask, mask_size); + if (groups < 0) { + av_free(mask); + return groups; + } + const int read_chunk = groups * par->read_size; + const int write_chunk = groups * par->write_size; + const int num_lanes = lane_aligned ? mmsize / 16 : 1; + const int in_total = num_lanes * read_chunk; + const int out_total = num_lanes * write_chunk; *out = (SwsCompiledOp) { - .priv = av_memdup(shuffle, sizeof(shuffle)), + .priv = mask, .free = av_free, .slice_align = 1, - .block_size = pixels * num_lanes, + .block_size = groups * uop->data.shuffle.pixels * num_lanes, .over_read = { movsize(in_total, mmsize) - in_total }, .over_write = { movsize(out_total, mmsize) - out_total }, .cpu_flags = mmsize > 32 ? AV_CPU_FLAG_AVX512 : @@ -544,35 +552,24 @@ static int solve_shuffle(const SwsOpList *ops, SwsCompiledOp *out) AV_CPU_FLAG_SSE4, }; - if (!out->priv) - return AVERROR(ENOMEM); - -#define ASSIGN_SHUFFLE_FUNC(IN, OUT, EXT) \ +#define ASSIGN_SHUFFLE_FUNC(EXT, NAME, ...) \ do { \ - SWS_DECL_FUNC(ff_packed_shuffle##IN##_##OUT##_##EXT); \ - if (in_total == IN && out_total == OUT) \ - out->func = ff_packed_shuffle##IN##_##OUT##_##EXT; \ -} while (0) - - ASSIGN_SHUFFLE_FUNC( 5, 15, sse4); - ASSIGN_SHUFFLE_FUNC( 4, 16, sse4); - ASSIGN_SHUFFLE_FUNC( 2, 12, sse4); - ASSIGN_SHUFFLE_FUNC(16, 8, sse4); - ASSIGN_SHUFFLE_FUNC(10, 15, sse4); - ASSIGN_SHUFFLE_FUNC( 8, 16, sse4); - ASSIGN_SHUFFLE_FUNC( 4, 12, sse4); - ASSIGN_SHUFFLE_FUNC(15, 5, sse4); - ASSIGN_SHUFFLE_FUNC(15, 15, sse4); - ASSIGN_SHUFFLE_FUNC(12, 16, sse4); - ASSIGN_SHUFFLE_FUNC( 6, 12, sse4); - ASSIGN_SHUFFLE_FUNC(16, 4, sse4); - ASSIGN_SHUFFLE_FUNC(16, 12, sse4); - ASSIGN_SHUFFLE_FUNC(16, 16, sse4); - ASSIGN_SHUFFLE_FUNC( 8, 12, sse4); - ASSIGN_SHUFFLE_FUNC(12, 12, sse4); - ASSIGN_SHUFFLE_FUNC(32, 32, avx2); - ASSIGN_SHUFFLE_FUNC(64, 64, avx512); - av_assert1(out->func); + const SwsUOpEntry *entry = &uop_##NAME##EXT; \ + if (!memcmp(&uop->par, &entry->par, sizeof(uop->par))) \ + out->func = (SwsOpFunc) entry->func; \ +} while (0); + + switch (mmsize) { + case 16: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _sse4); break; + case 32: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _avx2); break; + case 64: SWS_FOR(U8, RW_SHUFFLE, ASSIGN_SHUFFLE_FUNC, _avx512); break; + } + + if (!out->func) { + av_free(mask); + return AVERROR(ENOTSUP); + } + return 0; } @@ -600,6 +597,17 @@ static int compile_uops_x86(SwsContext *ctx, const SwsUOpList *uops, SwsCompiled if (mmsize < 0) return mmsize; + if (uops->num_ops == 1 && uops->ops[0].uop == SWS_UOP_RW_SHUFFLE) { + const SwsUOp *uop = &uops->ops[0]; + ret = translate_shuffle(uop, mmsize, out); + if (ret >= 0) { + char name[SWS_UOP_NAME_MAX]; + ff_sws_uop_name(uop, name); + av_log(ctx, AV_LOG_VERBOSE, "Using x86 packed shuffle fast path: %s\n", name); + } + return ret; + } + SwsOpChain *chain = ff_sws_op_chain_alloc(); if (!chain) return AVERROR(ENOMEM); @@ -670,16 +678,11 @@ static int compile_x86(SwsContext *ctx, const SwsOpList *ops, SwsCompiledOp *out if (EXTERNAL_FMA3(cpu_flags)) flags |= SWS_UOP_FLAG_FMA; - /* Special fast path for in-place packed shuffle */ - int ret = solve_shuffle(ops, out); - if (ret != AVERROR(ENOTSUP)) - return ret; - SwsUOpList *uops = ff_sws_uop_list_alloc(); if (!uops) return AVERROR(ENOMEM); - ret = ff_sws_ops_translate(ctx, ops, flags, uops); + int ret = ff_sws_ops_translate(ctx, ops, flags, uops); if (ret < 0) goto fail; diff --git a/libswscale/x86/ops_common.asm b/libswscale/x86/ops_common.asm index a5dd105da5..d54f7c6a13 100644 --- a/libswscale/x86/ops_common.asm +++ b/libswscale/x86/ops_common.asm @@ -116,17 +116,16 @@ process_fn 4 ; This is a special entry point for handling a subset of operation chains ; that can be reduced down to a single `pshufb` shuffle mask. For more details -; about when this works, refer to the documentation of `ff_sws_solve_shuffle`. +; about when this works, refer to `solve_shuffle()` in ops_optimizer.c. ; -; We specialize this function for every possible combination of pixel strides. -; For example, gray -> gray16 is classified as an "8, 16" operation because it -; takes 8 bytes and expands them out to 16 bytes in each application of the -; 128-bit shuffle mask. +; This macro gets instantiated for every parameter combination of +; SWS_UOP_RW_SHUFFLE, which embeds the clear value and read and write sizes. ; -; Since pshufb can't shuffle across lanes, we only instantiate SSE4 versions for -; all shuffles that are not a clean multiple of 128 bits (e.g. rgb24 -> rgb0). -; For the clean multiples (e.g. rgba -> argb), we also define AVX2 and AVX512 -; versions that can handle a larger number of bytes at once. +; Since pshufb can't shuffle across lanes, we only call SSE4 versions for +; all shuffles that are not a clean multiple of 128 bits (e.g. rgb24 -> rgb0), +; unless we have access to AVX-512 vpermb, or if the lanes are all independent, +; in which case we can also use `pshufb` on mmsize == 32/64. This is detected +; by the `LANE_ALIGNED` condition. %macro MOVSIZE 3 ; size, dst, src %if %1 <= 4 @@ -138,63 +137,100 @@ process_fn 4 %endif %endmacro -%macro packed_shuffle 2 ; size_in, size_out -cglobal packed_shuffle%1_%2, 6, 10, 2, \ - exec, shuffle, bx, y, bxend, yend, src, dst, src_stride, dst_stride +%macro RW_SHUFFLE 3 +%assign CLEAR_VALUE %1 +%assign READ_SIZE %2 +%assign WRITE_SIZE %3 + +%assign LANE_ALIGNED (READ_SIZE == WRITE_SIZE && 16 % READ_SIZE == 0) +%assign MAX_SIZE (READ_SIZE > WRITE_SIZE ? READ_SIZE : WRITE_SIZE) + +; Expand read/write sizes to the true number of groups, this matches +; logic on the C side in `translate_shuffle` / `ff_sws_shuffle_mask` +%assign GROUPS (mmsize / MAX_SIZE) +%assign READ_SIZE (READ_SIZE * GROUPS) +%assign WRITE_SIZE (WRITE_SIZE * GROUPS) + +cglobal NAME, 6, 10, 3, exec, shuffle, bx, y, bxend, yend, src, dst, src_stride, dst_stride +%if mmsize > 16 && !LANE_ALIGNED + ud2 ; runtime checks should prevent this variant from being called +%else mov srcq, [execq + SwsOpExec.in0] mov dstq, [execq + SwsOpExec.out0] mov src_strideq, [execq + SwsOpExec.in_stride0] mov dst_strideq, [execq + SwsOpExec.out_stride0] - VBROADCASTI128 m1, [shuffleq] + + ; setup shuffle mask + VBROADCASTI128 m0, [shuffleq] + %if cpuflag(avx512) && CLEAR_VALUE != 0 + vpmovb2m k1, m0 ; needed for vpblendmb + %endif + + ; setup clear value register if needed + %if CLEAR_VALUE == 0xFF + %if cpuflag(avx512) + vpternlogd m2, m2, m2, 0xff + %else + pcmpeqb m2, m2 + %endif + %elif CLEAR_VALUE != 0 ; clear-to-0 is implicitly handled by pshufb + mov shuffled, CLEAR_VALUE * 0x1010101 + movd xm2, shuffled + VPBROADCASTD m2, xm2 + %endif + + ; setup loop bounds and variables sub bxendd, bxd sub yendd, yd ; reuse now-unneeded regs - %define srcidxq execq - imul srcidxq, bxendq, -%1 -%if %1 = %2 - %define dstidxq srcidxq -%else - %define dstidxq shuffleq ; no longer needed reg - imul dstidxq, bxendq, -%2 -%endif + %define srcidxq execq + imul srcidxq, bxendq, -READ_SIZE + %if READ_SIZE == WRITE_SIZE + %define dstidxq srcidxq + %else + %define dstidxq shuffleq ; no longer needed reg + imul dstidxq, bxendq, -WRITE_SIZE + %endif sub srcq, srcidxq sub dstq, dstidxq + .loop: - MOVSIZE %1, m0, [srcq + srcidxq] - pshufb m0, m1 - MOVSIZE %2, [dstq + dstidxq], m0 - add srcidxq, %1 -IF %1 != %2,add dstidxq, %2 + MOVSIZE READ_SIZE, m1, [srcq + srcidxq] + pshufb m1, m0 + + %if CLEAR_VALUE != 0 + %if cpuflag(avx512) + vpblendmb m1{k1}, m1, m2 + %elif avx_enabled + vpblendvb m1, m1, m2, m0 + %else + pblendvb m1, m2 + %endif + %endif + + MOVSIZE WRITE_SIZE, [dstq + dstidxq], m1 + add srcidxq, READ_SIZE + %if READ_SIZE != WRITE_SIZE + add dstidxq, WRITE_SIZE + %endif jnz .loop add srcq, src_strideq add dstq, dst_strideq - imul srcidxq, bxendq, -%1 -IF %1 != %2,imul dstidxq, bxendq, -%2 + imul srcidxq, bxendq, -READ_SIZE + %if READ_SIZE != WRITE_SIZE + imul dstidxq, bxendq, -WRITE_SIZE + %endif dec yendd jnz .loop RET +%endif %endmacro INIT_XMM sse4 -packed_shuffle 5, 15 ; 8 -> 24 -packed_shuffle 4, 16 ; 8 -> 32, 16 -> 64 -packed_shuffle 2, 12 ; 8 -> 48 -packed_shuffle 16, 8 ; 16 -> 8 -packed_shuffle 10, 15 ; 16 -> 24 -packed_shuffle 8, 16 ; 16 -> 32, 32 -> 64 -packed_shuffle 4, 12 ; 16 -> 48 -packed_shuffle 15, 5 ; 24 -> 8 -packed_shuffle 15, 15 ; 24 -> 24 -packed_shuffle 12, 16 ; 24 -> 32 -packed_shuffle 6, 12 ; 24 -> 48 -packed_shuffle 16, 4 ; 32 -> 8, 64 -> 16 -packed_shuffle 16, 12 ; 32 -> 24, 64 -> 48 -packed_shuffle 16, 16 ; 32 -> 32, 64 -> 64 -packed_shuffle 8, 12 ; 32 -> 48 -packed_shuffle 12, 12 ; 48 -> 48 +DECL_U8_RW_SHUFFLE (RW_SHUFFLE) INIT_YMM avx2 -packed_shuffle 32, 32 +DECL_U8_RW_SHUFFLE (RW_SHUFFLE) INIT_ZMM avx512 -packed_shuffle 64, 64 +DECL_U8_RW_SHUFFLE (RW_SHUFFLE) diff --git a/libswscale/x86/uops_macros.asm.h b/libswscale/x86/uops_macros.asm.h index f855817760..24cf15ca7a 100644 --- a/libswscale/x86/uops_macros.asm.h +++ b/libswscale/x86/uops_macros.asm.h @@ -62,6 +62,7 @@ {DEF_MACRO(WRITE_NIBBLE, TYPE)}, \ {DEF_MACRO(WRITE_PACKED, TYPE)}, \ {DEF_MACRO(WRITE_PLANAR, TYPE)}, \ + {DEF_MACRO(RW_SHUFFLE, TYPE)}, \ {DEF_MACRO(PERMUTE, TYPE)}, \ {DEF_MACRO(COPY, TYPE)}, \ {DEF_MACRO(SWAP_BYTES, TYPE)}, \ _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
