This patches integrates so called double bufferring when we are loading
2 batch elements at a time and then processing them in parallel. On the
moden arm processors especially Apple Silicon it gives a visible
benefit, for subsampled pixel processing it is especially nice because
it allows to read elements w/ 2 instructions and write with a single one
(which is usually the slowest part).
Including the previous patch in a stack on macbook pro m4 max rgb_to_yuv_half
in checkasm goes up 2x of the c version
---
libswscale/aarch64/input.S | 332 ++++++++++++++++++++++++++++++++++---
1 file changed, 309 insertions(+), 23 deletions(-)
diff --git a/libswscale/aarch64/input.S b/libswscale/aarch64/input.S
index ee8eb24c14..59d66d0022 100644
--- a/libswscale/aarch64/input.S
+++ b/libswscale/aarch64/input.S
@@ -194,40 +194,94 @@ function ff_\fmt_rgb\()ToUV_half_neon, export=1
ldp w12, w13, [x6, #20] // w12: bu, w13: rv
ldp w14, w15, [x6, #28] // w14: gv, w15: bv
4:
- cmp w5, #8
rgb_set_uv_coeff half=1
- b.lt 2f
-1: // load 16 pixels and prefetch memory for the next block
+
+ cmp w5, #16
+ b.lt 2f // Go directly to scalar if <
16
+
+1:
.if \element == 3
- ld3 { v16.16b, v17.16b, v18.16b }, [x3], #48
- prfm pldl1strm, [x3, #48]
+ ld3 { v16.16b, v17.16b, v18.16b }, [x3], #48 // First 16
pixels
+ ld3 { v26.16b, v27.16b, v28.16b }, [x3], #48 // Second 16
pixels
+ prfm pldl1keep, [x3, #96]
.else
- ld4 { v16.16b, v17.16b, v18.16b, v19.16b }, [x3], #64
- prfm pldl1strm, [x3, #64]
+ ld4 { v16.16b, v17.16b, v18.16b, v19.16b }, [x3], #64 //
First 16 pixels
+ ld4 { v26.16b, v27.16b, v28.16b, v29.16b }, [x3], #64 //
Second 16 pixels
+ prfm pldl1keep, [x3, #128]
.endif
+ // **Sum adjacent pixel pairs**
.if \alpha_first
- uaddlp v21.8h, v19.16b // v21: summed b pairs
- uaddlp v20.8h, v18.16b // v20: summed g pairs
- uaddlp v19.8h, v17.16b // v19: summed r pairs
+ uaddlp v21.8h, v19.16b // Block 1: B sums
+ uaddlp v20.8h, v18.16b // Block 1: G sums
+ uaddlp v19.8h, v17.16b // Block 1: R sums
+ uaddlp v31.8h, v29.16b // Block 2: B sums
+ uaddlp v30.8h, v28.16b // Block 2: G sums
+ uaddlp v29.8h, v27.16b // Block 2: R sums
.else
- uaddlp v19.8h, v16.16b // v19: summed r pairs
- uaddlp v20.8h, v17.16b // v20: summed g pairs
- uaddlp v21.8h, v18.16b // v21: summed b pairs
+ uaddlp v19.8h, v16.16b // Block 1: R sums
+ uaddlp v20.8h, v17.16b // Block 1: G sums
+ uaddlp v21.8h, v18.16b // Block 1: B sums
+ uaddlp v29.8h, v26.16b // Block 2: R sums
+ uaddlp v30.8h, v27.16b // Block 2: G sums
+ uaddlp v31.8h, v28.16b // Block 2: B sums
.endif
- mov v22.16b, v6.16b // U first half
- mov v23.16b, v6.16b // U second half
- mov v24.16b, v6.16b // V first half
- mov v25.16b, v6.16b // V second half
-
- rgb_to_uv_interleaved_product v19, v20, v21, v0, v1, v2, v3, v4, v5,
v22, v23, v24, v25, v16, v17, #10
+ // init accumulatos for both blocks
+ mov v7.16b, v6.16b // U_low
+ mov v8.16b, v6.16b // U_high
+ mov v9.16b, v6.16b // V_low
+ mov v10.16b, v6.16b // V_high
+ mov v11.16b, v6.16b // U_low
+ mov v12.16b, v6.16b // U_high
+ mov v13.16b, v6.16b // V_low
+ mov v14.16b, v6.16b // V_high
+
+ smlal v7.4s, v0.4h, v19.4h // U += ru * r (0-3)
+ smlal v9.4s, v3.4h, v19.4h // V += rv * r (0-3)
+ smlal v11.4s, v0.4h, v29.4h // U += ru * r (0-3)
+ smlal v13.4s, v3.4h, v29.4h // V += rv * r (0-3)
+
+ smlal2 v8.4s, v0.8h, v19.8h // U += ru * r (4-7)
+ smlal2 v10.4s, v3.8h, v19.8h // V += rv * r (4-7)
+ smlal2 v12.4s, v0.8h, v29.8h // U += ru * r (4-7)
+ smlal2 v14.4s, v3.8h, v29.8h // V += rv * r (4-7)
+
+ smlal v7.4s, v1.4h, v20.4h // U += gu * g (0-3)
+ smlal v9.4s, v4.4h, v20.4h // V += gv * g (0-3)
+ smlal v11.4s, v1.4h, v30.4h // U += gu * g (0-3)
+ smlal v13.4s, v4.4h, v30.4h // V += gv * g (0-3)
+
+ smlal2 v8.4s, v1.8h, v20.8h // U += gu * g (4-7)
+ smlal2 v10.4s, v4.8h, v20.8h // V += gv * g (4-7)
+ smlal2 v12.4s, v1.8h, v30.8h // U += gu * g (4-7)
+ smlal2 v14.4s, v4.8h, v30.8h // V += gv * g (4-7)
+
+ smlal v7.4s, v2.4h, v21.4h // U += bu * b (0-3)
+ smlal v9.4s, v5.4h, v21.4h // V += bv * b (0-3)
+ smlal v11.4s, v2.4h, v31.4h // U += bu * b (0-3)
+ smlal v13.4s, v5.4h, v31.4h // V += bv * b (0-3)
+
+ smlal2 v8.4s, v2.8h, v21.8h // U += bu * b (4-7)
+ smlal2 v10.4s, v5.8h, v21.8h // V += bv * b (4-7)
+ smlal2 v12.4s, v2.8h, v31.8h // U += bu * b (4-7)
+ smlal2 v14.4s, v5.8h, v31.8h // V += bv * b (4-7)
+
+ sqshrn v16.4h, v7.4s, #10 // U (0-3)
+ sqshrn v17.4h, v9.4s, #10 // V (0-3)
+ sqshrn v22.4h, v11.4s, #10 // U (0-3)
+ sqshrn v23.4h, v13.4s, #10 // V (0-3)
+
+ sqshrn2 v16.8h, v8.4s, #10 // U (0-7)
+ sqshrn2 v17.8h, v10.4s, #10 // V (0-7)
+ sqshrn2 v22.8h, v12.4s, #10 // U (0-7)
+ sqshrn2 v23.8h, v14.4s, #10 // V (0-7)
- str q16, [x0], #16 // store dst_u
- str q17, [x1], #16 // store dst_v
+ stp q16, q22, [x0], #32 // Store all 16 U values
+ stp q17, q23, [x1], #32 // Store all 16 V values
- sub w5, w5, #8 // width -= 8
- cmp w5, #8 // width >= 8 ?
+ sub w5, w5, #16 // width -= 16
+ cmp w5, #16 // width >= 16 ?
b.ge 1b
cbz w5, 3f // No pixels left? Exit
@@ -459,3 +513,235 @@ endfunc
DISABLE_DOTPROD
#endif
+
+.macro rgbToUV_half_neon_double fmt_bgr, fmt_rgb, element, alpha_first=0
+function ff_\fmt_bgr\()ToUV_half_neon_double, export=1
+ cbz w5, 9f // exit immediately if width
is 0
+ cmp w5, #16 // check if we have at least
16 pixels
+ b.lt _ff_\fmt_bgr\()ToUV_half_neon
+
+ ldp w12, w11, [x6, #12] // w12: bu, w11: gu
+ ldp w10, w15, [x6, #20] // w10: ru, w15: bv
+ ldp w14, w13, [x6, #28] // w14: gv, w13: rv
+
+ b .Lcommon_uv_processing_\fmt_bgr
+endfunc
+
+function ff_\fmt_rgb\()ToUV_half_neon_double, export=1
+ cbz w5, 9f // exit immediately if width
is 0
+ cmp w5, #16 // check if we have at least
16 pixels
+ b.lt _ff_\fmt_rgb\()ToUV_half_neon
+
+ ldp w10, w11, [x6, #12] // w10: ru, w11: gu
+ ldp w12, w13, [x6, #20] // w12: bu, w13: rv
+ ldp w14, w15, [x6, #28] // w14: gv, w15: bv
+
+.Lcommon_uv_processing_\fmt_bgr:
+ rgb_set_uv_coeff half=1
+
+ zip1 v7.8h, v0.8h, v3.8h // [ru0, rv0, ru1, rv1, ru2,
rv2, ru3, rv3]
+ zip2 v8.8h, v0.8h, v3.8h // [ru4, rv4, ru5, rv5, ru6,
rv6, ru7, rv7]
+ zip1 v9.8h, v1.8h, v4.8h // [gu0, gv0, gu1, gv1, gu2,
gv2, gu3, gv3]
+ zip2 v10.8h, v1.8h, v4.8h // [gu4, gv4, gu5, gv5, gu6,
gv6, gu7, gv7]
+ zip1 v11.8h, v2.8h, v5.8h // [bu0, bv0, bu1, bv1, bu2,
bv2, bu3, bv3]
+ zip2 v12.8h, v2.8h, v5.8h // [bu4, bv4, bu5, bv5, bu6,
bv6, bu7, bv7]
+
+ zip1 v13.4s, v6.4s, v6.4s // [const, const, const,
const] for U/V pairs
+ zip2 v14.4s, v6.4s, v6.4s // [const, const, const,
const] for U/V pairs
+
+.Lprocess_16_\fmt_bgr:
+ // **OPTIMIZED LOADING: Load and sum with immediate interleaving**
+ .if \element == 3
+ ld3 { v16.16b, v17.16b, v18.16b }, [x3], #48
+ ld3 { v26.16b, v27.16b, v28.16b }, [x3], #48
+ prfm pldl1keep, [x3, #96] // Early prefetch
+
+ // Sum and immediately create interleaved RGB data
+ uaddlp v19.8h, v16.16b
+ uaddlp v20.8h, v17.16b
+ uaddlp v21.8h, v18.16b
+ uaddlp v29.8h, v26.16b
+ uaddlp v30.8h, v27.16b
+ uaddlp v31.8h, v28.16b
+ .else
+ ld4 { v16.16b, v17.16b, v18.16b, v19.16b }, [x3], #64
+ ld4 { v26.16b, v27.16b, v28.16b, v29.16b }, [x3], #64
+ prfm pldl1keep, [x3, #128] // Early prefetch
+
+ .if \alpha_first
+ uaddlp v21.8h, v19.16b
+ uaddlp v20.8h, v18.16b
+ uaddlp v19.8h, v17.16b
+ uaddlp v31.8h, v29.16b
+ uaddlp v30.8h, v28.16b
+ uaddlp v29.8h, v27.16b
+ .else
+ uaddlp v19.8h, v16.16b
+ uaddlp v20.8h, v17.16b
+ uaddlp v21.8h, v18.16b
+ uaddlp v29.8h, v26.16b
+ uaddlp v30.8h, v27.16b
+ uaddlp v31.8h, v28.16b
+ .endif
+ .endif
+
+ zip1 v22.8h, v19.8h, v19.8h // [r0, r0, r1, r1, r2, r2,
r3, r3] Block 1
+ zip2 v23.8h, v19.8h, v19.8h // [r4, r4, r5, r5, r6, r6,
r7, r7] Block 1
+ zip1 v24.8h, v20.8h, v20.8h // [g0, g0, g1, g1, g2, g2,
g3, g3] Block 1
+ zip2 v25.8h, v20.8h, v20.8h // [g4, g4, g5, g5, g6, g6,
g7, g7] Block 1
+
+ mov v0.16b, v13.16b // UV accumulator 1a
+ mov v1.16b, v13.16b // UV accumulator 1b
+ mov v2.16b, v14.16b // UV accumulator 1c
+ mov v3.16b, v14.16b // UV accumulator 1d
+
+ smlal v0.4s, v7.4h, v22.4h
+ smlal2 v1.4s, v7.8h, v22.8h
+ smlal v2.4s, v8.4h, v23.4h
+ smlal2 v3.4s, v8.8h, v23.8h
+
+ smlal v0.4s, v9.4h, v24.4h
+ smlal2 v1.4s, v9.8h, v24.8h
+ smlal v2.4s, v10.4h, v25.4h
+ smlal2 v3.4s, v10.8h, v25.8h
+
+ zip1 v22.8h, v21.8h, v21.8h // [b0, b0, b1, b1, b2, b2,
b3, b3] Block 1
+ zip2 v23.8h, v21.8h, v21.8h // [b4, b4, b5, b5, b6, b6,
b7, b7] Block 1
+
+ smlal2 v1.4s, v11.8h, v22.8h
+ smlal v2.4s, v12.4h, v23.4h
+ smlal2 v3.4s, v12.8h, v23.8h
+
+ zip1 v22.8h, v29.8h, v29.8h // [r0, r0, r1, r1, r2, r2,
r3, r3] Block 2
+ zip2 v23.8h, v29.8h, v29.8h // [r4, r4, r5, r5, r6, r6,
r7, r7] Block 2
+ zip1 v24.8h, v30.8h, v30.8h // [g0, g0, g1, g1, g2, g2,
g3, g3] Block 2
+ zip2 v25.8h, v30.8h, v30.8h // [g4, g4, g5, g5, g6, g6,
g7, g7] Block 2
+
+ mov v4.16b, v13.16b
+ mov v5.16b, v13.16b
+ mov v6.16b, v14.16b
+ mov v16.16b, v14.16b
+
+ smlal v4.4s, v7.4h, v22.4h
+ smlal2 v5.4s, v7.8h, v22.8h
+ smlal v6.4s, v8.4h, v23.4h
+ smlal2 v16.4s, v8.8h, v23.8h
+
+ smlal v4.4s, v9.4h, v24.4h
+ smlal2 v5.4s, v9.8h, v24.8h
+ smlal v6.4s, v10.4h, v25.4h
+ smlal2 v16.4s, v10.8h, v25.8h
+
+ zip1 v22.8h, v31.8h, v31.8h // [b0, b0, b1, b1, b2, b2,
b3, b3] Block 2
+ zip2 v23.8h, v31.8h, v31.8h // [b4, b4, b5, b5, b6, b6,
b7, b7] Block 2
+
+ smlal v4.4s, v11.4h, v22.4h // Process U&V for b0,b1
simultaneously
+ smlal2 v5.4s, v11.8h, v22.8h // Process U&V for b2,b3
simultaneously
+ smlal v6.4s, v12.4h, v23.4h // Process U&V for b4,b5
simultaneously
+ smlal2 v16.4s, v12.8h, v23.8h // Process U&V for b6,b7
simultaneously
+
+ uzp1 v17.4s, v0.4s, v1.4s
+ uzp2 v18.4s, v0.4s, v1.4s
+ uzp1 v19.4s, v2.4s, v3.4s
+ uzp2 v20.4s, v2.4s, v3.4s
+
+ uzp1 v21.4s, v4.4s, v5.4s // Extract U values (Block 2,
part 1)
+ uzp2 v22.4s, v4.4s, v5.4s // Extract V values (Block 2,
part 1)
+ uzp1 v23.4s, v6.4s, v16.4s // Extract U values (Block 2,
part 2)
+ uzp2 v24.4s, v6.4s, v16.4s // Extract V values (Block 2,
part 2)
+
+ sqshrn v25.4h, v17.4s, #10 // U1 (first 4)
+ sqshrn2 v25.8h, v19.4s, #10 // U1 (complete 8)
+ sqshrn v26.4h, v18.4s, #10 // V1 (first 4)
+ sqshrn2 v26.8h, v20.4s, #10 // V1 (complete 8)
+
+ sqshrn v27.4h, v21.4s, #10 // U2 (first 4)
+ sqshrn2 v27.8h, v23.4s, #10 // U2 (complete 8)
+ sqshrn v28.4h, v22.4s, #10 // V2 (first 4)
+ sqshrn2 v28.8h, v24.4s, #10 // V2 (complete 8)
+
+ // **EFFICIENT STORE: Use STP for optimal memory bandwidth**
+ stp q25, q27, [x0], #32 // Store U1, U2
+ stp q26, q28, [x1], #32 // Store V1, V2
+
+ sub w5, w5, #16 // Decrement counter
+ cmp w5, #16 // Check for more blocks
+ b.ge .Lprocess_16_\fmt_bgr
+
+ // Standard 8-pixel and scalar fallbacks (unchanged)
+ cmp w5, #8
+ b.lt .Lscalar_loop_init_\fmt_bgr
+
+.Lprocess_8_\fmt_bgr:
+ .if \element == 3
+ ld3 { v16.16b, v17.16b, v18.16b }, [x3], #48
+ uaddlp v19.8h, v16.16b
+ uaddlp v20.8h, v17.16b
+ uaddlp v21.8h, v18.16b
+ .else
+ ld4 { v16.16b, v17.16b, v18.16b, v19.16b }, [x3], #64
+ .if \alpha_first
+ uaddlp v21.8h, v19.16b
+ uaddlp v20.8h, v18.16b
+ uaddlp v19.8h, v17.16b
+ .else
+ uaddlp v19.8h, v16.16b
+ uaddlp v20.8h, v17.16b
+ uaddlp v21.8h, v18.16b
+ .endif
+ .endif
+
+ rgb_set_uv_coeff half=1
+ mov v22.16b, v6.16b
+ mov v23.16b, v6.16b
+ mov v24.16b, v6.16b
+ mov v25.16b, v6.16b
+
+ rgb_to_uv_interleaved_product v19, v20, v21, v0, v1, v2, v3, v4, v5,
v22, v23, v24, v25, v16, v17, #10
+
+ str q16, [x0], #16
+ str q17, [x1], #16
+
+ sub w5, w5, #8
+ cmp w5, #8
+ b.ge .Lprocess_8_\fmt_bgr
+
+.Lscalar_loop_init_\fmt_bgr:
+ cbz w5, 9f
+
+.Lscalar_loop_\fmt_bgr:
+ .if \alpha_first
+ rgb_load_add_half 1, 5, 2, 6, 3, 7
+ .else
+ .if \element == 3
+ rgb_load_add_half 0, 3, 1, 4, 2, 5
+ .else
+ rgb_load_add_half 0, 4, 1, 5, 2, 6
+ .endif
+ .endif
+
+ mov x8, x9
+ mov x17, x9
+ smaddl x8, w2, w10, x8
+ smaddl x17, w2, w13, x17
+ smaddl x8, w4, w11, x8
+ smaddl x17, w4, w14, x17
+ smaddl x8, w7, w12, x8
+ smaddl x17, w7, w15, x17
+ asr w8, w8, #10
+ asr w17, w17, #10
+
+ strh w8, [x0], #2
+ strh w17, [x1], #2
+
+ sub w5, w5, #2
+ add x3, x3, #(2*\element)
+ cbnz w5, .Lscalar_loop_\fmt_bgr
+
+9: ret
+endfunc
+.endm
+
+rgbToUV_half_neon_double bgra32, rgba32, element=4
+
+rgbToUV_half_neon_double abgr32, argb32, element=4, alpha_first=1
--
2.49.0
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
