PR #22750 opened by Lynne URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22750 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/22750.patch
This series of commits simply adds a version of OP_LINEAR which is not bitexact, using native matrix*vector + vector multiplication. It allows the compiler to fuse operations as it sees fit. It should be a bit faster on paper, but I have not found a difference when testing with low resolutions as dispatching still takes the majority of time. >From ad452e86d22ec825c1023f295c3b18e4611e51c6 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Wed, 8 Apr 2026 14:48:23 +0200 Subject: [PATCH 1/4] swscale/vulkan: move linear op handling to a separate function Sponsored-by: Sovereign Tech Fund --- libswscale/vulkan/ops.c | 88 +++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index 49a3d7827c..7ad2ae5ead 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -563,6 +563,54 @@ static void define_shader_bindings(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id, SpvStorageClassUniformConstant, 0); } +static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, + int data, int linear_ops_idx, int *nb_const_ids) +{ + int type_s = op->type == SWS_PIXEL_F32 ? id->f32_type : id->u32_type; + int type_v = op->type == SWS_PIXEL_F32 ? id->f32vec4_type : id->u32vec4_type; + + int tmp[4]; + tmp[0] = spi_OpCompositeExtract(spi, type_s, data, 0); + tmp[1] = spi_OpCompositeExtract(spi, type_s, data, 1); + tmp[2] = spi_OpCompositeExtract(spi, type_s, data, 2); + tmp[3] = spi_OpCompositeExtract(spi, type_s, data, 3); + + int off = spi_reserve(spi, 0); /* Current offset */ + spi->off = id->linear_deco_off[linear_ops_idx]; + for (int i = 0; i < id->linear_deco_ops[linear_ops_idx]; i++) + spi_OpDecorate(spi, spi->id + i, SpvDecorationNoContraction); + spi->off = off; + + int res[4]; + for (int j = 0; j < 4; j++) { + res[j] = op->type == SWS_PIXEL_F32 ? id->f32_0 : id->u32_cid[0]; + if (op->lin.m[j][0].num) + res[j] = spi_OpFMul(spi, type_s, tmp[0], + id->const_ids[(*nb_const_ids)++]); + + if (op->lin.m[j][0].num && op->lin.m[j][4].num) + res[j] = spi_OpFAdd(spi, type_s, + id->const_ids[(*nb_const_ids)++], res[j]); + else if (op->lin.m[j][4].num) + res[j] = id->const_ids[(*nb_const_ids)++]; + + for (int i = 1; i < 4; i++) { + if (!op->lin.m[j][i].num) + continue; + + int v = spi_OpFMul(spi, type_s, tmp[i], + id->const_ids[(*nb_const_ids)++]); + if (op->lin.m[j][0].num || op->lin.m[j][4].num) + res[j] = spi_OpFAdd(spi, type_s, res[j], v); + else + res[j] = v; + } + } + + return spi_OpCompositeConstruct(spi, type_v, + res[0], res[1], res[2], res[3]); +} + static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, SwsOpList *ops, FFVulkanShader *shd) { @@ -831,45 +879,7 @@ static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, break; } case SWS_OP_LINEAR: { - int tmp[4]; - tmp[0] = spi_OpCompositeExtract(spi, type_s, data, 0); - tmp[1] = spi_OpCompositeExtract(spi, type_s, data, 1); - tmp[2] = spi_OpCompositeExtract(spi, type_s, data, 2); - tmp[3] = spi_OpCompositeExtract(spi, type_s, data, 3); - - int off = spi_reserve(spi, 0); /* Current offset */ - spi->off = id->linear_deco_off[nb_linear_ops]; - for (int i = 0; i < id->linear_deco_ops[nb_linear_ops]; i++) - spi_OpDecorate(spi, spi->id + i, SpvDecorationNoContraction); - spi->off = off; - - int res[4]; - for (int j = 0; j < 4; j++) { - res[j] = op->type == SWS_PIXEL_F32 ? id->f32_0 : id->u32_cid[0]; - if (op->lin.m[j][0].num) - res[j] = spi_OpFMul(spi, type_s, tmp[0], - id->const_ids[nb_const_ids++]); - - if (op->lin.m[j][0].num && op->lin.m[j][4].num) - res[j] = spi_OpFAdd(spi, type_s, - id->const_ids[nb_const_ids++], res[j]); - else if (op->lin.m[j][4].num) - res[j] = id->const_ids[nb_const_ids++]; - - for (int i = 1; i < 4; i++) { - if (!op->lin.m[j][i].num) - continue; - - int v = spi_OpFMul(spi, type_s, tmp[i], - id->const_ids[nb_const_ids++]); - if (op->lin.m[j][0].num || op->lin.m[j][4].num) - res[j] = spi_OpFAdd(spi, type_s, res[j], v); - else - res[j] = v; - } - } - data = spi_OpCompositeConstruct(spi, type_v, - res[0], res[1], res[2], res[3]); + data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, &nb_const_ids); nb_linear_ops++; break; } -- 2.52.0 >From d3f6441b1ebba2ab1af14bc44b39df8247400c38 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Wed, 8 Apr 2026 15:11:25 +0200 Subject: [PATCH 2/4] swscale/vulkan: put entire linear matrix+vector as constant data Rather than only using what we need. The driver will remove any unused constants. Sponsored-by: Sovereign Tech Fund --- libswscale/vulkan/ops.c | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index 7ad2ae5ead..409bdf9831 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -460,26 +460,19 @@ static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) } break; case SWS_OP_LINEAR: { + float val; for (int i = 0; i < 4; i++) { - float val; - if (op->lin.m[i][0].num) { - val = op->lin.m[i][0].num/(float)op->lin.m[i][0].den; - id->const_ids[id->nb_const_ids++] = - spi_OpConstantFloat(spi, f32_type, val); - } - if (op->lin.m[i][4].num) { - val = op->lin.m[i][4].num/(float)op->lin.m[i][4].den; - id->const_ids[id->nb_const_ids++] = - spi_OpConstantFloat(spi, f32_type, val); - } - for (int j = 1; j < 4; j++) { - if (!op->lin.m[i][j].num) - continue; + for (int j = 0; j < 4; j++) { val = op->lin.m[i][j].num/(float)op->lin.m[i][j].den; id->const_ids[id->nb_const_ids++] = spi_OpConstantFloat(spi, f32_type, val); } } + for (int i = 0; i < 4; i++) { + val = op->lin.m[i][4].num/(float)op->lin.m[i][4].den; + id->const_ids[id->nb_const_ids++] = + spi_OpConstantFloat(spi, f32_type, val); + } break; } default: @@ -564,7 +557,7 @@ static void define_shader_bindings(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id, } static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, - int data, int linear_ops_idx, int *nb_const_ids) + int data, int linear_ops_idx, int const_off) { int type_s = op->type == SWS_PIXEL_F32 ? id->f32_type : id->u32_type; int type_v = op->type == SWS_PIXEL_F32 ? id->f32vec4_type : id->u32vec4_type; @@ -586,20 +579,20 @@ static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, res[j] = op->type == SWS_PIXEL_F32 ? id->f32_0 : id->u32_cid[0]; if (op->lin.m[j][0].num) res[j] = spi_OpFMul(spi, type_s, tmp[0], - id->const_ids[(*nb_const_ids)++]); + id->const_ids[const_off + j*4 + 0]); if (op->lin.m[j][0].num && op->lin.m[j][4].num) res[j] = spi_OpFAdd(spi, type_s, - id->const_ids[(*nb_const_ids)++], res[j]); + id->const_ids[const_off + 4*4 + j], res[j]); else if (op->lin.m[j][4].num) - res[j] = id->const_ids[(*nb_const_ids)++]; + res[j] = id->const_ids[const_off + 4*4 + j]; for (int i = 1; i < 4; i++) { if (!op->lin.m[j][i].num) continue; int v = spi_OpFMul(spi, type_s, tmp[i], - id->const_ids[(*nb_const_ids)++]); + id->const_ids[const_off + j*4 + i]); if (op->lin.m[j][0].num || op->lin.m[j][4].num) res[j] = spi_OpFAdd(spi, type_s, res[j], v); else @@ -879,8 +872,9 @@ static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, break; } case SWS_OP_LINEAR: { - data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, &nb_const_ids); + data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, nb_const_ids); nb_linear_ops++; + nb_const_ids += 4*5; break; } default: -- 2.52.0 >From 192bf916180fd232be455791a443e5feb9089124 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Wed, 8 Apr 2026 15:35:47 +0200 Subject: [PATCH 3/4] swscale/vulkan: create a constant matrix from linear op constants Sponsored-by: Sovereign Tech Fund --- libswscale/vulkan/ops.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index 409bdf9831..5a32d30cc3 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -242,6 +242,7 @@ typedef struct SPIRVIDs { int u32vec4_type; int f32vec4_type; + int f32mat4_type; /* Constants */ int u32_p; @@ -371,6 +372,7 @@ static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) id->u32vec4_type = spi_OpTypeVector(spi, u32_type, 4); id->f32vec4_type = spi_OpTypeVector(spi, f32_type, 4); + id->f32mat4_type = spi_OpTypeMatrix(spi, id->f32vec4_type, 4); /* Constants */ id->u32_p = spi_OpUndef(spi, u32_type); @@ -383,8 +385,8 @@ static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) id->nb_const_ids = 0; for (int n = 0; n < ops->num_ops; n++) { /* Make sure there's always enough space for the maximum number of - * constants a single operation needs (currently linear, 20 consts). */ - av_assert0((id->nb_const_ids + 20) <= FF_ARRAY_ELEMS(id->const_ids)); + * constants a single operation needs (currently linear, 31 consts). */ + av_assert0((id->nb_const_ids + 31) <= FF_ARRAY_ELEMS(id->const_ids)); const SwsOp *op = &ops->ops[n]; switch (op->op) { case SWS_OP_CONVERT: @@ -467,12 +469,32 @@ static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) id->const_ids[id->nb_const_ids++] = spi_OpConstantFloat(spi, f32_type, val); } + id->const_ids[id->nb_const_ids++] = + spi_OpConstantComposite(spi, id->f32vec4_type, + id->const_ids[id->nb_const_ids - 4], + id->const_ids[id->nb_const_ids - 3], + id->const_ids[id->nb_const_ids - 2], + id->const_ids[id->nb_const_ids - 1]); } + + id->const_ids[id->nb_const_ids++] = + spi_OpConstantComposite(spi, id->f32mat4_type, + id->const_ids[id->nb_const_ids - 5*4 + 4], + id->const_ids[id->nb_const_ids - 5*3 + 4], + id->const_ids[id->nb_const_ids - 5*2 + 4], + id->const_ids[id->nb_const_ids - 5*1 + 4]); + for (int i = 0; i < 4; i++) { val = op->lin.m[i][4].num/(float)op->lin.m[i][4].den; id->const_ids[id->nb_const_ids++] = spi_OpConstantFloat(spi, f32_type, val); } + id->const_ids[id->nb_const_ids++] = + spi_OpConstantComposite(spi, id->f32vec4_type, + id->const_ids[id->nb_const_ids - 4], + id->const_ids[id->nb_const_ids - 3], + id->const_ids[id->nb_const_ids - 2], + id->const_ids[id->nb_const_ids - 1]); break; } default: @@ -579,20 +601,20 @@ static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, res[j] = op->type == SWS_PIXEL_F32 ? id->f32_0 : id->u32_cid[0]; if (op->lin.m[j][0].num) res[j] = spi_OpFMul(spi, type_s, tmp[0], - id->const_ids[const_off + j*4 + 0]); + id->const_ids[const_off + j*5 + 0]); if (op->lin.m[j][0].num && op->lin.m[j][4].num) res[j] = spi_OpFAdd(spi, type_s, - id->const_ids[const_off + 4*4 + j], res[j]); + id->const_ids[const_off + 4*5 + 1 + j], res[j]); else if (op->lin.m[j][4].num) - res[j] = id->const_ids[const_off + 4*4 + j]; + res[j] = id->const_ids[const_off + 4*5 + 1 + j]; for (int i = 1; i < 4; i++) { if (!op->lin.m[j][i].num) continue; int v = spi_OpFMul(spi, type_s, tmp[i], - id->const_ids[const_off + j*4 + i]); + id->const_ids[const_off + j*5 + i]); if (op->lin.m[j][0].num || op->lin.m[j][4].num) res[j] = spi_OpFAdd(spi, type_s, res[j], v); else @@ -874,7 +896,7 @@ static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, case SWS_OP_LINEAR: { data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, nb_const_ids); nb_linear_ops++; - nb_const_ids += 4*5; + nb_const_ids += 5*5 + 1; break; } default: -- 2.52.0 >From 046cd1631fd4dc8aab8719a9ee167925858437cf Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Wed, 8 Apr 2026 15:59:49 +0200 Subject: [PATCH 4/4] swscale/vulkan: add a non-bitexact version of OP_LINEAR Uses matrix*vector + vector multiplication. Sponsored-by: Sovereign Tech Fund --- libswscale/vulkan/ops.c | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index 5a32d30cc3..c5ff6e3f4a 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -277,7 +277,7 @@ typedef struct SPIRVIDs { } SPIRVIDs; /* Section 1: Function to define all shader header data, and decorations */ -static void define_shader_header(FFVulkanShader *shd, SwsOpList *ops, +static void define_shader_header(SwsContext *sws, FFVulkanShader *shd, SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) { spi_OpCapability(spi, SpvCapabilityShader); /* Shader type */ @@ -326,6 +326,9 @@ static void define_shader_header(FFVulkanShader *shd, SwsOpList *ops, spi_OpDecorate(spi, id->dither[i].id, SpvDecorationBinding, i); } + if (!(sws->flags & SWS_BITEXACT)) + return; + /* All linear arithmetic ops must be decorated with NoContraction */ for (int n = 0; n < ops->num_ops; n++) { const SwsOp *op = &ops->ops[n]; @@ -351,7 +354,7 @@ static void define_shader_header(FFVulkanShader *shd, SwsOpList *ops, } /* Section 2: Define all types and constants */ -static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) +static void define_shader_consts(SwsContext *sws, SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) { /* Define scalar types */ id->void_type = spi_OpTypeVoid(spi); @@ -465,7 +468,9 @@ static void define_shader_consts(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id) float val; for (int i = 0; i < 4; i++) { for (int j = 0; j < 4; j++) { - val = op->lin.m[i][j].num/(float)op->lin.m[i][j].den; + int k = sws->flags & SWS_BITEXACT ? i : j; + int l = sws->flags & SWS_BITEXACT ? j : i; + val = op->lin.m[k][l].num/(float)op->lin.m[k][l].den; id->const_ids[id->nb_const_ids++] = spi_OpConstantFloat(spi, f32_type, val); } @@ -578,6 +583,16 @@ static void define_shader_bindings(SwsOpList *ops, SPICtx *spi, SPIRVIDs *id, SpvStorageClassUniformConstant, 0); } +static int insert_vmat_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, + int data, int const_off) +{ + data = spi_OpMatrixTimesVector(spi, id->f32vec4_type, + id->const_ids[const_off + 4*5], + data); + return spi_OpFAdd(spi, id->f32vec4_type, + id->const_ids[const_off + 4*5 + 1 + 4], data); +} + static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, int data, int linear_ops_idx, int const_off) { @@ -626,7 +641,7 @@ static int insert_bitexact_linear(const SwsOp *op, SPICtx *spi, SPIRVIDs *id, res[0], res[1], res[2], res[3]); } -static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, +static int add_ops_spirv(SwsContext *sws, VulkanPriv *p, FFVulkanOpsCtx *s, SwsOpList *ops, FFVulkanShader *shd) { uint8_t spvbuf[1024*16]; @@ -700,8 +715,8 @@ static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, id->nb_dither_bufs, 1, 0); /* Define shader header sections */ - define_shader_header(shd, ops, spi, id); - define_shader_consts(ops, spi, id); + define_shader_header(sws, shd, ops, spi, id); + define_shader_consts(sws, ops, spi, id); define_shader_bindings(ops, spi, id, in_img_count, out_img_count); /* Main function starts here */ @@ -894,7 +909,10 @@ static int add_ops_spirv(VulkanPriv *p, FFVulkanOpsCtx *s, break; } case SWS_OP_LINEAR: { - data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, nb_const_ids); + if (sws->flags & SWS_BITEXACT) + data = insert_bitexact_linear(op, spi, id, data, nb_linear_ops, nb_const_ids); + else + data = insert_vmat_linear(op, spi, id, data, nb_const_ids); nb_linear_ops++; nb_const_ids += 5*5 + 1; break; @@ -942,7 +960,7 @@ static void add_desc_read_write(FFVulkanDescriptorSetBinding *out_desc, #define QSTR "(%i/%i%s)" #define QTYPE(Q) (Q).num, (Q).den, cur_type == SWS_PIXEL_F32 ? ".0f" : "" -static int add_ops_glsl(VulkanPriv *p, FFVulkanOpsCtx *s, +static int add_ops_glsl(SwsContext *sws, VulkanPriv *p, FFVulkanOpsCtx *s, SwsOpList *ops, FFVulkanShader *shd) { int err; @@ -1193,12 +1211,12 @@ static int compile(SwsContext *sws, SwsOpList *ops, SwsCompiledOp *out, int glsl if (glsl) { err = AVERROR(ENOTSUP); #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG - err = add_ops_glsl(p, s, ops, &p->shd); + err = add_ops_glsl(sws, p, s, ops, &p->shd); #endif } else { err = AVERROR(ENOTSUP); #if HAVE_SPIRV_HEADERS_SPIRV_H || HAVE_SPIRV_UNIFIED1_SPIRV_H - err = add_ops_spirv(p, s, ops, &p->shd); + err = add_ops_spirv(sws, p, s, ops, &p->shd); #endif } if (err < 0) -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
