This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit a87d92d97523ef43354742c5f9cd12a59c904068 Author: Philip Langdale <[email protected]> AuthorDate: Sun Jun 21 10:45:46 2026 -0700 Commit: Philip Langdale <[email protected]> CommitDate: Sat Aug 29 17:07:53 2026 -0700 avfilter/fruc_vulkan: suppress spurious flow in featureless regions In textureless regions the optical flow engine has no data to track and its regularizer invents a smooth, often large, and internally self-consistent flow field. Backward warping then traces a background pixel along that invented flow onto a nearby moving object and pulls object pixels into what should be empty background, leaving a ghost outline at a distance from the object where nothing is actually moving. Forward/backward flow consistency does not catch this because the invented flow round-trips cleanly. I have no idea if this is a common limitation in optical flow implementations but it's definitely happening with the nvidia hardware, so this change attempts to introduce a heuristic to work around it. When the luma at a given location is sufficiently identical in both frames, and the optical flow magnitude is sufficiently large, we will now assume that the location is static and the flow is bogus. Genuine motion will usually lead to luma differences, and low magnitude flow will be respected either way. Again, this is all highly empirical, and the constants are justified purely by by fiddling around with specific samples. --- libavfilter/vf_fruc_vulkan.c | 2 ++ libavfilter/vulkan/fruc_interpolate.comp.glsl | 38 ++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/libavfilter/vf_fruc_vulkan.c b/libavfilter/vf_fruc_vulkan.c index 66d60eb615..df44ec3e7e 100644 --- a/libavfilter/vf_fruc_vulkan.c +++ b/libavfilter/vf_fruc_vulkan.c @@ -59,6 +59,7 @@ typedef struct GrayscalePushData { typedef struct InterpolatePushData { float t; int32_t planes; + float luma_weights[4]; ///< RGB->Y weights, matching the grayscale pass float plane_size[4][2]; ///< visible texel extent of each plane } InterpolatePushData; @@ -1001,6 +1002,7 @@ static int interpolate_frame(AVFilterContext *avctx, AVFrame *out, float t) .t = t, .planes = av_pix_fmt_count_planes(vkctx->output_format), }; + memcpy(pd.luma_weights, s->luma_weights, sizeof(pd.luma_weights)); /* The shader works in the visible frame's coordinate space; the source and * output images may each be allocated larger than that. */ for (int i = 0; i < pd.planes; i++) { diff --git a/libavfilter/vulkan/fruc_interpolate.comp.glsl b/libavfilter/vulkan/fruc_interpolate.comp.glsl index ec48074dfd..bb0b6d357d 100644 --- a/libavfilter/vulkan/fruc_interpolate.comp.glsl +++ b/libavfilter/vulkan/fruc_interpolate.comp.glsl @@ -38,6 +38,20 @@ layout (set = 0, binding = 4) uniform isampler2D flow_bwd; #define FLOW_FIXED_POINT_SCALE (1.0 / 32.0) +// Thresholds for the spurious-flow guard + +/* STATIC_LUMA_SIGMA is the per-pixel luma tolerance for calling a region "static" */ +#define STATIC_LUMA_SIGMA 0.03 +/* WARP_NEAR_PX is the threshold below which a warp is small enough that it won't + ever be considered "long range" */ +#define WARP_NEAR_PX 5.0 +/* WARP_FAR_PX is the threshold above which a warp is large enough that it will + always be considered "long range" */ +#define WARP_FAR_PX 20.0 +/* WARP_REACH_SCALE is the factor by which a warp's reach is scaled before the + * range test. Higher values will trigger the guard for smaller displacements. */ +#define WARP_REACH_SCALE 1.0 + /* These Picard values were established empirically on a couple of different * samples, but one could easily imagine reaching a different conclusion from * different data. */ @@ -47,6 +61,7 @@ layout (set = 0, binding = 4) uniform isampler2D flow_bwd; layout (push_constant, scalar) uniform pushConstants { float t; /* interpolation position in [0, 1] between f0 and f1 */ int planes; + vec4 luma_weights; /* RGB->Y weights, matching the grayscale pass */ /* Visible extent of each plane. Neither the source textures nor the output * storage images need match it: a hardware decoder allocates its frames * padded up to its alignment, and the output frames context may be the @@ -95,7 +110,28 @@ void main() vec4 c0 = texture(f0_img[i], f0_uv(s0, i)); vec4 c1 = texture(f1_img[i], f1_uv(s1, i)); + vec4 warped = mix(c0, c1, t); + + /* Spurious-flow guard. In textureless regions the flow + * engine invents bogus flows we need to ignore. */ + vec4 z0 = texture(f0_img[i], f0_uv(base, i)); + vec4 z1 = texture(f1_img[i], f1_uv(base, i)); + vec4 stat = mix(z0, z1, t); + + /* Key staticness off the same weighted luma the grayscale pass feeds the + * flow engine. */ + float lz = abs(dot(texture(f0_img[0], f0_uv(base, 0)), luma_weights) - + dot(texture(f1_img[0], f1_uv(base, 0)), luma_weights)); + float staticness = exp(-(lz * lz) / (STATIC_LUMA_SIGMA * STATIC_LUMA_SIGMA)); + + float disp = max(length((s0 - base) * luma_size), + length((s1 - base) * luma_size)); + /* smoothstep is used to transition between the two thresholds and avoid + * abrupt changes at the boundary. */ + float reach = smoothstep(WARP_NEAR_PX, WARP_FAR_PX, disp * WARP_REACH_SCALE); + + vec4 result = mix(warped, stat, staticness * reach); - imageStore(out_img[i], pos, mix(c0, c1, t)); + imageStore(out_img[i], pos, result); } } -- To stop receiving notification emails like this one, please contact [email protected]. _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
