This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit c3a1b5e0ada46dec6a5257748da15e763f4a4637 Author: Philip Langdale <[email protected]> AuthorDate: Wed Jun 24 07:59:46 2026 -0700 Commit: Philip Langdale <[email protected]> CommitDate: Sat Aug 29 16:38:05 2026 -0700 avfilter/fruc_vulkan: warp frames with the computed optical flow Replace the placeholder temporal blend in the interpolation shader with a motion-compensated algorithm. This is finally the point where we are actually using the optical flow information. For each output pixel the source position on the f0 and f1 grids is recovered by fixed-point Picard iteration before the samples are blended. The number of iterations and the relaxation factor are established through empirical testing on a couple of ad-hoc samples, representing highly uniform and non-uniform motion respectively. The approach here is hardly profound, and not meant to be. There are for sure many more advanced algorithms for doing motion interpolation from optical flow, and we can happily replace it with, or add, something more advanced once all the machinery is in place. --- doc/filters.texi | 6 ++--- libavfilter/vf_fruc_vulkan.c | 4 ++-- libavfilter/vulkan/fruc_interpolate.comp.glsl | 34 ++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index 4ba53ac5d9..2f3a6111d7 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -29680,9 +29680,9 @@ Frame rate up-conversion using the NVIDIA Vulkan optical flow extension (@code{VK_NV_optical_flow}), implemented on the GPU using Vulkan. For each pair of consecutive input frames the filter computes a forward and -backward optical flow field on the device's optical flow engine. Intermediate -frames at the requested output frame rate are produced by proportionally blending -the two source frames. The frame timing logic mirrors the @code{framerate} filter. +backward optical flow field on the device's optical flow engine, and uses those +fields to synthesise motion-compensated intermediate frames at the requested +output frame rate. The frame timing logic mirrors the @code{framerate} filter. Planar and semi-planar YUV, gray, and packed RGB whose sampled channels are already in component order are supported. Packed YUV, planar RGB, formats with diff --git a/libavfilter/vf_fruc_vulkan.c b/libavfilter/vf_fruc_vulkan.c index 4b9091b137..348b0daf33 100644 --- a/libavfilter/vf_fruc_vulkan.c +++ b/libavfilter/vf_fruc_vulkan.c @@ -702,7 +702,7 @@ static av_cold int init_filter(AVFilterContext *avctx) ff_fruc_grayscale_comp_spv_len, "main")); RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->grayscale)); - /* Interpolation shader. */ + /* Motion compensated interpolation shader. */ ff_vk_shader_load(&s->interpolate, VK_SHADER_STAGE_COMPUTE_BIT, NULL, (uint32_t []) { 32, 32, 1 }, 0); ff_vk_shader_add_push_const(&s->interpolate, 0, sizeof(InterpolatePushData), @@ -905,7 +905,7 @@ static void plane_wh(const AVPixFmtDescriptor *desc, int width, int height, *h = sub ? AV_CEIL_RSHIFT(height, desc->log2_chroma_h) : height; } -/* Produce the output frame at temporal position t. */ +/* Produce the motion compensated output frame at temporal position t. */ static int interpolate_frame(AVFilterContext *avctx, AVFrame *out, float t) { int err; diff --git a/libavfilter/vulkan/fruc_interpolate.comp.glsl b/libavfilter/vulkan/fruc_interpolate.comp.glsl index 3c702b8e7f..ec48074dfd 100644 --- a/libavfilter/vulkan/fruc_interpolate.comp.glsl +++ b/libavfilter/vulkan/fruc_interpolate.comp.glsl @@ -30,13 +30,20 @@ layout (set = 0, binding = 0) uniform sampler2D f0_img[]; layout (set = 0, binding = 1) uniform sampler2D f1_img[]; layout (set = 0, binding = 2) uniform writeonly image2D out_img[]; -/* Forward (f0 -> f1) and backward (f1 -> f0) flow fields. These are bound for - * descriptor-set compatibility with the C code but are not consulted yet: this - * initial implementation performs a plain temporal blend. Motion-compensated - * warping using these fields is introduced in a subsequent change. */ +/* Forward (f0 -> f1) and backward (f1 -> f0) flow fields. The NV optical flow + * engine writes signed fixed point (S10.5) vectors, sampled here as raw + * integers; dividing by 32 (2^5) yields correct values. */ layout (set = 0, binding = 3) uniform isampler2D flow_fwd; layout (set = 0, binding = 4) uniform isampler2D flow_bwd; +#define FLOW_FIXED_POINT_SCALE (1.0 / 32.0) + +/* These Picard values were established empirically on a couple of different + * samples, but one could easily imagine reaching a different conclusion from + * different data. */ +#define PICARD_ITERS 6 +#define PICARD_OMEGA 0.5 + layout (push_constant, scalar) uniform pushConstants { float t; /* interpolation position in [0, 1] between f0 and f1 */ int planes; @@ -50,6 +57,12 @@ layout (push_constant, scalar) uniform pushConstants { #define luma_size (plane_size[0]) /* what the flow vectors are expressed against */ +/* Normalized forward/backward flow at a normalized position. The flow vectors + * are relative to the original Luma, so must be converted to a resolution independent + * displacement that can be applied to other planes with different dimensions. */ +vec2 flow_at_fwd(vec2 p) { return vec2(texture(flow_fwd, p).xy) * FLOW_FIXED_POINT_SCALE / luma_size; } +vec2 flow_at_bwd(vec2 p) { return vec2(texture(flow_bwd, p).xy) * FLOW_FIXED_POINT_SCALE / luma_size; } + /* Convert a coordinate normalized against a plane's visible extent into the * source texture's own normalized space. The identity unless that source image * is padded. */ @@ -71,8 +84,17 @@ void main() vec2 base = (vec2(pos) + 0.5) / size; - vec4 c0 = texture(f0_img[i], f0_uv(base, i)); - vec4 c1 = texture(f1_img[i], f1_uv(base, i)); + /* The flow is anchored on the f0/f1 grids, not the intermediate frame. + * Recover the source position on each grid by Picard iteration. */ + vec2 s0 = base; + vec2 s1 = base; + for (int k = 0; k < PICARD_ITERS; k++) { + s0 = mix(s0, base - t * flow_at_fwd(s0), PICARD_OMEGA); + s1 = mix(s1, base - (1.0 - t) * flow_at_bwd(s1), PICARD_OMEGA); + } + + vec4 c0 = texture(f0_img[i], f0_uv(s0, i)); + vec4 c1 = texture(f1_img[i], f1_uv(s1, i)); imageStore(out_img[i], pos, mix(c0, c1, t)); } -- To stop receiving notification emails like this one, please contact [email protected]. _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
