This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit 09bf8dab5b8f5c9d1c9280af4dec7f84e8c0fe8b Author: Raja-89 <[email protected]> AuthorDate: Sat Aug 1 09:20:22 2026 +0530 Commit: guoyejun <[email protected]> CommitDate: Sun Aug 9 02:13:24 2026 +0000 avfilter/dnn: implement zero-copy CUDA tensor mapping for Torch backend When a CUDA frame with a supported RGB sw_format is received, map the GPU device pointer directly into a LibTorch tensor using torch::from_blob() with CUDA device options, bypassing cudaMemcpy entirely. Changes: - Add fill_model_input_cuda() that extracts the CUdeviceptr from the AVHWFramesContext and wraps it in a PyTorch GPU tensor with a no-op deleter (memory owned by FFmpeg AVBuffer ref-counting). - Handle GPU memory alignment padding via custom strides derived from AVFrame linesize. - Permute the mapped NHWC tensor to NCHW (PyTorch convention). - Manage CUDA context explicitly with cuCtxPushCurrent/cuCtxPopCurrent for thread safety in async filter graphs. - Implement zero-copy Device-to-Device output mapping that writes the model result directly into the output frame's VRAM, bypassing ff_proc_from_dnn_to_frame / sws_scale entirely for CUDA frames. - Add torch::cuda::synchronize() to prevent async race conditions with downstream encoders (e.g. NVENC reading uninitialized frames). - Add format validation in ff_dnn_zero_copy_supported_cuda() to reject unsupported sw_formats and batching with CUDA zero-copy early. - Register AV_PIX_FMT_CUDA in vf_dnn_processing pixel format list. - Gracefully fall back to CPU path when sw_format is not a supported RGB variant or when CONFIG_CUDA is not enabled. Signed-off-by: Raja-89 <[email protected]> --- doc/filters.texi | 25 +++++- libavfilter/dnn/dnn_backend_torch.cpp | 159 ++++++++++++++++++++++++++++++++-- libavfilter/dnn_filter_common.c | 43 +++++++++ libavfilter/dnn_filter_common.h | 3 + libavfilter/vf_dnn_processing.c | 12 +++ 5 files changed, 233 insertions(+), 9 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index 985a1615a9..4cedd53b26 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -12216,7 +12216,7 @@ need to build and install the OpenVINO for C library (see be needed if the header files and libraries are not installed into system path) @item torch -Libtorch backend. To enable this backend you need to build and install Libtroch +Libtorch backend. To enable this backend you need to build and install Libtorch for C++ library. Please download cxx11 ABI version (see @url{https://pytorch.org/get-started/locally}) and configure FFmpeg with @code{--enable-libtorch @@ -12224,6 +12224,16 @@ and configure FFmpeg with @code{--enable-libtorch --extra-cflags=-I/libtorch_root/libtorch/include/torch/csrc/api/include --extra-ldflags=-L/libtorch_root/libtorch/lib/} +When FFmpeg is built with CUDA support and the input frames use +@code{AV_PIX_FMT_CUDA} with an underlying pixel format of @code{rgb0}, +@code{bgr0}, @code{rgb24}, or @code{bgr24}, the Torch backend maps GPU +frame buffers directly into LibTorch CUDA tensors using zero-copy memory +mapping. This avoids host-device PCIe transfers and can significantly +reduce inference latency. The zero-copy path is selected automatically +when the input format qualifies; no additional options are required. +When the input format is not @code{AV_PIX_FMT_CUDA}, the backend uses +the standard CPU tensor path. + @item onnx ONNX Runtime backend. To enable this backend you need to install the ONNX Runtime library (see @url{https://onnxruntime.ai/}) and configure @@ -12306,6 +12316,19 @@ please use tools/python/tf_sess_config.py to get the configs of TensorFlow backe ./ffmpeg -i 480p.jpg -vf format=yuv420p,dnn_processing=dnn_backend=tensorflow:model=espcn.pb:input=x:output=y:backend_configs=sess_config=0x10022805320e09cdccccccccccec3f20012a01303801 -y tmp.espcn.jpg @end example +@item +Perform zero-copy GPU inference with the Libtorch backend. Because there +is currently no GPU-side filter to convert from YUV to RGB, +@code{hwupload} is used to upload CPU RGB frames into device memory. +Once a GPU-side YUV-to-RGB filter becomes available, the @code{hwupload} +step will no longer be necessary and all frames can stay in device memory +throughout the pipeline: +@example +./ffmpeg -init_hw_device cuda=cuda -filter_hw_device cuda -i input.mp4 \ + -vf "format=rgb24,hwupload,dnn_processing=dnn_backend=torch:model=my_model.pt,hwdownload,format=rgb0" \ + -y output.mp4 +@end example + @end itemize @section drawbox diff --git a/libavfilter/dnn/dnn_backend_torch.cpp b/libavfilter/dnn/dnn_backend_torch.cpp index 705327f4b1..1ecf95e938 100644 --- a/libavfilter/dnn/dnn_backend_torch.cpp +++ b/libavfilter/dnn/dnn_backend_torch.cpp @@ -27,11 +27,18 @@ #include <torch/script.h> extern "C" { +#include "config.h" #include "dnn_io_proc.h" #include "dnn_backend_common.h" #include "libavutil/opt.h" #include "libavutil/mem.h" #include "libavutil/cpu.h" +#if CONFIG_CUDA +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_cuda.h" +#include "libavutil/hwcontext_cuda_internal.h" +#include "libavutil/pixfmt.h" +#endif #include "queue.h" #include "safe_queue.h" } @@ -160,6 +167,125 @@ static void deleter(void *arg) av_freep(&arg); } +#if CONFIG_CUDA +static void cuda_tensor_deleter(void *arg) +{ + /* No-op: GPU memory is owned by FFmpeg AVBuffer ref-counting. + * LibTorch must not free it. */ + (void)arg; +} + +/** + * Map a CUDA frame's GPU pointer directly into a LibTorch tensor, + * bypassing any host-device memory copy. + * + * The resulting tensor is a zero-copy view over the frame's VRAM + * buffer; the AVBuffer reference keeps the memory alive. + */ +static int fill_model_input_th_cuda(THModel *th_model, THRequestItem *request) +{ + THInferRequest *infer_request = request->infer_request; + LastLevelTaskItem *lltask = request->lltasks[0]; + TaskItem *task = lltask->task; + AVFrame *frame = task->in_frame; + + + int height = frame->height; + int width = frame->width; + /* linesize[0] is in bytes; for packed RGB/BGR it equals width * channels + * plus alignment padding. Use it as the stride so PyTorch respects the + * actual memory layout. */ + int stride_bytes = frame->linesize[0]; + int channels = stride_bytes / width; /* 3 for RGB24, 4 for RGB0/BGR0 */ + + /* Wrap the GPU device pointer in a LibTorch tensor (no copy). */ + torch::Tensor byte_tensor = torch::from_blob( + frame->data[0], + {1, height, width, channels}, + {(long)(height * stride_bytes), (long)stride_bytes, + (long)channels, 1L}, + cuda_tensor_deleter, + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCUDA)); + + /* Convert NHWC uint8 → NCHW float32 in [0, 1] and keep on GPU. */ + *infer_request->input_tensor = + byte_tensor.to(torch::kFloat32).div(255.0f) + .permute({0, 3, 1, 2}) /* NHWC → NCHW */ + .slice(1, 0, 3) /* drop alpha if present */ + .contiguous(); + + return 0; +} + +static void fill_model_output_th_cuda(THModel *th_model, TaskItem *task, torch::Tensor &out_slice) +{ + AVHWFramesContext *hw_frames_ctx = + (AVHWFramesContext *)task->out_frame->hw_frames_ctx->data; + + /* Determine channel layout from sw_format. */ + int hw_channels = 3; + int rgb_start = 0; + bool needs_flip = false; + switch (hw_frames_ctx->sw_format) { + case AV_PIX_FMT_RGB24: + hw_channels = 3; rgb_start = 0; needs_flip = false; + break; + case AV_PIX_FMT_BGR24: + hw_channels = 3; rgb_start = 0; needs_flip = true; + break; + case AV_PIX_FMT_RGB0: + hw_channels = 4; rgb_start = 0; needs_flip = false; + break; + case AV_PIX_FMT_BGR0: + hw_channels = 4; rgb_start = 0; needs_flip = true; + break; + case AV_PIX_FMT_0RGB: + hw_channels = 4; rgb_start = 1; needs_flip = false; + break; + case AV_PIX_FMT_0BGR: + hw_channels = 4; rgb_start = 1; needs_flip = true; + break; + default: + av_log(th_model->ctx, AV_LOG_ERROR, + "Unsupported sw_format for CUDA zero-copy output\n"); + hw_channels = 3; + break; + } + + /* Convert model output: NCHW float [0,1] → NHWC uint8 [0,255]. */ + torch::Tensor out_u8 = + out_slice.mul(255.0f) + .permute({0, 2, 3, 1}) + .to(torch::kUInt8) + .contiguous(); + if (needs_flip) + out_u8 = out_u8.flip({3}); + + int out_h = (int)out_u8.size(1); + int out_w = (int)out_u8.size(2); + + /* Map the output frame's VRAM into a tensor with correct + * stride (linesize includes alignment padding). */ + torch::Tensor out_frame_tensor = torch::from_blob( + task->out_frame->data[0], + {1, out_h, out_w, hw_channels}, + {(long)(out_h * task->out_frame->linesize[0]), + (long)task->out_frame->linesize[0], + (long)hw_channels, 1L}, + cuda_tensor_deleter, + torch::TensorOptions() + .dtype(torch::kUInt8) + .device(torch::kCUDA)); + + /* Device-to-Device copy into the correct channel slice. */ + out_frame_tensor.slice(3, rgb_start, rgb_start + 3) + .copy_(out_u8); + + /* Flush the CUDA stream before the encoder reads the frame. */ + torch::cuda::synchronize(); +} +#endif /* CONFIG_CUDA */ + static int fill_model_input_th(THModel *th_model, THRequestItem *request) { LastLevelTaskItem *lltask = NULL; @@ -317,16 +443,25 @@ static void infer_completion_callback(void *args) { switch (th_model->model.func_type) { case DFT_PROCESS_FRAME: if (task->do_ioproc) { - // Post process can only deal with CPU memory. - if (out_slice.device() != torch::kCPU) - out_slice = out_slice.to(torch::kCPU); - outputs.scale = 255; - outputs.data = out_slice.data_ptr(); - if (th_model->model.frame_post_proc != NULL) { - th_model->model.frame_post_proc(task->out_frame, &outputs, th_model->model.filter_ctx); +#if CONFIG_CUDA + if (task->out_frame->format == AV_PIX_FMT_CUDA) { + fill_model_output_th_cuda(th_model, task, out_slice); } else { - ff_proc_from_dnn_to_frame(task->out_frame, &outputs, th_model->ctx); +#endif + if (out_slice.device() != torch::kCPU) + out_slice = out_slice.to(torch::kCPU); + outputs.scale = 255; + outputs.data = out_slice.data_ptr(); + if (th_model->model.frame_post_proc != NULL) { + th_model->model.frame_post_proc(task->out_frame, &outputs, + th_model->model.filter_ctx); + } else { + ff_proc_from_dnn_to_frame(task->out_frame, &outputs, + th_model->ctx); + } +#if CONFIG_CUDA } +#endif } else { task->out_frame->width = outputs.dims[dnn_get_width_idx_by_layout(outputs.layout)]; task->out_frame->height = outputs.dims[dnn_get_height_idx_by_layout(outputs.layout)]; @@ -374,7 +509,15 @@ static int execute_model_th(THRequestItem *request, Queue *lltask_queue) task = lltask->task; th_model = (THModel *)task->model; +#if CONFIG_CUDA + if (task->in_frame->format == AV_PIX_FMT_CUDA) { + ret = fill_model_input_th_cuda(th_model, request); + } else { + ret = fill_model_input_th(th_model, request); + } +#else ret = fill_model_input_th(th_model, request); +#endif if (ret != 0) { goto err; } diff --git a/libavfilter/dnn_filter_common.c b/libavfilter/dnn_filter_common.c index 73c5e6b33c..1e2bbd776e 100644 --- a/libavfilter/dnn_filter_common.c +++ b/libavfilter/dnn_filter_common.c @@ -16,10 +16,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include "config.h" #include "dnn_filter_common.h" #include "libavutil/avstring.h" #include "libavutil/mem.h" #include "libavutil/opt.h" +#include "libavutil/hwcontext.h" #define MAX_SUPPORTED_OUTPUTS_NB 4 @@ -240,3 +242,44 @@ void ff_dnn_uninit(DnnContext *ctx) av_freep(&ctx->model_outputnames); } } + +#if CONFIG_CUDA +int ff_dnn_zero_copy_supported_cuda(DnnContext *ctx, const AVFilterLink *inlink) +{ + AVBufferRef *hw_frames_ref = avfilter_link_get_hw_frames_ctx((AVFilterLink *)inlink); + AVHWFramesContext *hw_frames_ctx; + + if (!hw_frames_ref) + return 0; + + hw_frames_ctx = (AVHWFramesContext *)hw_frames_ref->data; + + if (inlink->format == AV_PIX_FMT_CUDA) { + if (ctx->batch_size > 1) { + av_log(inlink->dst, AV_LOG_ERROR, "CUDA zero-copy currently does not support batching.\n"); + av_buffer_unref(&hw_frames_ref); + return AVERROR(EINVAL); + } + + if (ctx->backend_type == DNN_TH) { + switch (hw_frames_ctx->sw_format) { + case AV_PIX_FMT_RGB24: + case AV_PIX_FMT_BGR24: + case AV_PIX_FMT_RGB0: + case AV_PIX_FMT_0RGB: + case AV_PIX_FMT_BGR0: + case AV_PIX_FMT_0BGR: + break; + default: + av_log(inlink->dst, AV_LOG_ERROR, + "Zero-copy CUDA path currently only supports RGB24/BGR24 or RGB0/BGR0 variants.\n"); + av_buffer_unref(&hw_frames_ref); + return AVERROR(EINVAL); + } + } + } + + av_buffer_unref(&hw_frames_ref); + return 0; +} +#endif diff --git a/libavfilter/dnn_filter_common.h b/libavfilter/dnn_filter_common.h index 42a4719997..1e89ab42ae 100644 --- a/libavfilter/dnn_filter_common.h +++ b/libavfilter/dnn_filter_common.h @@ -62,5 +62,8 @@ int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFr DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame); int ff_dnn_flush(DnnContext *ctx); void ff_dnn_uninit(DnnContext *ctx); +#if CONFIG_CUDA +int ff_dnn_zero_copy_supported_cuda(DnnContext *ctx, const AVFilterLink *inlink); +#endif #endif diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c index 4f1c05dc63..16904a2c20 100644 --- a/libavfilter/vf_dnn_processing.c +++ b/libavfilter/vf_dnn_processing.c @@ -23,11 +23,13 @@ * implementing a generic image processing filter using deep learning networks. */ +#include "config.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" #include "libavutil/avassert.h" #include "libavutil/imgutils.h" #include "filters.h" +#include "formats.h" #include "dnn_filter_common.h" #include "video.h" #include "libswscale/swscale.h" @@ -73,6 +75,9 @@ static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_NV12, +#if CONFIG_CUDA + AV_PIX_FMT_CUDA, +#endif AV_PIX_FMT_NONE }; @@ -132,6 +137,13 @@ static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLin return AVERROR(EIO); } return 0; +#if CONFIG_CUDA + case AV_PIX_FMT_CUDA: + { + DnnProcessingContext *dnn_ctx = ctx->priv; + return ff_dnn_zero_copy_supported_cuda(&dnn_ctx->dnnctx, inlink); + } +#endif default: avpriv_report_missing_feature(ctx, "%s", av_get_pix_fmt_name(fmt)); return AVERROR(EIO); _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
