On 12/09/2024 15:20, Benjamin Cheng wrote:
On Wed Sep 11, 2024 at 12:03 AM EDT, Lynne wrote:On 10/09/2024 15:29, Benjamin Cheng wrote:On Mon Sep 9, 2024 at 6:37 AM EDT, Lynne via ffmpeg-devel wrote:This commit adds the common Vulkan video encoding framework. It makes full use of the asynchronous features of our new common hardware encoding code, and of Vulkan. The code is able to handle anything from H264 to AV1 and MJPEG. --- configure | 2 + libavcodec/Makefile | 2 +- libavcodec/vulkan_encode.c | 979 +++++++++++++++++++++++++++++++++++++ libavcodec/vulkan_encode.h | 243 +++++++++ 4 files changed, 1225 insertions(+), 1 deletion(-) create mode 100644 libavcodec/vulkan_encode.c create mode 100644 libavcodec/vulkan_encode.hdiff --git a/configure b/configure index a8e67d230c..6cfb736a86 100755 --- a/configure +++ b/configure @@ -2638,6 +2638,7 @@ CONFIG_EXTRA=" vp3dsp vp56dsp vp8dsp + vulkan_encode wma_freqs wmv2dsp " @@ -3299,6 +3300,7 @@ qsvdec_select="qsv" qsvenc_select="qsv" qsvvpp_select="qsv" vaapi_encode_deps="vaapi" +vulkan_encode_deps="vulkan" v4l2_m2m_deps="linux_videodev2_h sem_timedwait"bilateral_cuda_filter_deps="ffnvcodec"diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 27ef4638ce..ff6a3c4efc 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1282,7 +1282,7 @@ SKIPHEADERS-$(CONFIG_QSVENC) += qsvenc.h SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h -SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_decode.h +SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_encode.h vulkan_decode.h SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.hdiff --git a/libavcodec/vulkan_encode.c b/libavcodec/vulkan_encode.cnew file mode 100644 index 0000000000..5e87d4c073 --- /dev/null +++ b/libavcodec/vulkan_encode.c @@ -0,0 +1,979 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/mem.h" +#include "libavutil/avassert.h" +#include "vulkan_encode.h" +#include "config.h" + +#include "libavutil/vulkan_loader.h" + +const AVCodecHWConfigInternal *const ff_vulkan_encode_hw_configs[] = { + HW_CONFIG_ENCODER_FRAMES(VULKAN, VULKAN), + NULL, +}; + +av_cold void ff_vulkan_encode_uninit(FFVulkanEncodeContext *ctx) +{ + FFVulkanContext *s = &ctx->s; + FFVulkanFunctions *vk = &s->vkfn; + + /* Wait on and free execution pool */ + ff_vk_exec_pool_free(s, &ctx->enc_pool); + + /* Destroy the session params */ + if (ctx->session_params) + vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev, + ctx->session_params, + s->hwctx->alloc); + + ff_hw_base_encode_close(&ctx->base); + + av_buffer_pool_uninit(&ctx->buf_pool); + + ff_vk_video_common_uninit(s, &ctx->common); + + ff_vk_uninit(s); +} + +static int vulkan_encode_init(AVCodecContext *avctx, FFHWBaseEncodePicture *pic) +{ + int err; + FFVulkanEncodeContext *ctx = avctx->priv_data; + FFVulkanEncodePicture *vp = pic->priv; + + AVFrame *f = pic->input_image; + AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; + AVVulkanFramesContext *vkfc = hwfc->hwctx; + AVVkFrame *vkf = (AVVkFrame *)f->data[0]; + + if (ctx->codec->picture_priv_data_size > 0) { + pic->codec_priv = av_mallocz(ctx->codec->picture_priv_data_size); + if (!pic->codec_priv) + return AVERROR(ENOMEM); + } + + /* Input image view */ + err = ff_vk_create_view(&ctx->s, &ctx->common, + &vp->in.view, &vp->in.aspect, + vkf, vkfc->format[0], 0); + if (err < 0) + return err; + + /* Reference view */ + if (!ctx->common.layered_dpb) { + AVFrame *rf = pic->recon_image; + AVVkFrame *rvkf = (AVVkFrame *)rf->data[0]; + err = ff_vk_create_view(&ctx->s, &ctx->common, + &vp->dpb.view, &vp->dpb.aspect, + rvkf, ctx->pic_format, 1); + if (err < 0) + return err; + } else { + vp->dpb.view = ctx->common.layered_view; + vp->dpb.aspect = ctx->common.layered_aspect; + } + + return 0; +} + +static int vulkan_encode_free(AVCodecContext *avctx, FFHWBaseEncodePicture *pic) +{ + FFVulkanEncodeContext *ctx = avctx->priv_data; + FFVulkanFunctions *vk = &ctx->s.vkfn; + + FFVulkanEncodePicture *vp = pic->priv; + + if (vp->in.view) + vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->in.view, + ctx->s.hwctx->alloc); + + if (!ctx->common.layered_dpb && vp->dpb.view) + vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->dpb.view, + ctx->s.hwctx->alloc); + + ctx->slots[vp->dpb_slot.slotIndex] = 0; + + return 0; +} + +static int init_pic_rc(AVCodecContext *avctx, FFHWBaseEncodePicture *pic, + VkVideoEncodeRateControlInfoKHR *rc_info, + VkVideoEncodeRateControlLayerInfoKHR *rc_layer /* Goes in ^ */) +{ + FFVulkanEncodeContext *ctx = avctx->priv_data; + + *rc_info = (VkVideoEncodeRateControlInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_INFO_KHR, + .rateControlMode = ctx->opts.rc_mode, + }; + + if (ctx->opts.rc_mode > VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) { + *rc_layer = (VkVideoEncodeRateControlLayerInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_LAYER_INFO_KHR, + .averageBitrate = avctx->bit_rate, + .maxBitrate = avctx->rc_max_rate ? avctx->rc_max_rate : avctx->bit_rate, + .frameRateNumerator = avctx->framerate.num, + .frameRateDenominator = avctx->framerate.den, + }; + rc_info->layerCount++; + rc_info->pLayers = rc_layer; + } + + return ctx->codec->init_pic_rc(avctx, pic, rc_info, rc_layer); +} + +static int vulkan_encode_issue(AVCodecContext *avctx, + FFHWBaseEncodePicture *base_pic) +{ + FFVulkanEncodeContext *ctx = avctx->priv_data; + FFVulkanFunctions *vk = &ctx->s.vkfn; + + const size_t size_align = ctx->caps.minBitstreamBufferSizeAlignment; + + FFVulkanEncodePicture *vp = base_pic->priv; + AVFrame *src = (AVFrame *)base_pic->input_image; + AVVkFrame *vkf = (AVVkFrame *)src->data[0]; + + int err, max_pkt_size; + + FFVkBuffer *sd_buf; + + int slot_index = -1; + FFVkExecContext *exec; + VkCommandBuffer cmd_buf; + VkImageMemoryBarrier2 img_bar[37]; + int nb_img_bar = 0; + + /* Coding start/end */ + VkVideoBeginCodingInfoKHR encode_start; + VkVideoEndCodingInfoKHR encode_end = { + .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR, + }; + + VkVideoEncodeRateControlLayerInfoKHR rc_layer; + VkVideoEncodeRateControlInfoKHR rc_info; + VkVideoEncodeQualityLevelInfoKHR q_info; + VkVideoCodingControlInfoKHR encode_ctrl; + + VkVideoReferenceSlotInfoKHR ref_slot[37]; + VkVideoEncodeInfoKHR encode_info; + + /* Create packet data buffer */ + max_pkt_size = FFALIGN(3 * ctx->base.surface_width * ctx->base.surface_height + (1 << 16), + ctx->caps.minBitstreamBufferSizeAlignment); + + err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &vp->pkt_buf, + VK_BUFFER_USAGE_VIDEO_ENCODE_DST_BIT_KHR, + &ctx->profile_list, max_pkt_size, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + if (err < 0) + return err; + + sd_buf = (FFVkBuffer *)vp->pkt_buf->data; + + /* Setup rate control */ + err = init_pic_rc(avctx, base_pic, &rc_info, &rc_layer); + if (err < 0) + return err; + + q_info = (VkVideoEncodeQualityLevelInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR, + .pNext = &rc_info, + .qualityLevel = ctx->opts.quality, + }; + encode_ctrl = (VkVideoCodingControlInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR, + .pNext = &q_info, + .flags = VK_VIDEO_CODING_CONTROL_ENCODE_QUALITY_LEVEL_BIT_KHR | + VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_BIT_KHR | + (base_pic->force_idr ? VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR : 0), + }; + + /* Current picture's ref slot */ + vp->dpb_res = (VkVideoPictureResourceInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR, + .pNext = NULL, + .codedOffset = { 0 }, + .codedExtent = (VkExtent2D){ ctx->base.surface_width, + ctx->base.surface_height }, + .baseArrayLayer = 0, + .imageViewBinding = vp->dpb.view, + }; + + for (int i = 0; i < ctx->caps.maxDpbSlots; i++) { + if (!ctx->slots[i]) { + slot_index = i; + ctx->slots[i] = 1; + break; + } + } + av_assert0(slot_index >= 0); + + vp->dpb_slot = (VkVideoReferenceSlotInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR, + .pNext = NULL, // Set later + .slotIndex = slot_index, + .pPictureResource = &vp->dpb_res, + }; + + encode_info = (VkVideoEncodeInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR, + .pNext = NULL, // Set later + .flags = 0x0, + .srcPictureResource = (VkVideoPictureResourceInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR, + .pNext = NULL, + .codedOffset = { 0, 0 }, + .codedExtent = (VkExtent2D){ base_pic->input_image->width, + base_pic->input_image->height }, + .baseArrayLayer = 0, + .imageViewBinding = vp->in.view, + }, + .pSetupReferenceSlot = &vp->dpb_slot, + .referenceSlotCount = 0, + .pReferenceSlots = ref_slot, + .dstBuffer = sd_buf->buf, + .dstBufferOffset = 0, + .dstBufferRange = sd_buf->size, + .precedingExternallyEncodedBytes = 0, + }; + + for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) { + for (int j = 0; j < base_pic->nb_refs[i]; j++) { + FFHWBaseEncodePicture *ref = base_pic->refs[i][j]; + FFVulkanEncodePicture *rvp = ref->priv; + ref_slot[encode_info.referenceSlotCount++] = rvp->dpb_slot; + } + } + + /* Setup picture parameters */ + err = ctx->codec->init_pic_params(avctx, base_pic, + &encode_info); + if (err < 0) + return err; + + encode_start = (VkVideoBeginCodingInfoKHR) { + .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR, + .pNext = !base_pic->force_idr ? &rc_info : NULL, + .videoSession = ctx->common.session, + .videoSessionParameters = ctx->session_params, + .referenceSlotCount = encode_info.referenceSlotCount, + .pReferenceSlots = ref_slot, + }; + + /* Calling vkCmdBeginVideoCodingKHR requires to declare all references + * being enabled upfront, including the current frame's output ref. + * If layered DBPs are used, make sure its not included twice. */ + if (!ctx->common.layered_dpb || !encode_info.referenceSlotCount) { + ref_slot[encode_info.referenceSlotCount] = vp->dpb_slot; + ref_slot[encode_info.referenceSlotCount].slotIndex = -1; + encode_start.referenceSlotCount++; + } + + /* Write header */ + if (base_pic->type == FF_HW_PICTURE_TYPE_IDR) { + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset; + size_t data_size = encode_info.dstBufferRange; + err = ctx->codec->write_sequence_headers(avctx, base_pic, hdr_dst, &data_size); + if (err < 0) + goto fail; + encode_info.dstBufferOffset += data_size; + encode_info.dstBufferRange -= data_size; + } + + /* Write extra units */ + if (ctx->codec->write_extra_headers) { + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset; + size_t data_size = encode_info.dstBufferRange; + err = ctx->codec->write_extra_headers(avctx, base_pic, hdr_dst, &data_size); + if (err < 0) + goto fail; + encode_info.dstBufferOffset += data_size; + encode_info.dstBufferRange -= data_size; + } + + /* Align buffer offset to the required value with filler units */ + if (ctx->codec->write_filler) { + uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset; + size_t data_size = encode_info.dstBufferRange; + + uint32_t offset = encode_info.dstBufferOffset; + size_t offset_align = ctx->caps.minBitstreamBufferOffsetAlignment; + + uint32_t filler_data = FFALIGN(offset, offset_align) - offset; + + if (filler_data) { + while (filler_data < ctx->codec->filler_header_size) + filler_data += offset_align; + + filler_data -= ctx->codec->filler_header_size; + + err = ctx->codec->write_filler(avctx, filler_data, + hdr_dst, &data_size); + if (err < 0) + goto fail; + + encode_info.dstBufferOffset += data_size; + encode_info.dstBufferRange -= data_size; + } + } + + vp->slices_offset = encode_info.dstBufferOffset; + + /* Align buffer size to the nearest lower alignment requirement. */ + encode_info.dstBufferRange -= size_align; + encode_info.dstBufferRange = FFALIGN(encode_info.dstBufferRange, + size_align); + + /* Start command buffer recording */ + exec = vp->exec = ff_vk_exec_get(&ctx->enc_pool); + ff_vk_exec_start(&ctx->s, exec); + cmd_buf = exec->buf; + + /* Output packet buffer */ + err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->pkt_buf, 1, 1); + if (err < 0) + goto fail; + + /* Source image */ + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, src, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + if (err < 0) + goto fail; + + /* Source image layout conversion */ + img_bar[nb_img_bar] = (VkImageMemoryBarrier2) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = NULL, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = vkf->access[0], + .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR, + .dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR, + .oldLayout = vkf->layout[0], + .newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR, + .srcQueueFamilyIndex = vkf->queue_family[0], + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vkf->img[0], + .subresourceRange = (VkImageSubresourceRange) { + .aspectMask = vp->in.aspect, + .layerCount = 1, + .levelCount = 1, + }, + }; + ff_vk_exec_update_frame(&ctx->s, exec, src, + &img_bar[nb_img_bar], &nb_img_bar); + + if (!ctx->common.layered_dpb) { + /* Source image's ref slot. + * No need to do a layout conversion, since the frames which are allocated + * with a DPB usage are automatically converted. */ + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, base_pic->recon_image, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + if (err < 0) + return err; + + /* All references */ + for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) { + for (int j = 0; j < base_pic->nb_refs[i]; j++) { + FFHWBaseEncodePicture *ref = base_pic->refs[i][j]; + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref->recon_image, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + if (err < 0) + return err; + } + } + } else { + err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + if (err < 0) + return err; + } + + /* Change image layout */ + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pImageMemoryBarriers = img_bar, + .imageMemoryBarrierCount = nb_img_bar, + }); + + /* Start, use parameters */ + vk->CmdBeginVideoCodingKHR(cmd_buf, &encode_start); + + /* Send control data */ + if (base_pic->force_idr) + vk->CmdControlVideoCodingKHR(cmd_buf, &encode_ctrl); + + /* Encode */ + vk->CmdBeginQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0, 0); + vk->CmdEncodeVideoKHR(cmd_buf, &encode_info); + vk->CmdEndQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0); + + /* End encoding */ + vk->CmdEndVideoCodingKHR(cmd_buf, &encode_end); + + /* End recording and submit for execution */ + err = ff_vk_exec_submit(&ctx->s, vp->exec); + if (err < 0) + goto fail; + + /* We don't need to keep the input image any longer, its already ref'd */ + av_frame_free(&base_pic->input_image); + + return 0; + +fail: + return err; +} + +static void vulkan_encode_wait(AVCodecContext *avctx, + FFHWBaseEncodePicture *base_pic) +{ + FFVulkanEncodeContext *ctx = avctx->priv_data; + FFVulkanEncodePicture *vp = base_pic->priv; + + av_assert0(base_pic->encode_issued); + + if (base_pic->encode_complete) + return; + + ff_vk_exec_wait(&ctx->s, vp->exec); + base_pic->encode_complete = 1; +} + +static int vulkan_encode_output(AVCodecContext *avctx, + FFHWBaseEncodePicture *base_pic, AVPacket *pkt) +{ + VkResult ret; + FFVulkanEncodePicture *vp = base_pic->priv; + FFVulkanEncodeContext *ctx = avctx->priv_data; + FFVkBuffer *sd_buf = (FFVkBuffer *)vp->pkt_buf->data; + uint32_t *query_data; + + vulkan_encode_wait(avctx, base_pic); + + ret = ff_vk_exec_get_query(&ctx->s, vp->exec, (void **)&query_data, 0); + if (ret == VK_NOT_READY) { + av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n", + ff_vk_ret2str(ret)); + return AVERROR(EINVAL); + } + + if (ret != VK_NOT_READY && ret != VK_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n", + ff_vk_ret2str(ret)); + return AVERROR_EXTERNAL; + } + if (query_data[2] != VK_QUERY_RESULT_STATUS_COMPLETE_KHR) { + av_log(avctx, AV_LOG_ERROR, "Unable to encode: %u\n", query_data[2]); + return AVERROR_EXTERNAL; + }The query pool is created with ctx->enc_caps.supportedEncodeFeedbackFlags. If the implementation reports all feedback flags defined by the spec, query_data[2] would refer to the HAS_OVERRIDES field of the feedback. In this case, the status would live in query_data[3].v2 sent to the ML which fixes both issues you mentioned, plus other bugs I and others encountered. The spec text says: > When retrieving the results of video encode feedback queries, the > values corresponding to each enabled video encode feedback are written > in the order of the bits defined above The order they're given in is: VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BUFFER_OFFSET_BIT_KHR = 0x00000001, VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_BYTES_WRITTEN_BIT_KHR = 0x00000002, VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_HAS_OVERRIDES_BIT_KHR = 0x00000004,Yes, so if all these feedback bits are reported as supported the query feedback buffer would look like:- query_data[0]: BUFFER_OFFSET- query_data[1]: BYTES_WRITTEN - query_data[2]: HAS_OVERRIDES Then, the result status appears at query_data[3] (if VK_QUERY_RESULT_WITH_STATUS_BIT_KHR is set).So I think the code is correct on this. Thanks for reviewing/testing
Fixed this in my branch, thanks
OpenPGP_0xA2FEA5F03F034464.asc
Description: OpenPGP public key
OpenPGP_signature.asc
Description: OpenPGP digital signature
_______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
