On Fri, 13 Nov 2015 19:44:24 -0500
Rick Kern <ker...@gmail.com> wrote:
> ---
> MAINTAINERS | 1 +
> configure | 14 +
> libavcodec/Makefile | 1 +
> libavcodec/allcodecs.c | 1 +
> libavcodec/vtenc.c | 1089
> ++++++++++++++++++++++++++++++++++++++++++++++++
> 5 files changed, 1106 insertions(+)
> create mode 100644 libavcodec/vtenc.c
I don't know nothing about VideotoolBox encoding or the libavcodec
encoding path, but I'll try to give a minimal review. I definitely
missed a bunch of things.
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 3735742..28782d2 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -286,6 +286,7 @@ Codecs:
> vp8 David Conrad, Jason Garrett-Glaser,
> Ronald Bultje
> vp9 Ronald Bultje, Clément Bœsch
> vqavideo.c Mike Melanson
> + vtenc.c Rick Kern
> wavpack.c Kostya Shishkov
> wmaprodec.c Sascha Sommer
> wmavoice.c Ronald S. Bultje
> diff --git a/configure b/configure
> index d5e76de..1cb56fc 100755
> --- a/configure
> +++ b/configure
> @@ -284,6 +284,7 @@ External library support:
> --disable-sdl disable sdl [autodetect]
> --disable-securetransport disable Secure Transport, needed for TLS support
> on OSX if openssl and gnutls are not used
> [autodetect]
> + --enable-vtenc enable VideoToolbox encoding support [no]
> --enable-x11grab enable X11 grabbing (legacy) [no]
> --disable-xlib disable xlib [autodetect]
> --disable-zlib disable zlib [autodetect]
> @@ -1470,6 +1471,7 @@ EXTERNAL_LIBRARY_LIST="
> openssl
> sdl
> securetransport
> + vtenc
> x11grab
> xlib
> zlib
> @@ -2606,6 +2608,7 @@ libzvbi_teletext_decoder_deps="libzvbi"
> nvenc_encoder_deps="nvenc"
> nvenc_h264_encoder_deps="nvenc"
> nvenc_hevc_encoder_deps="nvenc"
> +vtenc_h264_encoder_deps="vtenc"
>
> # demuxers / muxers
> ac3_demuxer_select="ac3_parser"
> @@ -5466,6 +5469,17 @@ enabled openssl && { check_lib openssl/ssl.h
> SSL_library_init -lssl -l
> check_lib openssl/ssl.h SSL_library_init
> -lssl32 -leay32 ||
> check_lib openssl/ssl.h SSL_library_init
> -lssl -lcrypto -lws2_32 -lgdi32 ||
> die "ERROR: openssl not found"; }
> +enabled vtenc && { { check_header VideoToolbox/VideoToolbox.h ||
> + die "ERROR: VideoToolbox/VideoToolbox.h not
> found."
> + } &&
> + { check_header "Availability.h" &&
> + { check_cpp_condition "Availability.h"
> "defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0" ||
> + check_cpp_condition "Availability.h"
> "defined(__MAC_10_9) && __MAC_OS_X_VERSION_MAX_ALLOWED >= __MAC_10_9"
> + } ||
> + die "VideoToolbox requires Mac OSX 10.9+
> or iOS 8.0+"
Wouldn't it be better and easier to just check for the API itself?
> + }
> + }
> +
> enabled qtkit_indev && { check_header_oc QTKit/QTKit.h || disable
> qtkit_indev; }
>
> if enabled gnutls; then
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 68a573f..afb38e4 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -116,6 +116,7 @@ OBJS-$(CONFIG_TEXTUREDSP) += texturedsp.o
> OBJS-$(CONFIG_TEXTUREDSPENC) += texturedspenc.o
> OBJS-$(CONFIG_TPELDSP) += tpeldsp.o
> OBJS-$(CONFIG_VIDEODSP) += videodsp.o
> +OBJS-$(CONFIG_VTENC) += vtenc.o
> OBJS-$(CONFIG_VP3DSP) += vp3dsp.o
> OBJS-$(CONFIG_VP56DSP) += vp56dsp.o
> OBJS-$(CONFIG_VP8DSP) += vp8dsp.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index 9f60d7c..6465df0 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -605,6 +605,7 @@ void avcodec_register_all(void)
> REGISTER_ENCODER(HEVC_QSV, hevc_qsv);
> REGISTER_ENCODER(LIBKVAZAAR, libkvazaar);
> REGISTER_ENCODER(MPEG2_QSV, mpeg2_qsv);
> + REGISTER_ENCODER(VTENC_H264, vtenc_h264);
>
> /* parsers */
> REGISTER_PARSER(AAC, aac);
> diff --git a/libavcodec/vtenc.c b/libavcodec/vtenc.c
> new file mode 100644
> index 0000000..06c5360
> --- /dev/null
> +++ b/libavcodec/vtenc.c
> @@ -0,0 +1,1089 @@
> +/*
> + * copyright (c) 2015 Rick Kern <ker...@gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
> USA
> + */
> +
> +#include <VideoToolbox/VideoToolbox.h>
> +#include <CoreVideo/CoreVideo.h>
> +#include <CoreMedia/CoreMedia.h>
> +#include <TargetConditionals.h>
> +#include <Availability.h>
> +#include "avcodec.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/avassert.h"
> +#include "libavutil/atomic.h"
> +#include "libavcodec/avcodec.h"
> +#include "internal.h"
> +#include "golomb.h"
> +#include <pthread.h>
> +
> +
> +const uint8_t start_code[] = {0, 0, 0, 1};
> +
> +typedef struct BufNode{
> + CMSampleBufferRef cm_buffer;
> + struct BufNode* next;
> + int error;
> +} BufNode;
> +
> +typedef struct VTEncContext{
> + AVClass* class;
> + VTCompressionSessionRef session;
> +
> + pthread_mutex_t lock;
> + pthread_cond_t cv_sample_sent;
> + int async_error;
> +
> + BufNode* q_head;
> + BufNode* q_tail;
> +
> + int64_t frame_ct_out;
> + int64_t frame_ct_in;
> +
> + int64_t first_pts;
> + int64_t dts_delta;
> +
> + char* profile;
> + char* level;
> +
> + bool flushing;
> + bool has_b_frames;
> + bool warned_color_range;
> +} VTEncContext;
> +
> +static void set_async_error(VTEncContext* vtctx, int err){
> + pthread_mutex_lock(&vtctx->lock);
> +
> + vtctx->async_error = err;
> +
> + BufNode* info = vtctx->q_head;
Apparently, our coding standard insists on not placing decorations
after statements (like C89 requires, though FFmpeg uses C99).
There are many more cases in this patch that need to be fixed.
> + vtctx->q_head = vtctx->q_tail = NULL;
> +
> + while(info){
> + CFRelease(info->cm_buffer);
> +
> + BufNode* next = info->next;
> + free(info);
> + info = next;
> + }
> +
> + pthread_mutex_unlock(&vtctx->lock);
> +}
> +
> +static int vtenc_q_pop(VTEncContext* vtctx, bool wait, CMSampleBufferRef*
> buf){
> + pthread_mutex_lock(&vtctx->lock);
> +
> + if(vtctx->async_error){
> + pthread_mutex_unlock(&vtctx->lock);
> + return vtctx->async_error;
> + }
> +
> + if(vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out){
> + *buf = NULL;
> +
> + pthread_mutex_unlock(&vtctx->lock);
> + return 0;
> + }
> +
> + if(!vtctx->q_head && wait){
> + pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
> + }
This should be a loop. Condition variables can unblock spuriously, and
it doesn't make sense to use them without checking the condition again.
> +
> + if(!vtctx->q_head){
> + pthread_mutex_unlock(&vtctx->lock);
> + *buf = NULL;
> + return 0;
> + }
> +
> + BufNode* info = vtctx->q_head;
> + vtctx->q_head = vtctx->q_head->next;
> + if(!vtctx->q_head){
> + vtctx->q_tail = NULL;
> + }
> +
> + pthread_mutex_unlock(&vtctx->lock);
> +
> + CMSampleBufferRef buffer = info->cm_buffer;
> + free(info);
> +
> + vtctx->frame_ct_out++;
> + *buf = buffer;
Why not assign *buf directly and go through the buffer variable?
> +
> + return 0;
> +}
> +
> +static void vtenc_q_push(VTEncContext* vtctx, CMSampleBufferRef buffer){
> + BufNode* info = (BufNode*)malloc(sizeof(BufNode));
> +
> + CFRetain(buffer);
> + info->cm_buffer = buffer;
> + info->next = NULL;
> +
> + pthread_mutex_lock(&vtctx->lock);
> + pthread_cond_signal(&vtctx->cv_sample_sent);
> +
> + if(!vtctx->q_head){
> + vtctx->q_head = vtctx->q_tail = info;
> + pthread_mutex_unlock(&vtctx->lock);
> + return;
Also seems a bit convoluted, can't the return and the second unlock be
avoided easily?
> + }
> +
> + vtctx->q_tail->next = info;
> + vtctx->q_tail = info;
> +
> + pthread_mutex_unlock(&vtctx->lock);
> +}
> +
> +static av_cold CMVideoCodecType get_cm_codec_type(enum AVCodecID id){
> + switch(id){
> + case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
> + default: return 0;
> + }
> +}
> +
> +static void vtenc_free_block(void* opaque, uint8_t* data){
> + CMBlockBufferRef block = (CMBlockBufferRef)opaque;
> + CFRelease(block);
> +}
> +
> +/**
> + * Get the parameter sets from a CMSampleBufferRef.
> + * @param dst If *dst isn't NULL, the parameters are copied into existing
> + * memory. *dst_size must be set accordingly when *dst != NULL.
> + * If *dst is NULL, it will be allocated.
> + * In all cases, *dst_size is set to the number of bytes used
> starting
> + * at *dst.
> + */
> +
> +static int get_params_info(
> + AVCodecContext* avctx,
> + CMVideoFormatDescriptionRef vid_fmt,
> + size_t* size)
> +{
> + size_t total_size = 0;
> + size_t ps_count;
> +
> + OSStatus status =
> CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL,
> &ps_count, NULL);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error getting parameter set count:
> %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + for(size_t i = 0; i < ps_count; i++){
> + const uint8_t* ps;
> + size_t ps_size;
> + status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
> i, &ps, &ps_size, NULL, NULL);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error getting parameter set size
> for index %zd: %d\n", i, status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + total_size += ps_size + sizeof(start_code);
> + }
> +
> + *size = total_size;
> + return 0;
> +}
> +
> +static int copy_param_sets(
> + AVCodecContext* avctx,
> + CMVideoFormatDescriptionRef vid_fmt,
> + uint8_t* dst,
> + size_t dst_size)
> +{
> + size_t ps_count;
> + OSStatus status =
> CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL,
> &ps_count, NULL);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error getting parameter set count for
> copying: %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + size_t offset = 0;
> + for(size_t i = 0; i < ps_count; i++){
> + const uint8_t* ps;
> + size_t ps_size;
> + status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
> i, &ps, &ps_size, NULL, NULL);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data
> for index %zd: %d\n", i, status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + size_t next_offset = offset + sizeof(start_code) + ps_size;
> + if(dst_size < next_offset){
> + av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for
> parameter sets.\n");
> + return AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + memcpy(dst + offset, start_code, sizeof(start_code));
> + offset += sizeof(start_code);
> +
> + memcpy(dst + offset, ps, ps_size);
> + offset = next_offset;
> + }
> +
> + return 0;
> +}
> +
> +static int set_extradata(AVCodecContext* avctx, CMSampleBufferRef
> sample_buffer){
> + CMVideoFormatDescriptionRef vid_fmt;
> + vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
> + if(!vid_fmt){
> + av_log(avctx, AV_LOG_ERROR, "No video format.\n");
> + return AVERROR_EXTERNAL;
> + }
> +
> + size_t total_size;
> + int status;
> + status = get_params_info(avctx, vid_fmt, &total_size);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
> + return status;
> + }
> +
> + avctx->extradata = malloc(total_size);
> + if(!avctx->extradata){
> + return AVERROR(ENOMEM);
> + }
> + avctx->extradata_size = total_size;
> +
> + status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
> + return status;
> + }
> +
> + return 0;
> +}
> +
> +static av_cold void vtenc_output_callback(
> + void* CM_NULLABLE ctx,
> + void* sourceFrameCtx,
> + OSStatus status,
> + VTEncodeInfoFlags flags,
> + CM_NULLABLE CMSampleBufferRef sample_buffer)
> +{
> + av_assert0(ctx);
> + AVCodecContext* avctx = (AVCodecContext*)ctx;
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + if(vtctx->async_error){
> + CFRelease(sample_buffer);
sample_buffer is marked as NULLABLE, and CFRelease(NULL) will crash.
Oversight?
> + return;
> + }
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", status);
> + set_async_error(vtctx, AVERROR_EXTERNAL);
> + return;
> + }
> +
> + if(!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)){
> + int set_status = set_extradata(avctx, sample_buffer);
> + if(set_status){
> + set_async_error(vtctx, set_status);
> + return;
> + }
> + }
> +
> + vtenc_q_push(vtctx, sample_buffer);
> +}
> +
> +static int get_length_code_size(
> + AVCodecContext* avctx,
> + CMSampleBufferRef sample_buffer,
> + size_t* size)
> +{
> + CMVideoFormatDescriptionRef vid_fmt;
> + vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
> + if(!vid_fmt){
> + av_log(avctx, AV_LOG_ERROR, "Error getting buffer format
> description.\n");
> + return AVERROR_EXTERNAL;
> + }
> +
> + int isize;
> + OSStatus status =
> CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt, 0, NULL, NULL,
> NULL, &isize);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n",
> status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + *size = isize;
> + return 0;
> +}
> +
> +static bool get_h264_profile(AVCodecContext* avctx, int* profile_num){
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + const char* profile = vtctx->profile;
> + if(!profile){
> + *profile_num = FF_PROFILE_UNKNOWN;
> + }
> + else if(!strcasecmp("baseline", profile)){
> + *profile_num = FF_PROFILE_H264_BASELINE;
> + }
> + else if(!strcasecmp("main", profile)){
> + *profile_num = FF_PROFILE_H264_MAIN;
> + }
> + else if(!strcasecmp("high", profile)){
> + *profile_num = FF_PROFILE_H264_HIGH;
> + }
> + else{
> + av_log(avctx, AV_LOG_ERROR, "Unknown profile '%s'\n", profile);
> + return false;
> + }
I wonder why we keep duplicating this across all h264 encoder wrappers,
but I guess it can't be avoided for now.
> +
> + return true;
> +}
> +
> +static bool get_h264_level(AVCodecContext* avctx, int* level_num){
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + if(!vtctx->level){
> + *level_num = FF_LEVEL_UNKNOWN;
> + return true;
> + }
> +
> + char* end;
> + unsigned long major = strtoul(vtctx->level, &end, 10);
> + unsigned long minor = 0;
> +
> + if(!major || (*end && *end != '.')){
> + av_log(avctx, AV_LOG_ERROR, "Error parsing level '%s'\n",
> vtctx->level);
> + return false;
> + }
> +
> + if(*end){
> + minor = strtoul(end + 1, &end, 10);
> + }
> +
> + if(*end){
> + av_log(avctx, AV_LOG_ERROR, "Error parsing level '%s'\n",
> vtctx->level);
> + return false;
> + }
> +
> + *level_num = (int)(major * 10 + minor);
> + return true;
> +}
Might be more robust to copy nvenc_h264_level_pairs[].
> +
> +/*
> + * Returns true on success.
> + *
> + * If profileLevelVal is NULL and this method returns true, don't specify the
> + * profile/level to the encoder.
> + */
> +static bool getVTProfileLevel(AVCodecContext* avctx, CFStringRef*
> profileLevelVal){
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + int profile;
> + int level;
> +
> + if(!get_h264_profile(avctx, &profile)){
> + return AVERROR(EINVAL);
> + }
> +
> + if(!get_h264_level(avctx, &level)){
> + return AVERROR(EINVAL);
> + }
> +
> + if(profile == FF_LEVEL_UNKNOWN &&
> + level != FF_LEVEL_UNKNOWN)
> + {
> + profile = vtctx->has_b_frames ? FF_PROFILE_H264_MAIN :
> FF_PROFILE_H264_BASELINE;
> + }
> +
> + switch(profile){
> + case FF_PROFILE_UNKNOWN:
> + *profileLevelVal = NULL;
> + return true;
> +
> + case FF_PROFILE_H264_BASELINE:
> + switch(level){
> + case FF_LEVEL_UNKNOWN: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_AutoLevel; break;
> + case 13: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_1_3; break;
> + case 30: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_3_0; break;
> + case 31: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_3_1; break;
> + case 32: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_3_2; break;
> + case 40: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_4_0; break;
> + case 41: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_4_1; break;
> + case 42: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_4_2; break;
> + case 50: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_5_0; break;
> + case 51: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_5_1; break;
> + case 52: *profileLevelVal =
> kVTProfileLevel_H264_Baseline_5_2; break;
> + default:
> + av_log(avctx, AV_LOG_ERROR, "Unrecognized level %s
> (%d)\n", vtctx->level, level);
> + return false;
> + }
> + return true;
> +
> + case FF_PROFILE_H264_MAIN:
> + switch(level){
> + case FF_LEVEL_UNKNOWN: *profileLevelVal =
> kVTProfileLevel_H264_Main_AutoLevel; break;
> + case 30: *profileLevelVal = kVTProfileLevel_H264_Main_3_0;
> break;
> + case 31: *profileLevelVal = kVTProfileLevel_H264_Main_3_1;
> break;
> + case 32: *profileLevelVal = kVTProfileLevel_H264_Main_3_2;
> break;
> + case 40: *profileLevelVal = kVTProfileLevel_H264_Main_4_0;
> break;
> + case 41: *profileLevelVal = kVTProfileLevel_H264_Main_4_1;
> break;
> + case 42: *profileLevelVal = kVTProfileLevel_H264_Main_4_2;
> break;
> + case 50: *profileLevelVal = kVTProfileLevel_H264_Main_5_0;
> break;
> + case 51: *profileLevelVal = kVTProfileLevel_H264_Main_5_1;
> break;
> + case 52: *profileLevelVal = kVTProfileLevel_H264_Main_5_2;
> break;
> + default:
> + av_log(avctx, AV_LOG_ERROR, "Unrecognized level %s
> (%d)\n", vtctx->level, level);
> + return false;
> + }
> + return true;
> +
> + case FF_PROFILE_H264_HIGH:
> + switch(level){
> + case FF_LEVEL_UNKNOWN: *profileLevelVal =
> kVTProfileLevel_H264_High_AutoLevel; break;
> + case 30: *profileLevelVal = kVTProfileLevel_H264_High_3_0;
> break;
> + case 31: *profileLevelVal = kVTProfileLevel_H264_High_3_1;
> break;
> + case 32: *profileLevelVal = kVTProfileLevel_H264_High_3_2;
> break;
> + case 40: *profileLevelVal = kVTProfileLevel_H264_High_4_0;
> break;
> + case 41: *profileLevelVal = kVTProfileLevel_H264_High_4_1;
> break;
> + case 42: *profileLevelVal = kVTProfileLevel_H264_High_4_2;
> break;
> + case 50: *profileLevelVal = kVTProfileLevel_H264_High_5_0;
> break;
> + case 51: *profileLevelVal = kVTProfileLevel_H264_High_5_1;
> break;
> + case 52: *profileLevelVal = kVTProfileLevel_H264_High_5_2;
> break;
> + default:
> + av_log(avctx, AV_LOG_ERROR, "Unrecognized level %s
> (%d)\n", vtctx->level, level);
> + return false;
> + }
> + return true;
> +
> + default:
> + av_log(avctx, AV_LOG_ERROR, "Unrecognized profile %s (%d)\n",
> vtctx->profile, profile);
> + return false;
> + }
> +}
> +
> +static av_cold int vtenc_init(AVCodecContext* avctx){
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> + CMVideoCodecType codec_type;
> + OSStatus status;
> +
> + codec_type = get_cm_codec_type(avctx->codec_id);
> + if(!codec_type){
> + av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n",
> avctx->codec_id);
> + return AVERROR(EINVAL);
> + }
> +
> + vtctx->has_b_frames = avctx->has_b_frames && avctx->max_b_frames > 0;
> +
> + CFStringRef profileLevel;
> + if(!getVTProfileLevel(avctx, &profileLevel)){
> + return AVERROR(EINVAL);
> + }
> +
> + vtctx->session = NULL;
> +
> + CFMutableDictionaryRef enc_info = CFDictionaryCreateMutable(
> + kCFAllocatorDefault,
> + 20,
> + &kCFCopyStringDictionaryKeyCallBacks,
> + &kCFTypeDictionaryValueCallBacks
> + );
No failure check?
> +
> + CFDictionarySetValue(enc_info,
> kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
> kCFBooleanTrue);
> +
> + status = VTCompressionSessionCreate(
> + kCFAllocatorDefault,
> + avctx->width,
> + avctx->height,
> + codec_type,
> + enc_info,
> + NULL,
> + kCFAllocatorDefault,
> + vtenc_output_callback,
> + avctx,
> + &vtctx->session
> + );
> +
> + if(status != 0 || !vtctx->session){
> + CFDictionaryRemoveValue(enc_info,
> kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder);
> +
> + status = VTCompressionSessionCreate(
> + kCFAllocatorDefault,
> + avctx->width,
> + avctx->height,
> + codec_type,
> + enc_info,
> + NULL,
> + kCFAllocatorDefault,
> + vtenc_output_callback,
> + avctx,
> + &vtctx->session
> + );
> + }
> +
> + if(status || !vtctx->session){
> + av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression
> session: %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + SInt32 bit_rate = avctx->bit_rate;
> + CFNumberRef bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
> kCFNumberSInt32Type, &bit_rate);
> + status = VTSessionSetProperty(vtctx->session,
> kVTCompressionPropertyKey_AverageBitRate, bit_rate_num);
> + CFRelease(bit_rate_num);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n",
> status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + if(profileLevel){
> + status = VTSessionSetProperty(vtctx->session,
> kVTCompressionPropertyKey_ProfileLevel, profileLevel);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error setting profile/level
> property: %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> + }
> +
> + if(avctx->gop_size > 0){
> + CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
> kCFNumberIntType, &avctx->gop_size);
> + status = VTSessionSetProperty(vtctx->session,
> kVTCompressionPropertyKey_MaxKeyFrameInterval, interval);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame
> interval' property: %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> + }
> +
> + if(!vtctx->has_b_frames){
> + status = VTSessionSetProperty(vtctx->session,
> kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame
> reordering' property: %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> + }
> +
> + status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n",
> status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + pthread_mutex_init(&vtctx->lock, NULL);
> + pthread_cond_init(&vtctx->cv_sample_sent, NULL);
> + vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
> +
> + return 0;
> +}
> +
> +static void vtenc_get_frame_info(
> + CMSampleBufferRef buffer,
> + bool* is_key_frame)
> +{
> + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(buffer,
> false);
> + CFIndex len = !attachments ? 0 : CFArrayGetCount(attachments);
> +
> + if(!len){
> + *is_key_frame = true;
> + return;
> + }
> +
> + CFDictionaryRef attachment =
> (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
> +
> + CFBooleanRef not_sync;
> + if(CFDictionaryGetValueIfPresent(attachment,
> kCMSampleAttachmentKey_NotSync, (const void**)¬_sync)){
> + *is_key_frame = !CFBooleanGetValue(not_sync);
> + }
> + else{
> + *is_key_frame = true;
> + }
> +}
> +
> +static int replace_length_codes(size_t length_code_size, uint8_t* data,
> size_t size){
> + if(length_code_size != sizeof(start_code)){
> + av_log(NULL, AV_LOG_ERROR, "lc != sizeof\n");
> + return AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + size_t remaining_size = size;
> + while(remaining_size > 0){
> + size_t box_len = 0;
> + for(size_t i = 0; i < length_code_size; i++){
> + box_len <<= 8;
> + box_len |= data[i];
> + }
> +
> + if(remaining_size < box_len + sizeof(start_code)){
> + av_log(NULL, AV_LOG_ERROR, "too small 2\n");
> + AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + memcpy(data, start_code, sizeof(start_code));
> + data += box_len + sizeof(start_code);
> + remaining_size -= box_len + sizeof(start_code);
> + }
> +
> + return 0;
> +}
> +
> +//expects zeroed memory
What is expected to be zeroed?
> +static int copy_replace_length_codes(
> + size_t length_code_size,
> + const uint8_t* src_data,
> + size_t src_size,
> + uint8_t* dst_data,
> + size_t dst_size)
> +{
> + if(length_code_size > 4){
> + return AVERROR_INVALIDDATA;
> + }
> +
> + size_t remaining_src_size = src_size;
> + size_t remaining_dst_size = dst_size;
> + while(remaining_src_size > 0){
> + size_t box_len = 0;
> + for(size_t i = 0; i < length_code_size; i++){
> + box_len <<= 8;
> + box_len |= src_data[i];
> + }
> +
> + size_t curr_src_len = box_len + length_code_size;
> + size_t curr_dst_len = box_len + sizeof(start_code);
> +
> + if(remaining_src_size < curr_src_len){
> + return AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + if(remaining_dst_size < curr_dst_len){
> + return AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + uint8_t* dst_box = dst_data + sizeof(start_code);
> + const uint8_t* src_box = src_data + length_code_size;
> +
> + memcpy(dst_data, start_code, sizeof(start_code));
> + memcpy(dst_box, src_box, box_len);
> +
> + src_data += curr_src_len;
> + dst_data += curr_dst_len;
> +
> + remaining_src_size -= curr_src_len;
> + remaining_dst_size -= curr_dst_len;
> + }
> +
> + return 0;
> +}
> +
> +static int vtenc_cm_to_avpacket(
> + AVCodecContext* avctx,
> + CMSampleBufferRef sample_buffer,
> + AVPacket* pkt)
> +{
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + bool is_key_frame;
> + bool add_header;
> + size_t length_code_size;
> + int status;
> +
> + vtenc_get_frame_info(sample_buffer, &is_key_frame);
> + status = get_length_code_size(avctx, sample_buffer, &length_code_size);
> + if(status) return status;
> +
> + add_header = is_key_frame && !(avctx->flags &
> AV_CODEC_FLAG_GLOBAL_HEADER);
> +
> + size_t header_size = 0;
> + CMVideoFormatDescriptionRef vid_fmt;
> + if(add_header){
> + vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
> + if(!vid_fmt){
> + av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
> + }
> +
> + int status = get_params_info(avctx, vid_fmt, &header_size);
> + if(status) return status;
> + }
> +
> + CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
> + if(!block){
> + av_log(avctx, AV_LOG_ERROR, "Could not get block buffer from sample
> buffer.\n");
> + return AVERROR_EXTERNAL;
> + }
> +
> + size_t in_buf_size;
> + char* buf_data;
> + status = CMBlockBufferGetDataPointer(block, 0, &in_buf_size, NULL,
> &buf_data);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error: cannot get data pointer: %d\n",
> status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + size_t out_buf_size = header_size + in_buf_size;
> + bool can_reuse_cmbuffer = !add_header &&
> + !pkt->data &&
> + length_code_size == sizeof(start_code);
> +
> + av_init_packet(pkt);
> +
> + if(can_reuse_cmbuffer){
> + CFRetain(block);
> +
> + AVBufferRef* buf_ref = av_buffer_create(
> + buf_data,
> + out_buf_size,
> + vtenc_free_block,
> + block,
> + 0
> + );
> +
> + if(!buf_ref){
> + CFRelease(block);
> + return AVERROR(ENOMEM);
> + }
> +
> + pkt->buf = buf_ref;
> + pkt->data = buf_data;
> + pkt->size = in_buf_size;
> +
> + status = replace_length_codes(length_code_size, pkt->data,
> pkt->size);
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error replacing length codes:
> %d\n", status);
> + return status;
> + }
> + }
> + else{
> + if(!pkt->data){
> + status = av_new_packet(pkt, out_buf_size);
> + if(status) return status;
> + }
> +
> + if(pkt->size < out_buf_size){
> + av_log(avctx, AV_LOG_ERROR, "Error: packet's buffer is too
> small.\n");
> + return AVERROR_BUFFER_TOO_SMALL;
> + }
> +
> + if(add_header){
> + status = copy_param_sets(avctx, vid_fmt, pkt->data,
> out_buf_size);
> + if(status) return status;
> + }
> +
> + status = copy_replace_length_codes(
> + length_code_size,
> + buf_data,
> + in_buf_size,
> + pkt->data + header_size,
> + pkt->size - header_size
> + );
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d",
> status);
> + return status;
> + }
> + }
> +
> + if(is_key_frame){
> + pkt->flags |= AV_PKT_FLAG_KEY;
> + }
> +
> + const CMTime pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
> + const CMTime dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
> +
> + int64_t dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
> + int64_t time_base_num = avctx->time_base.num;
> + pkt->pts = pts.value / time_base_num;
> + pkt->dts = dts.value / time_base_num - dts_delta;
> +
> + return 0;
> +}
> +
> +static int get_cv_pixel_info(
> + AVCodecContext* avctx,
> + const AVFrame* frame,
> + int* color,
> + int* plane_count,
> + size_t* widths,
> + size_t* heights,
> + size_t* strides)
> +{
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + int av_format = avctx->pix_fmt;
> + int av_color_range = avctx->color_range;
> +
> + switch(av_format){
> + case AV_PIX_FMT_NV12:
> + switch(av_color_range){
> + case AVCOL_RANGE_MPEG:
> + *color = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
> + break;
> +
> + case AVCOL_RANGE_JPEG:
> + *color = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
> + break;
> +
> + default:
> + if(!vtctx->warned_color_range){
> + vtctx->warned_color_range = true;
> + av_log(avctx, AV_LOG_WARNING, "Color range not set
> for NV12. Using MPEG range.\n");
> + }
> + *color = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
> + }
> +
> + *plane_count = 2;
> +
> + widths [0] = avctx->width;
> + heights[0] = avctx->height;
> + strides[0] = frame ? frame->linesize[0] : avctx->width;
> +
> + widths [1] = (avctx->width + 1) / 2;
> + heights[1] = (avctx->height + 1) / 2;
> + strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) &
> -2;
> + break;
> +
> + case AV_PIX_FMT_YUV420P:
> + switch(av_color_range){
> + case AVCOL_RANGE_MPEG:
> + *color = kCVPixelFormatType_420YpCbCr8Planar;
> + break;
> +
> + case AVCOL_RANGE_JPEG:
> + *color = kCVPixelFormatType_420YpCbCr8PlanarFullRange;
> + break;
> +
> + default:
> + if(!vtctx->warned_color_range){
> + vtctx->warned_color_range = true;
> + av_log(avctx, AV_LOG_WARNING, "Color range not set
> for YUV 4:2:0. Using MPEG range.\n");
> + }
> + *color = kCVPixelFormatType_420YpCbCr8Planar;
> + }
> +
> + *plane_count = 3;
> +
> + widths [0] = avctx->width;
> + heights[0] = avctx->height;
> + strides[0] = frame ? frame->linesize[0] : avctx->width;
> +
> + widths [1] = (avctx->width + 1) / 2;
> + heights[1] = (avctx->height + 1) / 2;
> + strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
> +
> + widths [2] = (avctx->width + 1) / 2;
> + heights[2] = (avctx->height + 1) / 2;
> + strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
> + break;
> +
> + case AV_PIX_FMT_YUVJ420P:
> + *color = kCVPixelFormatType_420YpCbCr8PlanarFullRange;
> + *plane_count = 3;
> +
> + widths [0] = avctx->width;
> + heights[0] = avctx->height;
> + strides[0] = frame ? frame->linesize[0] : avctx->width;
> +
> + widths [1] = (avctx->width + 1) / 2;
> + heights[1] = (avctx->height + 1) / 2;
> + strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
> +
> + widths [2] = (avctx->width + 1) / 2;
> + heights[2] = (avctx->height + 1) / 2;
> + strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
> + break;
> +
> + default: return AVERROR(EINVAL);
> + }
> +
> + return 0;
> +}
> +
> +static int vtenc_send_frame(AVCodecContext* avctx, VTEncContext* vtctx,
> const AVFrame* frame){
> + av_assert0(frame);
> +
> + int plane_count;
> + int color;
> + size_t widths [AV_NUM_DATA_POINTERS];
> + size_t heights[AV_NUM_DATA_POINTERS];
> + size_t strides[AV_NUM_DATA_POINTERS];
> + int status = get_cv_pixel_info(avctx, frame, &color, &plane_count,
> widths, heights, strides);
> + if(status){
> + av_log(
> + avctx,
> + AV_LOG_ERROR,
> + "Error: Cannot convert format %d color_range %d: %d\n",
> + frame->format,
> + frame->color_range,
> + status
> + );
> + return AVERROR_EXTERNAL;
> + }
> +
> + CVPixelBufferRef cv_img;
> + status = CVPixelBufferCreateWithPlanarBytes(
> + kCFAllocatorDefault,
> + frame->width,
> + frame->height,
> + color,
> + NULL,
> + 0,
> + plane_count,
> + (void**)frame->data,
> + widths,
> + heights,
> + strides,
> + NULL,
> + NULL,
> + NULL,
> + &cv_img
> + );
Does it copy the AVFrame data?
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error: cannot create CVPixelBufferRef:
> %d\n", status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + CMTime time = CMTimeMake(frame->pts * avctx->time_base.num,
> avctx->time_base.den);
> + VTEncodeInfoFlags flags = 0;
> + status = VTCompressionSessionEncodeFrame(
> + vtctx->session,
> + cv_img,
> + time,
> + kCMTimeInvalid,
> + NULL,
> + NULL,
> + &flags
> + );
> +
> + CFRelease(cv_img);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n",
> status);
> + return AVERROR_EXTERNAL;
> + }
> +
> + return 0;
> +}
> +
> +static av_cold int vtenc_frame(
> + AVCodecContext* avctx,
> + AVPacket* pkt,
> + const AVFrame* frame,
> + int* got_packet)
> +{
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> + int status;
> +
> + if(frame){
> + status = vtenc_send_frame(avctx, vtctx, frame);
> +
> + if(status){
> + status = AVERROR_EXTERNAL;
> + goto end_nopkt;
> + }
> +
> + if(vtctx->frame_ct_in == 0){
> + vtctx->first_pts = frame->pts;
> + }
> + else if(vtctx->frame_ct_in == 1 && vtctx->has_b_frames){
> + vtctx->dts_delta = frame->pts - vtctx->first_pts;
> + }
> +
> + vtctx->frame_ct_in++;
> + }
> + else if(!vtctx->flushing){
> + vtctx->flushing = true;
> +
> + status = VTCompressionSessionCompleteFrames(vtctx->session,
> kCMTimeIndefinite);
> +
> + if(status){
> + av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n",
> status);
> + status = AVERROR_EXTERNAL;
> + goto end_nopkt;
> + }
> + }
> +
> + *got_packet = 0;
> + bool get_frame = vtctx->dts_delta >= 0 || !frame;
> + if(!get_frame){
> + status = 0;
> + goto end_nopkt;
> + }
> +
> + CMSampleBufferRef buf = NULL;
> + status = vtenc_q_pop(vtctx, !frame, &buf);
> + if(status) goto end_nopkt;
> + if(!buf) goto end_nopkt;
> +
> + status = vtenc_cm_to_avpacket(avctx, buf, pkt);
> + CFRelease(buf);
> + if(status) goto end_nopkt;
> +
> + *got_packet = 1;
> + return 0;
> +
> +end_nopkt:
> + av_packet_unref(pkt);
> + return status;
> +}
> +
> +static av_cold int vtenc_close(AVCodecContext* avctx){
> + VTEncContext* vtctx = (VTEncContext*)avctx->priv_data;
> +
> + if(!vtctx->session) return 0;
> +
> + VTCompressionSessionInvalidate(vtctx->session);
> + pthread_cond_destroy(&vtctx->cv_sample_sent);
> + pthread_mutex_destroy(&vtctx->lock);
> + CFRelease(vtctx->session);
> + vtctx->session = NULL;
> +
> + return 0;
> +}
> +
> +static const enum AVPixelFormat pix_fmts[] = {
> + AV_PIX_FMT_NV12,
> + AV_PIX_FMT_YUV420P,
> + AV_PIX_FMT_YUVJ420P,
> + AV_PIX_FMT_NONE
> +};
> +
> +#define OFFSET(x) offsetof(VTEncContext, x)
> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
> +static const AVOption options[] = {
> + {"profile", "Profile", OFFSET(profile), AV_OPT_TYPE_STRING, { 0 },
> 0, 0, VE},
> + {"level", "Level", OFFSET(level), AV_OPT_TYPE_STRING, {.str=NULL},
> 0, 0, VE},
> + {NULL},
> +};
> +
> +static const AVClass vtenc_h264_class = {
> + .class_name = "vtenc_h264",
> + .item_name = av_default_item_name,
> + .option = options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
> +
> +AVCodec ff_vtenc_h264_encoder = {
> + .name = "vtenc_h264",
> + .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
> + .type = AVMEDIA_TYPE_VIDEO,
> + .id = AV_CODEC_ID_H264,
> + .priv_data_size = sizeof(VTEncContext),
> + .pix_fmts = pix_fmts,
> + .init = vtenc_init,
> + .encode2 = vtenc_frame,
> + .close = vtenc_close,
> + .capabilities = AV_CODEC_CAP_DELAY,
> + .priv_class = &vtenc_h264_class,
> + .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
> + FF_CODEC_CAP_INIT_CLEANUP,
> +};
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
