---
libavfilter/vf_libvmaf.c | 328 ++++++++++++++++++++++++++++++++++++++-
1 file changed, 326 insertions(+), 2 deletions(-)
diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c
index f655092b20..e6707aff53 100644
--- a/libavfilter/vf_libvmaf.c
+++ b/libavfilter/vf_libvmaf.c
@@ -27,8 +27,11 @@
#include "config_components.h"
#include <libvmaf.h>
+#include <libvmaf/version.h>
#include "libavutil/avstring.h"
+#include "libavutil/dict.h"
+#include "libavutil/frame.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/pixdesc.h"
@@ -46,6 +49,31 @@
#include "libavutil/hwcontext_cuda_internal.h"
#endif
+#define VMAF_VERSION_INT_VER(major, minor, patch) \
+ ((major) * 10000 + (minor) * 100 + (patch))
+
+#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR,
VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(3, 0, 0)
+#define CONFIG_LIBVMAF_METADATA_ENABLED 1
+#else
+#define CONFIG_LIBVMAF_METADATA_ENABLED 0
+#endif
+
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+#include <stdatomic.h>
+
+typedef struct FrameList {
+ AVFrame *frame;
+ unsigned frame_number;
+ unsigned propagated_handlers_cnt;
+ struct FrameList *next;
+} FrameList;
+
+typedef struct CallbackStruct {
+ struct LIBVMAFContext *s;
+ FrameList *frame_list;
+} CallbackStruct;
+#endif
+
typedef struct LIBVMAFContext {
const AVClass *class;
FFFrameSync fs;
@@ -56,8 +84,19 @@ typedef struct LIBVMAFContext {
int n_subsample;
char *model_cfg;
char *feature_cfg;
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ char *metadata_feature_cfg;
+ struct {
+ VmafMetadataConfiguration *metadata_cfgs;
+ unsigned metadata_cfg_cnt;
+ } metadata_cfg_list;
+ CallbackStruct *cb;
+ atomic_uint outlink_eof;
+ atomic_uint eof_frame;
+#endif
VmafContext *vmaf;
VmafModel **model;
+ int flushed;
unsigned model_cnt;
unsigned frame_cnt;
unsigned bpc;
@@ -77,6 +116,9 @@ static const AVOption libvmaf_options[] = {
{"n_subsample", "Set interval for frame subsampling used when computing
vmaf.", OFFSET(n_subsample), AV_OPT_TYPE_INT, {.i64=1}, 1, UINT_MAX, FLAGS},
{"model", "Set the model to be used for computing vmaf.",
OFFSET(model_cfg), AV_OPT_TYPE_STRING, {.str="version=vmaf_v0.6.1"},
0, 1, FLAGS},
{"feature", "Set the feature to be used for computing vmaf.",
OFFSET(feature_cfg), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS},
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ {"metadata_handler", "Set the feature to be propagated as metadata.",
OFFSET(metadata_feature_cfg), AV_OPT_TYPE_STRING, {.str="name=vmaf"},
0, 1, FLAGS},
+#endif
{ NULL }
};
@@ -105,6 +147,123 @@ static enum VmafPixelFormat pix_fmt_map(enum
AVPixelFormat av_pix_fmt)
}
}
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+static int add_to_frame_list(FrameList **head, AVFrame *frame, unsigned
frame_number)
+{
+ FrameList *new_frame = av_malloc(sizeof(FrameList));
+ if (!new_frame)
+ return AVERROR(ENOMEM);
+
+ new_frame->frame = frame;
+ new_frame->frame_number = frame_number;
+ new_frame->propagated_handlers_cnt = 0;
+ new_frame->next = NULL;
+
+ if (*head == NULL) {
+ *head = new_frame;
+ } else {
+ FrameList *current = *head;
+ while (current->next != NULL) {
+ current = current->next;
+ }
+ current->next = new_frame;
+ }
+
+ return 0;
+}
+
+static int remove_from_frame_list(FrameList **frame_list, unsigned
frame_number)
+{
+ FrameList *cur = *frame_list;
+ FrameList *prev = NULL;
+
+ while (cur) {
+ if (cur->frame_number == frame_number) {
+ if (prev)
+ prev->next = cur->next;
+ else
+ *frame_list = cur->next;
+ av_free(cur);
+ return 0;
+ }
+ prev = cur;
+ cur = cur->next;
+ }
+
+ return AVERROR(EINVAL);
+}
+
+static int free_frame_list(FrameList **frame_list)
+{
+ FrameList *cur = *frame_list;
+ while (cur) {
+ FrameList *next = cur->next;
+ av_frame_free(&cur->frame);
+ av_free(cur);
+ cur = next;
+ }
+ *frame_list = NULL;
+ return 0;
+}
+
+static FrameList* get_frame_from_frame_list(FrameList *frame_list,
+ unsigned frame_number)
+{
+ FrameList *cur = frame_list;
+ while (cur) {
+ if (cur->frame_number == frame_number)
+ return cur;
+ cur = cur->next;
+ }
+ return NULL;
+}
+
+static void set_meta(void *data, VmafMetadata *metadata)
+{
+ int err = 0;
+ FrameList *current_frame = NULL;
+ CallbackStruct *cb = data;
+ char value[128], key[128];
+ snprintf(value, sizeof(value), "%0.2f", metadata->score);
+ snprintf(key, sizeof(key), "%s.%d", metadata->feature_name,
metadata->picture_index);
+
+ current_frame = get_frame_from_frame_list(cb->frame_list,
metadata->picture_index);
+ if (!current_frame) {
+ av_log(NULL, AV_LOG_ERROR, "could not find frame with index: %d\n",
+ metadata->picture_index);
+ return;
+ }
+
+ err = av_dict_set(¤t_frame->frame->metadata, key, value, 0);
+ if (err < 0)
+ av_log(NULL, AV_LOG_ERROR, "could not set metadata: %s\n", key);
+
+ current_frame->propagated_handlers_cnt++;
+
+ if (current_frame->propagated_handlers_cnt ==
cb->s->metadata_cfg_list.metadata_cfg_cnt) {
+ FrameList *cur = cb->frame_list;
+ // This code block allows to send frames monotonically
+ while(cur && cur->frame_number <= metadata->picture_index) {
+ if (cur->propagated_handlers_cnt ==
cb->s->metadata_cfg_list.metadata_cfg_cnt) {
+ FrameList *next;
+ // Check outlink is closed
+ if (!cb->s->outlink_eof) {
+ av_log(cb->s->fs.parent, AV_LOG_DEBUG, "VMAF feature: %d,
score: %f\n", cur->frame_number, metadata->score);
+ cb->s->eof_frame = cur->frame_number;
+ if(ff_filter_frame(cb->s->fs.parent->outputs[0],
cur->frame))
+ return;
+ }
+ next = cur->next;
+ remove_from_frame_list(&cb->frame_list, cur->frame_number);
+ cur = next;
+ }
+ else
+ break;
+ }
+ }
+}
+#endif
+
static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc)
{
const int bytes_per_value = bpc > 8 ? 2 : 1;
@@ -160,13 +319,28 @@ static int do_vmaf(FFFrameSync *fs)
return AVERROR(ENOMEM);
}
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ err = add_to_frame_list(&s->cb->frame_list, dist, s->frame_cnt);
+ if (err) {
+ av_log(s, AV_LOG_ERROR, "problem during add_to_frame_list.\n");
+ return AVERROR(ENOMEM);
+ }
+#endif
+
err = vmaf_read_pictures(s->vmaf, &pic_ref, &pic_dist, s->frame_cnt++);
if (err) {
av_log(s, AV_LOG_ERROR, "problem during vmaf_read_pictures.\n");
return AVERROR(EINVAL);
}
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ if (s->metadata_cfg_list.metadata_cfg_cnt)
+ return 0;
+ else
+ return ff_filter_frame(ctx->outputs[0], dist);
+#else
return ff_filter_frame(ctx->outputs[0], dist);
+#endif
}
static AVDictionary **delimited_dict_parse(char *str, unsigned *cnt)
@@ -408,6 +582,83 @@ exit:
return err;
}
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+static int parse_metadata_handlers(AVFilterContext *ctx)
+{
+ LIBVMAFContext *s = ctx->priv;
+ AVDictionary **dict;
+ unsigned dict_cnt;
+ int err = 0;
+
+ if (!s->metadata_feature_cfg)
+ return 0;
+
+ dict_cnt = 0;
+ dict = delimited_dict_parse(s->metadata_feature_cfg, &dict_cnt);
+ if (!dict) {
+ av_log(ctx, AV_LOG_ERROR,
+ "could not parse metadata feature config: %s\n",
+ s->metadata_feature_cfg);
+ return AVERROR(EINVAL);
+ }
+
+ for (unsigned i = 0; i < dict_cnt; i++) {
+ VmafMetadataConfiguration *metadata_cfg = av_calloc(1,
sizeof(*metadata_cfg));
+ const AVDictionaryEntry *e = NULL;
+ char *feature_name = NULL;
+
+ while (e = av_dict_iterate(dict[i], e)) {
+ if (!strcmp(e->key, "name")) {
+ metadata_cfg->feature_name = av_strdup(e->value);
+ continue;
+ }
+ }
+
+ metadata_cfg->data = s->cb;
+ metadata_cfg->callback = &set_meta;
+
+ err = vmaf_register_metadata_handler(s->vmaf, *metadata_cfg);
+ if (err) {
+ av_log(ctx, AV_LOG_ERROR,
+ "problem during vmaf_register_metadata_handler: %s\n",
+ feature_name);
+ goto exit;
+ }
+
+ s->metadata_cfg_list.metadata_cfgs =
av_realloc(s->metadata_cfg_list.metadata_cfgs,
+
(s->metadata_cfg_list.metadata_cfg_cnt + 1) *
+
sizeof(*s->metadata_cfg_list.metadata_cfgs));
+ if (!s->metadata_cfg_list.metadata_cfgs) {
+ err = AVERROR(ENOMEM);
+ goto exit;
+ }
+
+
s->metadata_cfg_list.metadata_cfgs[s->metadata_cfg_list.metadata_cfg_cnt++] =
*metadata_cfg;
+ }
+
+exit:
+ for (unsigned i = 0; i < dict_cnt; i++) {
+ if (dict[i])
+ av_dict_free(&dict[i]);
+ }
+ av_free(dict);
+ return err;
+}
+
+static int init_metadata(AVFilterContext *ctx)
+{
+ LIBVMAFContext *s = ctx->priv;
+
+ s->cb = av_calloc(1, sizeof(CallbackStruct));
+ if (!s->cb)
+ return AVERROR(ENOMEM);
+
+ s->cb->s = s;
+
+ return 0;
+}
+#endif
+
static enum VmafLogLevel log_level_map(int log_level)
{
switch (log_level) {
@@ -441,6 +692,16 @@ static av_cold int init(AVFilterContext *ctx)
if (err)
return AVERROR(EINVAL);
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ err = init_metadata(ctx);
+ if (err)
+ return err;
+
+ err = parse_metadata_handlers(ctx);
+ if (err)
+ return err;
+#endif
+
err = parse_models(ctx);
if (err)
return err;
@@ -518,6 +779,38 @@ static int config_output(AVFilterLink *outlink)
static int activate(AVFilterContext *ctx)
{
LIBVMAFContext *s = ctx->priv;
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ // There are 2 cases for metadata propagation:
+ // 1. Where the case that outlink closes
+ // 2. Where inlink closes
+ // Case 1:
+ // In this case we need check outlink somehow for the status in every
iteration.
+ // If outlink is not wanting frame anymore, we need to proceed with
uninit with setting inlink.
+ // But nature of multithreading settting eof inside the activate call
can make sync issues and
+ // can lead to extra propagated frames. Atomic variables are used to
avoid this.
+ // Case 2:
+ // This case relatively easy to handle. Because of calculation of vmaf
score takes time
+ // So `do_vmaf` buffers many of frames before sending to outlink that
causes
+ // premature close of outlink.
+ // Checking inlink status is enough and if inlink == eof flushing vmaf
is enough for this.
+ int64_t pts;
+ int status, ret = 0;
+
+ if (ff_outlink_get_status(ctx->outputs[0]))
+ s->outlink_eof = 1;
+
+ if (ff_inlink_acknowledge_status(ctx->inputs[0], &status, &pts) &&
+ ff_inlink_acknowledge_status(ctx->inputs[1], &status, &pts)) {
+ if (!s->flushed) {
+ ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0);
+ if (ret)
+ av_log(ctx, AV_LOG_ERROR,
+ "problem flushing libvmaf context.\n");
+ else
+ s->flushed = 1;
+ }
+ }
+#endif
return ff_framesync_activate(&s->fs);
}
@@ -556,21 +849,52 @@ static av_cold void uninit(AVFilterContext *ctx)
LIBVMAFContext *s = ctx->priv;
int err = 0;
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ if (!s->outlink_eof)
+ s->outlink_eof = 1;
+#endif
+
ff_framesync_uninit(&s->fs);
if (!s->frame_cnt)
goto clean_up;
- err = vmaf_read_pictures(s->vmaf, NULL, NULL, 0);
+ if (!s->flushed) {
+ err = vmaf_read_pictures(s->vmaf, NULL, NULL, 0);
+ if (err) {
+ av_log(ctx, AV_LOG_ERROR,
+ "problem flushing libvmaf context.\n");
+ } else
+ s->flushed = 1;
+ }
+
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ if (s->metadata_cfg_list.metadata_cfgs) {
+ for (unsigned i = 0; i < s->metadata_cfg_list.metadata_cfg_cnt; i++) {
+ av_free(s->metadata_cfg_list.metadata_cfgs[i].feature_name);
+ }
+ av_free(s->metadata_cfg_list.metadata_cfgs);
+ }
+
+ err = free_frame_list(&s->cb->frame_list);
if (err) {
av_log(ctx, AV_LOG_ERROR,
- "problem flushing libvmaf context.\n");
+ "problem freeing frame list.\n");
}
+#endif
for (unsigned i = 0; i < s->model_cnt; i++) {
double vmaf_score;
+
+#if CONFIG_LIBVMAF_METADATA_ENABLED
+ err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool),
+ &vmaf_score, 0, s->eof_frame);
+ av_log(ctx, AV_LOG_DEBUG, "frame: %d frame_cnt %d\n", s->eof_frame,
s->frame_cnt - 1);
+#else
err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool),
&vmaf_score, 0, s->frame_cnt - 1);
+#endif
+
if (err) {
av_log(ctx, AV_LOG_ERROR,
"problem getting pooled vmaf score.\n");
--
2.45.2
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
