| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2017 Ronald S. Bultje <rsbultje@gmail.com> | ||
| 3 | * Copyright (c) 2017 Ashish Pratap Singh <ashk43712@gmail.com> | ||
| 4 | * | ||
| 5 | * This file is part of FFmpeg. | ||
| 6 | * | ||
| 7 | * FFmpeg is free software; you can redistribute it and/or | ||
| 8 | * modify it under the terms of the GNU Lesser General Public | ||
| 9 | * License as published by the Free Software Foundation; either | ||
| 10 | * version 2.1 of the License, or (at your option) any later version. | ||
| 11 | * | ||
| 12 | * FFmpeg is distributed in the hope that it will be useful, | ||
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| 15 | * Lesser General Public License for more details. | ||
| 16 | * | ||
| 17 | * You should have received a copy of the GNU Lesser General Public | ||
| 18 | * License along with FFmpeg; if not, write to the Free Software | ||
| 19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| 20 | */ | ||
| 21 | |||
| 22 | /** | ||
| 23 | * @file | ||
| 24 | * Calculate VMAF Motion score. | ||
| 25 | */ | ||
| 26 | |||
| 27 | #include "libavutil/file_open.h" | ||
| 28 | #include "libavutil/mem.h" | ||
| 29 | #include "libavutil/opt.h" | ||
| 30 | #include "libavutil/pixdesc.h" | ||
| 31 | #include "avfilter.h" | ||
| 32 | #include "formats.h" | ||
| 33 | #include "internal.h" | ||
| 34 | #include "video.h" | ||
| 35 | #include "vmaf_motion.h" | ||
| 36 | |||
| 37 | #define BIT_SHIFT 15 | ||
| 38 | |||
| 39 | static const float FILTER_5[5] = { | ||
| 40 | 0.054488685, | ||
| 41 | 0.244201342, | ||
| 42 | 0.402619947, | ||
| 43 | 0.244201342, | ||
| 44 | 0.054488685 | ||
| 45 | }; | ||
| 46 | |||
| 47 | typedef struct VMAFMotionContext { | ||
| 48 | const AVClass *class; | ||
| 49 | VMAFMotionData data; | ||
| 50 | FILE *stats_file; | ||
| 51 | char *stats_file_str; | ||
| 52 | } VMAFMotionContext; | ||
| 53 | |||
| 54 | #define OFFSET(x) offsetof(VMAFMotionContext, x) | ||
| 55 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM | ||
| 56 | |||
| 57 | static const AVOption vmafmotion_options[] = { | ||
| 58 | {"stats_file", "Set file where to store per-frame difference information", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS }, | ||
| 59 | { NULL } | ||
| 60 | }; | ||
| 61 | |||
| 62 | AVFILTER_DEFINE_CLASS(vmafmotion); | ||
| 63 | |||
| 64 | ✗ | static uint64_t image_sad(const uint16_t *img1, const uint16_t *img2, int w, | |
| 65 | int h, ptrdiff_t _img1_stride, ptrdiff_t _img2_stride) | ||
| 66 | { | ||
| 67 | ✗ | ptrdiff_t img1_stride = _img1_stride / sizeof(*img1); | |
| 68 | ✗ | ptrdiff_t img2_stride = _img2_stride / sizeof(*img2); | |
| 69 | ✗ | uint64_t sum = 0; | |
| 70 | int i, j; | ||
| 71 | |||
| 72 | ✗ | for (i = 0; i < h; i++) { | |
| 73 | ✗ | for (j = 0; j < w; j++) { | |
| 74 | ✗ | sum += abs(img1[j] - img2[j]); | |
| 75 | } | ||
| 76 | ✗ | img1 += img1_stride; | |
| 77 | ✗ | img2 += img2_stride; | |
| 78 | } | ||
| 79 | |||
| 80 | ✗ | return sum; | |
| 81 | } | ||
| 82 | |||
| 83 | ✗ | static void convolution_x(const uint16_t *filter, int filt_w, const uint16_t *src, | |
| 84 | uint16_t *dst, int w, int h, ptrdiff_t _src_stride, | ||
| 85 | ptrdiff_t _dst_stride) | ||
| 86 | { | ||
| 87 | ✗ | ptrdiff_t src_stride = _src_stride / sizeof(*src); | |
| 88 | ✗ | ptrdiff_t dst_stride = _dst_stride / sizeof(*dst); | |
| 89 | ✗ | int radius = filt_w / 2; | |
| 90 | ✗ | int borders_left = radius; | |
| 91 | ✗ | int borders_right = w - (filt_w - radius); | |
| 92 | int i, j, k; | ||
| 93 | ✗ | int sum = 0; | |
| 94 | |||
| 95 | ✗ | for (i = 0; i < h; i++) { | |
| 96 | ✗ | for (j = 0; j < borders_left; j++) { | |
| 97 | ✗ | sum = 0; | |
| 98 | ✗ | for (k = 0; k < filt_w; k++) { | |
| 99 | ✗ | int j_tap = FFABS(j - radius + k); | |
| 100 | ✗ | if (j_tap >= w) { | |
| 101 | ✗ | j_tap = w - (j_tap - w + 1); | |
| 102 | } | ||
| 103 | ✗ | sum += filter[k] * src[i * src_stride + j_tap]; | |
| 104 | } | ||
| 105 | ✗ | dst[i * dst_stride + j] = sum >> BIT_SHIFT; | |
| 106 | } | ||
| 107 | |||
| 108 | ✗ | for (j = borders_left; j < borders_right; j++) { | |
| 109 | ✗ | int sum = 0; | |
| 110 | ✗ | for (k = 0; k < filt_w; k++) { | |
| 111 | ✗ | sum += filter[k] * src[i * src_stride + j - radius + k]; | |
| 112 | } | ||
| 113 | ✗ | dst[i * dst_stride + j] = sum >> BIT_SHIFT; | |
| 114 | } | ||
| 115 | |||
| 116 | ✗ | for (j = borders_right; j < w; j++) { | |
| 117 | ✗ | sum = 0; | |
| 118 | ✗ | for (k = 0; k < filt_w; k++) { | |
| 119 | ✗ | int j_tap = FFABS(j - radius + k); | |
| 120 | ✗ | if (j_tap >= w) { | |
| 121 | ✗ | j_tap = w - (j_tap - w + 1); | |
| 122 | } | ||
| 123 | ✗ | sum += filter[k] * src[i * src_stride + j_tap]; | |
| 124 | } | ||
| 125 | ✗ | dst[i * dst_stride + j] = sum >> BIT_SHIFT; | |
| 126 | } | ||
| 127 | } | ||
| 128 | ✗ | } | |
| 129 | |||
| 130 | #define conv_y_fn(type, bits) \ | ||
| 131 | static void convolution_y_##bits##bit(const uint16_t *filter, int filt_w, \ | ||
| 132 | const uint8_t *_src, uint16_t *dst, \ | ||
| 133 | int w, int h, ptrdiff_t _src_stride, \ | ||
| 134 | ptrdiff_t _dst_stride) \ | ||
| 135 | { \ | ||
| 136 | const type *src = (const type *) _src; \ | ||
| 137 | ptrdiff_t src_stride = _src_stride / sizeof(*src); \ | ||
| 138 | ptrdiff_t dst_stride = _dst_stride / sizeof(*dst); \ | ||
| 139 | int radius = filt_w / 2; \ | ||
| 140 | int borders_top = radius; \ | ||
| 141 | int borders_bottom = h - (filt_w - radius); \ | ||
| 142 | int i, j, k; \ | ||
| 143 | int sum = 0; \ | ||
| 144 | \ | ||
| 145 | for (i = 0; i < borders_top; i++) { \ | ||
| 146 | for (j = 0; j < w; j++) { \ | ||
| 147 | sum = 0; \ | ||
| 148 | for (k = 0; k < filt_w; k++) { \ | ||
| 149 | int i_tap = FFABS(i - radius + k); \ | ||
| 150 | if (i_tap >= h) { \ | ||
| 151 | i_tap = h - (i_tap - h + 1); \ | ||
| 152 | } \ | ||
| 153 | sum += filter[k] * src[i_tap * src_stride + j]; \ | ||
| 154 | } \ | ||
| 155 | dst[i * dst_stride + j] = sum >> bits; \ | ||
| 156 | } \ | ||
| 157 | } \ | ||
| 158 | for (i = borders_top; i < borders_bottom; i++) { \ | ||
| 159 | for (j = 0; j < w; j++) { \ | ||
| 160 | sum = 0; \ | ||
| 161 | for (k = 0; k < filt_w; k++) { \ | ||
| 162 | sum += filter[k] * src[(i - radius + k) * src_stride + j]; \ | ||
| 163 | } \ | ||
| 164 | dst[i * dst_stride + j] = sum >> bits; \ | ||
| 165 | } \ | ||
| 166 | } \ | ||
| 167 | for (i = borders_bottom; i < h; i++) { \ | ||
| 168 | for (j = 0; j < w; j++) { \ | ||
| 169 | sum = 0; \ | ||
| 170 | for (k = 0; k < filt_w; k++) { \ | ||
| 171 | int i_tap = FFABS(i - radius + k); \ | ||
| 172 | if (i_tap >= h) { \ | ||
| 173 | i_tap = h - (i_tap - h + 1); \ | ||
| 174 | } \ | ||
| 175 | sum += filter[k] * src[i_tap * src_stride + j]; \ | ||
| 176 | } \ | ||
| 177 | dst[i * dst_stride + j] = sum >> bits; \ | ||
| 178 | } \ | ||
| 179 | } \ | ||
| 180 | } | ||
| 181 | |||
| 182 | ✗ | conv_y_fn(uint8_t, 8) | |
| 183 | ✗ | conv_y_fn(uint16_t, 10) | |
| 184 | |||
| 185 | ✗ | static void vmafmotiondsp_init(VMAFMotionDSPContext *dsp, int bpp) { | |
| 186 | ✗ | dsp->convolution_x = convolution_x; | |
| 187 | ✗ | dsp->convolution_y = bpp == 10 ? convolution_y_10bit : convolution_y_8bit; | |
| 188 | ✗ | dsp->sad = image_sad; | |
| 189 | ✗ | } | |
| 190 | |||
| 191 | ✗ | double ff_vmafmotion_process(VMAFMotionData *s, AVFrame *ref) | |
| 192 | { | ||
| 193 | double score; | ||
| 194 | |||
| 195 | ✗ | s->vmafdsp.convolution_y(s->filter, 5, ref->data[0], s->temp_data, | |
| 196 | ✗ | s->width, s->height, ref->linesize[0], s->stride); | |
| 197 | ✗ | s->vmafdsp.convolution_x(s->filter, 5, s->temp_data, s->blur_data[0], | |
| 198 | s->width, s->height, s->stride, s->stride); | ||
| 199 | |||
| 200 | ✗ | if (!s->nb_frames) { | |
| 201 | ✗ | score = 0.0; | |
| 202 | } else { | ||
| 203 | ✗ | uint64_t sad = s->vmafdsp.sad(s->blur_data[1], s->blur_data[0], | |
| 204 | s->width, s->height, s->stride, s->stride); | ||
| 205 | // the output score is always normalized to 8 bits | ||
| 206 | ✗ | score = (double) (sad * 1.0 / (s->width * s->height << (BIT_SHIFT - 8))); | |
| 207 | } | ||
| 208 | |||
| 209 | ✗ | FFSWAP(uint16_t *, s->blur_data[0], s->blur_data[1]); | |
| 210 | ✗ | s->nb_frames++; | |
| 211 | ✗ | s->motion_sum += score; | |
| 212 | |||
| 213 | ✗ | return score; | |
| 214 | } | ||
| 215 | |||
| 216 | ✗ | static void set_meta(AVDictionary **metadata, const char *key, float d) | |
| 217 | { | ||
| 218 | char value[128]; | ||
| 219 | ✗ | snprintf(value, sizeof(value), "%0.2f", d); | |
| 220 | ✗ | av_dict_set(metadata, key, value, 0); | |
| 221 | ✗ | } | |
| 222 | |||
| 223 | ✗ | static void do_vmafmotion(AVFilterContext *ctx, AVFrame *ref) | |
| 224 | { | ||
| 225 | ✗ | VMAFMotionContext *s = ctx->priv; | |
| 226 | double score; | ||
| 227 | |||
| 228 | ✗ | score = ff_vmafmotion_process(&s->data, ref); | |
| 229 | ✗ | set_meta(&ref->metadata, "lavfi.vmafmotion.score", score); | |
| 230 | ✗ | if (s->stats_file) { | |
| 231 | ✗ | fprintf(s->stats_file, | |
| 232 | "n:%"PRId64" motion:%0.2lf\n", s->data.nb_frames, score); | ||
| 233 | } | ||
| 234 | ✗ | } | |
| 235 | |||
| 236 | |||
| 237 | ✗ | int ff_vmafmotion_init(VMAFMotionData *s, | |
| 238 | int w, int h, enum AVPixelFormat fmt) | ||
| 239 | { | ||
| 240 | size_t data_sz; | ||
| 241 | int i; | ||
| 242 | ✗ | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); | |
| 243 | |||
| 244 | ✗ | if (w < 3 || h < 3) | |
| 245 | ✗ | return AVERROR(EINVAL); | |
| 246 | |||
| 247 | ✗ | s->width = w; | |
| 248 | ✗ | s->height = h; | |
| 249 | ✗ | s->stride = FFALIGN(w * sizeof(uint16_t), 32); | |
| 250 | |||
| 251 | ✗ | data_sz = (size_t) s->stride * h; | |
| 252 | ✗ | if (!(s->blur_data[0] = av_malloc(data_sz)) || | |
| 253 | ✗ | !(s->blur_data[1] = av_malloc(data_sz)) || | |
| 254 | ✗ | !(s->temp_data = av_malloc(data_sz))) { | |
| 255 | ✗ | return AVERROR(ENOMEM); | |
| 256 | } | ||
| 257 | |||
| 258 | ✗ | for (i = 0; i < 5; i++) { | |
| 259 | ✗ | s->filter[i] = lrint(FILTER_5[i] * (1 << BIT_SHIFT)); | |
| 260 | } | ||
| 261 | |||
| 262 | ✗ | vmafmotiondsp_init(&s->vmafdsp, desc->comp[0].depth); | |
| 263 | |||
| 264 | ✗ | return 0; | |
| 265 | } | ||
| 266 | |||
| 267 | ✗ | static int query_formats(AVFilterContext *ctx) | |
| 268 | { | ||
| 269 | ✗ | AVFilterFormats *fmts_list = NULL; | |
| 270 | int format, ret; | ||
| 271 | |||
| 272 | ✗ | for (format = 0; av_pix_fmt_desc_get(format); format++) { | |
| 273 | ✗ | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format); | |
| 274 | ✗ | if (!(desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_HWACCEL | AV_PIX_FMT_FLAG_BITSTREAM | AV_PIX_FMT_FLAG_PAL)) && | |
| 275 | ✗ | (desc->flags & AV_PIX_FMT_FLAG_PLANAR || desc->nb_components == 1) && | |
| 276 | ✗ | (!(desc->flags & AV_PIX_FMT_FLAG_BE) == !HAVE_BIGENDIAN || desc->comp[0].depth == 8) && | |
| 277 | ✗ | (desc->comp[0].depth == 8 || desc->comp[0].depth == 10) && | |
| 278 | ✗ | (ret = ff_add_format(&fmts_list, format)) < 0) | |
| 279 | ✗ | return ret; | |
| 280 | } | ||
| 281 | |||
| 282 | ✗ | return ff_set_common_formats(ctx, fmts_list); | |
| 283 | } | ||
| 284 | |||
| 285 | ✗ | static int config_input_ref(AVFilterLink *inlink) | |
| 286 | { | ||
| 287 | ✗ | AVFilterContext *ctx = inlink->dst; | |
| 288 | ✗ | VMAFMotionContext *s = ctx->priv; | |
| 289 | |||
| 290 | ✗ | return ff_vmafmotion_init(&s->data, ctx->inputs[0]->w, | |
| 291 | ✗ | ctx->inputs[0]->h, ctx->inputs[0]->format); | |
| 292 | } | ||
| 293 | |||
| 294 | ✗ | double ff_vmafmotion_uninit(VMAFMotionData *s) | |
| 295 | { | ||
| 296 | ✗ | av_free(s->blur_data[0]); | |
| 297 | ✗ | av_free(s->blur_data[1]); | |
| 298 | ✗ | av_free(s->temp_data); | |
| 299 | |||
| 300 | ✗ | return s->nb_frames > 0 ? s->motion_sum / s->nb_frames : 0.0; | |
| 301 | } | ||
| 302 | |||
| 303 | ✗ | static int filter_frame(AVFilterLink *inlink, AVFrame *ref) | |
| 304 | { | ||
| 305 | ✗ | AVFilterContext *ctx = inlink->dst; | |
| 306 | ✗ | do_vmafmotion(ctx, ref); | |
| 307 | ✗ | return ff_filter_frame(ctx->outputs[0], ref); | |
| 308 | } | ||
| 309 | |||
| 310 | ✗ | static av_cold int init(AVFilterContext *ctx) | |
| 311 | { | ||
| 312 | ✗ | VMAFMotionContext *s = ctx->priv; | |
| 313 | |||
| 314 | ✗ | if (s->stats_file_str) { | |
| 315 | ✗ | if (!strcmp(s->stats_file_str, "-")) { | |
| 316 | ✗ | s->stats_file = stdout; | |
| 317 | } else { | ||
| 318 | ✗ | s->stats_file = avpriv_fopen_utf8(s->stats_file_str, "w"); | |
| 319 | ✗ | if (!s->stats_file) { | |
| 320 | ✗ | int err = AVERROR(errno); | |
| 321 | char buf[128]; | ||
| 322 | ✗ | av_strerror(err, buf, sizeof(buf)); | |
| 323 | ✗ | av_log(ctx, AV_LOG_ERROR, "Could not open stats file %s: %s\n", | |
| 324 | s->stats_file_str, buf); | ||
| 325 | ✗ | return err; | |
| 326 | } | ||
| 327 | } | ||
| 328 | } | ||
| 329 | |||
| 330 | ✗ | return 0; | |
| 331 | } | ||
| 332 | |||
| 333 | ✗ | static av_cold void uninit(AVFilterContext *ctx) | |
| 334 | { | ||
| 335 | ✗ | VMAFMotionContext *s = ctx->priv; | |
| 336 | ✗ | double avg_motion = ff_vmafmotion_uninit(&s->data); | |
| 337 | |||
| 338 | ✗ | if (s->data.nb_frames > 0) { | |
| 339 | ✗ | av_log(ctx, AV_LOG_INFO, "VMAF Motion avg: %.3f\n", avg_motion); | |
| 340 | } | ||
| 341 | |||
| 342 | ✗ | if (s->stats_file && s->stats_file != stdout) | |
| 343 | ✗ | fclose(s->stats_file); | |
| 344 | ✗ | } | |
| 345 | |||
| 346 | static const AVFilterPad vmafmotion_inputs[] = { | ||
| 347 | { | ||
| 348 | .name = "reference", | ||
| 349 | .type = AVMEDIA_TYPE_VIDEO, | ||
| 350 | .filter_frame = filter_frame, | ||
| 351 | .config_props = config_input_ref, | ||
| 352 | }, | ||
| 353 | }; | ||
| 354 | |||
| 355 | const AVFilter ff_vf_vmafmotion = { | ||
| 356 | .name = "vmafmotion", | ||
| 357 | .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF Motion score."), | ||
| 358 | .init = init, | ||
| 359 | .uninit = uninit, | ||
| 360 | .priv_size = sizeof(VMAFMotionContext), | ||
| 361 | .priv_class = &vmafmotion_class, | ||
| 362 | .flags = AVFILTER_FLAG_METADATA_ONLY, | ||
| 363 | FILTER_INPUTS(vmafmotion_inputs), | ||
| 364 | FILTER_OUTPUTS(ff_video_default_filterpad), | ||
| 365 | FILTER_QUERY_FUNC(query_formats), | ||
| 366 | }; | ||
| 367 |