FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* This file is part of FFmpeg.
3			*
4			* FFmpeg is free software; you can redistribute it and/or
5			* modify it under the terms of the GNU Lesser General Public
6			* License as published by the Free Software Foundation; either
7			* version 2.1 of the License, or (at your option) any later version.
8			*
9			* FFmpeg is distributed in the hope that it will be useful,
10			* but WITHOUT ANY WARRANTY; without even the implied warranty of
11			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12			* Lesser General Public License for more details.
13			*
14			* You should have received a copy of the GNU Lesser General Public
15			* License along with FFmpeg; if not, write to the Free Software
16			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17			*/
18
19			/**
20			* @file
21			* Calculate the correlation between two input videos.
22			*/
23
24			#include "libavutil/avstring.h"
25			#include "libavutil/mem.h"
26			#include "libavutil/opt.h"
27			#include "libavutil/pixdesc.h"
28			#include "avfilter.h"
29			#include "drawutils.h"
30			#include "filters.h"
31			#include "framesync.h"
32
33			typedef struct Sums {
34			uint64_t s[2];
35			} Sums;
36
37			typedef struct QSums {
38			float s[3];
39			} QSums;
40
41			typedef struct CorrContext {
42			const AVClass *class;
43			FFFrameSync fs;
44			double score, min_score, max_score, score_comp[4];
45			uint64_t nb_frames;
46			int nb_threads;
47			int is_rgb;
48			uint8_t rgba_map[4];
49			int max[4];
50			char comps[4];
51			float mean[4][2];
52			Sums *sums;
53			QSums *qsums;
54			int nb_components;
55			int planewidth[4];
56			int planeheight[4];
57			int (*sum_slice)(AVFilterContext *ctx, void *arg,
58			int jobnr, int nb_jobs);
59			int (*corr_slice)(AVFilterContext *ctx, void *arg,
60			int jobnr, int nb_jobs);
61			} CorrContext;
62
63			typedef struct ThreadData {
64			AVFrame *master, *ref;
65			} ThreadData;
66
67			#define OFFSET(x) offsetof(CorrContext, x)
68			#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
69
70		✗	static void set_meta(AVFilterContext *ctx,
71			AVDictionary **metadata, const char *key, char comp, float d)
72			{
73			char value[128];
74		✗	snprintf(value, sizeof(value), "%f", d);
75		✗	if (comp) {
76			char key2[128];
77		✗	snprintf(key2, sizeof(key2), "lavfi.%s.%s%s%c",
78		✗	ctx->filter->name, ctx->filter->name, key, comp);
79		✗	av_dict_set(metadata, key2, value, 0);
80			} else {
81			char key2[128];
82		✗	snprintf(key2, sizeof(key2), "lavfi.%s.%s%s",
83		✗	ctx->filter->name, ctx->filter->name, key);
84		✗	av_dict_set(metadata, key2, value, 0);
85			}
86		✗	}
87
88			#define SUM(type, name) \
89			static int sum_##name(AVFilterContext *ctx, void *arg, \
90			int jobnr, int nb_jobs) \
91			{ \
92			CorrContext *s = ctx->priv; \
93			ThreadData *td = arg; \
94			AVFrame *master = td->master; \
95			AVFrame *ref = td->ref; \
96			\
97			for (int c = 0; c < s->nb_components; c++) { \
98			const ptrdiff_t linesize1 = master->linesize[c] / \
99			sizeof(type); \
100			const ptrdiff_t linesize2 = ref->linesize[c] / \
101			sizeof(type); \
102			const int h = s->planeheight[c]; \
103			const int w = s->planewidth[c]; \
104			const int slice_start = (h * jobnr) / nb_jobs; \
105			const int slice_end = (h * (jobnr+1)) / nb_jobs; \
106			const type *src1 = (const type *)master->data[c] + \
107			linesize1 * slice_start; \
108			const type *src2 = (const type *)ref->data[c] + \
109			linesize2 * slice_start; \
110			uint64_t sum1 = 0, sum2 = 0; \
111			\
112			for (int y = slice_start; y < slice_end; y++) { \
113			for (int x = 0; x < w; x++) { \
114			sum1 += src1[x]; \
115			sum2 += src2[x]; \
116			} \
117			\
118			src1 += linesize1; \
119			src2 += linesize2; \
120			} \
121			\
122			s->sums[jobnr * s->nb_components + c].s[0] = sum1; \
123			s->sums[jobnr * s->nb_components + c].s[1] = sum2; \
124			} \
125			\
126			return 0; \
127			}
128
129		✗	SUM(uint8_t, slice8)
130		✗	SUM(uint16_t, slice16)
131
132			#define CORR(type, name) \
133			static int corr_##name(AVFilterContext *ctx, void *arg, \
134			int jobnr, int nb_jobs) \
135			{ \
136			CorrContext *s = ctx->priv; \
137			ThreadData *td = arg; \
138			AVFrame *master = td->master; \
139			AVFrame *ref = td->ref; \
140			\
141			for (int c = 0; c < s->nb_components; c++) { \
142			const ptrdiff_t linesize1 = master->linesize[c] / \
143			sizeof(type); \
144			const ptrdiff_t linesize2 = ref->linesize[c] / \
145			sizeof(type); \
146			const type *src1 = (const type *)master->data[c]; \
147			const type *src2 = (const type *)ref->data[c]; \
148			const int h = s->planeheight[c]; \
149			const int w = s->planewidth[c]; \
150			const int slice_start = (h * jobnr) / nb_jobs; \
151			const int slice_end = (h * (jobnr+1)) / nb_jobs; \
152			const float scale = 1.f / s->max[c]; \
153			const float mean1 = s->mean[c][0]; \
154			const float mean2 = s->mean[c][1]; \
155			float sum12 = 0.f, sum1q = 0.f, sum2q = 0.f; \
156			\
157			src1 = (const type *)master->data[c] + \
158			slice_start * linesize1; \
159			src2 = (const type *)ref->data[c] + \
160			slice_start * linesize2; \
161			\
162			for (int y = slice_start; y < slice_end; y++) { \
163			for (int x = 0; x < w; x++) { \
164			const float f1 = scale * src1[x] - mean1; \
165			const float f2 = scale * src2[x] - mean2; \
166			\
167			sum12 += f1 * f2; \
168			sum1q += f1 * f1; \
169			sum2q += f2 * f2; \
170			} \
171			\
172			src1 += linesize1; \
173			src2 += linesize2; \
174			} \
175			\
176			s->qsums[jobnr * s->nb_components + c].s[0] = sum12; \
177			s->qsums[jobnr * s->nb_components + c].s[1] = sum1q; \
178			s->qsums[jobnr * s->nb_components + c].s[2] = sum2q; \
179			} \
180			\
181			return 0; \
182			}
183
184		✗	CORR(uint8_t, slice8)
185		✗	CORR(uint16_t, slice16)
186
187		✗	static int do_corr(FFFrameSync *fs)
188			{
189		✗	AVFilterContext *ctx = fs->parent;
190		✗	CorrContext *s = ctx->priv;
191			AVFrame *master, *ref;
192		✗	double comp_score[4], score = 0.;
193			AVDictionary **metadata;
194			ThreadData td;
195			int ret;
196
197		✗	ret = ff_framesync_dualinput_get(fs, &master, &ref);
198		✗	if (ret < 0)
199		✗	return ret;
200		✗	if (ctx->is_disabled || !ref)
201		✗	return ff_filter_frame(ctx->outputs[0], master);
202		✗	metadata = &master->metadata;
203
204		✗	td.master = master;
205		✗	td.ref = ref;
206		✗	ff_filter_execute(ctx, s->sum_slice, &td, NULL,
207		✗	FFMIN(s->planeheight[1], s->nb_threads));
208
209		✗	for (int c = 0; c < s->nb_components; c++) {
210		✗	const double scale = 1.f / s->max[c];
211		✗	uint64_t sum1 = 0, sum2 = 0;
212
213		✗	for (int n = 0; n < s->nb_threads; n++) {
214		✗	sum1 += s->sums[n * s->nb_components + c].s[0];
215		✗	sum2 += s->sums[n * s->nb_components + c].s[1];
216			}
217
218		✗	s->mean[c][0] = scale * (sum1 /(double)(s->planewidth[c] * s->planeheight[c]));
219		✗	s->mean[c][1] = scale * (sum2 /(double)(s->planewidth[c] * s->planeheight[c]));
220			}
221
222		✗	ff_filter_execute(ctx, s->corr_slice, &td, NULL,
223		✗	FFMIN(s->planeheight[1], s->nb_threads));
224
225		✗	for (int c = 0; c < s->nb_components; c++) {
226		✗	double sumq, sum12 = 0.0, sum1q = 0.0, sum2q = 0.0;
227
228		✗	for (int n = 0; n < s->nb_threads; n++) {
229		✗	sum12 += s->qsums[n * s->nb_components + c].s[0];
230		✗	sum1q += s->qsums[n * s->nb_components + c].s[1];
231		✗	sum2q += s->qsums[n * s->nb_components + c].s[2];
232			}
233
234		✗	sumq = sqrt(sum1q * sum2q);
235		✗	if (sumq > 0.0) {
236		✗	comp_score[c] = av_clipd(sum12 / sumq,-1.0,1.0);
237			} else {
238		✗	comp_score[c] = 0.f;
239			}
240			}
241
242		✗	for (int c = 0; c < s->nb_components; c++)
243		✗	score += comp_score[c];
244		✗	score /= s->nb_components;
245		✗	s->score += score;
246
247		✗	s->min_score = fmin(s->min_score, score);
248		✗	s->max_score = fmax(s->max_score, score);
249
250		✗	for (int c = 0; c < s->nb_components; c++)
251		✗	s->score_comp[c] += comp_score[c];
252		✗	s->nb_frames++;
253
254		✗	for (int j = 0; j < s->nb_components; j++) {
255		✗	int c = s->is_rgb ? s->rgba_map[j] : j;
256		✗	set_meta(ctx, metadata, ".", s->comps[j], comp_score[c]);
257			}
258		✗	set_meta(ctx, metadata, "_avg", 0, score);
259
260		✗	return ff_filter_frame(ctx->outputs[0], master);
261			}
262
263		✗	static av_cold int init(AVFilterContext *ctx)
264			{
265		✗	CorrContext *s = ctx->priv;
266
267		✗	s->fs.on_event = do_corr;
268
269		✗	return 0;
270			}
271
272			static const enum AVPixelFormat pix_fmts[] = {
273			AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9, AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14, AV_PIX_FMT_GRAY16,
274			#define PF_NOALPHA(suf) AV_PIX_FMT_YUV420##suf, AV_PIX_FMT_YUV422##suf, AV_PIX_FMT_YUV444##suf
275			#define PF_ALPHA(suf) AV_PIX_FMT_YUVA420##suf, AV_PIX_FMT_YUVA422##suf, AV_PIX_FMT_YUVA444##suf
276			#define PF(suf) PF_NOALPHA(suf), PF_ALPHA(suf)
277			PF(P), PF(P9), PF(P10), PF_NOALPHA(P12), PF_NOALPHA(P14), PF(P16),
278			AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_YUV410P,
279			AV_PIX_FMT_YUVJ411P, AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
280			AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
281			AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRP9, AV_PIX_FMT_GBRP10,
282			AV_PIX_FMT_GBRP12, AV_PIX_FMT_GBRP14, AV_PIX_FMT_GBRP16,
283			AV_PIX_FMT_GBRAP, AV_PIX_FMT_GBRAP10, AV_PIX_FMT_GBRAP12, AV_PIX_FMT_GBRAP16,
284			AV_PIX_FMT_NONE
285			};
286
287		✗	static int config_input_ref(AVFilterLink *inlink)
288			{
289		✗	const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
290		✗	AVFilterContext *ctx = inlink->dst;
291		✗	CorrContext *s = ctx->priv;
292
293		✗	s->nb_threads = ff_filter_get_nb_threads(ctx);
294		✗	s->nb_components = desc->nb_components;
295		✗	if (ctx->inputs[0]->w != ctx->inputs[1]->w ||
296		✗	ctx->inputs[0]->h != ctx->inputs[1]->h) {
297		✗	av_log(ctx, AV_LOG_ERROR, "Width and height of input videos must be same.\n");
298		✗	return AVERROR(EINVAL);
299			}
300
301		✗	s->is_rgb = ff_fill_rgba_map(s->rgba_map, inlink->format) >= 0;
302		✗	s->comps[0] = s->is_rgb ? 'R' : 'Y' ;
303		✗	s->comps[1] = s->is_rgb ? 'G' : 'U' ;
304		✗	s->comps[2] = s->is_rgb ? 'B' : 'V' ;
305		✗	s->comps[3] = 'A';
306
307		✗	s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
308		✗	s->planeheight[0] = s->planeheight[3] = inlink->h;
309		✗	s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
310		✗	s->planewidth[0] = s->planewidth[3] = inlink->w;
311
312		✗	s->sums = av_calloc(s->nb_threads * s->nb_components, sizeof(*s->sums));
313		✗	s->qsums = av_calloc(s->nb_threads * s->nb_components, sizeof(*s->qsums));
314		✗	if (!s->qsums || !s->sums)
315		✗	return AVERROR(ENOMEM);
316
317		✗	s->min_score = +INFINITY;
318		✗	s->max_score = -INFINITY;
319
320		✗	s->max[0] = (1 << desc->comp[0].depth) - 1;
321		✗	s->max[1] = (1 << desc->comp[1].depth) - 1;
322		✗	s->max[2] = (1 << desc->comp[2].depth) - 1;
323		✗	s->max[3] = (1 << desc->comp[3].depth) - 1;
324
325		✗	s->sum_slice = desc->comp[0].depth > 8 ? sum_slice16 : sum_slice8;
326		✗	s->corr_slice = desc->comp[0].depth > 8 ? corr_slice16 : corr_slice8;
327
328		✗	return 0;
329			}
330
331		✗	static int config_output(AVFilterLink *outlink)
332			{
333		✗	FilterLink *outl = ff_filter_link(outlink);
334		✗	AVFilterContext *ctx = outlink->src;
335		✗	CorrContext *s = ctx->priv;
336		✗	AVFilterLink *mainlink = ctx->inputs[0];
337		✗	FilterLink *ml = ff_filter_link(mainlink);
338			int ret;
339
340		✗	ret = ff_framesync_init_dualinput(&s->fs, ctx);
341		✗	if (ret < 0)
342		✗	return ret;
343		✗	outlink->w = mainlink->w;
344		✗	outlink->h = mainlink->h;
345		✗	outlink->time_base = mainlink->time_base;
346		✗	outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio;
347		✗	outl->frame_rate = ml->frame_rate;
348		✗	if ((ret = ff_framesync_configure(&s->fs)) < 0)
349		✗	return ret;
350
351		✗	outlink->time_base = s->fs.time_base;
352
353		✗	if (av_cmp_q(mainlink->time_base, outlink->time_base) ||
354		✗	av_cmp_q(ctx->inputs[1]->time_base, outlink->time_base))
355		✗	av_log(ctx, AV_LOG_WARNING, "not matching timebases found between first input: %d/%d and second input %d/%d, results may be incorrect!\n",
356			mainlink->time_base.num, mainlink->time_base.den,
357		✗	ctx->inputs[1]->time_base.num, ctx->inputs[1]->time_base.den);
358
359		✗	return 0;
360			}
361
362		✗	static int activate(AVFilterContext *ctx)
363			{
364		✗	CorrContext *s = ctx->priv;
365		✗	return ff_framesync_activate(&s->fs);
366			}
367
368		✗	static av_cold void uninit(AVFilterContext *ctx)
369			{
370		✗	CorrContext *s = ctx->priv;
371
372		✗	if (s->nb_frames > 0) {
373			char buf[256];
374
375		✗	buf[0] = 0;
376		✗	for (int j = 0; j < s->nb_components; j++) {
377		✗	int c = s->is_rgb ? s->rgba_map[j] : j;
378		✗	av_strlcatf(buf, sizeof(buf), " %c:%f", s->comps[j], s->score_comp[c] / s->nb_frames);
379			}
380
381		✗	av_log(ctx, AV_LOG_INFO, "%s%s average:%f min:%f max:%f\n",
382		✗	ctx->filter->name,
383			buf,
384		✗	s->score / s->nb_frames,
385			s->min_score,
386			s->max_score);
387			}
388
389		✗	ff_framesync_uninit(&s->fs);
390		✗	av_freep(&s->qsums);
391		✗	av_freep(&s->sums);
392		✗	}
393
394			static const AVFilterPad corr_inputs[] = {
395			{
396			.name = "main",
397			.type = AVMEDIA_TYPE_VIDEO,
398			},{
399			.name = "reference",
400			.type = AVMEDIA_TYPE_VIDEO,
401			.config_props = config_input_ref,
402			},
403			};
404
405			static const AVFilterPad corr_outputs[] = {
406			{
407			.name = "default",
408			.type = AVMEDIA_TYPE_VIDEO,
409			.config_props = config_output,
410			},
411			};
412
413			static const AVOption options[] = {
414			{ NULL }
415			};
416
417			#define corr_options options
418		✗	FRAMESYNC_DEFINE_CLASS(corr, CorrContext, fs);
419
420			const FFFilter ff_vf_corr = {
421			.p.name = "corr",
422			.p.description = NULL_IF_CONFIG_SMALL("Calculate the correlation between two video streams."),
423			.p.priv_class = &corr_class,
424			.p.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL |
425			AVFILTER_FLAG_SLICE_THREADS |
426			AVFILTER_FLAG_METADATA_ONLY,
427			.preinit = corr_framesync_preinit,
428			.init = init,
429			.uninit = uninit,
430			.activate = activate,
431			.priv_size = sizeof(CorrContext),
432			FILTER_INPUTS(corr_inputs),
433			FILTER_OUTPUTS(corr_outputs),
434			FILTER_PIXFMTS_ARRAY(pix_fmts),
435			};
436