FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2017 Gerion Entrup
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or modify
7			* it under the terms of the GNU General Public License as published by
8			* the Free Software Foundation; either version 2 of the License, or
9			* (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14			* GNU General Public License for more details.
15			*
16			* You should have received a copy of the GNU General Public License along
17			* with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18			* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19			*/
20
21			/**
22			* @file
23			* MPEG-7 video signature calculation and lookup filter
24			* @see http://epubs.surrey.ac.uk/531590/1/MPEG-7%20Video%20Signature%20Author%27s%20Copy.pdf
25			*/
26
27			#include "libavcodec/put_bits.h"
28			#include "libavformat/avformat.h"
29			#include "libavutil/opt.h"
30			#include "libavutil/avstring.h"
31			#include "libavutil/file_open.h"
32			#include "avfilter.h"
33			#include "internal.h"
34			#include "signature.h"
35			#include "signature_lookup.c"
36
37			#define OFFSET(x) offsetof(SignatureContext, x)
38			#define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
39			#define BLOCK_LCM (int64_t) 476985600
40
41			static const AVOption signature_options[] = {
42			{ "detectmode", "set the detectmode",
43			OFFSET(mode), AV_OPT_TYPE_INT, {.i64 = MODE_OFF}, 0, NB_LOOKUP_MODE-1, FLAGS, "mode" },
44			{ "off", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_OFF}, 0, 0, .flags = FLAGS, "mode" },
45			{ "full", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FULL}, 0, 0, .flags = FLAGS, "mode" },
46			{ "fast", NULL, 0, AV_OPT_TYPE_CONST, {.i64 = MODE_FAST}, 0, 0, .flags = FLAGS, "mode" },
47			{ "nb_inputs", "number of inputs",
48			OFFSET(nb_inputs), AV_OPT_TYPE_INT, {.i64 = 1}, 1, INT_MAX, FLAGS },
49			{ "filename", "filename for output files",
50			OFFSET(filename), AV_OPT_TYPE_STRING, {.str = ""}, 0, NB_FORMATS-1, FLAGS },
51			{ "format", "set output format",
52			OFFSET(format), AV_OPT_TYPE_INT, {.i64 = FORMAT_BINARY}, 0, 1, FLAGS , "format" },
53			{ "binary", 0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_BINARY}, 0, 0, FLAGS, "format" },
54			{ "xml", 0, 0, AV_OPT_TYPE_CONST, {.i64=FORMAT_XML}, 0, 0, FLAGS, "format" },
55			{ "th_d", "threshold to detect one word as similar",
56			OFFSET(thworddist), AV_OPT_TYPE_INT, {.i64 = 9000}, 1, INT_MAX, FLAGS },
57			{ "th_dc", "threshold to detect all words as similar",
58			OFFSET(thcomposdist), AV_OPT_TYPE_INT, {.i64 = 60000}, 1, INT_MAX, FLAGS },
59			{ "th_xh", "threshold to detect frames as similar",
60			OFFSET(thl1), AV_OPT_TYPE_INT, {.i64 = 116}, 1, INT_MAX, FLAGS },
61			{ "th_di", "minimum length of matching sequence in frames",
62			OFFSET(thdi), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, FLAGS },
63			{ "th_it", "threshold for relation of good to all frames",
64			OFFSET(thit), AV_OPT_TYPE_DOUBLE, {.dbl = 0.5}, 0.0, 1.0, FLAGS },
65			{ NULL }
66			};
67
68			AVFILTER_DEFINE_CLASS(signature);
69
70			/* all formats with a separate gray value */
71			static const enum AVPixelFormat pix_fmts[] = {
72			AV_PIX_FMT_GRAY8,
73			AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
74			AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
75			AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
76			AV_PIX_FMT_YUVJ411P, AV_PIX_FMT_YUVJ420P,
77			AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P,
78			AV_PIX_FMT_YUVJ440P,
79			AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,
80			AV_PIX_FMT_NONE
81			};
82
83		✗	static int config_input(AVFilterLink *inlink)
84			{
85		✗	AVFilterContext *ctx = inlink->dst;
86		✗	SignatureContext *sic = ctx->priv;
87		✗	StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);
88
89		✗	sc->time_base = inlink->time_base;
90			/* test for overflow */
91		✗	sc->divide = (((uint64_t) inlink->w/32) * (inlink->w/32 + 1) * (inlink->h/32 * inlink->h/32 + 1) > INT64_MAX / (BLOCK_LCM * 255));
92		✗	if (sc->divide) {
93		✗	av_log(ctx, AV_LOG_WARNING, "Input dimension too high for precise calculation, numbers will be rounded.\n");
94			}
95		✗	sc->w = inlink->w;
96		✗	sc->h = inlink->h;
97		✗	return 0;
98			}
99
100		✗	static int get_block_size(const Block *b)
101			{
102		✗	return (b->to.y - b->up.y + 1) * (b->to.x - b->up.x + 1);
103			}
104
105		✗	static uint64_t get_block_sum(StreamContext *sc, uint64_t intpic[32][32], const Block *b)
106			{
107		✗	uint64_t sum = 0;
108
109			int x0, y0, x1, y1;
110
111		✗	x0 = b->up.x;
112		✗	y0 = b->up.y;
113		✗	x1 = b->to.x;
114		✗	y1 = b->to.y;
115
116		✗	if (x0-1 >= 0 && y0-1 >= 0) {
117		✗	sum = intpic[y1][x1] + intpic[y0-1][x0-1] - intpic[y1][x0-1] - intpic[y0-1][x1];
118		✗	} else if (x0-1 >= 0) {
119		✗	sum = intpic[y1][x1] - intpic[y1][x0-1];
120		✗	} else if (y0-1 >= 0) {
121		✗	sum = intpic[y1][x1] - intpic[y0-1][x1];
122			} else {
123		✗	sum = intpic[y1][x1];
124			}
125		✗	return sum;
126			}
127
128		✗	static int cmp(const void *x, const void *y)
129			{
130		✗	const uint64_t *a = x, *b = y;
131		✗	return *a < *b ? -1 : ( *a > *b ? 1 : 0 );
132			}
133
134			/**
135			* sets the bit at position pos to 1 in data
136			*/
137		✗	static void set_bit(uint8_t* data, size_t pos)
138			{
139		✗	uint8_t mask = 1 << 7-(pos%8);
140		✗	data[pos/8] |= mask;
141		✗	}
142
143		✗	static int filter_frame(AVFilterLink *inlink, AVFrame *picref)
144			{
145		✗	AVFilterContext *ctx = inlink->dst;
146		✗	SignatureContext *sic = ctx->priv;
147		✗	StreamContext *sc = &(sic->streamcontexts[FF_INLINK_IDX(inlink)]);
148			FineSignature* fs;
149
150			static const uint8_t pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };
151			/* indexes of words : 210,217,219,274,334 44,175,233,270,273 57,70,103,237,269 100,285,295,337,354 101,102,111,275,296
152			s2usw = sorted to unsorted wordvec: 44 is at index 5, 57 at index 10...
153			*/
154			static const unsigned int wordvec[25] = {44,57,70,100,101,102,103,111,175,210,217,219,233,237,269,270,273,274,275,285,295,296,334,337,354};
155			static const uint8_t s2usw[25] = { 5,10,11, 15, 20, 21, 12, 22, 6, 0, 1, 2, 7, 13, 14, 8, 9, 3, 23, 16, 17, 24, 4, 18, 19};
156
157		✗	uint8_t wordt2b[5] = { 0, 0, 0, 0, 0 }; /* word ternary to binary */
158			uint64_t intpic[32][32];
159			uint64_t rowcount;
160		✗	uint8_t *p = picref->data[0];
161			int inti, intj;
162			int *intjlut;
163
164			uint64_t conflist[DIFFELEM_SIZE];
165		✗	int f = 0, g = 0, w = 0;
166		✗	int32_t dh1 = 1, dh2 = 1, dw1 = 1, dw2 = 1, a, b;
167			int64_t denom;
168			int i, j, k, ternary;
169			uint64_t blocksum;
170			int blocksize;
171			int64_t th; /* threshold */
172			int64_t sum;
173
174		✗	int64_t precfactor = (sc->divide) ? 65536 : BLOCK_LCM;
175
176			/* initialize fs */
177		✗	if (sc->curfinesig) {
178		✗	fs = av_mallocz(sizeof(FineSignature));
179		✗	if (!fs)
180		✗	return AVERROR(ENOMEM);
181		✗	sc->curfinesig->next = fs;
182		✗	fs->prev = sc->curfinesig;
183		✗	sc->curfinesig = fs;
184			} else {
185		✗	fs = sc->curfinesig = sc->finesiglist;
186		✗	sc->curcoarsesig1->first = fs;
187			}
188
189		✗	fs->pts = picref->pts;
190		✗	fs->index = sc->lastindex++;
191
192		✗	memset(intpic, 0, sizeof(uint64_t)*32*32);
193		✗	intjlut = av_malloc_array(inlink->w, sizeof(int));
194		✗	if (!intjlut)
195		✗	return AVERROR(ENOMEM);
196		✗	for (i = 0; i < inlink->w; i++) {
197		✗	intjlut[i] = (i*32)/inlink->w;
198			}
199
200		✗	for (i = 0; i < inlink->h; i++) {
201		✗	inti = (i*32)/inlink->h;
202		✗	for (j = 0; j < inlink->w; j++) {
203		✗	intj = intjlut[j];
204		✗	intpic[inti][intj] += p[j];
205			}
206		✗	p += picref->linesize[0];
207			}
208		✗	av_freep(&intjlut);
209
210			/* The following calculates a summed area table (intpic) and brings the numbers
211			* in intpic to the same denominator.
212			* So you only have to handle the numinator in the following sections.
213			*/
214		✗	dh1 = inlink->h / 32;
215		✗	if (inlink->h % 32)
216		✗	dh2 = dh1 + 1;
217		✗	dw1 = inlink->w / 32;
218		✗	if (inlink->w % 32)
219		✗	dw2 = dw1 + 1;
220		✗	denom = (sc->divide) ? dh1 * (int64_t)dh2 * dw1 * dw2 : 1;
221
222		✗	for (i = 0; i < 32; i++) {
223		✗	rowcount = 0;
224		✗	a = 1;
225		✗	if (dh2 > 1) {
226		✗	a = ((inlink->h*(i+1))%32 == 0) ? (inlink->h*(i+1))/32 - 1 : (inlink->h*(i+1))/32;
227		✗	a -= ((inlink->h*i)%32 == 0) ? (inlink->h*i)/32 - 1 : (inlink->h*i)/32;
228		✗	a = (a == dh1)? dh2 : dh1;
229			}
230		✗	for (j = 0; j < 32; j++) {
231		✗	b = 1;
232		✗	if (dw2 > 1) {
233		✗	b = ((inlink->w*(j+1))%32 == 0) ? (inlink->w*(j+1))/32 - 1 : (inlink->w*(j+1))/32;
234		✗	b -= ((inlink->w*j)%32 == 0) ? (inlink->w*j)/32 - 1 : (inlink->w*j)/32;
235		✗	b = (b == dw1)? dw2 : dw1;
236			}
237		✗	rowcount += intpic[i][j] * a * b * precfactor / denom;
238		✗	if (i > 0) {
239		✗	intpic[i][j] = intpic[i-1][j] + rowcount;
240			} else {
241		✗	intpic[i][j] = rowcount;
242			}
243			}
244			}
245
246		✗	denom = (sc->divide) ? 1 : dh1 * (int64_t)dh2 * dw1 * dw2;
247
248		✗	for (i = 0; i < ELEMENT_COUNT; i++) {
249		✗	const ElemCat* elemcat = elements[i];
250			int64_t* elemsignature;
251			uint64_t* sortsignature;
252
253		✗	elemsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));
254		✗	if (!elemsignature)
255		✗	return AVERROR(ENOMEM);
256		✗	sortsignature = av_malloc_array(elemcat->elem_count, sizeof(int64_t));
257		✗	if (!sortsignature) {
258		✗	av_freep(&elemsignature);
259		✗	return AVERROR(ENOMEM);
260			}
261
262		✗	for (j = 0; j < elemcat->elem_count; j++) {
263		✗	blocksum = 0;
264		✗	blocksize = 0;
265		✗	for (k = 0; k < elemcat->left_count; k++) {
266		✗	blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);
267		✗	blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);
268			}
269		✗	sum = blocksum / blocksize;
270		✗	if (elemcat->av_elem) {
271		✗	sum -= 128 * precfactor * denom;
272			} else {
273		✗	blocksum = 0;
274		✗	blocksize = 0;
275		✗	for (; k < elemcat->block_count; k++) {
276		✗	blocksum += get_block_sum(sc, intpic, &elemcat->blocks[j*elemcat->block_count+k]);
277		✗	blocksize += get_block_size(&elemcat->blocks[j*elemcat->block_count+k]);
278			}
279		✗	sum -= blocksum / blocksize;
280		✗	conflist[g++] = FFABS(sum * 8 / (precfactor * denom));
281			}
282
283		✗	elemsignature[j] = sum;
284		✗	sortsignature[j] = FFABS(sum);
285			}
286
287			/* get threshold */
288		✗	qsort(sortsignature, elemcat->elem_count, sizeof(uint64_t), cmp);
289		✗	th = sortsignature[(int) (elemcat->elem_count*0.333)];
290
291			/* ternarize */
292		✗	for (j = 0; j < elemcat->elem_count; j++) {
293		✗	if (elemsignature[j] < -th) {
294		✗	ternary = 0;
295		✗	} else if (elemsignature[j] <= th) {
296		✗	ternary = 1;
297			} else {
298		✗	ternary = 2;
299			}
300		✗	fs->framesig[f/5] += ternary * pot3[f%5];
301
302		✗	if (f == wordvec[w]) {
303		✗	fs->words[s2usw[w]/5] += ternary * pot3[wordt2b[s2usw[w]/5]++];
304		✗	if (w < 24)
305		✗	w++;
306			}
307		✗	f++;
308			}
309		✗	av_freep(&elemsignature);
310		✗	av_freep(&sortsignature);
311			}
312
313			/* confidence */
314		✗	qsort(conflist, DIFFELEM_SIZE, sizeof(uint64_t), cmp);
315		✗	fs->confidence = FFMIN(conflist[DIFFELEM_SIZE/2], 255);
316
317			/* coarsesignature */
318		✗	if (sc->coarsecount == 0) {
319		✗	if (sc->curcoarsesig2) {
320		✗	sc->curcoarsesig1 = av_mallocz(sizeof(CoarseSignature));
321		✗	if (!sc->curcoarsesig1)
322		✗	return AVERROR(ENOMEM);
323		✗	sc->curcoarsesig1->first = fs;
324		✗	sc->curcoarsesig2->next = sc->curcoarsesig1;
325		✗	sc->coarseend = sc->curcoarsesig1;
326			}
327			}
328		✗	if (sc->coarsecount == 45) {
329		✗	sc->midcoarse = 1;
330		✗	sc->curcoarsesig2 = av_mallocz(sizeof(CoarseSignature));
331		✗	if (!sc->curcoarsesig2)
332		✗	return AVERROR(ENOMEM);
333		✗	sc->curcoarsesig2->first = fs;
334		✗	sc->curcoarsesig1->next = sc->curcoarsesig2;
335		✗	sc->coarseend = sc->curcoarsesig2;
336			}
337		✗	for (i = 0; i < 5; i++) {
338		✗	set_bit(sc->curcoarsesig1->data[i], fs->words[i]);
339			}
340			/* assuming the actual frame is the last */
341		✗	sc->curcoarsesig1->last = fs;
342		✗	if (sc->midcoarse) {
343		✗	for (i = 0; i < 5; i++) {
344		✗	set_bit(sc->curcoarsesig2->data[i], fs->words[i]);
345			}
346		✗	sc->curcoarsesig2->last = fs;
347			}
348
349		✗	sc->coarsecount = (sc->coarsecount+1)%90;
350
351			/* debug printing finesignature */
352		✗	if (av_log_get_level() == AV_LOG_DEBUG) {
353		✗	av_log(ctx, AV_LOG_DEBUG, "input %d, confidence: %d\n", FF_INLINK_IDX(inlink), fs->confidence);
354
355		✗	av_log(ctx, AV_LOG_DEBUG, "words:");
356		✗	for (i = 0; i < 5; i++) {
357		✗	av_log(ctx, AV_LOG_DEBUG, " %d:", fs->words[i] );
358		✗	av_log(ctx, AV_LOG_DEBUG, " %d", fs->words[i] / pot3[0] );
359		✗	for (j = 1; j < 5; j++)
360		✗	av_log(ctx, AV_LOG_DEBUG, ",%d", fs->words[i] % pot3[j-1] / pot3[j] );
361		✗	av_log(ctx, AV_LOG_DEBUG, ";");
362			}
363		✗	av_log(ctx, AV_LOG_DEBUG, "\n");
364
365		✗	av_log(ctx, AV_LOG_DEBUG, "framesignature:");
366		✗	for (i = 0; i < SIGELEM_SIZE/5; i++) {
367		✗	av_log(ctx, AV_LOG_DEBUG, " %d", fs->framesig[i] / pot3[0] );
368		✗	for (j = 1; j < 5; j++)
369		✗	av_log(ctx, AV_LOG_DEBUG, ",%d", fs->framesig[i] % pot3[j-1] / pot3[j] );
370			}
371		✗	av_log(ctx, AV_LOG_DEBUG, "\n");
372			}
373
374		✗	if (FF_INLINK_IDX(inlink) == 0)
375		✗	return ff_filter_frame(inlink->dst->outputs[0], picref);
376		✗	return 1;
377			}
378
379		✗	static int xml_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)
380			{
381			FineSignature* fs;
382			CoarseSignature* cs;
383			int i, j;
384			FILE* f;
385		✗	unsigned int pot3[5] = { 3*3*3*3, 3*3*3, 3*3, 3, 1 };
386
387		✗	f = avpriv_fopen_utf8(filename, "w");
388		✗	if (!f) {
389		✗	int err = AVERROR(EINVAL);
390			char buf[128];
391		✗	av_strerror(err, buf, sizeof(buf));
392		✗	av_log(ctx, AV_LOG_ERROR, "cannot open xml file %s: %s\n", filename, buf);
393		✗	return err;
394			}
395
396			/* header */
397		✗	fprintf(f, "<?xml version='1.0' encoding='ASCII' ?>\n");
398		✗	fprintf(f, "<Mpeg7 xmlns=\"urn:mpeg:mpeg7:schema:2001\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"urn:mpeg:mpeg7:schema:2001 schema/Mpeg7-2001.xsd\">\n");
399		✗	fprintf(f, " <DescriptionUnit xsi:type=\"DescriptorCollectionType\">\n");
400		✗	fprintf(f, " <Descriptor xsi:type=\"VideoSignatureType\">\n");
401		✗	fprintf(f, " <VideoSignatureRegion>\n");
402		✗	fprintf(f, " <VideoSignatureSpatialRegion>\n");
403		✗	fprintf(f, " <Pixel>0 0 </Pixel>\n");
404		✗	fprintf(f, " <Pixel>%d %d </Pixel>\n", sc->w - 1, sc->h - 1);
405		✗	fprintf(f, " </VideoSignatureSpatialRegion>\n");
406		✗	fprintf(f, " <StartFrameOfSpatialRegion>0</StartFrameOfSpatialRegion>\n");
407			/* hoping num is 1, other values are vague */
408		✗	fprintf(f, " <MediaTimeUnit>%d</MediaTimeUnit>\n", sc->time_base.den / sc->time_base.num);
409		✗	fprintf(f, " <MediaTimeOfSpatialRegion>\n");
410		✗	fprintf(f, " <StartMediaTimeOfSpatialRegion>0</StartMediaTimeOfSpatialRegion>\n");
411		✗	fprintf(f, " <EndMediaTimeOfSpatialRegion>%" PRIu64 "</EndMediaTimeOfSpatialRegion>\n", sc->coarseend->last->pts);
412		✗	fprintf(f, " </MediaTimeOfSpatialRegion>\n");
413
414			/* coarsesignatures */
415		✗	for (cs = sc->coarsesiglist; cs; cs = cs->next) {
416		✗	fprintf(f, " <VSVideoSegment>\n");
417		✗	fprintf(f, " <StartFrameOfSegment>%" PRIu32 "</StartFrameOfSegment>\n", cs->first->index);
418		✗	fprintf(f, " <EndFrameOfSegment>%" PRIu32 "</EndFrameOfSegment>\n", cs->last->index);
419		✗	fprintf(f, " <MediaTimeOfSegment>\n");
420		✗	fprintf(f, " <StartMediaTimeOfSegment>%" PRIu64 "</StartMediaTimeOfSegment>\n", cs->first->pts);
421		✗	fprintf(f, " <EndMediaTimeOfSegment>%" PRIu64 "</EndMediaTimeOfSegment>\n", cs->last->pts);
422		✗	fprintf(f, " </MediaTimeOfSegment>\n");
423		✗	for (i = 0; i < 5; i++) {
424		✗	fprintf(f, " <BagOfWords>");
425		✗	for (j = 0; j < 31; j++) {
426		✗	uint8_t n = cs->data[i][j];
427		✗	if (j < 30) {
428		✗	fprintf(f, "%d %d %d %d %d %d %d %d ", (n & 0x80) >> 7,
429		✗	(n & 0x40) >> 6,
430		✗	(n & 0x20) >> 5,
431		✗	(n & 0x10) >> 4,
432		✗	(n & 0x08) >> 3,
433		✗	(n & 0x04) >> 2,
434		✗	(n & 0x02) >> 1,
435			(n & 0x01));
436			} else {
437			/* print only 3 bit in last byte */
438		✗	fprintf(f, "%d %d %d ", (n & 0x80) >> 7,
439		✗	(n & 0x40) >> 6,
440		✗	(n & 0x20) >> 5);
441			}
442			}
443		✗	fprintf(f, "</BagOfWords>\n");
444			}
445		✗	fprintf(f, " </VSVideoSegment>\n");
446			}
447
448			/* finesignatures */
449		✗	for (fs = sc->finesiglist; fs; fs = fs->next) {
450		✗	fprintf(f, " <VideoFrame>\n");
451		✗	fprintf(f, " <MediaTimeOfFrame>%" PRIu64 "</MediaTimeOfFrame>\n", fs->pts);
452			/* confidence */
453		✗	fprintf(f, " <FrameConfidence>%d</FrameConfidence>\n", fs->confidence);
454			/* words */
455		✗	fprintf(f, " <Word>");
456		✗	for (i = 0; i < 5; i++) {
457		✗	fprintf(f, "%d ", fs->words[i]);
458		✗	if (i < 4) {
459		✗	fprintf(f, " ");
460			}
461			}
462		✗	fprintf(f, "</Word>\n");
463			/* framesignature */
464		✗	fprintf(f, " <FrameSignature>");
465		✗	for (i = 0; i< SIGELEM_SIZE/5; i++) {
466		✗	if (i > 0) {
467		✗	fprintf(f, " ");
468			}
469		✗	fprintf(f, "%d ", fs->framesig[i] / pot3[0]);
470		✗	for (j = 1; j < 5; j++)
471		✗	fprintf(f, " %d ", fs->framesig[i] % pot3[j-1] / pot3[j] );
472			}
473		✗	fprintf(f, "</FrameSignature>\n");
474		✗	fprintf(f, " </VideoFrame>\n");
475			}
476		✗	fprintf(f, " </VideoSignatureRegion>\n");
477		✗	fprintf(f, " </Descriptor>\n");
478		✗	fprintf(f, " </DescriptionUnit>\n");
479		✗	fprintf(f, "</Mpeg7>\n");
480
481		✗	fclose(f);
482		✗	return 0;
483			}
484
485		✗	static int binary_export(AVFilterContext *ctx, StreamContext *sc, const char* filename)
486			{
487			FILE* f;
488			FineSignature* fs;
489			CoarseSignature* cs;
490		✗	uint32_t numofsegments = (sc->lastindex + 44)/45;
491			int i, j;
492			PutBitContext buf;
493			/* buffer + header + coarsesignatures + finesignature */
494		✗	int len = (512 + 6 * 32 + 3*16 + 2 +
495		✗	numofsegments * (4*32 + 1 + 5*243) +
496		✗	sc->lastindex * (2 + 32 + 6*8 + 608)) / 8;
497		✗	uint8_t* buffer = av_malloc_array(len, sizeof(uint8_t));
498		✗	if (!buffer)
499		✗	return AVERROR(ENOMEM);
500
501		✗	f = avpriv_fopen_utf8(filename, "wb");
502		✗	if (!f) {
503		✗	int err = AVERROR(EINVAL);
504			char buf[128];
505		✗	av_strerror(err, buf, sizeof(buf));
506		✗	av_log(ctx, AV_LOG_ERROR, "cannot open file %s: %s\n", filename, buf);
507		✗	av_freep(&buffer);
508		✗	return err;
509			}
510		✗	init_put_bits(&buf, buffer, len);
511
512		✗	put_bits32(&buf, 1); /* NumOfSpatial Regions, only 1 supported */
513		✗	put_bits(&buf, 1, 1); /* SpatialLocationFlag, always the whole image */
514		✗	put_bits32(&buf, 0); /* PixelX,1 PixelY,1, 0,0 */
515		✗	put_bits(&buf, 16, sc->w-1 & 0xFFFF); /* PixelX,2 */
516		✗	put_bits(&buf, 16, sc->h-1 & 0xFFFF); /* PixelY,2 */
517		✗	put_bits32(&buf, 0); /* StartFrameOfSpatialRegion */
518		✗	put_bits32(&buf, sc->lastindex); /* NumOfFrames */
519			/* hoping num is 1, other values are vague */
520			/* den/num might be greater than 16 bit, so cutting it */
521		✗	put_bits(&buf, 16, 0xFFFF & (sc->time_base.den / sc->time_base.num)); /* MediaTimeUnit */
522		✗	put_bits(&buf, 1, 1); /* MediaTimeFlagOfSpatialRegion */
523		✗	put_bits32(&buf, 0); /* StartMediaTimeOfSpatialRegion */
524		✗	put_bits32(&buf, 0xFFFFFFFF & sc->coarseend->last->pts); /* EndMediaTimeOfSpatialRegion */
525		✗	put_bits32(&buf, numofsegments); /* NumOfSegments */
526			/* coarsesignatures */
527		✗	for (cs = sc->coarsesiglist; cs; cs = cs->next) {
528		✗	put_bits32(&buf, cs->first->index); /* StartFrameOfSegment */
529		✗	put_bits32(&buf, cs->last->index); /* EndFrameOfSegment */
530		✗	put_bits(&buf, 1, 1); /* MediaTimeFlagOfSegment */
531		✗	put_bits32(&buf, 0xFFFFFFFF & cs->first->pts); /* StartMediaTimeOfSegment */
532		✗	put_bits32(&buf, 0xFFFFFFFF & cs->last->pts); /* EndMediaTimeOfSegment */
533		✗	for (i = 0; i < 5; i++) {
534			/* put 243 bits ( = 7 * 32 + 19 = 8 * 28 + 19) into buffer */
535		✗	for (j = 0; j < 30; j++) {
536		✗	put_bits(&buf, 8, cs->data[i][j]);
537			}
538		✗	put_bits(&buf, 3, cs->data[i][30] >> 5);
539			}
540			}
541			/* finesignatures */
542		✗	put_bits(&buf, 1, 0); /* CompressionFlag, only 0 supported */
543		✗	for (fs = sc->finesiglist; fs; fs = fs->next) {
544		✗	put_bits(&buf, 1, 1); /* MediaTimeFlagOfFrame */
545		✗	put_bits32(&buf, 0xFFFFFFFF & fs->pts); /* MediaTimeOfFrame */
546		✗	put_bits(&buf, 8, fs->confidence); /* FrameConfidence */
547		✗	for (i = 0; i < 5; i++) {
548		✗	put_bits(&buf, 8, fs->words[i]); /* Words */
549			}
550			/* framesignature */
551		✗	for (i = 0; i < SIGELEM_SIZE/5; i++) {
552		✗	put_bits(&buf, 8, fs->framesig[i]);
553			}
554			}
555
556		✗	flush_put_bits(&buf);
557		✗	fwrite(buffer, 1, put_bytes_output(&buf), f);
558		✗	fclose(f);
559		✗	av_freep(&buffer);
560		✗	return 0;
561			}
562
563		✗	static int export(AVFilterContext *ctx, StreamContext *sc, int input)
564			{
565		✗	SignatureContext* sic = ctx->priv;
566			char filename[1024];
567
568		✗	if (sic->nb_inputs > 1) {
569			/* error already handled */
570		✗	av_assert0(av_get_frame_filename(filename, sizeof(filename), sic->filename, input) == 0);
571			} else {
572		✗	if (av_strlcpy(filename, sic->filename, sizeof(filename)) >= sizeof(filename))
573		✗	return AVERROR(EINVAL);
574			}
575		✗	if (sic->format == FORMAT_XML) {
576		✗	return xml_export(ctx, sc, filename);
577			} else {
578		✗	return binary_export(ctx, sc, filename);
579			}
580			}
581
582		✗	static int request_frame(AVFilterLink *outlink)
583			{
584		✗	AVFilterContext *ctx = outlink->src;
585		✗	SignatureContext *sic = ctx->priv;
586			StreamContext *sc, *sc2;
587			MatchingInfo match;
588			int i, j, ret;
589		✗	int lookup = 1; /* indicates wheather EOF of all files is reached */
590
591			/* process all inputs */
592		✗	for (i = 0; i < sic->nb_inputs; i++){
593		✗	sc = &(sic->streamcontexts[i]);
594
595		✗	ret = ff_request_frame(ctx->inputs[i]);
596
597			/* return if unexpected error occurs in input stream */
598		✗	if (ret < 0 && ret != AVERROR_EOF)
599		✗	return ret;
600
601			/* export signature at EOF */
602		✗	if (ret == AVERROR_EOF && !sc->exported) {
603			/* export if wanted */
604		✗	if (strlen(sic->filename) > 0) {
605		✗	if (export(ctx, sc, i) < 0)
606		✗	return ret;
607			}
608		✗	sc->exported = 1;
609			}
610		✗	lookup &= sc->exported;
611			}
612
613			/* signature lookup */
614		✗	if (lookup && sic->mode != MODE_OFF) {
615			/* iterate over every pair */
616		✗	for (i = 0; i < sic->nb_inputs; i++) {
617		✗	sc = &(sic->streamcontexts[i]);
618		✗	for (j = i+1; j < sic->nb_inputs; j++) {
619		✗	sc2 = &(sic->streamcontexts[j]);
620		✗	match = lookup_signatures(ctx, sic, sc, sc2, sic->mode);
621		✗	if (match.score != 0) {
622		✗	av_log(ctx, AV_LOG_INFO, "matching of video %d at %f and %d at %f, %d frames matching\n",
623		✗	i, ((double) match.first->pts * sc->time_base.num) / sc->time_base.den,
624		✗	j, ((double) match.second->pts * sc2->time_base.num) / sc2->time_base.den,
625			match.matchframes);
626		✗	if (match.whole)
627		✗	av_log(ctx, AV_LOG_INFO, "whole video matching\n");
628			} else {
629		✗	av_log(ctx, AV_LOG_INFO, "no matching of video %d and %d\n", i, j);
630			}
631			}
632			}
633			}
634
635		✗	return ret;
636			}
637
638		✗	static av_cold int init(AVFilterContext *ctx)
639			{
640
641		✗	SignatureContext *sic = ctx->priv;
642			StreamContext *sc;
643			int i, ret;
644			char tmp[1024];
645
646		✗	sic->streamcontexts = av_mallocz(sic->nb_inputs * sizeof(StreamContext));
647		✗	if (!sic->streamcontexts)
648		✗	return AVERROR(ENOMEM);
649
650		✗	for (i = 0; i < sic->nb_inputs; i++) {
651		✗	AVFilterPad pad = {
652			.type = AVMEDIA_TYPE_VIDEO,
653		✗	.name = av_asprintf("in%d", i),
654			.config_props = config_input,
655			.filter_frame = filter_frame,
656			};
657
658		✗	if (!pad.name)
659		✗	return AVERROR(ENOMEM);
660		✗	if ((ret = ff_append_inpad_free_name(ctx, &pad)) < 0)
661		✗	return ret;
662
663		✗	sc = &(sic->streamcontexts[i]);
664
665		✗	sc->lastindex = 0;
666		✗	sc->finesiglist = av_mallocz(sizeof(FineSignature));
667		✗	if (!sc->finesiglist)
668		✗	return AVERROR(ENOMEM);
669		✗	sc->curfinesig = NULL;
670
671		✗	sc->coarsesiglist = av_mallocz(sizeof(CoarseSignature));
672		✗	if (!sc->coarsesiglist)
673		✗	return AVERROR(ENOMEM);
674		✗	sc->curcoarsesig1 = sc->coarsesiglist;
675		✗	sc->coarseend = sc->coarsesiglist;
676		✗	sc->coarsecount = 0;
677		✗	sc->midcoarse = 0;
678			}
679
680			/* check filename */
681		✗	if (sic->nb_inputs > 1 && strlen(sic->filename) > 0 && av_get_frame_filename(tmp, sizeof(tmp), sic->filename, 0) == -1) {
682		✗	av_log(ctx, AV_LOG_ERROR, "The filename must contain %%d or %%0nd, if you have more than one input.\n");
683		✗	return AVERROR(EINVAL);
684			}
685
686		✗	return 0;
687			}
688
689
690
691		✗	static av_cold void uninit(AVFilterContext *ctx)
692			{
693		✗	SignatureContext *sic = ctx->priv;
694			StreamContext *sc;
695			void* tmp;
696			FineSignature* finsig;
697			CoarseSignature* cousig;
698			int i;
699
700
701			/* free the lists */
702		✗	if (sic->streamcontexts != NULL) {
703		✗	for (i = 0; i < sic->nb_inputs; i++) {
704		✗	sc = &(sic->streamcontexts[i]);
705		✗	finsig = sc->finesiglist;
706		✗	cousig = sc->coarsesiglist;
707
708		✗	while (finsig) {
709		✗	tmp = finsig;
710		✗	finsig = finsig->next;
711		✗	av_freep(&tmp);
712			}
713		✗	sc->finesiglist = NULL;
714
715		✗	while (cousig) {
716		✗	tmp = cousig;
717		✗	cousig = cousig->next;
718		✗	av_freep(&tmp);
719			}
720		✗	sc->coarsesiglist = NULL;
721			}
722		✗	av_freep(&sic->streamcontexts);
723			}
724		✗	}
725
726		✗	static int config_output(AVFilterLink *outlink)
727			{
728		✗	AVFilterContext *ctx = outlink->src;
729		✗	AVFilterLink *inlink = ctx->inputs[0];
730
731		✗	outlink->time_base = inlink->time_base;
732		✗	outlink->frame_rate = inlink->frame_rate;
733		✗	outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
734		✗	outlink->w = inlink->w;
735		✗	outlink->h = inlink->h;
736
737		✗	return 0;
738			}
739
740			static const AVFilterPad signature_outputs[] = {
741			{
742			.name = "default",
743			.type = AVMEDIA_TYPE_VIDEO,
744			.request_frame = request_frame,
745			.config_props = config_output,
746			},
747			};
748
749			const AVFilter ff_vf_signature = {
750			.name = "signature",
751			.description = NULL_IF_CONFIG_SMALL("Calculate the MPEG-7 video signature"),
752			.priv_size = sizeof(SignatureContext),
753			.priv_class = &signature_class,
754			.init = init,
755			.uninit = uninit,
756			FILTER_OUTPUTS(signature_outputs),
757			.inputs = NULL,
758			FILTER_PIXFMTS_ARRAY(pix_fmts),
759			.flags = AVFILTER_FLAG_DYNAMIC_INPUTS,
760			};
761