FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Copyright (c) 2017 Paul B Mahol
3			*
4			* This file is part of FFmpeg.
5			*
6			* FFmpeg is free software; you can redistribute it and/or
7			* modify it under the terms of the GNU Lesser General Public
8			* License as published by the Free Software Foundation; either
9			* version 2.1 of the License, or (at your option) any later version.
10			*
11			* FFmpeg is distributed in the hope that it will be useful,
12			* but WITHOUT ANY WARRANTY; without even the implied warranty of
13			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14			* Lesser General Public License for more details.
15			*
16			* You should have received a copy of the GNU Lesser General Public
17			* License along with FFmpeg; if not, write to the Free Software
18			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19			*/
20
21			/**
22			* @file
23			* Filter for reading closed captioning data (EIA-608).
24			* See also https://en.wikipedia.org/wiki/EIA-608
25			*/
26
27			#include <string.h>
28
29			#include "libavutil/internal.h"
30			#include "libavutil/mem.h"
31			#include "libavutil/opt.h"
32			#include "libavutil/pixdesc.h"
33
34			#include "avfilter.h"
35			#include "filters.h"
36			#include "video.h"
37
38			#define LAG 25
39			#define CLOCK_BITSIZE_MIN 0.2f
40			#define CLOCK_BITSIZE_MAX 1.5f
41			#define SYNC_BITSIZE_MIN 12.f
42			#define SYNC_BITSIZE_MAX 15.f
43
44			typedef struct LineItem {
45			int input;
46			int output;
47
48			float unfiltered;
49			float filtered;
50			float average;
51			float deviation;
52			} LineItem;
53
54			typedef struct CodeItem {
55			uint8_t bit;
56			int size;
57			} CodeItem;
58
59			typedef struct ScanItem {
60			int nb_line;
61			int found;
62			int white;
63			int black;
64			uint64_t *histogram;
65			uint8_t byte[2];
66
67			CodeItem *code;
68			LineItem *line;
69			} ScanItem;
70
71			typedef struct ReadEIA608Context {
72			const AVClass *class;
73
74			int start, end;
75			float spw;
76			int chp;
77			int lp;
78
79			int depth;
80			int max;
81			int nb_allocated;
82			ScanItem *scan;
83
84			void (*read_line[2])(AVFrame *in, int nb_line,
85			LineItem *line, int lp, int w);
86			} ReadEIA608Context;
87
88			#define OFFSET(x) offsetof(ReadEIA608Context, x)
89			#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
90
91			static const AVOption readeia608_options[] = {
92			{ "scan_min", "set from which line to scan for codes", OFFSET(start), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS },
93			{ "scan_max", "set to which line to scan for codes", OFFSET(end), AV_OPT_TYPE_INT, {.i64=29}, 0, INT_MAX, FLAGS },
94			{ "spw", "set ratio of width reserved for sync code detection", OFFSET(spw), AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1, 0.7, FLAGS },
95			{ "chp", "check and apply parity bit", OFFSET(chp), AV_OPT_TYPE_BOOL, {.i64= 0}, 0, 1, FLAGS },
96			{ "lp", "lowpass line prior to processing", OFFSET(lp), AV_OPT_TYPE_BOOL, {.i64= 1}, 0, 1, FLAGS },
97			{ NULL }
98			};
99
100			AVFILTER_DEFINE_CLASS(readeia608);
101
102			static const enum AVPixelFormat pixel_fmts[] = {
103			AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9,
104			AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14,
105			AV_PIX_FMT_GRAY16,
106			AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
107			AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
108			AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
109			AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
110			AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
111			AV_PIX_FMT_YUVJ411P,
112			AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9,
113			AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10,
114			AV_PIX_FMT_YUV440P10,
115			AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12,
116			AV_PIX_FMT_YUV440P12,
117			AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14,
118			AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16,
119			AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
120			AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16,
121			AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16,
122			AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16,
123			AV_PIX_FMT_NONE
124			};
125
126		✗	static int config_filter(AVFilterContext *ctx, int start, int end)
127			{
128		✗	ReadEIA608Context *s = ctx->priv;
129		✗	AVFilterLink *inlink = ctx->inputs[0];
130		✗	int size = inlink->w + LAG;
131
132		✗	if (end >= inlink->h) {
133		✗	av_log(ctx, AV_LOG_WARNING, "Last line to scan too large, clipping.\n");
134		✗	end = inlink->h - 1;
135			}
136
137		✗	if (start > end) {
138		✗	av_log(ctx, AV_LOG_ERROR, "Invalid range.\n");
139		✗	return AVERROR(EINVAL);
140			}
141
142		✗	if (s->nb_allocated < end - start + 1) {
143		✗	const int diff = end - start + 1 - s->nb_allocated;
144
145		✗	s->scan = av_realloc_f(s->scan, end - start + 1, sizeof(*s->scan));
146		✗	if (!s->scan)
147		✗	return AVERROR(ENOMEM);
148		✗	memset(&s->scan[s->nb_allocated], 0, diff * sizeof(*s->scan));
149		✗	s->nb_allocated = end - start + 1;
150			}
151
152		✗	for (int i = 0; i < s->nb_allocated; i++) {
153		✗	ScanItem *scan = &s->scan[i];
154
155		✗	if (!scan->histogram)
156		✗	scan->histogram = av_calloc(s->max + 1, sizeof(*scan->histogram));
157		✗	if (!scan->line)
158		✗	scan->line = av_calloc(size, sizeof(*scan->line));
159		✗	if (!scan->code)
160		✗	scan->code = av_calloc(size, sizeof(*scan->code));
161		✗	if (!scan->line || !scan->code || !scan->histogram)
162		✗	return AVERROR(ENOMEM);
163			}
164
165		✗	s->start = start;
166		✗	s->end = end;
167
168		✗	return 0;
169			}
170
171		✗	static void build_histogram(ReadEIA608Context *s, ScanItem *scan, const LineItem *line, int len)
172			{
173		✗	memset(scan->histogram, 0, (s->max + 1) * sizeof(*scan->histogram));
174
175		✗	for (int i = LAG; i < len + LAG; i++)
176		✗	scan->histogram[line[i].input]++;
177		✗	}
178
179		✗	static void find_black_and_white(ReadEIA608Context *s, ScanItem *scan)
180			{
181		✗	const int max = s->max;
182		✗	int start = 0, end = 0, middle;
183		✗	int black = 0, white = 0;
184			int cnt;
185
186		✗	for (int i = 0; i <= max; i++) {
187		✗	if (scan->histogram[i]) {
188		✗	start = i;
189		✗	break;
190			}
191			}
192
193		✗	for (int i = max; i >= 0; i--) {
194		✗	if (scan->histogram[i]) {
195		✗	end = i;
196		✗	break;
197			}
198			}
199
200		✗	middle = start + (end - start) / 2;
201
202		✗	cnt = 0;
203		✗	for (int i = start; i <= middle; i++) {
204		✗	if (scan->histogram[i] > cnt) {
205		✗	cnt = scan->histogram[i];
206		✗	black = i;
207			}
208			}
209
210		✗	cnt = 0;
211		✗	for (int i = end; i >= middle; i--) {
212		✗	if (scan->histogram[i] > cnt) {
213		✗	cnt = scan->histogram[i];
214		✗	white = i;
215			}
216			}
217
218		✗	scan->black = black;
219		✗	scan->white = white;
220		✗	}
221
222		✗	static float meanf(const LineItem *line, int len)
223			{
224		✗	float sum = 0.0, mean = 0.0;
225
226		✗	for (int i = 0; i < len; i++)
227		✗	sum += line[i].filtered;
228
229		✗	mean = sum / len;
230
231		✗	return mean;
232			}
233
234		✗	static float stddevf(const LineItem *line, int len)
235			{
236		✗	float m = meanf(line, len);
237		✗	float standard_deviation = 0.f;
238
239		✗	for (int i = 0; i < len; i++)
240		✗	standard_deviation += (line[i].filtered - m) * (line[i].filtered - m);
241
242		✗	return sqrtf(standard_deviation / (len - 1));
243			}
244
245		✗	static void thresholding(ReadEIA608Context *s, ScanItem *scan, LineItem *line,
246			int lag, float threshold, float influence, int len)
247			{
248		✗	for (int i = lag; i < len + lag; i++) {
249		✗	line[i].unfiltered = line[i].input / 255.f;
250		✗	line[i].filtered = line[i].unfiltered;
251			}
252
253		✗	for (int i = 0; i < lag; i++) {
254		✗	line[i].unfiltered = meanf(line, len * s->spw);
255		✗	line[i].filtered = line[i].unfiltered;
256			}
257
258		✗	line[lag - 1].average = meanf(line, lag);
259		✗	line[lag - 1].deviation = stddevf(line, lag);
260
261		✗	for (int i = lag; i < len + lag; i++) {
262		✗	if (fabsf(line[i].unfiltered - line[i-1].average) > threshold * line[i-1].deviation) {
263		✗	if (line[i].unfiltered > line[i-1].average) {
264		✗	line[i].output = 255;
265			} else {
266		✗	line[i].output = 0;
267			}
268
269		✗	line[i].filtered = influence * line[i].unfiltered + (1.f - influence) * line[i-1].filtered;
270			} else {
271			int distance_from_black, distance_from_white;
272
273		✗	distance_from_black = FFABS(line[i].input - scan->black);
274		✗	distance_from_white = FFABS(line[i].input - scan->white);
275
276		✗	line[i].output = distance_from_black <= distance_from_white ? 0 : 255;
277			}
278
279		✗	line[i].average = meanf(line + i - lag, lag);
280		✗	line[i].deviation = stddevf(line + i - lag, lag);
281			}
282		✗	}
283
284		✗	static int periods(const LineItem *line, CodeItem *code, int len)
285			{
286		✗	int hold = line[LAG].output, cnt = 0;
287		✗	int last = LAG;
288
289		✗	memset(code, 0, len * sizeof(*code));
290
291		✗	for (int i = LAG + 1; i < len + LAG; i++) {
292		✗	if (line[i].output != hold) {
293		✗	code[cnt].size = i - last;
294		✗	code[cnt].bit = hold;
295		✗	hold = line[i].output;
296		✗	last = i;
297		✗	cnt++;
298			}
299			}
300
301		✗	code[cnt].size = LAG + len - last;
302		✗	code[cnt].bit = hold;
303
304		✗	return cnt + 1;
305			}
306
307		✗	static void dump_code(AVFilterContext *ctx, ScanItem *scan, int len, int item)
308			{
309		✗	av_log(ctx, AV_LOG_DEBUG, "%d:", item);
310		✗	for (int i = 0; i < len; i++) {
311		✗	av_log(ctx, AV_LOG_DEBUG, " %03d", scan->code[i].size);
312			}
313		✗	av_log(ctx, AV_LOG_DEBUG, "\n");
314		✗	}
315
316			#define READ_LINE(type, name) \
317			static void read_##name(AVFrame *in, int nb_line, LineItem *line, int lp, int w) \
318			{ \
319			const type *src = (const type *)(&in->data[0][nb_line * in->linesize[0]]);\
320			\
321			if (lp) { \
322			for (int i = 0; i < w; i++) { \
323			int a = FFMAX(i - 3, 0); \
324			int b = FFMAX(i - 2, 0); \
325			int c = FFMAX(i - 1, 0); \
326			int d = FFMIN(i + 3, w-1); \
327			int e = FFMIN(i + 2, w-1); \
328			int f = FFMIN(i + 1, w-1); \
329			\
330			line[LAG + i].input = (src[a] + src[b] + src[c] + src[i] + \
331			src[d] + src[e] + src[f] + 6) / 7; \
332			} \
333			} else { \
334			for (int i = 0; i < w; i++) { \
335			line[LAG + i].input = src[i]; \
336			} \
337			} \
338			}
339
340		✗	READ_LINE(uint8_t, byte)
341		✗	READ_LINE(uint16_t, word)
342
343		✗	static int config_input(AVFilterLink *inlink)
344			{
345		✗	AVFilterContext *ctx = inlink->dst;
346		✗	ReadEIA608Context *s = ctx->priv;
347		✗	const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
348
349		✗	if (!desc)
350		✗	return AVERROR_BUG;
351		✗	s->depth = desc->comp[0].depth;
352		✗	s->max = (1 << desc->comp[0].depth) - 1;
353		✗	s->read_line[0] = read_byte;
354		✗	s->read_line[1] = read_word;
355
356		✗	return config_filter(ctx, s->start, s->end);
357			}
358
359		✗	static void extract_line(AVFilterContext *ctx, AVFrame *in, ScanItem *scan, int w, int nb_line)
360			{
361		✗	ReadEIA608Context *s = ctx->priv;
362		✗	LineItem *line = scan->line;
363			int i, j, ch, len;
364		✗	uint8_t codes[19] = { 0 };
365		✗	float bit_size = 0.f;
366			int parity;
367
368		✗	memset(line, 0, (w + LAG) * sizeof(*line));
369		✗	scan->byte[0] = scan->byte[1] = 0;
370		✗	scan->found = 0;
371
372		✗	s->read_line[s->depth > 8](in, nb_line, line, s->lp, w);
373
374		✗	build_histogram(s, scan, line, w);
375		✗	find_black_and_white(s, scan);
376		✗	if (scan->white - scan->black < 5)
377		✗	return;
378
379		✗	thresholding(s, scan, line, LAG, 1, 0, w);
380		✗	len = periods(line, scan->code, w);
381		✗	dump_code(ctx, scan, len, nb_line);
382		✗	if (len < 15 ||
383		✗	scan->code[14].bit != 0 ||
384		✗	w / (float)scan->code[14].size < SYNC_BITSIZE_MIN ||
385		✗	w / (float)scan->code[14].size > SYNC_BITSIZE_MAX) {
386		✗	return;
387			}
388
389		✗	for (i = 14; i < len; i++) {
390		✗	bit_size += scan->code[i].size;
391			}
392
393		✗	bit_size /= 19.f;
394		✗	for (i = 1; i < 14; i++) {
395		✗	if (scan->code[i].size / bit_size > CLOCK_BITSIZE_MAX ||
396		✗	scan->code[i].size / bit_size < CLOCK_BITSIZE_MIN) {
397		✗	return;
398			}
399			}
400
401		✗	if (scan->code[15].size / bit_size < 0.45f) {
402		✗	return;
403			}
404
405		✗	for (j = 0, i = 14; i < len; i++) {
406			int run, bit;
407
408		✗	run = lrintf(scan->code[i].size / bit_size);
409		✗	bit = scan->code[i].bit;
410
411		✗	for (int k = 0; j < 19 && k < run; k++) {
412		✗	codes[j++] = bit;
413			}
414
415		✗	if (j >= 19)
416		✗	break;
417			}
418
419		✗	for (ch = 0; ch < 2; ch++) {
420		✗	for (parity = 0, i = 0; i < 8; i++) {
421		✗	int b = codes[3 + ch * 8 + i];
422
423		✗	if (b == 255) {
424		✗	parity++;
425		✗	b = 1;
426			} else {
427		✗	b = 0;
428			}
429		✗	scan->byte[ch] |= b << i;
430			}
431
432		✗	if (s->chp) {
433		✗	if (!(parity & 1)) {
434		✗	scan->byte[ch] = 0x7F;
435			}
436			}
437			}
438
439		✗	scan->nb_line = nb_line;
440		✗	scan->found = 1;
441			}
442
443		✗	static int extract_lines(AVFilterContext *ctx, void *arg,
444			int job, int nb_jobs)
445			{
446		✗	ReadEIA608Context *s = ctx->priv;
447		✗	AVFilterLink *inlink = ctx->inputs[0];
448		✗	const int h = s->end - s->start + 1;
449		✗	const int start = (h * job) / nb_jobs;
450		✗	const int end = (h * (job+1)) / nb_jobs;
451		✗	AVFrame *in = arg;
452
453		✗	for (int i = start; i < end; i++) {
454		✗	ScanItem *scan = &s->scan[i];
455
456		✗	extract_line(ctx, in, scan, inlink->w, s->start + i);
457			}
458
459		✗	return 0;
460			}
461
462		✗	static int filter_frame(AVFilterLink *inlink, AVFrame *in)
463			{
464		✗	AVFilterContext *ctx = inlink->dst;
465		✗	AVFilterLink *outlink = ctx->outputs[0];
466		✗	ReadEIA608Context *s = ctx->priv;
467			int nb_found;
468
469		✗	ff_filter_execute(ctx, extract_lines, in, NULL,
470		✗	FFMIN(FFMAX(s->end - s->start + 1, 1), ff_filter_get_nb_threads(ctx)));
471
472		✗	nb_found = 0;
473		✗	for (int i = 0; i < s->end - s->start + 1; i++) {
474		✗	ScanItem *scan = &s->scan[i];
475			uint8_t key[128], value[128];
476
477		✗	if (!scan->found)
478		✗	continue;
479
480			//snprintf(key, sizeof(key), "lavfi.readeia608.%d.bits", nb_found);
481			//snprintf(value, sizeof(value), "0b%d%d%d%d%d%d%d%d 0b%d%d%d%d%d%d%d%d", codes[3]==255,codes[4]==255,codes[5]==255,codes[6]==255,codes[7]==255,codes[8]==255,codes[9]==255,codes[10]==255,codes[11]==255,codes[12]==255,codes[13]==255,codes[14]==255,codes[15]==255,codes[16]==255,codes[17]==255,codes[18]==255);
482			//av_dict_set(&in->metadata, key, value, 0);
483
484		✗	snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", nb_found);
485		✗	snprintf(value, sizeof(value), "0x%02X%02X", scan->byte[0], scan->byte[1]);
486		✗	av_dict_set(&in->metadata, key, value, 0);
487
488		✗	snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", nb_found);
489		✗	av_dict_set_int(&in->metadata, key, scan->nb_line, 0);
490
491		✗	nb_found++;
492			}
493
494		✗	return ff_filter_frame(outlink, in);
495			}
496
497		✗	static av_cold void uninit(AVFilterContext *ctx)
498			{
499		✗	ReadEIA608Context *s = ctx->priv;
500
501		✗	for (int i = 0; i < s->nb_allocated; i++) {
502		✗	ScanItem *scan = &s->scan[i];
503
504		✗	av_freep(&scan->histogram);
505		✗	av_freep(&scan->code);
506		✗	av_freep(&scan->line);
507			}
508
509		✗	s->nb_allocated = 0;
510		✗	av_freep(&s->scan);
511		✗	}
512
513		✗	static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
514			char *res, int res_len, int flags)
515			{
516		✗	ReadEIA608Context *s = ctx->priv;
517		✗	int ret, start = s->start, end = s->end;
518
519		✗	ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
520		✗	if (ret < 0)
521		✗	return ret;
522
523		✗	ret = config_filter(ctx, s->start, s->end);
524		✗	if (ret < 0) {
525		✗	s->start = start;
526		✗	s->end = end;
527			}
528
529		✗	return 0;
530			}
531
532			static const AVFilterPad readeia608_inputs[] = {
533			{
534			.name = "default",
535			.type = AVMEDIA_TYPE_VIDEO,
536			.filter_frame = filter_frame,
537			.config_props = config_input,
538			},
539			};
540
541			const FFFilter ff_vf_readeia608 = {
542			.p.name = "readeia608",
543			.p.description = NULL_IF_CONFIG_SMALL("Read EIA-608 Closed Caption codes from input video and write them to frame metadata."),
544			.p.priv_class = &readeia608_class,
545			.p.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC |
546			AVFILTER_FLAG_SLICE_THREADS |
547			AVFILTER_FLAG_METADATA_ONLY,
548			.priv_size = sizeof(ReadEIA608Context),
549			FILTER_INPUTS(readeia608_inputs),
550			FILTER_OUTPUTS(ff_video_default_filterpad),
551			FILTER_PIXFMTS_ARRAY(pixel_fmts),
552			.uninit = uninit,
553			.process_command = process_command,
554			};
555