FFmpeg coverage


Directory: ../../../ffmpeg/
File: src/libavfilter/vf_readeia608.c
Date: 2024-02-16 17:37:06
Exec Total Coverage
Lines: 0 227 0.0%
Functions: 0 16 0.0%
Branches: 0 140 0.0%

Line Branch Exec Source
1 /*
2 * Copyright (c) 2017 Paul B Mahol
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * Filter for reading closed captioning data (EIA-608).
24 * See also https://en.wikipedia.org/wiki/EIA-608
25 */
26
27 #include <string.h>
28
29 #include "libavutil/internal.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/pixdesc.h"
32
33 #include "avfilter.h"
34 #include "internal.h"
35 #include "video.h"
36
37 #define LAG 25
38 #define CLOCK_BITSIZE_MIN 0.2f
39 #define CLOCK_BITSIZE_MAX 1.5f
40 #define SYNC_BITSIZE_MIN 12.f
41 #define SYNC_BITSIZE_MAX 15.f
42
43 typedef struct LineItem {
44 int input;
45 int output;
46
47 float unfiltered;
48 float filtered;
49 float average;
50 float deviation;
51 } LineItem;
52
53 typedef struct CodeItem {
54 uint8_t bit;
55 int size;
56 } CodeItem;
57
58 typedef struct ScanItem {
59 int nb_line;
60 int found;
61 int white;
62 int black;
63 uint64_t *histogram;
64 uint8_t byte[2];
65
66 CodeItem *code;
67 LineItem *line;
68 } ScanItem;
69
70 typedef struct ReadEIA608Context {
71 const AVClass *class;
72
73 int start, end;
74 float spw;
75 int chp;
76 int lp;
77
78 int depth;
79 int max;
80 int nb_allocated;
81 ScanItem *scan;
82
83 void (*read_line[2])(AVFrame *in, int nb_line,
84 LineItem *line, int lp, int w);
85 } ReadEIA608Context;
86
87 #define OFFSET(x) offsetof(ReadEIA608Context, x)
88 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
89
90 static const AVOption readeia608_options[] = {
91 { "scan_min", "set from which line to scan for codes", OFFSET(start), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX, FLAGS },
92 { "scan_max", "set to which line to scan for codes", OFFSET(end), AV_OPT_TYPE_INT, {.i64=29}, 0, INT_MAX, FLAGS },
93 { "spw", "set ratio of width reserved for sync code detection", OFFSET(spw), AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1, 0.7, FLAGS },
94 { "chp", "check and apply parity bit", OFFSET(chp), AV_OPT_TYPE_BOOL, {.i64= 0}, 0, 1, FLAGS },
95 { "lp", "lowpass line prior to processing", OFFSET(lp), AV_OPT_TYPE_BOOL, {.i64= 1}, 0, 1, FLAGS },
96 { NULL }
97 };
98
99 AVFILTER_DEFINE_CLASS(readeia608);
100
101 static const enum AVPixelFormat pixel_fmts[] = {
102 AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9,
103 AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14,
104 AV_PIX_FMT_GRAY16,
105 AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
106 AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
107 AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
108 AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
109 AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
110 AV_PIX_FMT_YUVJ411P,
111 AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9,
112 AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10,
113 AV_PIX_FMT_YUV440P10,
114 AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12,
115 AV_PIX_FMT_YUV440P12,
116 AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14,
117 AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16,
118 AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P,
119 AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16,
120 AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16,
121 AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16,
122 AV_PIX_FMT_NONE
123 };
124
125 static int config_filter(AVFilterContext *ctx, int start, int end)
126 {
127 ReadEIA608Context *s = ctx->priv;
128 AVFilterLink *inlink = ctx->inputs[0];
129 int size = inlink->w + LAG;
130
131 if (end >= inlink->h) {
132 av_log(ctx, AV_LOG_WARNING, "Last line to scan too large, clipping.\n");
133 end = inlink->h - 1;
134 }
135
136 if (start > end) {
137 av_log(ctx, AV_LOG_ERROR, "Invalid range.\n");
138 return AVERROR(EINVAL);
139 }
140
141 if (s->nb_allocated < end - start + 1) {
142 const int diff = end - start + 1 - s->nb_allocated;
143
144 s->scan = av_realloc_f(s->scan, end - start + 1, sizeof(*s->scan));
145 if (!s->scan)
146 return AVERROR(ENOMEM);
147 memset(&s->scan[s->nb_allocated], 0, diff * sizeof(*s->scan));
148 s->nb_allocated = end - start + 1;
149 }
150
151 for (int i = 0; i < s->nb_allocated; i++) {
152 ScanItem *scan = &s->scan[i];
153
154 if (!scan->histogram)
155 scan->histogram = av_calloc(s->max + 1, sizeof(*scan->histogram));
156 if (!scan->line)
157 scan->line = av_calloc(size, sizeof(*scan->line));
158 if (!scan->code)
159 scan->code = av_calloc(size, sizeof(*scan->code));
160 if (!scan->line || !scan->code || !scan->histogram)
161 return AVERROR(ENOMEM);
162 }
163
164 s->start = start;
165 s->end = end;
166
167 return 0;
168 }
169
170 static void build_histogram(ReadEIA608Context *s, ScanItem *scan, const LineItem *line, int len)
171 {
172 memset(scan->histogram, 0, (s->max + 1) * sizeof(*scan->histogram));
173
174 for (int i = LAG; i < len + LAG; i++)
175 scan->histogram[line[i].input]++;
176 }
177
178 static void find_black_and_white(ReadEIA608Context *s, ScanItem *scan)
179 {
180 const int max = s->max;
181 int start = 0, end = 0, middle;
182 int black = 0, white = 0;
183 int cnt;
184
185 for (int i = 0; i <= max; i++) {
186 if (scan->histogram[i]) {
187 start = i;
188 break;
189 }
190 }
191
192 for (int i = max; i >= 0; i--) {
193 if (scan->histogram[i]) {
194 end = i;
195 break;
196 }
197 }
198
199 middle = start + (end - start) / 2;
200
201 cnt = 0;
202 for (int i = start; i <= middle; i++) {
203 if (scan->histogram[i] > cnt) {
204 cnt = scan->histogram[i];
205 black = i;
206 }
207 }
208
209 cnt = 0;
210 for (int i = end; i >= middle; i--) {
211 if (scan->histogram[i] > cnt) {
212 cnt = scan->histogram[i];
213 white = i;
214 }
215 }
216
217 scan->black = black;
218 scan->white = white;
219 }
220
221 static float meanf(const LineItem *line, int len)
222 {
223 float sum = 0.0, mean = 0.0;
224
225 for (int i = 0; i < len; i++)
226 sum += line[i].filtered;
227
228 mean = sum / len;
229
230 return mean;
231 }
232
233 static float stddevf(const LineItem *line, int len)
234 {
235 float m = meanf(line, len);
236 float standard_deviation = 0.f;
237
238 for (int i = 0; i < len; i++)
239 standard_deviation += (line[i].filtered - m) * (line[i].filtered - m);
240
241 return sqrtf(standard_deviation / (len - 1));
242 }
243
244 static void thresholding(ReadEIA608Context *s, ScanItem *scan, LineItem *line,
245 int lag, float threshold, float influence, int len)
246 {
247 for (int i = lag; i < len + lag; i++) {
248 line[i].unfiltered = line[i].input / 255.f;
249 line[i].filtered = line[i].unfiltered;
250 }
251
252 for (int i = 0; i < lag; i++) {
253 line[i].unfiltered = meanf(line, len * s->spw);
254 line[i].filtered = line[i].unfiltered;
255 }
256
257 line[lag - 1].average = meanf(line, lag);
258 line[lag - 1].deviation = stddevf(line, lag);
259
260 for (int i = lag; i < len + lag; i++) {
261 if (fabsf(line[i].unfiltered - line[i-1].average) > threshold * line[i-1].deviation) {
262 if (line[i].unfiltered > line[i-1].average) {
263 line[i].output = 255;
264 } else {
265 line[i].output = 0;
266 }
267
268 line[i].filtered = influence * line[i].unfiltered + (1.f - influence) * line[i-1].filtered;
269 } else {
270 int distance_from_black, distance_from_white;
271
272 distance_from_black = FFABS(line[i].input - scan->black);
273 distance_from_white = FFABS(line[i].input - scan->white);
274
275 line[i].output = distance_from_black <= distance_from_white ? 0 : 255;
276 }
277
278 line[i].average = meanf(line + i - lag, lag);
279 line[i].deviation = stddevf(line + i - lag, lag);
280 }
281 }
282
283 static int periods(const LineItem *line, CodeItem *code, int len)
284 {
285 int hold = line[LAG].output, cnt = 0;
286 int last = LAG;
287
288 memset(code, 0, len * sizeof(*code));
289
290 for (int i = LAG + 1; i < len + LAG; i++) {
291 if (line[i].output != hold) {
292 code[cnt].size = i - last;
293 code[cnt].bit = hold;
294 hold = line[i].output;
295 last = i;
296 cnt++;
297 }
298 }
299
300 code[cnt].size = LAG + len - last;
301 code[cnt].bit = hold;
302
303 return cnt + 1;
304 }
305
306 static void dump_code(AVFilterContext *ctx, ScanItem *scan, int len, int item)
307 {
308 av_log(ctx, AV_LOG_DEBUG, "%d:", item);
309 for (int i = 0; i < len; i++) {
310 av_log(ctx, AV_LOG_DEBUG, " %03d", scan->code[i].size);
311 }
312 av_log(ctx, AV_LOG_DEBUG, "\n");
313 }
314
315 #define READ_LINE(type, name) \
316 static void read_##name(AVFrame *in, int nb_line, LineItem *line, int lp, int w) \
317 { \
318 const type *src = (const type *)(&in->data[0][nb_line * in->linesize[0]]);\
319 \
320 if (lp) { \
321 for (int i = 0; i < w; i++) { \
322 int a = FFMAX(i - 3, 0); \
323 int b = FFMAX(i - 2, 0); \
324 int c = FFMAX(i - 1, 0); \
325 int d = FFMIN(i + 3, w-1); \
326 int e = FFMIN(i + 2, w-1); \
327 int f = FFMIN(i + 1, w-1); \
328 \
329 line[LAG + i].input = (src[a] + src[b] + src[c] + src[i] + \
330 src[d] + src[e] + src[f] + 6) / 7; \
331 } \
332 } else { \
333 for (int i = 0; i < w; i++) { \
334 line[LAG + i].input = src[i]; \
335 } \
336 } \
337 }
338
339 READ_LINE(uint8_t, byte)
340 READ_LINE(uint16_t, word)
341
342 static int config_input(AVFilterLink *inlink)
343 {
344 AVFilterContext *ctx = inlink->dst;
345 ReadEIA608Context *s = ctx->priv;
346 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
347
348 if (!desc)
349 return AVERROR_BUG;
350 s->depth = desc->comp[0].depth;
351 s->max = (1 << desc->comp[0].depth) - 1;
352 s->read_line[0] = read_byte;
353 s->read_line[1] = read_word;
354
355 return config_filter(ctx, s->start, s->end);
356 }
357
358 static void extract_line(AVFilterContext *ctx, AVFrame *in, ScanItem *scan, int w, int nb_line)
359 {
360 ReadEIA608Context *s = ctx->priv;
361 LineItem *line = scan->line;
362 int i, j, ch, len;
363 uint8_t codes[19] = { 0 };
364 float bit_size = 0.f;
365 int parity;
366
367 memset(line, 0, (w + LAG) * sizeof(*line));
368 scan->byte[0] = scan->byte[1] = 0;
369 scan->found = 0;
370
371 s->read_line[s->depth > 8](in, nb_line, line, s->lp, w);
372
373 build_histogram(s, scan, line, w);
374 find_black_and_white(s, scan);
375 if (scan->white - scan->black < 5)
376 return;
377
378 thresholding(s, scan, line, LAG, 1, 0, w);
379 len = periods(line, scan->code, w);
380 dump_code(ctx, scan, len, nb_line);
381 if (len < 15 ||
382 scan->code[14].bit != 0 ||
383 w / (float)scan->code[14].size < SYNC_BITSIZE_MIN ||
384 w / (float)scan->code[14].size > SYNC_BITSIZE_MAX) {
385 return;
386 }
387
388 for (i = 14; i < len; i++) {
389 bit_size += scan->code[i].size;
390 }
391
392 bit_size /= 19.f;
393 for (i = 1; i < 14; i++) {
394 if (scan->code[i].size / bit_size > CLOCK_BITSIZE_MAX ||
395 scan->code[i].size / bit_size < CLOCK_BITSIZE_MIN) {
396 return;
397 }
398 }
399
400 if (scan->code[15].size / bit_size < 0.45f) {
401 return;
402 }
403
404 for (j = 0, i = 14; i < len; i++) {
405 int run, bit;
406
407 run = lrintf(scan->code[i].size / bit_size);
408 bit = scan->code[i].bit;
409
410 for (int k = 0; j < 19 && k < run; k++) {
411 codes[j++] = bit;
412 }
413
414 if (j >= 19)
415 break;
416 }
417
418 for (ch = 0; ch < 2; ch++) {
419 for (parity = 0, i = 0; i < 8; i++) {
420 int b = codes[3 + ch * 8 + i];
421
422 if (b == 255) {
423 parity++;
424 b = 1;
425 } else {
426 b = 0;
427 }
428 scan->byte[ch] |= b << i;
429 }
430
431 if (s->chp) {
432 if (!(parity & 1)) {
433 scan->byte[ch] = 0x7F;
434 }
435 }
436 }
437
438 scan->nb_line = nb_line;
439 scan->found = 1;
440 }
441
442 static int extract_lines(AVFilterContext *ctx, void *arg,
443 int job, int nb_jobs)
444 {
445 ReadEIA608Context *s = ctx->priv;
446 AVFilterLink *inlink = ctx->inputs[0];
447 const int h = s->end - s->start + 1;
448 const int start = (h * job) / nb_jobs;
449 const int end = (h * (job+1)) / nb_jobs;
450 AVFrame *in = arg;
451
452 for (int i = start; i < end; i++) {
453 ScanItem *scan = &s->scan[i];
454
455 extract_line(ctx, in, scan, inlink->w, s->start + i);
456 }
457
458 return 0;
459 }
460
461 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
462 {
463 AVFilterContext *ctx = inlink->dst;
464 AVFilterLink *outlink = ctx->outputs[0];
465 ReadEIA608Context *s = ctx->priv;
466 int nb_found;
467
468 ff_filter_execute(ctx, extract_lines, in, NULL,
469 FFMIN(FFMAX(s->end - s->start + 1, 1), ff_filter_get_nb_threads(ctx)));
470
471 nb_found = 0;
472 for (int i = 0; i < s->end - s->start + 1; i++) {
473 ScanItem *scan = &s->scan[i];
474 uint8_t key[128], value[128];
475
476 if (!scan->found)
477 continue;
478
479 //snprintf(key, sizeof(key), "lavfi.readeia608.%d.bits", nb_found);
480 //snprintf(value, sizeof(value), "0b%d%d%d%d%d%d%d%d 0b%d%d%d%d%d%d%d%d", codes[3]==255,codes[4]==255,codes[5]==255,codes[6]==255,codes[7]==255,codes[8]==255,codes[9]==255,codes[10]==255,codes[11]==255,codes[12]==255,codes[13]==255,codes[14]==255,codes[15]==255,codes[16]==255,codes[17]==255,codes[18]==255);
481 //av_dict_set(&in->metadata, key, value, 0);
482
483 snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", nb_found);
484 snprintf(value, sizeof(value), "0x%02X%02X", scan->byte[0], scan->byte[1]);
485 av_dict_set(&in->metadata, key, value, 0);
486
487 snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", nb_found);
488 av_dict_set_int(&in->metadata, key, scan->nb_line, 0);
489
490 nb_found++;
491 }
492
493 return ff_filter_frame(outlink, in);
494 }
495
496 static av_cold void uninit(AVFilterContext *ctx)
497 {
498 ReadEIA608Context *s = ctx->priv;
499
500 for (int i = 0; i < s->nb_allocated; i++) {
501 ScanItem *scan = &s->scan[i];
502
503 av_freep(&scan->histogram);
504 av_freep(&scan->code);
505 av_freep(&scan->line);
506 }
507
508 s->nb_allocated = 0;
509 av_freep(&s->scan);
510 }
511
512 static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
513 char *res, int res_len, int flags)
514 {
515 ReadEIA608Context *s = ctx->priv;
516 int ret, start = s->start, end = s->end;
517
518 ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
519 if (ret < 0)
520 return ret;
521
522 ret = config_filter(ctx, s->start, s->end);
523 if (ret < 0) {
524 s->start = start;
525 s->end = end;
526 }
527
528 return 0;
529 }
530
531 static const AVFilterPad readeia608_inputs[] = {
532 {
533 .name = "default",
534 .type = AVMEDIA_TYPE_VIDEO,
535 .filter_frame = filter_frame,
536 .config_props = config_input,
537 },
538 };
539
540 const AVFilter ff_vf_readeia608 = {
541 .name = "readeia608",
542 .description = NULL_IF_CONFIG_SMALL("Read EIA-608 Closed Caption codes from input video and write them to frame metadata."),
543 .priv_size = sizeof(ReadEIA608Context),
544 .priv_class = &readeia608_class,
545 FILTER_INPUTS(readeia608_inputs),
546 FILTER_OUTPUTS(ff_video_default_filterpad),
547 FILTER_PIXFMTS_ARRAY(pixel_fmts),
548 .uninit = uninit,
549 .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC |
550 AVFILTER_FLAG_SLICE_THREADS |
551 AVFILTER_FLAG_METADATA_ONLY,
552 .process_command = process_command,
553 };
554