FFmpeg coverage

Directory: ../../../ffmpeg/
File: src/libavfilter/vf_thumbnail.c
Date: 2025-07-28 20:30:09
Exec Total Coverage
Lines: 134 189 70.9%
Functions: 10 11 90.9%
Branches: 66 104 63.5%
Line Branch Exec Source
1 /*
2 * Copyright (c) 2011 Smartjog S.A.S, Clément Bœsch <clement.boesch@smartjog.com>
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * Potential thumbnail lookup filter to reduce the risk of an inappropriate
24 * selection (such as a black frame) we could get with an absolute seek.
25 *
26 * Simplified version of algorithm by Vadim Zaliva <lord@crocodile.org>.
27 * @see http://notbrainsurgery.livejournal.com/29773.html
28 */
29
30 #include "libavutil/intreadwrite.h"
31 #include "libavutil/mem.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/pixdesc.h"
34 #include "avfilter.h"
35 #include "filters.h"
36 #include "formats.h"
37
38 #define HIST_SIZE (3*256)
39
40 struct thumb_frame {
41 AVFrame *buf; ///< cached frame
42 int histogram[HIST_SIZE]; ///< RGB color distribution histogram of the frame
43 };
44
45 typedef struct ThumbContext {
46 const AVClass *class;
47 int n; ///< current frame
48 int loglevel;
49 int n_frames; ///< number of frames for analysis
50 struct thumb_frame *frames; ///< the n_frames frames
51 AVRational tb; ///< copy of the input timebase to ease access
52
53 int nb_threads;
54 int *thread_histogram;
55
56 int planewidth[4];
57 int planeheight[4];
58 int planes;
59 int bitdepth;
60 } ThumbContext;
61
62 #define OFFSET(x) offsetof(ThumbContext, x)
63 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
64
65 static const AVOption thumbnail_options[] = {
66 { "n", "set the frames batch size", OFFSET(n_frames), AV_OPT_TYPE_INT, {.i64=100}, 2, INT_MAX, FLAGS },
67 { "log", "force stats logging level", OFFSET(loglevel), AV_OPT_TYPE_INT, {.i64 = AV_LOG_INFO}, INT_MIN, INT_MAX, FLAGS, .unit = "level" },
68 { "quiet", "logging disabled", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_QUIET}, 0, 0, FLAGS, .unit = "level" },
69 { "info", "information logging level", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_INFO}, 0, 0, FLAGS, .unit = "level" },
70 { "verbose", "verbose logging level", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_VERBOSE}, 0, 0, FLAGS, .unit = "level" },
71 { NULL }
72 };
73
74 AVFILTER_DEFINE_CLASS(thumbnail);
75
76 2 static av_cold int init(AVFilterContext *ctx)
77 {
78 2 ThumbContext *s = ctx->priv;
79
80 2 s->frames = av_calloc(s->n_frames, sizeof(*s->frames));
81
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (!s->frames) {
82 ✗ av_log(ctx, AV_LOG_ERROR,
83 "Allocation failure, try to lower the number of frames\n");
84 ✗ return AVERROR(ENOMEM);
85 }
86 2 av_log(ctx, AV_LOG_VERBOSE, "batch size: %d frames\n", s->n_frames);
87 2 return 0;
88 }
89
90 /**
91 * @brief Compute Sum-square deviation to estimate "closeness".
92 * @param hist color distribution histogram
93 * @param median average color distribution histogram
94 * @return sum of squared errors
95 */
96 50 static double frame_sum_square_err(const int *hist, const double *median)
97 {
98 int i;
99 50 double err, sum_sq_err = 0;
100
101
2/2
✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 50 times.
 38450 for (i = 0; i < HIST_SIZE; i++) {
102 38400 err = median[i] - (double)hist[i];
103 38400 sum_sq_err += err*err;
104 }
105 50 return sum_sq_err;
106 }
107
108 5 static AVFrame *get_best_frame(AVFilterContext *ctx)
109 {
110 AVFrame *picref;
111 5 ThumbContext *s = ctx->priv;
112 5 int i, j, best_frame_idx = 0;
113 5 int nb_frames = s->n;
114 5 double avg_hist[HIST_SIZE] = {0}, sq_err, min_sq_err = -1;
115
116 // average histogram of the N frames
117
2/2
✓ Branch 0 taken 3840 times.
✓ Branch 1 taken 5 times.
 3845 for (j = 0; j < FF_ARRAY_ELEMS(avg_hist); j++) {
118
2/2
✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 3840 times.
 42240 for (i = 0; i < nb_frames; i++)
119 38400 avg_hist[j] += (double)s->frames[i].histogram[j];
120 3840 avg_hist[j] /= nb_frames;
121 }
122
123 // find the frame closer to the average using the sum of squared errors
124
2/2
✓ Branch 0 taken 50 times.
✓ Branch 1 taken 5 times.
 55 for (i = 0; i < nb_frames; i++) {
125 50 sq_err = frame_sum_square_err(s->frames[i].histogram, avg_hist);
126
4/4
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 12 times.
✓ Branch 3 taken 33 times.
 50 if (i == 0 || sq_err < min_sq_err)
127 17 best_frame_idx = i, min_sq_err = sq_err;
128 }
129
130 // free and reset everything (except the best frame buffer)
131
2/2
✓ Branch 0 taken 50 times.
✓ Branch 1 taken 5 times.
 55 for (i = 0; i < nb_frames; i++) {
132 50 memset(s->frames[i].histogram, 0, sizeof(s->frames[i].histogram));
133
2/2
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
 50 if (i != best_frame_idx)
134 45 av_frame_free(&s->frames[i].buf);
135 }
136 5 s->n = 0;
137
138 // raise the chosen one
139 5 picref = s->frames[best_frame_idx].buf;
140
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (s->loglevel != AV_LOG_QUIET)
141 5 av_log(ctx, s->loglevel, "frame id #%d (pts_time=%f) selected "
142 "from a set of %d images\n", best_frame_idx,
143 5 picref->pts * av_q2d(s->tb), nb_frames);
144 5 s->frames[best_frame_idx].buf = NULL;
145
146 5 return picref;
147 }
148
149 150 static void get_hist8(int *hist, const uint8_t *p, ptrdiff_t stride,
150 ptrdiff_t width, ptrdiff_t height)
151 {
152 150 int shist[4][256] = {0};
153
154 150 const int width4 = width & ~3;
155
2/2
✓ Branch 0 taken 28800 times.
✓ Branch 1 taken 150 times.
 28950 while (height--) {
156
2/2
✓ Branch 0 taken 1900800 times.
✓ Branch 1 taken 28800 times.
 1929600 for (int x = 0; x < width4; x += 4) {
157 1900800 const uint32_t v = AV_RN32(&p[x]);
158 1900800 shist[0][(uint8_t) (v >> 0)]++;
159 1900800 shist[1][(uint8_t) (v >> 8)]++;
160 1900800 shist[2][(uint8_t) (v >> 16)]++;
161 1900800 shist[3][(uint8_t) (v >> 24)]++;
162 }
163 /* handle tail */
164
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28800 times.
 28800 for (int x = width4; x < width; x++)
165 ✗ hist[p[x]]++;
166 28800 p += stride;
167 }
168
169
2/2
✓ Branch 0 taken 600 times.
✓ Branch 1 taken 150 times.
 750 for (int i = 0; i < 4; i++) {
170
2/2
✓ Branch 0 taken 153600 times.
✓ Branch 1 taken 600 times.
 154200 for (int j = 0; j < 256; j++)
171 153600 hist[j] += shist[i][j];
172 }
173 150 }
174
175 ✗ static void get_hist16(int *hist, const uint8_t *p, ptrdiff_t stride,
176 ptrdiff_t width, ptrdiff_t height, int shift)
177 {
178 ✗ int shist[4][256] = {0};
179
180 ✗ const int width4 = width & ~3;
181 ✗ while (height--) {
182 ✗ const uint16_t *p16 = (const uint16_t *) p;
183 ✗ for (int x = 0; x < width4; x += 4) {
184 ✗ const uint64_t v = AV_RN64(&p16[x]);
185 ✗ shist[0][(uint8_t) (v >> (shift + 0))]++;
186 ✗ shist[1][(uint8_t) (v >> (shift + 16))]++;
187 ✗ shist[2][(uint8_t) (v >> (shift + 32))]++;
188 ✗ shist[3][(uint8_t) (v >> (shift + 48))]++;
189 }
190 /* handle tail */
191 ✗ for (int x = width4; x < width; x++)
192 ✗ hist[p16[x]]++;
193 ✗ p += stride;
194 }
195
196 ✗ for (int i = 0; i < 4; i++) {
197 ✗ for (int j = 0; j < 256; j++)
198 ✗ hist[j] += shist[i][j];
199 }
200 ✗ }
201
202 50 static int do_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
203 {
204 50 ThumbContext *s = ctx->priv;
205 50 AVFrame *frame = arg;
206 50 int *hist = s->thread_histogram + HIST_SIZE * jobnr;
207 50 const int h = frame->height;
208 50 const int w = frame->width;
209 50 const int slice_start = (h * jobnr) / nb_jobs;
210 50 const int slice_end = (h * (jobnr+1)) / nb_jobs;
211 50 const uint8_t *p = frame->data[0] + slice_start * frame->linesize[0];
212
213 50 memset(hist, 0, sizeof(*hist) * HIST_SIZE);
214
215
1/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 50 times.
 50 switch (frame->format) {
216 ✗ case AV_PIX_FMT_RGB24:
217 case AV_PIX_FMT_BGR24:
218 ✗ for (int j = slice_start; j < slice_end; j++) {
219 ✗ for (int i = 0; i < w; i++) {
220 ✗ hist[0*256 + p[i*3 ]]++;
221 ✗ hist[1*256 + p[i*3 + 1]]++;
222 ✗ hist[2*256 + p[i*3 + 2]]++;
223 }
224 ✗ p += frame->linesize[0];
225 }
226 ✗ break;
227 ✗ case AV_PIX_FMT_RGB0:
228 case AV_PIX_FMT_BGR0:
229 case AV_PIX_FMT_RGBA:
230 case AV_PIX_FMT_BGRA:
231 ✗ for (int j = slice_start; j < slice_end; j++) {
232 ✗ for (int i = 0; i < w; i++) {
233 ✗ hist[0*256 + p[i*4 ]]++;
234 ✗ hist[1*256 + p[i*4 + 1]]++;
235 ✗ hist[2*256 + p[i*4 + 2]]++;
236 }
237 ✗ p += frame->linesize[0];
238 }
239 ✗ break;
240 ✗ case AV_PIX_FMT_0RGB:
241 case AV_PIX_FMT_0BGR:
242 case AV_PIX_FMT_ARGB:
243 case AV_PIX_FMT_ABGR:
244 ✗ for (int j = slice_start; j < slice_end; j++) {
245 ✗ for (int i = 0; i < w; i++) {
246 ✗ hist[0*256 + p[i*4 + 1]]++;
247 ✗ hist[1*256 + p[i*4 + 2]]++;
248 ✗ hist[2*256 + p[i*4 + 3]]++;
249 }
250 ✗ p += frame->linesize[0];
251 }
252 ✗ break;
253 50 default:
254
2/2
✓ Branch 0 taken 150 times.
✓ Branch 1 taken 50 times.
 200 for (int plane = 0; plane < s->planes; plane++) {
255 150 const int slice_start = (s->planeheight[plane] * jobnr) / nb_jobs;
256 150 const int slice_end = (s->planeheight[plane] * (jobnr+1)) / nb_jobs;
257 150 const uint8_t *p = frame->data[plane] + slice_start * frame->linesize[plane];
258 150 const ptrdiff_t linesize = frame->linesize[plane];
259 150 const int planewidth = s->planewidth[plane];
260 150 int *hhist = hist + 256 * plane;
261
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 150 times.
 150 if (s->bitdepth > 8) {
262 ✗ get_hist16(hhist, p, linesize, planewidth, slice_end - slice_start,
263 ✗ s->bitdepth - 8);
264 } else {
265 150 get_hist8(hhist, p, linesize, planewidth, slice_end - slice_start);
266 }
267 }
268 50 break;
269 }
270
271 50 return 0;
272 }
273
274 50 static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
275 {
276 50 AVFilterContext *ctx = inlink->dst;
277 50 ThumbContext *s = ctx->priv;
278 50 AVFilterLink *outlink = ctx->outputs[0];
279 50 int *hist = s->frames[s->n].histogram;
280
281 // keep a reference of each frame
282 50 s->frames[s->n].buf = frame;
283
284 50 ff_filter_execute(ctx, do_slice, frame, NULL,
285 50 FFMIN(frame->height, s->nb_threads));
286
287 // update current frame histogram
288
2/2
✓ Branch 0 taken 50 times.
✓ Branch 1 taken 50 times.
 100 for (int j = 0; j < FFMIN(frame->height, s->nb_threads); j++) {
289 50 int *thread_histogram = s->thread_histogram + HIST_SIZE * j;
290
291
2/2
✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 50 times.
 38450 for (int i = 0; i < HIST_SIZE; i++)
292 38400 hist[i] += thread_histogram[i];
293 }
294
295 // no selection until the buffer of N frames is filled up
296 50 s->n++;
297
2/2
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
 50 if (s->n < s->n_frames)
298 45 return 0;
299
300 5 return ff_filter_frame(outlink, get_best_frame(ctx));
301 }
302
303 2 static av_cold void uninit(AVFilterContext *ctx)
304 {
305 int i;
306 2 ThumbContext *s = ctx->priv;
307
3/6
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2 times.
 2 for (i = 0; i < s->n_frames && s->frames && s->frames[i].buf; i++)
308 ✗ av_frame_free(&s->frames[i].buf);
309 2 av_freep(&s->frames);
310 2 av_freep(&s->thread_histogram);
311 2 }
312
313 95 static int request_frame(AVFilterLink *link)
314 {
315 95 AVFilterContext *ctx = link->src;
316 95 ThumbContext *s = ctx->priv;
317 95 int ret = ff_request_frame(ctx->inputs[0]);
318
319
3/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 94 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
 95 if (ret == AVERROR_EOF && s->n) {
320 ✗ ret = ff_filter_frame(link, get_best_frame(ctx));
321 ✗ if (ret < 0)
322 ✗ return ret;
323 ✗ ret = AVERROR_EOF;
324 }
325
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 94 times.
 95 if (ret < 0)
326 1 return ret;
327 94 return 0;
328 }
329
330 1 static int config_props(AVFilterLink *inlink)
331 {
332 1 AVFilterContext *ctx = inlink->dst;
333 1 ThumbContext *s = ctx->priv;
334 1 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
335
336 1 s->nb_threads = ff_filter_get_nb_threads(ctx);
337 1 s->thread_histogram = av_calloc(HIST_SIZE, s->nb_threads * sizeof(*s->thread_histogram));
338
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (!s->thread_histogram)
339 ✗ return AVERROR(ENOMEM);
340
341 1 s->tb = inlink->time_base;
342 1 s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
343 1 s->planewidth[0] = s->planewidth[3] = inlink->w;
344 1 s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
345 1 s->planeheight[0] = s->planeheight[3] = inlink->h;
346 1 s->planes = av_pix_fmt_count_planes(inlink->format) - !!(desc->flags & AV_PIX_FMT_FLAG_ALPHA);
347 1 s->bitdepth = desc->comp[0].depth;
348
349 1 return 0;
350 }
351
352 static const enum AVPixelFormat packed_rgb_fmts[] = {
353 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
354 AV_PIX_FMT_RGBA, AV_PIX_FMT_BGRA,
355 AV_PIX_FMT_RGB0, AV_PIX_FMT_BGR0,
356 AV_PIX_FMT_ABGR, AV_PIX_FMT_ARGB,
357 AV_PIX_FMT_0BGR, AV_PIX_FMT_0RGB,
358 AV_PIX_FMT_NONE
359 };
360
361 1 static int query_formats(const AVFilterContext *ctx,
362 AVFilterFormatsConfig **cfg_in,
363 AVFilterFormatsConfig **cfg_out)
364 {
365 1 const AVPixFmtDescriptor *desc = NULL;
366 AVFilterFormats *formats;
367
368 1 formats = ff_make_format_list(packed_rgb_fmts);
369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (!formats)
370 ✗ return AVERROR(ENOMEM);
371
372
373
2/2
✓ Branch 1 taken 267 times.
✓ Branch 2 taken 1 times.
 268 while ((desc = av_pix_fmt_desc_next(desc))) {
374 267 int color_comps = desc->nb_components - !!(desc->flags & AV_PIX_FMT_FLAG_ALPHA);
375
4/4
✓ Branch 0 taken 241 times.
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 133 times.
✓ Branch 3 taken 108 times.
 267 if ((color_comps == 1 || (desc->flags & AV_PIX_FMT_FLAG_PLANAR)) &&
376
2/2
✓ Branch 0 taken 141 times.
✓ Branch 1 taken 18 times.
 159 !(desc->flags & (AV_PIX_FMT_FLAG_FLOAT | AV_PIX_FMT_FLAG_BITSTREAM)) &&
377
4/4
✓ Branch 0 taken 118 times.
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 59 times.
✓ Branch 3 taken 59 times.
 141 (desc->comp[0].depth <= 8 || HAVE_BIGENDIAN == !!(desc->flags & AV_PIX_FMT_FLAG_BE)) &&
378
4/4
✓ Branch 0 taken 73 times.
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 58 times.
✓ Branch 3 taken 15 times.
 82 (desc->nb_components < 3 || desc->comp[1].plane != desc->comp[2].plane) &&
379
2/2
✓ Branch 0 taken 65 times.
✓ Branch 1 taken 2 times.
 67 desc->comp[0].depth <= 16)
380 {
381 65 int ret = ff_add_format(&formats, av_pix_fmt_desc_get_id(desc));
382
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 65 times.
 65 if (ret < 0)
383 ✗ return ret;
384 }
385 }
386
387 1 return ff_set_common_formats2(ctx, cfg_in, cfg_out, formats);
388 }
389
390 static const AVFilterPad thumbnail_inputs[] = {
391 {
392 .name = "default",
393 .type = AVMEDIA_TYPE_VIDEO,
394 .config_props = config_props,
395 .filter_frame = filter_frame,
396 },
397 };
398
399 static const AVFilterPad thumbnail_outputs[] = {
400 {
401 .name = "default",
402 .type = AVMEDIA_TYPE_VIDEO,
403 .request_frame = request_frame,
404 },
405 };
406
407 const FFFilter ff_vf_thumbnail = {
408 .p.name = "thumbnail",
409 .p.description = NULL_IF_CONFIG_SMALL("Select the most representative frame in a given sequence of consecutive frames."),
410 .p.priv_class = &thumbnail_class,
411 .p.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC |
412 AVFILTER_FLAG_SLICE_THREADS,
413 .priv_size = sizeof(ThumbContext),
414 .init = init,
415 .uninit = uninit,
416 FILTER_INPUTS(thumbnail_inputs),
417 FILTER_OUTPUTS(thumbnail_outputs),
418 FILTER_QUERY_FUNC2(query_formats),
419 };
420