| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2011 Smartjog S.A.S, Clément Bœsch <clement.boesch@smartjog.com> | ||
| 3 | * | ||
| 4 | * This file is part of FFmpeg. | ||
| 5 | * | ||
| 6 | * FFmpeg is free software; you can redistribute it and/or | ||
| 7 | * modify it under the terms of the GNU Lesser General Public | ||
| 8 | * License as published by the Free Software Foundation; either | ||
| 9 | * version 2.1 of the License, or (at your option) any later version. | ||
| 10 | * | ||
| 11 | * FFmpeg is distributed in the hope that it will be useful, | ||
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| 14 | * Lesser General Public License for more details. | ||
| 15 | * | ||
| 16 | * You should have received a copy of the GNU Lesser General Public | ||
| 17 | * License along with FFmpeg; if not, write to the Free Software | ||
| 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| 19 | */ | ||
| 20 | |||
| 21 | /** | ||
| 22 | * @file | ||
| 23 | * Potential thumbnail lookup filter to reduce the risk of an inappropriate | ||
| 24 | * selection (such as a black frame) we could get with an absolute seek. | ||
| 25 | * | ||
| 26 | * Simplified version of algorithm by Vadim Zaliva <lord@crocodile.org>. | ||
| 27 | * @see http://notbrainsurgery.livejournal.com/29773.html | ||
| 28 | */ | ||
| 29 | |||
| 30 | #include "libavutil/mem.h" | ||
| 31 | #include "libavutil/opt.h" | ||
| 32 | #include "libavutil/pixdesc.h" | ||
| 33 | #include "avfilter.h" | ||
| 34 | #include "filters.h" | ||
| 35 | |||
| 36 | #define HIST_SIZE (3*256) | ||
| 37 | |||
| 38 | struct thumb_frame { | ||
| 39 | AVFrame *buf; ///< cached frame | ||
| 40 | int histogram[HIST_SIZE]; ///< RGB color distribution histogram of the frame | ||
| 41 | }; | ||
| 42 | |||
| 43 | typedef struct ThumbContext { | ||
| 44 | const AVClass *class; | ||
| 45 | int n; ///< current frame | ||
| 46 | int loglevel; | ||
| 47 | int n_frames; ///< number of frames for analysis | ||
| 48 | struct thumb_frame *frames; ///< the n_frames frames | ||
| 49 | AVRational tb; ///< copy of the input timebase to ease access | ||
| 50 | |||
| 51 | int nb_threads; | ||
| 52 | int *thread_histogram; | ||
| 53 | |||
| 54 | int planewidth[4]; | ||
| 55 | int planeheight[4]; | ||
| 56 | } ThumbContext; | ||
| 57 | |||
| 58 | #define OFFSET(x) offsetof(ThumbContext, x) | ||
| 59 | #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM | ||
| 60 | |||
| 61 | static const AVOption thumbnail_options[] = { | ||
| 62 | { "n", "set the frames batch size", OFFSET(n_frames), AV_OPT_TYPE_INT, {.i64=100}, 2, INT_MAX, FLAGS }, | ||
| 63 | { "log", "force stats logging level", OFFSET(loglevel), AV_OPT_TYPE_INT, {.i64 = AV_LOG_INFO}, INT_MIN, INT_MAX, FLAGS, .unit = "level" }, | ||
| 64 | { "quiet", "logging disabled", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_QUIET}, 0, 0, FLAGS, .unit = "level" }, | ||
| 65 | { "info", "information logging level", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_INFO}, 0, 0, FLAGS, .unit = "level" }, | ||
| 66 | { "verbose", "verbose logging level", 0, AV_OPT_TYPE_CONST, {.i64 = AV_LOG_VERBOSE}, 0, 0, FLAGS, .unit = "level" }, | ||
| 67 | { NULL } | ||
| 68 | }; | ||
| 69 | |||
| 70 | AVFILTER_DEFINE_CLASS(thumbnail); | ||
| 71 | |||
| 72 | 2 | static av_cold int init(AVFilterContext *ctx) | |
| 73 | { | ||
| 74 | 2 | ThumbContext *s = ctx->priv; | |
| 75 | |||
| 76 | 2 | s->frames = av_calloc(s->n_frames, sizeof(*s->frames)); | |
| 77 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
|
2 | if (!s->frames) { |
| 78 | ✗ | av_log(ctx, AV_LOG_ERROR, | |
| 79 | "Allocation failure, try to lower the number of frames\n"); | ||
| 80 | ✗ | return AVERROR(ENOMEM); | |
| 81 | } | ||
| 82 | 2 | av_log(ctx, AV_LOG_VERBOSE, "batch size: %d frames\n", s->n_frames); | |
| 83 | 2 | return 0; | |
| 84 | } | ||
| 85 | |||
| 86 | /** | ||
| 87 | * @brief Compute Sum-square deviation to estimate "closeness". | ||
| 88 | * @param hist color distribution histogram | ||
| 89 | * @param median average color distribution histogram | ||
| 90 | * @return sum of squared errors | ||
| 91 | */ | ||
| 92 | 50 | static double frame_sum_square_err(const int *hist, const double *median) | |
| 93 | { | ||
| 94 | int i; | ||
| 95 | 50 | double err, sum_sq_err = 0; | |
| 96 | |||
| 97 |
2/2✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 50 times.
|
38450 | for (i = 0; i < HIST_SIZE; i++) { |
| 98 | 38400 | err = median[i] - (double)hist[i]; | |
| 99 | 38400 | sum_sq_err += err*err; | |
| 100 | } | ||
| 101 | 50 | return sum_sq_err; | |
| 102 | } | ||
| 103 | |||
| 104 | 5 | static AVFrame *get_best_frame(AVFilterContext *ctx) | |
| 105 | { | ||
| 106 | AVFrame *picref; | ||
| 107 | 5 | ThumbContext *s = ctx->priv; | |
| 108 | 5 | int i, j, best_frame_idx = 0; | |
| 109 | 5 | int nb_frames = s->n; | |
| 110 | 5 | double avg_hist[HIST_SIZE] = {0}, sq_err, min_sq_err = -1; | |
| 111 | |||
| 112 | // average histogram of the N frames | ||
| 113 |
2/2✓ Branch 0 taken 3840 times.
✓ Branch 1 taken 5 times.
|
3845 | for (j = 0; j < FF_ARRAY_ELEMS(avg_hist); j++) { |
| 114 |
2/2✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 3840 times.
|
42240 | for (i = 0; i < nb_frames; i++) |
| 115 | 38400 | avg_hist[j] += (double)s->frames[i].histogram[j]; | |
| 116 | 3840 | avg_hist[j] /= nb_frames; | |
| 117 | } | ||
| 118 | |||
| 119 | // find the frame closer to the average using the sum of squared errors | ||
| 120 |
2/2✓ Branch 0 taken 50 times.
✓ Branch 1 taken 5 times.
|
55 | for (i = 0; i < nb_frames; i++) { |
| 121 | 50 | sq_err = frame_sum_square_err(s->frames[i].histogram, avg_hist); | |
| 122 |
4/4✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
✓ Branch 2 taken 12 times.
✓ Branch 3 taken 33 times.
|
50 | if (i == 0 || sq_err < min_sq_err) |
| 123 | 17 | best_frame_idx = i, min_sq_err = sq_err; | |
| 124 | } | ||
| 125 | |||
| 126 | // free and reset everything (except the best frame buffer) | ||
| 127 |
2/2✓ Branch 0 taken 50 times.
✓ Branch 1 taken 5 times.
|
55 | for (i = 0; i < nb_frames; i++) { |
| 128 | 50 | memset(s->frames[i].histogram, 0, sizeof(s->frames[i].histogram)); | |
| 129 |
2/2✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
|
50 | if (i != best_frame_idx) |
| 130 | 45 | av_frame_free(&s->frames[i].buf); | |
| 131 | } | ||
| 132 | 5 | s->n = 0; | |
| 133 | |||
| 134 | // raise the chosen one | ||
| 135 | 5 | picref = s->frames[best_frame_idx].buf; | |
| 136 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (s->loglevel != AV_LOG_QUIET) |
| 137 | 5 | av_log(ctx, s->loglevel, "frame id #%d (pts_time=%f) selected " | |
| 138 | "from a set of %d images\n", best_frame_idx, | ||
| 139 | 5 | picref->pts * av_q2d(s->tb), nb_frames); | |
| 140 | 5 | s->frames[best_frame_idx].buf = NULL; | |
| 141 | |||
| 142 | 5 | return picref; | |
| 143 | } | ||
| 144 | |||
| 145 | 50 | static int do_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) | |
| 146 | { | ||
| 147 | 50 | ThumbContext *s = ctx->priv; | |
| 148 | 50 | AVFrame *frame = arg; | |
| 149 | 50 | int *hist = s->thread_histogram + HIST_SIZE * jobnr; | |
| 150 | 50 | const int h = frame->height; | |
| 151 | 50 | const int w = frame->width; | |
| 152 | 50 | const int slice_start = (h * jobnr) / nb_jobs; | |
| 153 | 50 | const int slice_end = (h * (jobnr+1)) / nb_jobs; | |
| 154 | 50 | const uint8_t *p = frame->data[0] + slice_start * frame->linesize[0]; | |
| 155 | |||
| 156 | 50 | memset(hist, 0, sizeof(*hist) * HIST_SIZE); | |
| 157 | |||
| 158 |
1/4✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 50 times.
|
50 | switch (frame->format) { |
| 159 | ✗ | case AV_PIX_FMT_RGB24: | |
| 160 | case AV_PIX_FMT_BGR24: | ||
| 161 | ✗ | for (int j = slice_start; j < slice_end; j++) { | |
| 162 | ✗ | for (int i = 0; i < w; i++) { | |
| 163 | ✗ | hist[0*256 + p[i*3 ]]++; | |
| 164 | ✗ | hist[1*256 + p[i*3 + 1]]++; | |
| 165 | ✗ | hist[2*256 + p[i*3 + 2]]++; | |
| 166 | } | ||
| 167 | ✗ | p += frame->linesize[0]; | |
| 168 | } | ||
| 169 | ✗ | break; | |
| 170 | ✗ | case AV_PIX_FMT_RGB0: | |
| 171 | case AV_PIX_FMT_BGR0: | ||
| 172 | case AV_PIX_FMT_RGBA: | ||
| 173 | case AV_PIX_FMT_BGRA: | ||
| 174 | ✗ | for (int j = slice_start; j < slice_end; j++) { | |
| 175 | ✗ | for (int i = 0; i < w; i++) { | |
| 176 | ✗ | hist[0*256 + p[i*4 ]]++; | |
| 177 | ✗ | hist[1*256 + p[i*4 + 1]]++; | |
| 178 | ✗ | hist[2*256 + p[i*4 + 2]]++; | |
| 179 | } | ||
| 180 | ✗ | p += frame->linesize[0]; | |
| 181 | } | ||
| 182 | ✗ | break; | |
| 183 | ✗ | case AV_PIX_FMT_0RGB: | |
| 184 | case AV_PIX_FMT_0BGR: | ||
| 185 | case AV_PIX_FMT_ARGB: | ||
| 186 | case AV_PIX_FMT_ABGR: | ||
| 187 | ✗ | for (int j = slice_start; j < slice_end; j++) { | |
| 188 | ✗ | for (int i = 0; i < w; i++) { | |
| 189 | ✗ | hist[0*256 + p[i*4 + 1]]++; | |
| 190 | ✗ | hist[1*256 + p[i*4 + 2]]++; | |
| 191 | ✗ | hist[2*256 + p[i*4 + 3]]++; | |
| 192 | } | ||
| 193 | ✗ | p += frame->linesize[0]; | |
| 194 | } | ||
| 195 | ✗ | break; | |
| 196 | 50 | default: | |
| 197 |
2/2✓ Branch 0 taken 150 times.
✓ Branch 1 taken 50 times.
|
200 | for (int plane = 0; plane < 3; plane++) { |
| 198 | 150 | const int slice_start = (s->planeheight[plane] * jobnr) / nb_jobs; | |
| 199 | 150 | const int slice_end = (s->planeheight[plane] * (jobnr+1)) / nb_jobs; | |
| 200 | 150 | const uint8_t *p = frame->data[plane] + slice_start * frame->linesize[plane]; | |
| 201 | 150 | const ptrdiff_t linesize = frame->linesize[plane]; | |
| 202 | 150 | const int planewidth = s->planewidth[plane]; | |
| 203 | 150 | int *hhist = hist + 256 * plane; | |
| 204 | |||
| 205 |
2/2✓ Branch 0 taken 28800 times.
✓ Branch 1 taken 150 times.
|
28950 | for (int j = slice_start; j < slice_end; j++) { |
| 206 |
2/2✓ Branch 0 taken 7603200 times.
✓ Branch 1 taken 28800 times.
|
7632000 | for (int i = 0; i < planewidth; i++) |
| 207 | 7603200 | hhist[p[i]]++; | |
| 208 | 28800 | p += linesize; | |
| 209 | } | ||
| 210 | } | ||
| 211 | 50 | break; | |
| 212 | } | ||
| 213 | |||
| 214 | 50 | return 0; | |
| 215 | } | ||
| 216 | |||
| 217 | 50 | static int filter_frame(AVFilterLink *inlink, AVFrame *frame) | |
| 218 | { | ||
| 219 | 50 | AVFilterContext *ctx = inlink->dst; | |
| 220 | 50 | ThumbContext *s = ctx->priv; | |
| 221 | 50 | AVFilterLink *outlink = ctx->outputs[0]; | |
| 222 | 50 | int *hist = s->frames[s->n].histogram; | |
| 223 | |||
| 224 | // keep a reference of each frame | ||
| 225 | 50 | s->frames[s->n].buf = frame; | |
| 226 | |||
| 227 | 50 | ff_filter_execute(ctx, do_slice, frame, NULL, | |
| 228 | 50 | FFMIN(frame->height, s->nb_threads)); | |
| 229 | |||
| 230 | // update current frame histogram | ||
| 231 |
2/2✓ Branch 0 taken 50 times.
✓ Branch 1 taken 50 times.
|
100 | for (int j = 0; j < FFMIN(frame->height, s->nb_threads); j++) { |
| 232 | 50 | int *thread_histogram = s->thread_histogram + HIST_SIZE * j; | |
| 233 | |||
| 234 |
2/2✓ Branch 0 taken 38400 times.
✓ Branch 1 taken 50 times.
|
38450 | for (int i = 0; i < HIST_SIZE; i++) |
| 235 | 38400 | hist[i] += thread_histogram[i]; | |
| 236 | } | ||
| 237 | |||
| 238 | // no selection until the buffer of N frames is filled up | ||
| 239 | 50 | s->n++; | |
| 240 |
2/2✓ Branch 0 taken 45 times.
✓ Branch 1 taken 5 times.
|
50 | if (s->n < s->n_frames) |
| 241 | 45 | return 0; | |
| 242 | |||
| 243 | 5 | return ff_filter_frame(outlink, get_best_frame(ctx)); | |
| 244 | } | ||
| 245 | |||
| 246 | 2 | static av_cold void uninit(AVFilterContext *ctx) | |
| 247 | { | ||
| 248 | int i; | ||
| 249 | 2 | ThumbContext *s = ctx->priv; | |
| 250 |
3/6✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2 times.
|
2 | for (i = 0; i < s->n_frames && s->frames && s->frames[i].buf; i++) |
| 251 | ✗ | av_frame_free(&s->frames[i].buf); | |
| 252 | 2 | av_freep(&s->frames); | |
| 253 | 2 | av_freep(&s->thread_histogram); | |
| 254 | 2 | } | |
| 255 | |||
| 256 | 49 | static int request_frame(AVFilterLink *link) | |
| 257 | { | ||
| 258 | 49 | AVFilterContext *ctx = link->src; | |
| 259 | 49 | ThumbContext *s = ctx->priv; | |
| 260 | 49 | int ret = ff_request_frame(ctx->inputs[0]); | |
| 261 | |||
| 262 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
49 | if (ret == AVERROR_EOF && s->n) { |
| 263 | ✗ | ret = ff_filter_frame(link, get_best_frame(ctx)); | |
| 264 | ✗ | if (ret < 0) | |
| 265 | ✗ | return ret; | |
| 266 | ✗ | ret = AVERROR_EOF; | |
| 267 | } | ||
| 268 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (ret < 0) |
| 269 | ✗ | return ret; | |
| 270 | 49 | return 0; | |
| 271 | } | ||
| 272 | |||
| 273 | 1 | static int config_props(AVFilterLink *inlink) | |
| 274 | { | ||
| 275 | 1 | AVFilterContext *ctx = inlink->dst; | |
| 276 | 1 | ThumbContext *s = ctx->priv; | |
| 277 | 1 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); | |
| 278 | |||
| 279 | 1 | s->nb_threads = ff_filter_get_nb_threads(ctx); | |
| 280 | 1 | s->thread_histogram = av_calloc(HIST_SIZE, s->nb_threads * sizeof(*s->thread_histogram)); | |
| 281 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
|
1 | if (!s->thread_histogram) |
| 282 | ✗ | return AVERROR(ENOMEM); | |
| 283 | |||
| 284 | 1 | s->tb = inlink->time_base; | |
| 285 | 1 | s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w); | |
| 286 | 1 | s->planewidth[0] = s->planewidth[3] = inlink->w; | |
| 287 | 1 | s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h); | |
| 288 | 1 | s->planeheight[0] = s->planeheight[3] = inlink->h; | |
| 289 | |||
| 290 | 1 | return 0; | |
| 291 | } | ||
| 292 | |||
| 293 | static const enum AVPixelFormat pix_fmts[] = { | ||
| 294 | AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, | ||
| 295 | AV_PIX_FMT_RGBA, AV_PIX_FMT_BGRA, | ||
| 296 | AV_PIX_FMT_RGB0, AV_PIX_FMT_BGR0, | ||
| 297 | AV_PIX_FMT_ABGR, AV_PIX_FMT_ARGB, | ||
| 298 | AV_PIX_FMT_0BGR, AV_PIX_FMT_0RGB, | ||
| 299 | AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, | ||
| 300 | AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, | ||
| 301 | AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P, | ||
| 302 | AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, | ||
| 303 | AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P, | ||
| 304 | AV_PIX_FMT_YUVJ411P, | ||
| 305 | AV_PIX_FMT_YUVA420P, AV_PIX_FMT_YUVA422P, AV_PIX_FMT_YUVA444P, | ||
| 306 | AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRAP, | ||
| 307 | AV_PIX_FMT_NONE | ||
| 308 | }; | ||
| 309 | |||
| 310 | static const AVFilterPad thumbnail_inputs[] = { | ||
| 311 | { | ||
| 312 | .name = "default", | ||
| 313 | .type = AVMEDIA_TYPE_VIDEO, | ||
| 314 | .config_props = config_props, | ||
| 315 | .filter_frame = filter_frame, | ||
| 316 | }, | ||
| 317 | }; | ||
| 318 | |||
| 319 | static const AVFilterPad thumbnail_outputs[] = { | ||
| 320 | { | ||
| 321 | .name = "default", | ||
| 322 | .type = AVMEDIA_TYPE_VIDEO, | ||
| 323 | .request_frame = request_frame, | ||
| 324 | }, | ||
| 325 | }; | ||
| 326 | |||
| 327 | const AVFilter ff_vf_thumbnail = { | ||
| 328 | .name = "thumbnail", | ||
| 329 | .description = NULL_IF_CONFIG_SMALL("Select the most representative frame in a given sequence of consecutive frames."), | ||
| 330 | .priv_size = sizeof(ThumbContext), | ||
| 331 | .init = init, | ||
| 332 | .uninit = uninit, | ||
| 333 | FILTER_INPUTS(thumbnail_inputs), | ||
| 334 | FILTER_OUTPUTS(thumbnail_outputs), | ||
| 335 | FILTER_PIXFMTS_ARRAY(pix_fmts), | ||
| 336 | .priv_class = &thumbnail_class, | ||
| 337 | .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC | | ||
| 338 | AVFILTER_FLAG_SLICE_THREADS, | ||
| 339 | }; | ||
| 340 |