| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2018 Sergey Lavrushkin | ||
| 3 | * | ||
| 4 | * This file is part of FFmpeg. | ||
| 5 | * | ||
| 6 | * FFmpeg is free software; you can redistribute it and/or | ||
| 7 | * modify it under the terms of the GNU Lesser General Public | ||
| 8 | * License as published by the Free Software Foundation; either | ||
| 9 | * version 2.1 of the License, or (at your option) any later version. | ||
| 10 | * | ||
| 11 | * FFmpeg is distributed in the hope that it will be useful, | ||
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| 14 | * Lesser General Public License for more details. | ||
| 15 | * | ||
| 16 | * You should have received a copy of the GNU Lesser General Public | ||
| 17 | * License along with FFmpeg; if not, write to the Free Software | ||
| 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| 19 | */ | ||
| 20 | |||
| 21 | /** | ||
| 22 | * @file | ||
| 23 | * Filter implementing image super-resolution using deep convolutional networks. | ||
| 24 | * https://arxiv.org/abs/1501.00092 | ||
| 25 | * https://arxiv.org/abs/1609.05158 | ||
| 26 | */ | ||
| 27 | |||
| 28 | #include "avfilter.h" | ||
| 29 | #include "internal.h" | ||
| 30 | #include "video.h" | ||
| 31 | #include "libavutil/opt.h" | ||
| 32 | #include "libavutil/pixdesc.h" | ||
| 33 | #include "libswscale/swscale.h" | ||
| 34 | #include "dnn_filter_common.h" | ||
| 35 | |||
| 36 | typedef struct SRContext { | ||
| 37 | const AVClass *class; | ||
| 38 | DnnContext dnnctx; | ||
| 39 | int scale_factor; | ||
| 40 | struct SwsContext *sws_uv_scale; | ||
| 41 | int sws_uv_height; | ||
| 42 | struct SwsContext *sws_pre_scale; | ||
| 43 | } SRContext; | ||
| 44 | |||
| 45 | #define OFFSET(x) offsetof(SRContext, x) | ||
| 46 | #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM | ||
| 47 | static const AVOption sr_options[] = { | ||
| 48 | { "dnn_backend", "DNN backend used for model execution", OFFSET(dnnctx.backend_type), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, FLAGS, .unit = "backend" }, | ||
| 49 | #if (CONFIG_LIBTENSORFLOW == 1) | ||
| 50 | { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, .unit = "backend" }, | ||
| 51 | #endif | ||
| 52 | { "scale_factor", "scale factor for SRCNN model", OFFSET(scale_factor), AV_OPT_TYPE_INT, { .i64 = 2 }, 2, 4, FLAGS }, | ||
| 53 | { "model", "path to model file specifying network architecture and its parameters", OFFSET(dnnctx.model_filename), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS }, | ||
| 54 | { "input", "input name of the model", OFFSET(dnnctx.model_inputname), AV_OPT_TYPE_STRING, { .str = "x" }, 0, 0, FLAGS }, | ||
| 55 | { "output", "output name of the model", OFFSET(dnnctx.model_outputnames_string), AV_OPT_TYPE_STRING, { .str = "y" }, 0, 0, FLAGS }, | ||
| 56 | { NULL } | ||
| 57 | }; | ||
| 58 | |||
| 59 | AVFILTER_DEFINE_CLASS(sr); | ||
| 60 | |||
| 61 | ✗ | static av_cold int init(AVFilterContext *context) | |
| 62 | { | ||
| 63 | ✗ | SRContext *sr_context = context->priv; | |
| 64 | ✗ | return ff_dnn_init(&sr_context->dnnctx, DFT_PROCESS_FRAME, context); | |
| 65 | } | ||
| 66 | |||
| 67 | static const enum AVPixelFormat pixel_formats[] = { | ||
| 68 | AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, | ||
| 69 | AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, AV_PIX_FMT_GRAY8, | ||
| 70 | AV_PIX_FMT_NONE | ||
| 71 | }; | ||
| 72 | |||
| 73 | ✗ | static int config_output(AVFilterLink *outlink) | |
| 74 | { | ||
| 75 | ✗ | AVFilterContext *context = outlink->src; | |
| 76 | ✗ | SRContext *ctx = context->priv; | |
| 77 | int result; | ||
| 78 | ✗ | AVFilterLink *inlink = context->inputs[0]; | |
| 79 | int out_width, out_height; | ||
| 80 | |||
| 81 | // have a try run in case that the dnn model resize the frame | ||
| 82 | ✗ | result = ff_dnn_get_output(&ctx->dnnctx, inlink->w, inlink->h, &out_width, &out_height); | |
| 83 | ✗ | if (result != 0) { | |
| 84 | ✗ | av_log(ctx, AV_LOG_ERROR, "could not get output from the model\n"); | |
| 85 | ✗ | return result; | |
| 86 | } | ||
| 87 | |||
| 88 | ✗ | if (inlink->w != out_width || inlink->h != out_height) { | |
| 89 | //espcn | ||
| 90 | ✗ | outlink->w = out_width; | |
| 91 | ✗ | outlink->h = out_height; | |
| 92 | ✗ | if (inlink->format != AV_PIX_FMT_GRAY8){ | |
| 93 | ✗ | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); | |
| 94 | ✗ | int sws_src_h = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h); | |
| 95 | ✗ | int sws_src_w = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w); | |
| 96 | ✗ | int sws_dst_h = AV_CEIL_RSHIFT(outlink->h, desc->log2_chroma_h); | |
| 97 | ✗ | int sws_dst_w = AV_CEIL_RSHIFT(outlink->w, desc->log2_chroma_w); | |
| 98 | ✗ | ctx->sws_uv_scale = sws_getContext(sws_src_w, sws_src_h, AV_PIX_FMT_GRAY8, | |
| 99 | sws_dst_w, sws_dst_h, AV_PIX_FMT_GRAY8, | ||
| 100 | SWS_BICUBIC, NULL, NULL, NULL); | ||
| 101 | ✗ | ctx->sws_uv_height = sws_src_h; | |
| 102 | } | ||
| 103 | } else { | ||
| 104 | //srcnn | ||
| 105 | ✗ | outlink->w = out_width * ctx->scale_factor; | |
| 106 | ✗ | outlink->h = out_height * ctx->scale_factor; | |
| 107 | ✗ | ctx->sws_pre_scale = sws_getContext(inlink->w, inlink->h, inlink->format, | |
| 108 | ✗ | outlink->w, outlink->h, outlink->format, | |
| 109 | SWS_BICUBIC, NULL, NULL, NULL); | ||
| 110 | } | ||
| 111 | |||
| 112 | ✗ | return 0; | |
| 113 | } | ||
| 114 | |||
| 115 | ✗ | static int filter_frame(AVFilterLink *inlink, AVFrame *in) | |
| 116 | { | ||
| 117 | ✗ | DNNAsyncStatusType async_state = 0; | |
| 118 | ✗ | AVFilterContext *context = inlink->dst; | |
| 119 | ✗ | SRContext *ctx = context->priv; | |
| 120 | ✗ | AVFilterLink *outlink = context->outputs[0]; | |
| 121 | ✗ | AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h); | |
| 122 | int dnn_result; | ||
| 123 | |||
| 124 | ✗ | if (!out){ | |
| 125 | ✗ | av_log(context, AV_LOG_ERROR, "could not allocate memory for output frame\n"); | |
| 126 | ✗ | av_frame_free(&in); | |
| 127 | ✗ | return AVERROR(ENOMEM); | |
| 128 | } | ||
| 129 | ✗ | av_frame_copy_props(out, in); | |
| 130 | |||
| 131 | ✗ | if (ctx->sws_pre_scale) { | |
| 132 | ✗ | sws_scale(ctx->sws_pre_scale, | |
| 133 | ✗ | (const uint8_t **)in->data, in->linesize, 0, in->height, | |
| 134 | ✗ | out->data, out->linesize); | |
| 135 | ✗ | dnn_result = ff_dnn_execute_model(&ctx->dnnctx, out, out); | |
| 136 | } else { | ||
| 137 | ✗ | dnn_result = ff_dnn_execute_model(&ctx->dnnctx, in, out); | |
| 138 | } | ||
| 139 | |||
| 140 | ✗ | if (dnn_result != 0){ | |
| 141 | ✗ | av_log(ctx, AV_LOG_ERROR, "failed to execute loaded model\n"); | |
| 142 | ✗ | av_frame_free(&in); | |
| 143 | ✗ | av_frame_free(&out); | |
| 144 | ✗ | return dnn_result; | |
| 145 | } | ||
| 146 | |||
| 147 | do { | ||
| 148 | ✗ | async_state = ff_dnn_get_result(&ctx->dnnctx, &in, &out); | |
| 149 | ✗ | } while (async_state == DAST_NOT_READY); | |
| 150 | |||
| 151 | ✗ | if (async_state != DAST_SUCCESS) | |
| 152 | ✗ | return AVERROR(EINVAL); | |
| 153 | |||
| 154 | ✗ | if (ctx->sws_uv_scale) { | |
| 155 | ✗ | sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 1), in->linesize + 1, | |
| 156 | ✗ | 0, ctx->sws_uv_height, out->data + 1, out->linesize + 1); | |
| 157 | ✗ | sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 2), in->linesize + 2, | |
| 158 | ✗ | 0, ctx->sws_uv_height, out->data + 2, out->linesize + 2); | |
| 159 | } | ||
| 160 | ✗ | if (in != out) { | |
| 161 | ✗ | av_frame_free(&in); | |
| 162 | } | ||
| 163 | ✗ | return ff_filter_frame(outlink, out); | |
| 164 | } | ||
| 165 | |||
| 166 | ✗ | static av_cold void uninit(AVFilterContext *context) | |
| 167 | { | ||
| 168 | ✗ | SRContext *sr_context = context->priv; | |
| 169 | |||
| 170 | ✗ | ff_dnn_uninit(&sr_context->dnnctx); | |
| 171 | ✗ | sws_freeContext(sr_context->sws_uv_scale); | |
| 172 | ✗ | sws_freeContext(sr_context->sws_pre_scale); | |
| 173 | ✗ | } | |
| 174 | |||
| 175 | static const AVFilterPad sr_inputs[] = { | ||
| 176 | { | ||
| 177 | .name = "default", | ||
| 178 | .type = AVMEDIA_TYPE_VIDEO, | ||
| 179 | .filter_frame = filter_frame, | ||
| 180 | }, | ||
| 181 | }; | ||
| 182 | |||
| 183 | static const AVFilterPad sr_outputs[] = { | ||
| 184 | { | ||
| 185 | .name = "default", | ||
| 186 | .config_props = config_output, | ||
| 187 | .type = AVMEDIA_TYPE_VIDEO, | ||
| 188 | }, | ||
| 189 | }; | ||
| 190 | |||
| 191 | const AVFilter ff_vf_sr = { | ||
| 192 | .name = "sr", | ||
| 193 | .description = NULL_IF_CONFIG_SMALL("Apply DNN-based image super resolution to the input."), | ||
| 194 | .priv_size = sizeof(SRContext), | ||
| 195 | .init = init, | ||
| 196 | .uninit = uninit, | ||
| 197 | FILTER_INPUTS(sr_inputs), | ||
| 198 | FILTER_OUTPUTS(sr_outputs), | ||
| 199 | FILTER_PIXFMTS_ARRAY(pixel_formats), | ||
| 200 | .priv_class = &sr_class, | ||
| 201 | }; | ||
| 202 |