GCC Code Coverage Report
Directory: ../../../ffmpeg/ Exec Total Coverage
File: src/libavfilter/dnn/dnn_backend_native.c Lines: 11 190 5.8 %
Date: 2020-08-14 10:39:37 Branches: 5 94 5.3 %

Line Branch Exec Source
1
/*
2
 * Copyright (c) 2018 Sergey Lavrushkin
3
 *
4
 * This file is part of FFmpeg.
5
 *
6
 * FFmpeg is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU Lesser General Public
8
 * License as published by the Free Software Foundation; either
9
 * version 2.1 of the License, or (at your option) any later version.
10
 *
11
 * FFmpeg is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
 * Lesser General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with FFmpeg; if not, write to the Free Software
18
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
 */
20
21
/**
22
 * @file
23
 * DNN native backend implementation.
24
 */
25
26
#include "dnn_backend_native.h"
27
#include "libavutil/avassert.h"
28
#include "dnn_backend_native_layer_conv2d.h"
29
#include "dnn_backend_native_layers.h"
30
31
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
32
{
33
    ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
34
35
    for (int i = 0; i < network->operands_num; ++i) {
36
        DnnOperand *oprd = &network->operands[i];
37
        if (strcmp(oprd->name, input_name) == 0) {
38
            if (oprd->type != DOT_INPUT)
39
                return DNN_ERROR;
40
            input->dt = oprd->data_type;
41
            av_assert0(oprd->dims[0] == 1);
42
            input->height = oprd->dims[1];
43
            input->width = oprd->dims[2];
44
            input->channels = oprd->dims[3];
45
            return DNN_SUCCESS;
46
        }
47
    }
48
49
    // do not find the input operand
50
    return DNN_ERROR;
51
}
52
53
static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
54
{
55
    ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
56
    DnnOperand *oprd = NULL;
57
58
    if (network->layers_num <= 0 || network->operands_num <= 0)
59
        return DNN_ERROR;
60
61
    /* inputs */
62
    for (int i = 0; i < network->operands_num; ++i) {
63
        oprd = &network->operands[i];
64
        if (strcmp(oprd->name, input_name) == 0) {
65
            if (oprd->type != DOT_INPUT)
66
                return DNN_ERROR;
67
            break;
68
        }
69
        oprd = NULL;
70
    }
71
72
    if (!oprd)
73
        return DNN_ERROR;
74
75
    oprd->dims[0] = 1;
76
    oprd->dims[1] = input->height;
77
    oprd->dims[2] = input->width;
78
    oprd->dims[3] = input->channels;
79
80
    av_freep(&oprd->data);
81
    oprd->length = calculate_operand_data_length(oprd);
82
    if (oprd->length <= 0)
83
        return DNN_ERROR;
84
    oprd->data = av_malloc(oprd->length);
85
    if (!oprd->data)
86
        return DNN_ERROR;
87
88
    input->data = oprd->data;
89
90
    /* outputs */
91
    network->nb_output = 0;
92
    av_freep(&network->output_indexes);
93
    network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes));
94
    if (!network->output_indexes)
95
        return DNN_ERROR;
96
97
    for (uint32_t i = 0; i < nb_output; ++i) {
98
        const char *output_name = output_names[i];
99
        for (int j = 0; j < network->operands_num; ++j) {
100
            oprd = &network->operands[j];
101
            if (strcmp(oprd->name, output_name) == 0) {
102
                network->output_indexes[network->nb_output++] = j;
103
                break;
104
            }
105
        }
106
    }
107
108
    if (network->nb_output != nb_output)
109
        return DNN_ERROR;
110
111
    return DNN_SUCCESS;
112
}
113
114
// Loads model and its parameters that are stored in a binary file with following structure:
115
// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
116
// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
117
// For DEPTH_TO_SPACE layer: block_size
118
DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options)
119
{
120
    DNNModel *model = NULL;
121
    char header_expected[] = "FFMPEGDNNNATIVE";
122
    char *buf;
123
    size_t size;
124
    int version, header_size, major_version_expected = 1;
125
    ConvolutionalNetwork *network = NULL;
126
    AVIOContext *model_file_context;
127
    int file_size, dnn_size, parsed_size;
128
    int32_t layer;
129
    DNNLayerType layer_type;
130
131
    if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
132
        return NULL;
133
    }
134
    file_size = avio_size(model_file_context);
135
136
    model = av_mallocz(sizeof(DNNModel));
137
    if (!model){
138
        goto fail;
139
    }
140
141
    /**
142
     * check file header with string and version
143
     */
144
    size = sizeof(header_expected);
145
    buf = av_malloc(size);
146
    if (!buf) {
147
        goto fail;
148
    }
149
150
    // size - 1 to skip the ending '\0' which is not saved in file
151
    avio_get_str(model_file_context, size - 1, buf, size);
152
    dnn_size = size - 1;
153
    if (strncmp(buf, header_expected, size) != 0) {
154
        av_freep(&buf);
155
        goto fail;
156
    }
157
    av_freep(&buf);
158
159
    version = (int32_t)avio_rl32(model_file_context);
160
    dnn_size += 4;
161
    if (version != major_version_expected) {
162
        goto fail;
163
    }
164
165
    // currently no need to check minor version
166
    version = (int32_t)avio_rl32(model_file_context);
167
    dnn_size += 4;
168
    header_size = dnn_size;
169
170
    network = av_mallocz(sizeof(ConvolutionalNetwork));
171
    if (!network){
172
        goto fail;
173
    }
174
    model->model = (void *)network;
175
176
    avio_seek(model_file_context, file_size - 8, SEEK_SET);
177
    network->layers_num = (int32_t)avio_rl32(model_file_context);
178
    network->operands_num = (int32_t)avio_rl32(model_file_context);
179
    dnn_size += 8;
180
    avio_seek(model_file_context, header_size, SEEK_SET);
181
182
    network->layers = av_mallocz(network->layers_num * sizeof(Layer));
183
    if (!network->layers){
184
        goto fail;
185
    }
186
187
    network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
188
    if (!network->operands){
189
        goto fail;
190
    }
191
192
    for (layer = 0; layer < network->layers_num; ++layer){
193
        layer_type = (int32_t)avio_rl32(model_file_context);
194
        dnn_size += 4;
195
196
        if (layer_type >= DLT_COUNT) {
197
            goto fail;
198
        }
199
200
        network->layers[layer].type = layer_type;
201
        parsed_size = layer_funcs[layer_type].pf_load(&network->layers[layer], model_file_context, file_size, network->operands_num);
202
        if (!parsed_size) {
203
            goto fail;
204
        }
205
        dnn_size += parsed_size;
206
    }
207
208
    for (int32_t i = 0; i < network->operands_num; ++i){
209
        DnnOperand *oprd;
210
        int32_t name_len;
211
        int32_t operand_index = (int32_t)avio_rl32(model_file_context);
212
        dnn_size += 4;
213
214
        if (operand_index >= network->operands_num) {
215
            goto fail;
216
        }
217
218
        oprd = &network->operands[operand_index];
219
        name_len = (int32_t)avio_rl32(model_file_context);
220
        dnn_size += 4;
221
222
        avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
223
        dnn_size += name_len;
224
225
        oprd->type = (int32_t)avio_rl32(model_file_context);
226
        dnn_size += 4;
227
228
        oprd->data_type = (int32_t)avio_rl32(model_file_context);
229
        dnn_size += 4;
230
231
        for (int32_t dim = 0; dim < 4; ++dim) {
232
            oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
233
            dnn_size += 4;
234
        }
235
236
        oprd->isNHWC = 1;
237
    }
238
239
    avio_closep(&model_file_context);
240
241
    if (dnn_size != file_size){
242
        ff_dnn_free_model_native(&model);
243
        return NULL;
244
    }
245
246
    model->set_input_output = &set_input_output_native;
247
    model->get_input = &get_input_native;
248
    model->options = options;
249
250
    return model;
251
252
fail:
253
    ff_dnn_free_model_native(&model);
254
    avio_closep(&model_file_context);
255
    return NULL;
256
}
257
258
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
259
{
260
    ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
261
    int32_t layer;
262
    uint32_t nb = FFMIN(nb_output, network->nb_output);
263
264
    if (network->layers_num <= 0 || network->operands_num <= 0)
265
        return DNN_ERROR;
266
    if (!network->operands[0].data)
267
        return DNN_ERROR;
268
269
    for (layer = 0; layer < network->layers_num; ++layer){
270
        DNNLayerType layer_type = network->layers[layer].type;
271
        layer_funcs[layer_type].pf_exec(network->operands,
272
                                  network->layers[layer].input_operand_indexes,
273
                                  network->layers[layer].output_operand_index,
274
                                  network->layers[layer].params);
275
    }
276
277
    for (uint32_t i = 0; i < nb; ++i) {
278
        DnnOperand *oprd = &network->operands[network->output_indexes[i]];
279
        outputs[i].data = oprd->data;
280
        outputs[i].height = oprd->dims[1];
281
        outputs[i].width = oprd->dims[2];
282
        outputs[i].channels = oprd->dims[3];
283
        outputs[i].dt = oprd->data_type;
284
    }
285
286
    return DNN_SUCCESS;
287
}
288
289
32
int32_t calculate_operand_dims_count(const DnnOperand *oprd)
290
{
291
32
    int32_t result = 1;
292
160
    for (int i = 0; i < 4; ++i)
293
128
        result *= oprd->dims[i];
294
295
32
    return result;
296
}
297
298
40
int32_t calculate_operand_data_length(const DnnOperand* oprd)
299
{
300
    // currently, we just support DNN_FLOAT
301
40
    uint64_t len = sizeof(float);
302
200
    for (int i = 0; i < 4; i++) {
303
160
        len *= oprd->dims[i];
304
160
        if (len > INT32_MAX)
305
            return 0;
306
    }
307
40
    return len;
308
}
309
310
void ff_dnn_free_model_native(DNNModel **model)
311
{
312
    ConvolutionalNetwork *network;
313
    ConvolutionalParams *conv_params;
314
    int32_t layer;
315
316
    if (*model)
317
    {
318
        if ((*model)->model) {
319
            network = (ConvolutionalNetwork *)(*model)->model;
320
            if (network->layers) {
321
                for (layer = 0; layer < network->layers_num; ++layer){
322
                    if (network->layers[layer].type == DLT_CONV2D){
323
                        conv_params = (ConvolutionalParams *)network->layers[layer].params;
324
                        av_freep(&conv_params->kernel);
325
                        av_freep(&conv_params->biases);
326
                    }
327
                    av_freep(&network->layers[layer].params);
328
                }
329
                av_freep(&network->layers);
330
            }
331
332
            if (network->operands) {
333
                for (uint32_t operand = 0; operand < network->operands_num; ++operand)
334
                    av_freep(&network->operands[operand].data);
335
                av_freep(&network->operands);
336
            }
337
338
            av_freep(&network->output_indexes);
339
            av_freep(&network);
340
        }
341
        av_freep(model);
342
    }
343
}