FFmpeg coverage


Directory: ../../../ffmpeg/
File: src/libavfilter/dnn_interface.h
Date: 2024-04-19 17:50:32
Exec Total Coverage
Lines: 0 6 0.0%
Functions: 0 3 0.0%
Branches: 0 6 0.0%

Line Branch Exec Source
1 /*
2 * Copyright (c) 2018 Sergey Lavrushkin
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21 /**
22 * @file
23 * DNN inference engine interface.
24 */
25
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
28
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
32
33 #define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
34
35 typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType;
36
37 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
38
39 typedef enum {
40 DCO_NONE,
41 DCO_BGR,
42 DCO_RGB,
43 } DNNColorOrder;
44
45 typedef enum {
46 DAST_FAIL, // something wrong
47 DAST_EMPTY_QUEUE, // no more inference result to get
48 DAST_NOT_READY, // all queued inferences are not finished
49 DAST_SUCCESS // got a result frame successfully
50 } DNNAsyncStatusType;
51
52 typedef enum {
53 DFT_NONE,
54 DFT_PROCESS_FRAME, // process the whole frame
55 DFT_ANALYTICS_DETECT, // detect from the whole frame
56 DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
57 }DNNFunctionType;
58
59 typedef enum {
60 DL_NONE,
61 DL_NCHW,
62 DL_NHWC,
63 } DNNLayout;
64
65 typedef struct DNNData{
66 void *data;
67 int dims[4];
68 // dt and order together decide the color format
69 DNNDataType dt;
70 DNNColorOrder order;
71 DNNLayout layout;
72 float scale;
73 float mean;
74 } DNNData;
75
76 typedef struct DNNExecBaseParams {
77 const char *input_name;
78 const char **output_names;
79 uint32_t nb_output;
80 AVFrame *in_frame;
81 AVFrame *out_frame;
82 } DNNExecBaseParams;
83
84 typedef struct DNNExecClassificationParams {
85 DNNExecBaseParams base;
86 const char *target;
87 } DNNExecClassificationParams;
88
89 typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
90 typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
91 typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
92
93 typedef struct DNNModel{
94 // Stores model that can be different for different backends.
95 void *model;
96 // Stores options when the model is executed by the backend
97 const char *options;
98 // Stores FilterContext used for the interaction between AVFrame and DNNData
99 AVFilterContext *filter_ctx;
100 // Stores function type of the model
101 DNNFunctionType func_type;
102 // Gets model input information
103 // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
104 int (*get_input)(void *model, DNNData *input, const char *input_name);
105 // Gets model output width/height with given input w/h
106 int (*get_output)(void *model, const char *input_name, int input_width, int input_height,
107 const char *output_name, int *output_width, int *output_height);
108 // set the pre process to transfer data from AVFrame to DNNData
109 // the default implementation within DNN is used if it is not provided by the filter
110 FramePrePostProc frame_pre_proc;
111 // set the post process to transfer data from DNNData to AVFrame
112 // the default implementation within DNN is used if it is not provided by the filter
113 FramePrePostProc frame_post_proc;
114 // set the post process to interpret detect result from DNNData
115 DetectPostProc detect_post_proc;
116 // set the post process to interpret classify result from DNNData
117 ClassifyPostProc classify_post_proc;
118 } DNNModel;
119
120 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
121 typedef struct DNNModule{
122 // Loads model and parameters from given file. Returns NULL if it is not possible.
123 DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
124 // Executes model with specified input and output. Returns the error code otherwise.
125 int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
126 // Retrieve inference result.
127 DNNAsyncStatusType (*get_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
128 // Flush all the pending tasks.
129 int (*flush)(const DNNModel *model);
130 // Frees memory allocated for model.
131 void (*free_model)(DNNModel **model);
132 } DNNModule;
133
134 // Initializes DNNModule depending on chosen backend.
135 const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
136
137 static inline int dnn_get_width_idx_by_layout(DNNLayout layout)
138 {
139 return layout == DL_NHWC ? 2 : 3;
140 }
141
142 static inline int dnn_get_height_idx_by_layout(DNNLayout layout)
143 {
144 return layout == DL_NHWC ? 1 : 2;
145 }
146
147 static inline int dnn_get_channel_idx_by_layout(DNNLayout layout)
148 {
149 return layout == DL_NHWC ? 3 : 1;
150 }
151
152 #endif
153