| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /* | ||
| 2 | * Copyright (c) 2018 Sergey Lavrushkin | ||
| 3 | * | ||
| 4 | * This file is part of FFmpeg. | ||
| 5 | * | ||
| 6 | * FFmpeg is free software; you can redistribute it and/or | ||
| 7 | * modify it under the terms of the GNU Lesser General Public | ||
| 8 | * License as published by the Free Software Foundation; either | ||
| 9 | * version 2.1 of the License, or (at your option) any later version. | ||
| 10 | * | ||
| 11 | * FFmpeg is distributed in the hope that it will be useful, | ||
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| 14 | * Lesser General Public License for more details. | ||
| 15 | * | ||
| 16 | * You should have received a copy of the GNU Lesser General Public | ||
| 17 | * License along with FFmpeg; if not, write to the Free Software | ||
| 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
| 19 | */ | ||
| 20 | |||
| 21 | /** | ||
| 22 | * @file | ||
| 23 | * DNN inference engine interface. | ||
| 24 | */ | ||
| 25 | |||
| 26 | #ifndef AVFILTER_DNN_INTERFACE_H | ||
| 27 | #define AVFILTER_DNN_INTERFACE_H | ||
| 28 | |||
| 29 | #include <stdint.h> | ||
| 30 | #include "libavutil/frame.h" | ||
| 31 | #include "avfilter.h" | ||
| 32 | |||
| 33 | #define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!') | ||
| 34 | |||
| 35 | typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType; | ||
| 36 | |||
| 37 | typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType; | ||
| 38 | |||
| 39 | typedef enum { | ||
| 40 | DCO_NONE, | ||
| 41 | DCO_BGR, | ||
| 42 | DCO_RGB, | ||
| 43 | } DNNColorOrder; | ||
| 44 | |||
| 45 | typedef enum { | ||
| 46 | DAST_FAIL, // something wrong | ||
| 47 | DAST_EMPTY_QUEUE, // no more inference result to get | ||
| 48 | DAST_NOT_READY, // all queued inferences are not finished | ||
| 49 | DAST_SUCCESS // got a result frame successfully | ||
| 50 | } DNNAsyncStatusType; | ||
| 51 | |||
| 52 | typedef enum { | ||
| 53 | DFT_NONE, | ||
| 54 | DFT_PROCESS_FRAME, // process the whole frame | ||
| 55 | DFT_ANALYTICS_DETECT, // detect from the whole frame | ||
| 56 | DFT_ANALYTICS_CLASSIFY, // classify for each bounding box | ||
| 57 | }DNNFunctionType; | ||
| 58 | |||
| 59 | typedef enum { | ||
| 60 | DL_NONE, | ||
| 61 | DL_NCHW, | ||
| 62 | DL_NHWC, | ||
| 63 | } DNNLayout; | ||
| 64 | |||
| 65 | typedef struct DNNData{ | ||
| 66 | void *data; | ||
| 67 | int dims[4]; | ||
| 68 | // dt and order together decide the color format | ||
| 69 | DNNDataType dt; | ||
| 70 | DNNColorOrder order; | ||
| 71 | DNNLayout layout; | ||
| 72 | float scale; | ||
| 73 | float mean; | ||
| 74 | } DNNData; | ||
| 75 | |||
| 76 | typedef struct DNNExecBaseParams { | ||
| 77 | const char *input_name; | ||
| 78 | const char **output_names; | ||
| 79 | uint32_t nb_output; | ||
| 80 | AVFrame *in_frame; | ||
| 81 | AVFrame *out_frame; | ||
| 82 | } DNNExecBaseParams; | ||
| 83 | |||
| 84 | typedef struct DNNExecClassificationParams { | ||
| 85 | DNNExecBaseParams base; | ||
| 86 | const char *target; | ||
| 87 | } DNNExecClassificationParams; | ||
| 88 | |||
| 89 | typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx); | ||
| 90 | typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx); | ||
| 91 | typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx); | ||
| 92 | |||
| 93 | typedef struct DNNModel{ | ||
| 94 | // Stores model that can be different for different backends. | ||
| 95 | void *model; | ||
| 96 | // Stores options when the model is executed by the backend | ||
| 97 | const char *options; | ||
| 98 | // Stores FilterContext used for the interaction between AVFrame and DNNData | ||
| 99 | AVFilterContext *filter_ctx; | ||
| 100 | // Stores function type of the model | ||
| 101 | DNNFunctionType func_type; | ||
| 102 | // Gets model input information | ||
| 103 | // Just reuse struct DNNData here, actually the DNNData.data field is not needed. | ||
| 104 | int (*get_input)(void *model, DNNData *input, const char *input_name); | ||
| 105 | // Gets model output width/height with given input w/h | ||
| 106 | int (*get_output)(void *model, const char *input_name, int input_width, int input_height, | ||
| 107 | const char *output_name, int *output_width, int *output_height); | ||
| 108 | // set the pre process to transfer data from AVFrame to DNNData | ||
| 109 | // the default implementation within DNN is used if it is not provided by the filter | ||
| 110 | FramePrePostProc frame_pre_proc; | ||
| 111 | // set the post process to transfer data from DNNData to AVFrame | ||
| 112 | // the default implementation within DNN is used if it is not provided by the filter | ||
| 113 | FramePrePostProc frame_post_proc; | ||
| 114 | // set the post process to interpret detect result from DNNData | ||
| 115 | DetectPostProc detect_post_proc; | ||
| 116 | // set the post process to interpret classify result from DNNData | ||
| 117 | ClassifyPostProc classify_post_proc; | ||
| 118 | } DNNModel; | ||
| 119 | |||
| 120 | // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. | ||
| 121 | typedef struct DNNModule{ | ||
| 122 | // Loads model and parameters from given file. Returns NULL if it is not possible. | ||
| 123 | DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx); | ||
| 124 | // Executes model with specified input and output. Returns the error code otherwise. | ||
| 125 | int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params); | ||
| 126 | // Retrieve inference result. | ||
| 127 | DNNAsyncStatusType (*get_result)(const DNNModel *model, AVFrame **in, AVFrame **out); | ||
| 128 | // Flush all the pending tasks. | ||
| 129 | int (*flush)(const DNNModel *model); | ||
| 130 | // Frees memory allocated for model. | ||
| 131 | void (*free_model)(DNNModel **model); | ||
| 132 | } DNNModule; | ||
| 133 | |||
| 134 | // Initializes DNNModule depending on chosen backend. | ||
| 135 | const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx); | ||
| 136 | |||
| 137 | ✗ | static inline int dnn_get_width_idx_by_layout(DNNLayout layout) | |
| 138 | { | ||
| 139 | ✗ | return layout == DL_NHWC ? 2 : 3; | |
| 140 | } | ||
| 141 | |||
| 142 | ✗ | static inline int dnn_get_height_idx_by_layout(DNNLayout layout) | |
| 143 | { | ||
| 144 | ✗ | return layout == DL_NHWC ? 1 : 2; | |
| 145 | } | ||
| 146 | |||
| 147 | ✗ | static inline int dnn_get_channel_idx_by_layout(DNNLayout layout) | |
| 148 | { | ||
| 149 | ✗ | return layout == DL_NHWC ? 3 : 1; | |
| 150 | } | ||
| 151 | |||
| 152 | #endif | ||
| 153 |