FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Packetization for RTP Payload Format For AV1 (v1.0)
3			* https://aomediacodec.github.io/av1-rtp-spec/
4			* Copyright (c) 2024 Axis Communications
5			*
6			* This file is part of FFmpeg.
7			*
8			* FFmpeg is free software; you can redistribute it and/or
9			* modify it under the terms of the GNU Lesser General Public
10			* License as published by the Free Software Foundation; either
11			* version 2.1 of the License, or (at your option) any later version.
12			*
13			* FFmpeg is distributed in the hope that it will be useful,
14			* but WITHOUT ANY WARRANTY; without even the implied warranty of
15			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16			* Lesser General Public License for more details.
17			*
18			* You should have received a copy of the GNU Lesser General Public
19			* License along with FFmpeg; if not, write to the Free Software
20			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21			*/
22
23			/**
24			* @file
25			* @brief AV1 / RTP packetization code (RTP Payload Format For AV1 (v1.0))
26			* @author Chris Hodges <chris.hodges@axis.com>
27			* @note This will remove TDs and OBU size fields
28			*/
29
30			#include "avformat.h"
31			#include "rtpenc.h"
32			#include "libavcodec/av1.h"
33			#include "rtp_av1.h"
34
35			// enable tracing of packet data
36			//#define RTPENC_AV1_VERBOSE_TRACE
37
38			// enable searching for sequence header as workaround for AV1 encoders
39			// that do not set AV_PKT_FLAG_KEY correctly
40			#define RTPENC_AV1_SEARCH_SEQ_HEADER 1
41
42		✗	void ff_rtp_send_av1(AVFormatContext *ctx, const uint8_t *frame_buf, int frame_size, int is_keyframe) {
43		✗	uint8_t aggr_hdr = 0;
44		✗	int last_packet_of_frame = 0;
45		✗	RTPMuxContext *rtp_ctx = ctx->priv_data;
46		✗	const uint8_t *obu_ptr = frame_buf;
47		✗	int start_new_packet = 0;
48		✗	unsigned int num_obus = 0;
49		✗	unsigned int rem_pkt_size = rtp_ctx->max_payload_size - 1;
50		✗	uint8_t *pkt_ptr = NULL;
51
52		✗	const uint8_t *curr_obu_ptr = NULL;
53		✗	uint32_t curr_elem_size = 0;
54		✗	int curr_obu_hdr = -1;
55		✗	int curr_obu_ext = -1;
56		✗	const uint8_t *last_obu_ptr = NULL;
57		✗	uint32_t last_elem_size = 0;
58		✗	int last_obu_hdr = -1;
59		✗	int last_obu_ext = -1;
60
61		✗	rtp_ctx->timestamp = rtp_ctx->cur_timestamp;
62
63			/* The payload structure is supposed to be straight-forward, but there are a
64			* couple of edge cases to be tackled and make things very complex.
65			* These are mainly due to:
66			* - the OBU element size being optional for the last element, but MANDATORY
67			* if there are more than 3 elements
68			* - the size field of the element is made up of a variable number of
69			* LEB bytes
70			* - the latter in combination with the desire to fill the max packet size
71			* could cause a catch22
72			* - if there's less than 2 bytes remaining (depending on the required LEB),
73			* one would not have space for the payload of an element and must instead
74			* start the next packet
75			* - if there's less than 3 bytes remaining, the header byte plus the
76			* optional extension byte will not fit in the fragment making the
77			* handling even more complicated
78			* - as some OBU types are supposed to be filtered out, it is hard to decide
79			* via the remaining length whether the outputted OBU element will
80			* actually be the last one
81			*
82			* There are two major ways to tackle that: Pre-parsing of all OBUs within a
83			* frame (adds memory complexity) or lazy copying of the prior element.
84			* Here, the latter is implemented.
85			*/
86
87		✗	if (is_keyframe) {
88			#if RTPENC_AV1_SEARCH_SEQ_HEADER
89			/* search for OBU_SEQUENCE_HEADER to get a better indication that
90			* the frame was marked as keyframe is really a KEY_FRAME and not
91			* a INTRA_ONLY frame. This might be unnecessary if the AV1 parser/
92			* encoder always correctly specifies AV_PKT_FLAG_KEY.
93			*
94			* Note: Spec does NOT prohibit resending bit-identical
95			* OBU_SEQUENCE_HEADER for ANY kind of frame, though!
96			*/
97		✗	int rem_size = frame_size;
98		✗	const uint8_t *buf_ptr = frame_buf;
99		✗	while (rem_size > 0) {
100			uint32_t obu_size;
101		✗	uint8_t obu_hdr = *buf_ptr++;
102		✗	uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
103			int num_lebs;
104
105		✗	if (obu_type == AV1_OBU_SEQUENCE_HEADER) {
106		✗	av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
107		✗	aggr_hdr |= AV1F_AGGR_HDR_FIRST_PKT;
108		✗	break;
109			}
110		✗	if (!(obu_hdr & AV1F_OBU_HAS_SIZE_FIELD)) {
111		✗	break;
112			}
113		✗	rem_size--;
114			// read out explicit OBU size
115		✗	num_lebs = parse_leb(ctx, buf_ptr, rem_size, &obu_size);
116		✗	if (!num_lebs) {
117		✗	break;
118			}
119		✗	buf_ptr += num_lebs + obu_size;
120		✗	rem_size -= num_lebs + obu_size;
121			}
122			#else // RTPENC_AV1_SEARCH_SEQ_HEADER
123			av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
124			aggr_hdr |= AV1F_AGGR_HDR_FIRST_PKT;
125			#endif // RTPENC_AV1_SEARCH_SEQ_HEADER
126			}
127		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
128		✗	pkt_ptr = rtp_ctx->buf + 1;
129
130			#ifdef RTPENC_AV1_VERBOSE_TRACE
131			av_log(ctx, AV_LOG_TRACE, "AV1 Frame %d in (%x), size=%d:\n",
132			rtp_ctx->seq, rtp_ctx->flags, frame_size);
133			av_hex_dump_log(ctx, AV_LOG_TRACE, frame_buf, FFMIN(frame_size, 128));
134			#endif
135
136		✗	while (frame_size) {
137			uint32_t obu_size;
138		✗	int num_lebs = 0;
139		✗	int ext_byte = -1;
140
141		✗	uint8_t obu_hdr = *obu_ptr++;
142		✗	uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
143		✗	frame_size--;
144
145		✗	if (obu_hdr & AV1F_OBU_FORBIDDEN) {
146		✗	av_log(ctx, AV_LOG_ERROR, "Forbidden bit set in AV1 OBU header (0x%02x)\n", obu_hdr);
147		✗	return;
148			}
149
150		✗	if (obu_hdr & AV1F_OBU_EXTENSION_FLAG) {
151		✗	if (!frame_size) {
152		✗	av_log(ctx, AV_LOG_ERROR, "Out of data for AV1 OBU header extension byte\n");
153		✗	return;
154			}
155		✗	ext_byte = *obu_ptr++;
156		✗	frame_size--;
157			}
158
159		✗	if (obu_hdr & AV1F_OBU_HAS_SIZE_FIELD) {
160		✗	obu_hdr &= ~AV1F_OBU_HAS_SIZE_FIELD; // remove size field
161			// read out explicit OBU size
162		✗	num_lebs = parse_leb(ctx, obu_ptr, frame_size, &obu_size);
163		✗	if (!num_lebs) {
164		✗	return;
165			}
166		✗	obu_ptr += num_lebs;
167		✗	frame_size -= num_lebs;
168			} else {
169		✗	av_log(ctx, AV_LOG_ERROR, "Cannot handle AV1 OBUs without size fields\n");
170		✗	return;
171			}
172
173		✗	if ((long) obu_size > frame_size) {
174		✗	av_log(ctx, AV_LOG_ERROR, "AV1 OBU size %d larger than remaining frame size %d\n", obu_size, frame_size);
175		✗	return;
176			}
177
178		✗	if (obu_size > 0xfffffffd) {
179		✗	av_log(ctx, AV_LOG_ERROR, "AV1 OBU size 0x%x might overflow (attack?)\n", obu_size);
180		✗	return;
181			}
182
183		✗	frame_size -= obu_size;
184
185		✗	if ((obu_type == AV1_OBU_TEMPORAL_DELIMITER) ||
186		✗	(obu_type == AV1_OBU_TILE_LIST) ||
187			(obu_type == AV1_OBU_PADDING)) {
188			// ignore and remove according to spec (note that OBU_PADDING is not
189			// mentioned in spec, but it does not make sense to transmit it).
190		✗	obu_ptr += obu_size;
191			// additional handling if the ignored OBU was the last one
192		✗	if (!frame_size) {
193			// we're done, flush the last packet, set RTP marker bit
194		✗	last_packet_of_frame = 1;
195		✗	goto flush_last_packet;
196			}
197		✗	continue;
198			}
199
200			/* if the last OBU had a temporal or spatial ID, they need to match to
201			* current; otherwise start new packet */
202		✗	if ((last_obu_ext >= 0) && (curr_obu_ext != last_obu_ext)) {
203		✗	start_new_packet = 1;
204			}
205
206		✗	flush_last_packet:
207		✗	last_obu_ptr = curr_obu_ptr;
208		✗	last_elem_size = curr_elem_size;
209		✗	last_obu_hdr = curr_obu_hdr;
210		✗	last_obu_ext = curr_obu_ext;
211
212		✗	curr_obu_ptr = obu_ptr; // behind header
213		✗	curr_elem_size = obu_size + 1 + ((ext_byte >= 0) ? 1 : 0);
214		✗	curr_obu_hdr = obu_hdr;
215		✗	curr_obu_ext = ext_byte;
216
217		✗	obu_ptr += obu_size;
218
219		✗	if (last_obu_ptr) {
220		✗	unsigned int first_elem_with_size = last_elem_size + calc_leb_size(last_elem_size);
221			// check if last packet fits completely and has reasonable space for
222			// at least a fragment of the next
223		✗	if (!last_packet_of_frame && (first_elem_with_size + 10 < rem_pkt_size)) {
224		✗	num_lebs = write_leb(pkt_ptr, last_elem_size);
225		✗	pkt_ptr += num_lebs;
226		✗	rem_pkt_size -= num_lebs;
227			} else {
228		✗	if ((num_obus >= 3) && (last_packet_of_frame || (first_elem_with_size <= rem_pkt_size))) {
229			// last fits with forced size, but nothing else
230		✗	num_lebs = write_leb(pkt_ptr, last_elem_size);
231		✗	pkt_ptr += num_lebs;
232		✗	rem_pkt_size -= num_lebs;
233			}
234			// force new packet
235		✗	start_new_packet = 1;
236			}
237
238			// write header and optional extension byte (if not a continued fragment)
239		✗	if (last_obu_hdr >= 0) {
240		✗	*pkt_ptr++ = last_obu_hdr;
241		✗	last_elem_size--;
242		✗	rem_pkt_size--;
243		✗	if (last_obu_ext >= 0) {
244		✗	*pkt_ptr++ = last_obu_ext;
245		✗	last_elem_size--;
246		✗	rem_pkt_size--;
247			}
248			}
249			// copy payload
250		✗	memcpy(pkt_ptr, last_obu_ptr, last_elem_size);
251		✗	pkt_ptr += last_elem_size;
252		✗	rem_pkt_size -= last_elem_size;
253		✗	num_obus++;
254			}
255
256		✗	if (start_new_packet || last_packet_of_frame) {
257		✗	if (num_obus < 4) {
258		✗	aggr_hdr |= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
259			}
260		✗	rtp_ctx->buf[0] = aggr_hdr;
261
262			#ifdef RTPENC_AV1_VERBOSE_TRACE
263			av_log(ctx, AV_LOG_TRACE, "Sending NON-FRAG packet no %d, %ld/%d, %d OBUs (marker=%d)\n",
264			((RTPMuxContext *) ctx->priv_data)->seq,
265			pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus, last_packet_of_frame);
266			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
267			av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
268			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
269			#endif
270
271		✗	ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, last_packet_of_frame);
272
273		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
274		✗	pkt_ptr = rtp_ctx->buf + 1;
275		✗	aggr_hdr = 0;
276		✗	num_obus = 0;
277			}
278
279		✗	if (last_packet_of_frame) {
280		✗	break;
281			}
282
283			// check if element needs to be fragmented, otherwise we will deal with
284			// it in the next iteration
285		✗	if ((curr_elem_size > rem_pkt_size) ||
286		✗	((num_obus >= 3) && (curr_elem_size + calc_leb_size(curr_elem_size)) > rem_pkt_size)) {
287		✗	uint32_t frag_size = rem_pkt_size;
288
289			// if there are going more than 3 OBU elements, we are obliged to
290			// have the length field for the last
291		✗	if (num_obus >= 3) {
292			// that's an upper limit of LEBs
293		✗	num_lebs = calc_leb_size(rem_pkt_size - 1);
294		✗	frag_size -= num_lebs;
295
296			// write a fixed number of LEBs, in case the frag_size could
297			// now be specified with one less byte
298		✗	write_leb_n(pkt_ptr, frag_size, num_lebs);
299		✗	pkt_ptr += num_lebs;
300		✗	rem_pkt_size -= num_lebs;
301			}
302
303			// write header and optional extension byte
304		✗	*pkt_ptr++ = curr_obu_hdr;
305		✗	curr_elem_size--;
306		✗	rem_pkt_size--;
307		✗	if (curr_obu_ext >= 0) {
308		✗	*pkt_ptr++ = curr_obu_ext;
309		✗	curr_elem_size--;
310		✗	rem_pkt_size--;
311			}
312
313			// disable header writing for final fragment
314		✗	curr_obu_hdr = -1;
315		✗	curr_obu_ext = -1;
316
317			// send more full packet sized fragments
318			do {
319			// copy payload
320		✗	memcpy(pkt_ptr, curr_obu_ptr, rem_pkt_size);
321		✗	pkt_ptr += rem_pkt_size;
322		✗	curr_obu_ptr += rem_pkt_size;
323		✗	curr_elem_size -= rem_pkt_size;
324		✗	num_obus++;
325
326		✗	aggr_hdr |= AV1F_AGGR_HDR_LAST_FRAG;
327		✗	if (num_obus < 4) {
328		✗	aggr_hdr |= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
329			}
330		✗	rtp_ctx->buf[0] = aggr_hdr;
331
332			#ifdef RTPENC_AV1_VERBOSE_TRACE
333			av_log(ctx, AV_LOG_DEBUG, "Sending FRAG packet no %d, %ld/%d, %d OBUs\n",
334			((RTPMuxContext *) ctx->priv_data)->seq,
335			pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus);
336			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
337			av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
338			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
339			#endif
340
341		✗	ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, 0);
342		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
343		✗	pkt_ptr = rtp_ctx->buf + 1;
344
345		✗	aggr_hdr = AV1F_AGGR_HDR_FRAG_CONT;
346		✗	num_obus = 0;
347		✗	} while (curr_elem_size > rem_pkt_size);
348		✗	start_new_packet = 0;
349			}
350
351		✗	if (!frame_size) {
352			// we're done, flush the last packet, set RTP marker bit
353		✗	last_packet_of_frame = 1;
354		✗	goto flush_last_packet;
355			}
356			}
357			}
358